diff --git a/WebContent/WEB-INF/lib/questdb-jdk8-6.0.1.jar b/WebContent/WEB-INF/lib/questdb-jdk8-6.0.1.jar new file mode 100644 index 0000000000..9ef91e861d Binary files /dev/null and b/WebContent/WEB-INF/lib/questdb-jdk8-6.0.1.jar differ diff --git a/build.xml b/build.xml index 07f1c09d44..e69e30344e 100644 --- a/build.xml +++ b/build.xml @@ -249,6 +249,9 @@ haltonfailure="yes"> + + @@ -660,6 +663,7 @@ + diff --git a/scadalts-ui/src/locales/de.json b/scadalts-ui/src/locales/de.json index 55e27a1e3b..2ca2eebcf5 100644 --- a/scadalts-ui/src/locales/de.json +++ b/scadalts-ui/src/locales/de.json @@ -337,6 +337,7 @@ "systemsettings.notification.save.systemevent": "System Event Types Settings saved!", "systemsettings.notification.save.systeminfo": "Systeminfo-Einstellungen gespeichert!", "systemsettings.notification.send.email": "E-Mail wurde gesendet an ", + "systemsettings.notification.save.dbquery": "DbQuery Einstellungen gespeichert!", "systemsettings.scadaconf.aclServer": "Zugriffskontrolllistenserver", "systemsettings.scadaconf.apiReplaceAlertOnView": "Ersetzen Sie die angezeigte Warnung", "systemsettings.scadaconf.cacheEnable": "Event cache", @@ -356,6 +357,8 @@ "systemsettings.scadaconf.useCacheDataSourcePointsWhenSystemIsReady": "Verwenden Sie Cache-Datenquellenpunkte, wenn das System bereit ist", "systemsettings.smsdomain.label": "SMS-Gateway-Serveradresse", "systemsettings.smsdomain.title": "SMS Domain", + "systemsettings.dbquery.title": "DBQuery-Einstellungen", + "systemsettings.dbquery.readEnabled": "Lesen aktivieren", "systemsettings.title": "Systemeinstellungen", "systemsettings.tooltip.derbydatabase": "Die Derby-Datenbank ist in dieser ScadaLTS-Version nicht verfügbar!", "systemsettings.tooltip.purgedata": "Löschen Sie die Zustimmung zum historischen Datenpunkt", diff --git a/scadalts-ui/src/locales/en.json b/scadalts-ui/src/locales/en.json index f8f5e54568..fb26a7d725 100644 --- a/scadalts-ui/src/locales/en.json +++ b/scadalts-ui/src/locales/en.json @@ -787,6 +787,7 @@ "systemsettings.notification.save.systemevent": "System Event Types settings saved!", "systemsettings.notification.save.systeminfo": "System info settings saved!", "systemsettings.notification.send.email": "Email has been sent to ", + "systemsettings.notification.save.dbquery": "DbQuery settings saved!", "systemsettings.scadaconf.aclServer": "Access Control List server", "systemsettings.scadaconf.apiReplaceAlertOnView": "Replace alert on view", "systemsettings.scadaconf.cacheEnable": "Event cache", @@ -806,6 +807,8 @@ "systemsettings.scadaconf.useCacheDataSourcePointsWhenSystemIsReady": "Use cache data sources points when the system is ready", "systemsettings.smsdomain.label": "SMS-Gateway server address", "systemsettings.smsdomain.title": "SMS Domain", + "systemsettings.dbquery.title": "DBQuery settings", + "systemsettings.dbquery.readEnabled": "Enable read", "systemsettings.title": "System settings", "systemsettings.tooltip.derbydatabase": "Derby database is unavailable in this ScadaLTS version!", "systemsettings.tooltip.purgedata": "Delete all historical datapoint values", diff --git a/scadalts-ui/src/store/systemSettings/index.js b/scadalts-ui/src/store/systemSettings/index.js index 559e932f4e..c5fd3b85ac 100644 --- a/scadalts-ui/src/store/systemSettings/index.js +++ b/scadalts-ui/src/store/systemSettings/index.js @@ -23,6 +23,7 @@ const storeSystemSettings = { schemaVersion: undefined, scadaConfig: undefined, defaultLoggingType: undefined, + dbQuerySettings: undefined, }, mutations: { setDatabaseType(state, databaseType) { @@ -71,6 +72,9 @@ const storeSystemSettings = { setDefaultLoggingType(state, defaultLoggingType) { state.defaultLoggingType = defaultLoggingType; }, + setDbQuerySettings(state, dbQuerySettings) { + state.dbQuerySettings = dbQuerySettings; + }, }, actions: { getDatabaseType({ commit, dispatch }) { @@ -272,6 +276,20 @@ const storeSystemSettings = { }); }, + getDbQuerySettings({ commit, dispatch }) { + return dispatch('requestGet', '/systemSettings/getDbQuerySettings').then((r) => { + commit('setDbQuerySettings', r); + return r; + }); + }, + + saveDbQuerySettings({ state, dispatch }) { + return dispatch('requestPost', { + url: '/systemSettings/saveDbQuerySettings', + data: state.dbQuerySettings, + }); + }, + purgeData(context) { return new Promise((resolve, reject) => { axios diff --git a/scadalts-ui/src/views/System/SystemSettings/DbQuerySettingsComponent.vue b/scadalts-ui/src/views/System/SystemSettings/DbQuerySettingsComponent.vue new file mode 100644 index 0000000000..e497d49a12 --- /dev/null +++ b/scadalts-ui/src/views/System/SystemSettings/DbQuerySettingsComponent.vue @@ -0,0 +1,130 @@ + + + diff --git a/scadalts-ui/src/views/System/SystemSettings/index.vue b/scadalts-ui/src/views/System/SystemSettings/index.vue index f1475381e8..b078aa5726 100644 --- a/scadalts-ui/src/views/System/SystemSettings/index.vue +++ b/scadalts-ui/src/views/System/SystemSettings/index.vue @@ -67,6 +67,11 @@ + + @@ -281,6 +286,7 @@ import SmsDomainSettingsComponent from './SmsDomainSettingsComponent'; import ScadaConfigurationComponent from './ScadaConfigurationComponent'; import AmChartSettingsComponent from './AmChartSettingsComponent'; import DataBaseInfoComponent from './DataBaseInfoComponent'; +import DbQuerySettingsComponent from './DbQuerySettingsComponent'; export default { el: '#systemsettings', @@ -298,6 +304,7 @@ export default { ScadaConfigurationComponent, AmChartSettingsComponent, DataBaseInfoComponent, + DbQuerySettingsComponent, }, filters: { blank: function (value) { @@ -347,7 +354,7 @@ export default { setTimeout(() => { let el = document.getElementById(element); if(!!el) { - el.scrollIntoView(); + el.scrollIntoView(); } }, 1000); } diff --git a/src/com/serotonin/mango/Common.java b/src/com/serotonin/mango/Common.java index 7289022818..7ed90c8a96 100644 --- a/src/com/serotonin/mango/Common.java +++ b/src/com/serotonin/mango/Common.java @@ -117,6 +117,7 @@ public interface ContextKeys { String HTTP_RECEIVER_MULTICASTER = "HTTP_RECEIVER_MULTICASTER"; String DOCUMENTATION_MANIFEST = "DOCUMENTATION_MANIFEST"; String DATA_POINTS_NAME_ID_MAPPING = "DATAPOINTS_NAME_ID_MAPPING"; + String DATABASE_QUERY_ACCESS = "DATABASE_QUERY_ACCESS"; } public interface TimePeriods { diff --git a/src/com/serotonin/mango/MangoContextListener.java b/src/com/serotonin/mango/MangoContextListener.java index f839cb0a37..b007665bab 100644 --- a/src/com/serotonin/mango/MangoContextListener.java +++ b/src/com/serotonin/mango/MangoContextListener.java @@ -414,13 +414,31 @@ private void databaseInitialize(ServletContext ctx) { DatabaseAccess databaseAccess = DatabaseAccess .createDatabaseAccess(ctx); ctx.setAttribute(Common.ContextKeys.DATABASE_ACCESS, databaseAccess); - databaseAccess.initialize(); + boolean dbQueryEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.enabled", + false); + if(dbQueryEnabled) { + DatabaseAccess databaseQueryAccess = DatabaseAccess + .createDatabaseAccess(ctx, "dbquery."); + ctx.setAttribute(Common.ContextKeys.DATABASE_QUERY_ACCESS, databaseQueryAccess); + + databaseAccess.initOnlyDatasourceJdbc(); + databaseQueryAccess.initOnlyDatasourceJdbc(); + + databaseAccess.initialize(); + databaseQueryAccess.initialize(); + } else { + databaseAccess.initialize(); + } } private void databaseTerminate(ContextWrapper ctx) { DatabaseAccess databaseAccess = ctx.getDatabaseAccess(); if (databaseAccess != null) databaseAccess.terminate(); + + DatabaseAccess databaseQueryAccess = ctx.getDatabaseQueryAccess(); + if(databaseQueryAccess != null) + databaseQueryAccess.terminate(); } // diff --git a/src/com/serotonin/mango/db/BasePooledAccess.java b/src/com/serotonin/mango/db/BasePooledAccess.java index 8d923c1faa..720bb52d48 100644 --- a/src/com/serotonin/mango/db/BasePooledAccess.java +++ b/src/com/serotonin/mango/db/BasePooledAccess.java @@ -50,18 +50,23 @@ public BasePooledAccess(ServletContext ctx) super(ctx); } + public BasePooledAccess(ServletContext ctx, String dbPrefix) + { + super(ctx, dbPrefix); + } + @SuppressWarnings("deprecation") @Override protected void initializeImpl(String propertyPrefix, String dataSourceName) { log.info("Initializing pooled connection manager"); - if(Common.getEnvironmentProfile().getString(propertyPrefix + "db.datasource", "false").equals("true")) + if(Common.getEnvironmentProfile().getString(propertyPrefix + getDbPrefix() + "datasource", "false").equals("true")) { try { - log.info("Looking for Datasource: " + Common.getEnvironmentProfile().getString(propertyPrefix + "db.datasourceName")); - dataSource = (DataSource) new InitialContext().lookup(Common.getEnvironmentProfile().getString(propertyPrefix + "db.datasourceName")); + log.info("Looking for Datasource: " + Common.getEnvironmentProfile().getString(propertyPrefix + getDbPrefix() + "datasourceName")); + dataSource = (DataSource) new InitialContext().lookup(Common.getEnvironmentProfile().getString(propertyPrefix + getDbPrefix() + "datasourceName")); Connection conn = dataSource.getConnection(); log.info("DataSource meta: " + conn.getMetaData().getDatabaseProductName() + " " + conn.getMetaData().getDatabaseProductVersion()); dataSourceFound = true; @@ -84,15 +89,15 @@ protected void initializeImpl(String propertyPrefix) dataSource = new BasicDataSource(); ((BasicDataSource) dataSource).setDriverClassName(getDriverClassName()); ((BasicDataSource) dataSource).setUrl(getUrl(propertyPrefix)); - ((BasicDataSource) dataSource).setUsername(Common.getEnvironmentProfile().getString(propertyPrefix + "db.username")); + ((BasicDataSource) dataSource).setUsername(Common.getEnvironmentProfile().getString(propertyPrefix + getDbPrefix() + "username")); ((BasicDataSource) dataSource).setPassword(getDatabasePassword(propertyPrefix)); - ((BasicDataSource) dataSource).setMaxActive(Common.getEnvironmentProfile().getInt(propertyPrefix + "db.pool.maxActive", 10)); - ((BasicDataSource) dataSource).setMaxIdle(Common.getEnvironmentProfile().getInt(propertyPrefix + "db.pool.maxIdle", 10)); + ((BasicDataSource) dataSource).setMaxActive(Common.getEnvironmentProfile().getInt(propertyPrefix + getDbPrefix() + "pool.maxActive", 10)); + ((BasicDataSource) dataSource).setMaxIdle(Common.getEnvironmentProfile().getInt(propertyPrefix + getDbPrefix() + "pool.maxIdle", 10)); } protected String getUrl(String propertyPrefix) { - return Common.getEnvironmentProfile().getString(propertyPrefix + "db.url"); + return Common.getEnvironmentProfile().getString(propertyPrefix + getDbPrefix() + "url"); } abstract protected String getDriverClassName(); diff --git a/src/com/serotonin/mango/db/DatabaseAccess.java b/src/com/serotonin/mango/db/DatabaseAccess.java index 7bac36ff90..fc843df5b4 100644 --- a/src/com/serotonin/mango/db/DatabaseAccess.java +++ b/src/com/serotonin/mango/db/DatabaseAccess.java @@ -24,9 +24,12 @@ import javax.servlet.ServletContext; import javax.sql.DataSource; +import net.bull.javamelody.internal.common.LOG; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.flywaydb.core.Flyway; import org.scada_lts.dao.SystemSettingsDAO; +import org.springframework.dao.DataAccessException; import org.springframework.jdbc.CannotGetJdbcConnectionException; import org.springframework.jdbc.datasource.DataSourceUtils; @@ -49,33 +52,81 @@ public enum DatabaseType { DatabaseAccess getImpl(ServletContext ctx) { return new DerbyAccess(ctx); } + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return new DerbyAccess(ctx, dbPrefix); + } }, MSSQL { @Override DatabaseAccess getImpl(ServletContext ctx) { return new MSSQLAccess(ctx); } + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return new MSSQLAccess(ctx, dbPrefix); + } }, MYSQL { @Override DatabaseAccess getImpl(ServletContext ctx) { return new MySQLAccess(ctx); } + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return new MySQLAccess(ctx, dbPrefix); + } }, POSTGRES { @Override DatabaseAccess getImpl(ServletContext ctx) { return new PostgreSQLAccess(ctx); } + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return new PostgreSQLAccess(ctx, dbPrefix); + } }, ORACLE11G { @Override DatabaseAccess getImpl(ServletContext ctx) { return new Oracle11GAccess(ctx); } - }; + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return new Oracle11GAccess(ctx, dbPrefix); + } + }, + QUESTDB { + @Override + DatabaseAccess getImpl(ServletContext ctx) { + return new QuestDbAccess(ctx); + } + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return new QuestDbAccess(ctx, dbPrefix); + } + }, + NONE { + @Override + DatabaseAccess getImpl(ServletContext ctx) { + return null; + } + + @Override + DatabaseAccess getImpl(ServletContext ctx, String dbPrefix) { + return null; + } + }; abstract DatabaseAccess getImpl(ServletContext ctx); + abstract DatabaseAccess getImpl(ServletContext ctx, String dbPrefix); } public static DatabaseAccess createDatabaseAccess(ServletContext ctx) { @@ -90,26 +141,38 @@ public static DatabaseAccess createDatabaseAccess(ServletContext ctx) { return dt.getImpl(ctx); } + public static DatabaseAccess createDatabaseAccess(ServletContext ctx, String dbPrefix) { + + String type = Common.getEnvironmentProfile().getString(dbPrefix + "type", + "derby"); + DatabaseType dt = DatabaseType.valueOf(type.toUpperCase()); + + if (dt == null) + throw new IllegalArgumentException("Unknown database type: " + type); + + return dt.getImpl(ctx, dbPrefix); + } + public static DatabaseAccess getDatabaseAccess() { return Common.ctx.getDatabaseAccess(); } protected final ServletContext ctx; + private final String dbPrefix; protected DatabaseAccess(ServletContext ctx) { this.ctx = ctx; + this.dbPrefix = "db."; + } + + public DatabaseAccess(ServletContext ctx, String dbPrefix) { + this.ctx = ctx; + this.dbPrefix = dbPrefix; } public void initialize() { - if (Common.getEnvironmentProfile().getString("db.datasource", "false") - .equals("true")) { - initializeImpl( - "", - Common.getEnvironmentProfile().getString( - "db.datasourceName")); - } else { - initializeImpl(""); - } + if(getDataSource() == null) + initOnlyDatasourceJdbc(); ExtendedJdbcTemplate ejt = new ExtendedJdbcTemplate(); ejt.setDataSource(getDataSource()); @@ -120,7 +183,7 @@ public void initialize() { String convertTypeStr = null; try { convertTypeStr = Common.getEnvironmentProfile().getString( - "convert.db.type"); + "convert." + dbPrefix + ".type"); } catch (MissingResourceException e) { // no op } @@ -171,7 +234,7 @@ public void initialize() { // matches // // the application version. if (Common.getEnvironmentProfile() - .getString("db.upgrade.check", "false").equals("true")) { + .getString(dbPrefix + "upgrade.check", "false").equals("true")) { DBUpgrade.checkUpgrade(); } } @@ -184,6 +247,18 @@ public void initialize() { postInitialize(ejt); } + public void initOnlyDatasourceJdbc() { + if (Common.getEnvironmentProfile().getString(dbPrefix + "datasource", "false") + .equals("true")) { + initializeImpl( + "", + Common.getEnvironmentProfile().getString( + dbPrefix + "datasourceName")); + } else { + initializeImpl(""); + } + } + abstract public DatabaseType getType(); abstract public void terminate(); @@ -206,7 +281,73 @@ protected void postInitialize( // no op - override as necessary } - abstract protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt); + protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { + boolean shemaExist = true; + boolean baseLineNotExist = false; + + String tableToCheck = Common.getEnvironmentProfile().getString(getDbPrefix() + "table.tocheck", "users"); + try { + ejt.execute("select count(*) from " + tableToCheck); + LOG.info("schemaExist:"+shemaExist); + } catch (DataAccessException e) { + shemaExist = false; + LOG.info("schemaExist:"+shemaExist); + } + + try { + ejt.execute("select count(*) from schema_version"); + LOG.info("BaseLineNotExist:"+baseLineNotExist); + } catch (DataAccessException e) { + baseLineNotExist = true; + LOG.info("BaseLineNotExist:"+baseLineNotExist); + } + + String migrationPackage = Common.getEnvironmentProfile().getString(getDbPrefix() + "migration.package"); + try { + Flyway flyway = null; + + if (shemaExist) { + // old shema without flayway + if (baseLineNotExist) { + flyway = Flyway.configure() + .baselineOnMigrate(true) + .dataSource(getDataSource()) + .locations(migrationPackage) + .table("schema_version") + .load(); + + flyway.baseline(); + flyway.migrate(); + } + } else { + //shema not exist + if (baseLineNotExist) { + flyway = Flyway.configure() + .baselineOnMigrate(true) + .dataSource(getDataSource()) + .locations(migrationPackage) + .table("schema_version") + .load(); + //flayway.baseline(); + flyway.migrate(); + } + } + if (flyway == null) { + flyway = Flyway.configure() + .dataSource(getDataSource()) + .locations(migrationPackage) + .table("schema_version") + .load(); + } + //flyway.repair(); + flyway.migrate(); + } catch (Exception e) { + log.error(e.getMessage(), e); + //Need stop scada + } + + return false; + } abstract public void runScript(String[] script, final OutputStream out) throws Exception; @@ -237,7 +378,7 @@ public void doInConnection(ConnectionCallbackVoid callback) { public String getDatabasePassword(String propertyPrefix) { String input = Common.getEnvironmentProfile().getString( - propertyPrefix + "db.password"); + propertyPrefix + getDbPrefix() + "password"); return new DatabaseAccessUtils().decrypt(input); } @@ -262,4 +403,8 @@ public PreparedStatement prepareStatement(Connection connection, return connection.prepareStatement(sql, 1); } + + public String getDbPrefix() { + return dbPrefix; + } } diff --git a/src/com/serotonin/mango/db/DerbyAccess.java b/src/com/serotonin/mango/db/DerbyAccess.java index d09a12c689..edb050c1f7 100644 --- a/src/com/serotonin/mango/db/DerbyAccess.java +++ b/src/com/serotonin/mango/db/DerbyAccess.java @@ -62,6 +62,10 @@ public DerbyAccess(ServletContext ctx) { super(ctx); } + public DerbyAccess(ServletContext ctx, String dbPrefix) { + super(ctx, dbPrefix); + } + @Override public DatabaseType getType() { return DatabaseType.DERBY; diff --git a/src/com/serotonin/mango/db/MSSQLAccess.java b/src/com/serotonin/mango/db/MSSQLAccess.java index 93fba4fce9..97e01104b1 100644 --- a/src/com/serotonin/mango/db/MSSQLAccess.java +++ b/src/com/serotonin/mango/db/MSSQLAccess.java @@ -31,6 +31,10 @@ public MSSQLAccess(ServletContext ctx) { super(ctx); } + public MSSQLAccess(ServletContext ctx, String dbPrefix) { + super(ctx, dbPrefix); + } + @Override public DatabaseType getType() { return DatabaseType.MSSQL; diff --git a/src/com/serotonin/mango/db/MySQLAccess.java b/src/com/serotonin/mango/db/MySQLAccess.java index 8c0b331385..c6d2b9032b 100644 --- a/src/com/serotonin/mango/db/MySQLAccess.java +++ b/src/com/serotonin/mango/db/MySQLAccess.java @@ -37,6 +37,10 @@ public MySQLAccess(ServletContext ctx) { super(ctx); } + public MySQLAccess(ServletContext ctx, String dbPrefix) { + super(ctx, dbPrefix); + } + @Override protected void initializeImpl(String propertyPrefix) { super.initializeImpl(propertyPrefix); @@ -115,6 +119,7 @@ protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { LOG.info("BaseLineNotExist:"+baseLineNotExist); } + String migrationPackage = Common.getEnvironmentProfile().getString(getDbPrefix() + "migration.package"); try { Flyway flyway = null; @@ -124,7 +129,7 @@ protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { flyway = Flyway.configure() .baselineOnMigrate(true) .dataSource(getDataSource()) - .locations("org.scada_lts.dao.migration.mysql") + .locations(migrationPackage) .table("schema_version") .load(); @@ -137,7 +142,7 @@ protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { flyway = Flyway.configure() .baselineOnMigrate(true) .dataSource(getDataSource()) - .locations("org.scada_lts.dao.migration.mysql") + .locations(migrationPackage) .table("schema_version") .load(); //flayway.baseline(); @@ -147,7 +152,7 @@ protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { if (flyway == null) { flyway = Flyway.configure() .dataSource(getDataSource()) - .locations("org.scada_lts.dao.migration.mysql") + .locations(migrationPackage) .table("schema_version") .load(); } diff --git a/src/com/serotonin/mango/db/Oracle11GAccess.java b/src/com/serotonin/mango/db/Oracle11GAccess.java index 4574f92f2c..6dbaf6f26c 100644 --- a/src/com/serotonin/mango/db/Oracle11GAccess.java +++ b/src/com/serotonin/mango/db/Oracle11GAccess.java @@ -55,6 +55,10 @@ public Oracle11GAccess(ServletContext ctx) { super(ctx); } + public Oracle11GAccess(ServletContext ctx, String dbPrefix) { + super(ctx, dbPrefix); + } + @Override protected void initializeImpl(String propertyPrefix) { super.initializeImpl(propertyPrefix); @@ -92,9 +96,9 @@ protected void createSchema(String scriptFile) { } try { Connection con = DriverManager.getConnection(Common - .getEnvironmentProfile().getString("db.url"), Common - .getEnvironmentProfile().getString("db.username"), Common - .getEnvironmentProfile().getString("db.password")); + .getEnvironmentProfile().getString(getDbPrefix() + "url"), Common + .getEnvironmentProfile().getString(getDbPrefix() + "username"), Common + .getEnvironmentProfile().getString(getDbPrefix() + "password")); // Initialize object for ScripRunner ScriptRunner sr = new ScriptRunner(con, false, false); diff --git a/src/com/serotonin/mango/db/PostgreSQLAccess.java b/src/com/serotonin/mango/db/PostgreSQLAccess.java index 1015c206b0..33cd6b12b7 100644 --- a/src/com/serotonin/mango/db/PostgreSQLAccess.java +++ b/src/com/serotonin/mango/db/PostgreSQLAccess.java @@ -28,16 +28,25 @@ import javax.servlet.ServletContext; import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.dao.DataAccessException; import com.serotonin.db.spring.ExtendedJdbcTemplate; import com.serotonin.mango.Common; public class PostgreSQLAccess extends BasePooledAccess { + + private static final Log LOG = LogFactory.getLog(PostgreSQLAccess.class); + public PostgreSQLAccess(ServletContext ctx) { super(ctx); } + public PostgreSQLAccess(ServletContext ctx, String dbPrefix) { + super(ctx, dbPrefix); + } + @Override protected void initializeImpl(String propertyPrefix) { super.initializeImpl(propertyPrefix); @@ -74,7 +83,7 @@ public DatabaseAccess.DatabaseType getType() { protected String getDriverClassName() { return "org.postgresql.Driver"; } - +/* @Override protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { try { @@ -99,10 +108,10 @@ protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { Common.getEnvironmentProfile().getString("db.password")); createSchema("/WEB-INF/db/createTables-postgresql.sql"); conn.close(); - return true; + return true; } } catch (SQLException ex) { - //Logger.getLogger(PostgreSQLAccess.class.getName()).log(Level.SEVERE, null, ex); + //Logger.getLogger(PostgreSQLAccess.class.getName()).log(Level.SEVERE, null, ex); } catch (ClassNotFoundException ex) { Logger.getLogger(PostgreSQLAccess.class.getName()).log(Level.SEVERE, null, ex); } @@ -119,7 +128,7 @@ protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { } return false; } - +*/ @Override public double applyBounds(double value) { if (Double.isNaN(value)) diff --git a/src/com/serotonin/mango/db/QuestDbAccess.java b/src/com/serotonin/mango/db/QuestDbAccess.java new file mode 100644 index 0000000000..21263de09f --- /dev/null +++ b/src/com/serotonin/mango/db/QuestDbAccess.java @@ -0,0 +1,32 @@ +package com.serotonin.mango.db; + +import com.serotonin.db.spring.ExtendedJdbcTemplate; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.scada_lts.dao.migration.query.questdb.V2_8_0_0__CreatePointValuesDenormalized; + +import javax.servlet.ServletContext; + +public class QuestDbAccess extends PostgreSQLAccess { + + private static final Log LOG = LogFactory.getLog(QuestDbAccess.class); + + public QuestDbAccess(ServletContext ctx) { + super(ctx); + } + + public QuestDbAccess(ServletContext ctx, String dbPrefix) { + super(ctx, dbPrefix); + } + + @Override + protected boolean newDatabaseCheck(ExtendedJdbcTemplate ejt) { + try { + new V2_8_0_0__CreatePointValuesDenormalized().migrate(null); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + return false; + } +} diff --git a/src/com/serotonin/mango/db/dao/PointValueDao.java b/src/com/serotonin/mango/db/dao/PointValueDao.java index 6a0b4e0721..e6f1c55e7d 100644 --- a/src/com/serotonin/mango/db/dao/PointValueDao.java +++ b/src/com/serotonin/mango/db/dao/PointValueDao.java @@ -56,7 +56,7 @@ public PointValueTime savePointValueSync(int pointId, PointValueTime pointValue, SetPointSource source) { long id = savePointValueImpl(pointId, pointValue, source, false); - PointValueTime savedPointValue; + /*PointValueTime savedPointValue; int retries = 5; while (true) { try { @@ -67,9 +67,9 @@ public PointValueTime savePointValueSync(int pointId, throw e; retries--; } - } + }*/ - return savedPointValue; + return pointValue; } /** diff --git a/src/com/serotonin/mango/web/ContextWrapper.java b/src/com/serotonin/mango/web/ContextWrapper.java index 0bc9f837e2..e16264f818 100644 --- a/src/com/serotonin/mango/web/ContextWrapper.java +++ b/src/com/serotonin/mango/web/ContextWrapper.java @@ -55,6 +55,11 @@ public DatabaseAccess getDatabaseAccess() { .getAttribute(Common.ContextKeys.DATABASE_ACCESS); } + public DatabaseAccess getDatabaseQueryAccess() { + return (DatabaseAccess) ctx + .getAttribute(Common.ContextKeys.DATABASE_QUERY_ACCESS); + } + @SuppressWarnings("unchecked") public List getImageSets() { return (List) ctx.getAttribute(Common.ContextKeys.IMAGE_SETS); diff --git a/src/org/scada_lts/dao/DAO.java b/src/org/scada_lts/dao/DAO.java index 0d17782b56..b12e359573 100644 --- a/src/org/scada_lts/dao/DAO.java +++ b/src/org/scada_lts/dao/DAO.java @@ -23,6 +23,7 @@ import javax.sql.DataSource; +import com.serotonin.mango.db.DatabaseAccess; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.jdbc.core.JdbcTemplate; @@ -41,8 +42,9 @@ public class DAO { private static final Log LOG = LogFactory.getLog(DAO.class); private NamedParameterJdbcTemplate namedParamJdbcTemplate; - private JdbcTemplate jdbcTemplate; + private JdbcTemplate jdbcTemplate; private static DAO instance; + private static DAO query; private boolean test =false; private DAO() { @@ -55,7 +57,13 @@ private DAO() { LOG.error(e); } } - + + private DAO(NamedParameterJdbcTemplate namedParamJdbcTemplate, JdbcTemplate jdbcTemplate) { + LOG.trace("Create DAO"); + this.namedParamJdbcTemplate = namedParamJdbcTemplate; + this.jdbcTemplate = jdbcTemplate; + } + /** * Get id. * @return Method queryForObject() can also return "null" @@ -70,6 +78,24 @@ public static DAO getInstance() { } return instance; } + + public static DAO query() { + if (query == null) { + boolean dbQueryEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.enabled", false); + if(dbQueryEnabled) { + DatabaseAccess databaseAccess = Common.ctx.getDatabaseQueryAccess(); + if (databaseAccess == null) { + throw new IllegalStateException("dbquery.enabled is true, you must definition datasource for dbquery or disabled in env.properties"); + } else { + DataSource ds = databaseAccess.getDataSource(); + NamedParameterJdbcTemplate namedParamJdbcTemplate = new NamedParameterJdbcTemplate(ds); + JdbcTemplate jdbcTemplate = new JdbcTemplate(ds); + query = new DAO(namedParamJdbcTemplate, jdbcTemplate); + } + } + } + return query; + } /** * Method utility needed because of mango compatibility. @@ -97,7 +123,7 @@ public static String boolToChar(boolean b) { public JdbcTemplate getJdbcTemp() { return jdbcTemplate; } - + /** * Set jdbcTemplate * @see TestDAO diff --git a/src/org/scada_lts/dao/SystemSettingsDAO.java b/src/org/scada_lts/dao/SystemSettingsDAO.java index e041129873..162947f4fb 100644 --- a/src/org/scada_lts/dao/SystemSettingsDAO.java +++ b/src/org/scada_lts/dao/SystemSettingsDAO.java @@ -27,6 +27,7 @@ import org.scada_lts.utils.ApplicationBeans; import org.scada_lts.utils.ColorUtils; import org.scada_lts.web.mvc.api.AggregateSettings; +import org.scada_lts.web.mvc.api.DbQuerySettings; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Propagation; @@ -122,6 +123,9 @@ public class SystemSettingsDAO { public static final String AGGREGATION_VALUES_LIMIT = "aggregationValuesLimit"; public static final String AGGREGATION_LIMIT_FACTOR = "aggregationLimitFactor"; + //QueryDB + public static final String DB_QUERY_READ_ENABLED = "dbQueryReadEnabled"; + private static final String DELETE_WATCH_LISTS = "delete from watchLists"; private static final String DELETE_MANGO_VIEWS = "delete from mangoViews"; private static final String DELETE_POINT_EVENT_DETECTORS = "delete from pointEventDetectors"; @@ -374,6 +378,9 @@ public String getDatabaseSchemaVersion(String key, String defaultValue) { DEFAULT_VALUES.put(AGGREGATION_LIMIT_FACTOR, String.valueOf(aggregateSettings.getLimitFactor())); DEFAULT_VALUES.put(AGGREGATION_VALUES_LIMIT, aggregateSettings.getValuesLimit()); + DbQuerySettings dbQuerySettings = DbQuerySettings.fromEnvProperties(); + DEFAULT_VALUES.put(DB_QUERY_READ_ENABLED, dbQuerySettings.isReadEnabled()); + DEFAULT_VALUES.put(VALUES_LIMIT_FOR_PURGE, 100); } diff --git a/src/org/scada_lts/dao/migration/mysql/V2_8_0_0__CreateViewPointValuesDenormalized.java b/src/org/scada_lts/dao/migration/mysql/V2_8_0_0__CreateViewPointValuesDenormalized.java new file mode 100644 index 0000000000..6f7e91ec48 --- /dev/null +++ b/src/org/scada_lts/dao/migration/mysql/V2_8_0_0__CreateViewPointValuesDenormalized.java @@ -0,0 +1,41 @@ +package org.scada_lts.dao.migration.mysql; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.scada_lts.dao.DAO; +import org.springframework.jdbc.core.JdbcTemplate; + + +public class V2_8_0_0__CreateViewPointValuesDenormalized extends BaseJavaMigration { + + + @Override + public void migrate(Context context) throws Exception { + final JdbcTemplate mysqlJdbcTmp = DAO.getInstance().getJdbcTemp(); + createViewForPointValues(mysqlJdbcTmp); + } + + + public void createViewForPointValues(JdbcTemplate jdbcTmp) { + + String createViewSQL = "CREATE OR REPLACE VIEW pointValuesDenormalized AS " + + "SELECT " + + "pv.dataPointId, " + + "pvm.pointValue, " + + "pv.dataType, " + + "CONCAT(DATE(FROM_UNIXTIME(pv.ts * 0.001)), \"T\", TIME(FROM_UNIXTIME(pv.ts * 0.001)), \"Z\") AS timestamp, " + + "pv.ts, " + + "(SELECT REPLACE(JSON_OBJECT('sourceType', pva.sourceType, " + + " 'sourceId', pva.sourceId, 'username', us.username), '\"', '')) AS metaData " + + "FROM " + + " (SELECT id, CONCAT_WS('', IF(dataType IN (4,5), null, pointValue), pva.textPointValueShort, pva.textPointValueLong) AS pointValue " + + "FROM pointValues pv LEFT JOIN pointValueAnnotations pva ON pv.id = pva.pointValueId) AS pvm, pointValues pv " + + "LEFT JOIN pointValueAnnotations pva ON pv.id = pva.pointValueId " + + "LEFT JOIN users us ON pva.sourceId = us.id " + + "WHERE pvm.id = pv.id;"; + + jdbcTmp.execute(createViewSQL); + } + + +} \ No newline at end of file diff --git a/src/org/scada_lts/dao/migration/query/mysql/V2_8_0_0__CreatePointValuesDenormalized.java b/src/org/scada_lts/dao/migration/query/mysql/V2_8_0_0__CreatePointValuesDenormalized.java new file mode 100644 index 0000000000..06f29bd568 --- /dev/null +++ b/src/org/scada_lts/dao/migration/query/mysql/V2_8_0_0__CreatePointValuesDenormalized.java @@ -0,0 +1,48 @@ +/* + * (c) 2016 Abil'I.T. http://abilit.eu/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package org.scada_lts.dao.migration.query.mysql; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.scada_lts.dao.DAO; +import org.springframework.jdbc.core.JdbcTemplate; + +public class V2_8_0_0__CreatePointValuesDenormalized extends BaseJavaMigration { + + @Override + public void migrate(Context context) throws Exception { + + final JdbcTemplate jdbcTemplate = DAO.query().getJdbcTemp(); + + final String pointValueAnnotations = "" + + "create table IF NOT EXISTS pointValuesDenormalized (" + + "dataPointId int," + + "dataType int," + + "pointValue double," + + "ts bigint," + + "textPointValueShort varchar(128)," + + "textPointValueLong longtext," + + "sourceType smallint," + + "sourceId int," + + "username varchar(40)) ENGINE=InnoDB;"; + + jdbcTemplate.execute(pointValueAnnotations); + + } + +} diff --git a/src/org/scada_lts/dao/migration/query/postgres/V2_8_0_0__CreatePointValuesDenormalized.java b/src/org/scada_lts/dao/migration/query/postgres/V2_8_0_0__CreatePointValuesDenormalized.java new file mode 100644 index 0000000000..cba4c805c7 --- /dev/null +++ b/src/org/scada_lts/dao/migration/query/postgres/V2_8_0_0__CreatePointValuesDenormalized.java @@ -0,0 +1,48 @@ +/* + * (c) 2016 Abil'I.T. http://abilit.eu/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package org.scada_lts.dao.migration.query.postgres; + + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.scada_lts.dao.DAO; +import org.springframework.jdbc.core.JdbcTemplate; + +public class V2_8_0_0__CreatePointValuesDenormalized extends BaseJavaMigration { + + @Override + public void migrate(Context context) throws Exception { + + final JdbcTemplate jdbcTemplate = DAO.query().getJdbcTemp(); + + final String pointValueAnnotations = "" + + "create table IF NOT EXISTS pointValuesDenormalized (" + + "dataPointId int," + + "dataType int," + + "pointValue DOUBLE PRECISION," + + "ts bigint," + + "textPointValueShort varchar(128)," + + "textPointValueLong text," + + "sourceType smallint," + + "sourceId int," + + "username varchar(40));"; + + jdbcTemplate.execute(pointValueAnnotations); + + } +} diff --git a/src/org/scada_lts/dao/migration/query/questdb/V2_8_0_0__CreatePointValuesDenormalized.java b/src/org/scada_lts/dao/migration/query/questdb/V2_8_0_0__CreatePointValuesDenormalized.java new file mode 100644 index 0000000000..03f9df0158 --- /dev/null +++ b/src/org/scada_lts/dao/migration/query/questdb/V2_8_0_0__CreatePointValuesDenormalized.java @@ -0,0 +1,335 @@ +/* + * (c) 2016 Abil'I.T. http://abilit.eu/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package org.scada_lts.dao.migration.query.questdb; + +import com.serotonin.mango.Common; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.multipart.FilePart; +import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; +import org.apache.commons.httpclient.methods.multipart.Part; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.scada_lts.dao.DAO; +import org.springframework.jdbc.core.JdbcOperations; + +import java.io.*; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class V2_8_0_0__CreatePointValuesDenormalized extends BaseJavaMigration { + + + private static final Log LOG = LogFactory.getLog(V2_8_0_0__CreatePointValuesDenormalized.class); + + @Override + public void migrate(Context context) throws Exception { + + final JdbcOperations questdb = DAO.query().getJdbcTemp(); + final JdbcOperations mysql = DAO.getInstance().getJdbcTemp(); + + URL resource = V2_8_0_0__CreatePointValuesDenormalized.class.getClassLoader().getResource("questdb-schema.json"); + File schema = Paths.get(Objects.requireNonNull(resource).toURI()).toFile(); + int limit = Common.getEnvironmentProfile().getInt("db.values.export.limit", 1001); + boolean overwrite = Common.getEnvironmentProfile().getBoolean("dbquery.import.overwrite", false); + boolean valuesImportEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.values.import.enabled", true); + + List dataPoints = mysql.queryForList("select dataPointId from pointValues group by dataPointId order by dataPointId", Integer.class); + + if(overwrite) { + try { + List tables = questdb.queryForList("SHOW TABLES;", String.class); + + for(String table: tables) { + if(!table.equalsIgnoreCase("telemetry") + && !table.equalsIgnoreCase("telemetry_config") + && table.contains("pointValues")) { + String dropQuery = "DROP TABLE " + table + ";"; + questdb.execute(dropQuery); + } + } + + } catch (Exception ex) { + LOG.warn(ex.getMessage()); + } + } + + if(valuesImportEnabled) { + + MigrationSettings migrationSettings = MigrationSettings.builder() + .toRead(mysql) + .overwrite(false) + .schema(schema) + .build(); + + for (int dataPoint : dataPoints) { + LOG.info("Datapoint: " + dataPoint); + for (int i = 0; migrationNext(PaginationParams.params(i, limit, dataPoint), migrationSettings); i += limit) { + } + } + } +/* + final String createEmptyTableQuery = "CREATE TABLE IF NOT EXISTS pointValuesDenormalized (" + + "dataPointId INT, " + + "dataType INT, " + + "pointValue DOUBLE, " + + "ts LONG, " + + "timestamp TIMESTAMP, " + + "textPointValueShort SYMBOL, " + + "textPointValueLong SYMBOL, " + + "sourceType INT, " + + "sourceId INT, " + + "username SYMBOL) timestamp(timestamp) PARTITION BY DAY;"; + + questdb.execute(createEmptyTableQuery);*/ + + } + + private static boolean migrationNext(PaginationParams paginationParams, + MigrationSettings migrationSettings) { + if(migrationSettings.getSchema() == null + || !migrationSettings.getSchema().exists() + || paginationParams.getOffset() < 0 + || paginationParams.getLimit() <= 0 + || migrationSettings.getToRead() == null) { + LOG.warn(migrationSettings); + return false; + } + AtomicBoolean exportedToCsv = new AtomicBoolean(false); + AtomicInteger rowsImported = new AtomicInteger(0); + createAccessDir(migrationSettings.getToRead()) + .flatMap(dir -> exportToCsv(dir, paginationParams, migrationSettings.getToRead())) + .ifPresent(csv -> { + exportedToCsv.set(true); + try { + if (csv.length() > 0) { + int imported = importToQuestDb(csv, migrationSettings, paginationParams.getDataPointId()); + rowsImported.set(imported); + } + } finally { + delete(csv); + } + }); + LOG.info("last offset: " + (paginationParams.getOffset() + rowsImported.get())); + return exportedToCsv.get() && rowsImported.get() == paginationParams.getLimit(); + } + + + private static Optional createAccessDir(JdbcOperations jdbcOperations) { + try { + String secureFilePrivQuery = "SHOW VARIABLES LIKE \"secure_file_priv\""; + String secureFilePriv = jdbcOperations.query(secureFilePrivQuery, (resultSet, i) -> resultSet.getString("Value")).get(0); + if(secureFilePriv == null || "NULL".equalsIgnoreCase(secureFilePriv)) { + LOG.warn("The secure_file_priv constant in the mysql database has an empty null value set, this blocked the database from being exported to a file, set a value for this constant other than null:\n" + + "1) Setting the value to empty will make it possible to export to any location;\n" + + "2) Setting this parameter to a specific path will force the use of that particular path;\n" + + "\n" + + "The change requires a mysql server restart. Applies to mysql version 5.7."); + return Optional.empty(); + } + String tempDir = System.getProperty("java.io.tmpdir"); + if((tempDir != null && tempDir.equals(secureFilePriv)) || secureFilePriv.isEmpty()) { + File file = File.createTempFile("pointValues", ".csv"); + delete(file); + return Optional.of(changePath(file)); + } + File file = new File(secureFilePriv + File.separator + "pointValues" + System.currentTimeMillis() + ".csv"); + return Optional.of(changePath(file)); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return Optional.empty(); + } + } + + private static String changePath(File file) { + if(file.getAbsolutePath().contains("\\")) { + return file.getAbsolutePath().replace("\\", "\\\\"); + } + return file.getAbsolutePath(); + } + + public static Optional exportToCsv(String csv, + PaginationParams paginationParams, + JdbcOperations jdbcOperations) { + String query = "SELECT * FROM (" + + " SELECT 'timestamp', 'ts', 'pointValue', 'metaData', 'dataType' UNION ALL" + + " (" + + " SELECT timestamp, ts, pointValue, metaData, dataType " + + " FROM pointValuesDenormalized WHERE dataPointId = " + paginationParams.getDataPointId() + " LIMIT " + paginationParams.getLimit() + " OFFSET " + paginationParams.getOffset() + " " + + " )" + + ") result INTO OUTFILE '" + csv + "' FIELDS TERMINATED BY '\t' ENCLOSED BY '' LINES TERMINATED BY '\r\n';"; + + try { + jdbcOperations.execute(query); + LOG.info("MySql export finished: OK"); + return Optional.of(new File(csv)); + } catch (Exception ex) { + LOG.info("MySql export finished: ERROR"); + LOG.error(ex.getMessage(), ex); + return Optional.empty(); + } + } + + public static int importToQuestDb(File csv, MigrationSettings migrationSettings, int dataPointId) { + try { + PostMethod postMethod = new PostMethod("http://localhost:9000/imp?name=pointValues" + dataPointId + "×tamp=timestamp&partitionBy=MONTH&overwrite=" + migrationSettings.isOverwrite()); + MultipartRequestEntity entity = new MultipartRequestEntity(new Part[]{new FilePart("schema", migrationSettings.getSchema()), new FilePart("data", csv)}, postMethod.getParams()); + postMethod.setRequestEntity(entity); + MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager(); + HttpClient httpClient = new HttpClient(manager); + httpClient.executeMethod(postMethod); + String response = postMethod.getResponseBodyAsString(); + LOG.info("QuestDb import finished: \r\n" + response); + return parseRowsImported(response); + } catch (Exception ex) { + LOG.error(ex.getMessage(), ex); + return -3; + } + } + + private static int parseRowsImported(String response) { + String[] lines = response.split("\r\n"); + for(String line: lines) { + if(line.contains("Rows imported")) { + LOG.info(line); + String[] cols = line.split("([|])"); + try { + return Integer.parseInt(cols[2].trim()); + } catch (Exception ex) { + return -1; + } + } + } + return -2; + } + + public static void delete(File csv) { + try { + Files.delete(csv.toPath()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + private static class PaginationParams { + int offset; + int limit; + int dataPointId; + + private PaginationParams(int offset, int limit, int dataPointId) { + this.offset = offset; + this.limit = limit; + this.dataPointId = dataPointId; + } + + public static PaginationParams params(int offset, int limit, int dataPointId) { + return new PaginationParams(offset, limit, dataPointId); + } + + public int getOffset() { + return offset; + } + + public int getLimit() { + return limit; + } + + public int getDataPointId() { + return dataPointId; + } + + @Override + public String toString() { + return "PaginationParams{" + + "offset=" + offset + + ", limit=" + limit + + '}'; + } + } + + private static class MigrationSettings { + File schema; + JdbcOperations toRead; + boolean overwrite; + + private MigrationSettings(File schema, JdbcOperations toRead, boolean overwrite) { + this.schema = schema; + this.toRead = toRead; + this.overwrite = overwrite; + } + + public static MigrationSettings.Builder builder() { + return new MigrationSettings.Builder(); + } + + private static class Builder { + File schema; + JdbcOperations toRead; + boolean overwrite; + + public Builder schema(File schema) { + this.schema = schema; + return this; + } + + public Builder toRead(JdbcOperations toRead) { + this.toRead = toRead; + return this; + } + + public Builder overwrite(boolean overwrite) { + this.overwrite = overwrite; + return this; + } + + public MigrationSettings build() { + return new MigrationSettings(schema, toRead, overwrite); + } + } + + public File getSchema() { + return schema; + } + + public JdbcOperations getToRead() { + return toRead; + } + + public boolean isOverwrite() { + return overwrite; + } + + @Override + public String toString() { + return "MigrationSettings{" + + "schema=" + schema + + ", jdbcOperations=" + toRead.getClass().getSimpleName() + + ", overwrite=" + overwrite + + '}'; + } + } +} diff --git a/src/org/scada_lts/dao/model/point/PointValueAdnnotation.java b/src/org/scada_lts/dao/model/point/PointValueAdnnotation.java index 205aaaeb18..b0365e1ad6 100644 --- a/src/org/scada_lts/dao/model/point/PointValueAdnnotation.java +++ b/src/org/scada_lts/dao/model/point/PointValueAdnnotation.java @@ -17,7 +17,9 @@ */ package org.scada_lts.dao.model.point; -/** +import com.serotonin.mango.rt.dataImage.SetPointSource; + +/** * Bean * * @author grzegorz bylica Abil'I.T. development team, sdt@abilit.eu @@ -45,6 +47,29 @@ public PointValueAdnnotation(long id, String shortString, String longString, Int this.sourceType = sourceType; this.sourceId = sourceId; } + + public static PointValueAdnnotation newInstance(String svalue, SetPointSource source, long pointValueId) { + Integer sourceType = null, sourceId = null; + if (source != null) { + sourceType = source.getSetPointSourceType(); + sourceId = source.getSetPointSourceId(); + } else { + sourceType = SetPointSource.Types.UNKNOWN; + sourceId = 1; + } + + String shortString = null; + String longString = null; + if (svalue != null) { + if (svalue.length() > 128) + longString = svalue; + else + shortString = svalue; + } + return new PointValueAdnnotation(pointValueId, shortString, longString, sourceType, sourceId); + } + + public String getChangeOwner() { return changeOwner; } diff --git a/src/org/scada_lts/dao/pointvalues/DataPointSimpleValue.java b/src/org/scada_lts/dao/pointvalues/DataPointSimpleValue.java new file mode 100644 index 0000000000..25197b406b --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/DataPointSimpleValue.java @@ -0,0 +1,17 @@ +package org.scada_lts.dao.pointvalues; + +public class DataPointSimpleValue { + int pointId; + double value; + long timestamp; + + DataPointSimpleValue(int pointId, double value, long timestamp) { + this.pointId = pointId; + this.value = value; + this.timestamp = timestamp; + } + + public long getTimestamp() { + return timestamp; + } +} \ No newline at end of file diff --git a/src/org/scada_lts/dao/pointvalues/IAmChartDAO.java b/src/org/scada_lts/dao/pointvalues/IAmChartDAO.java new file mode 100644 index 0000000000..0489869357 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IAmChartDAO.java @@ -0,0 +1,14 @@ +package org.scada_lts.dao.pointvalues; + +import com.serotonin.mango.vo.DataPointVO; + +import java.util.List; +import java.util.Map; + +public interface IAmChartDAO { + List> getPointValuesFromRange(int[] pointIds, long startTs, long endTs); + List> getPointValuesToCompareFromRange(int[] pointIds, long startTs, long endTs); + List getPointValuesFromRangeWithLimit(int[] pointIds, long startTs, long endTs, int limit); + List aggregatePointValues(DataPointVO dataPoint, long startTs, long endTs, + long intervalMs, int limit); +} diff --git a/src/org/scada_lts/dao/pointvalues/ILimitConst.java b/src/org/scada_lts/dao/pointvalues/ILimitConst.java new file mode 100644 index 0000000000..af706e55ea --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/ILimitConst.java @@ -0,0 +1,6 @@ +package org.scada_lts.dao.pointvalues; + +public interface ILimitConst { + String LIMIT = " LIMIT "; + int NO_LIMIT = 0; +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsCommand.java b/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsCommand.java new file mode 100644 index 0000000000..3ab40ef8f8 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsCommand.java @@ -0,0 +1,12 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.model.point.PointValue; +import org.scada_lts.dao.model.point.PointValueAdnnotation; + +import java.util.List; + +public interface IPointValueAdnnotationsCommand { + Object[] create(PointValueAdnnotation entity); + void update(int userId); + void updateAnnotations(List values); +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsDAO.java b/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsDAO.java new file mode 100644 index 0000000000..1ee245451c --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsDAO.java @@ -0,0 +1,11 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.DAO; + +public interface IPointValueAdnnotationsDAO extends IPointValueAdnnotationsQuery, + IPointValueAdnnotationsCommand, ILimitConst { + + static IPointValueAdnnotationsDAO newCommandRepository() { + return new PointValueAdnnotationsDAO(DAO.getInstance().getJdbcTemp()); + } +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsQuery.java b/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsQuery.java new file mode 100644 index 0000000000..ccad7d3081 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueAdnnotationsQuery.java @@ -0,0 +1,12 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.model.point.PointValueAdnnotation; + +import java.util.List; + +public interface IPointValueAdnnotationsQuery { + List findAllPointValueAdnnotations(); + List filteredPointValueAdnnotations(String filter, Object[] argsFilter, long limit); + List findAllWithUserNamePointValueAdnnotations(); + //PointValueAdnnotation findByIdPointValueAdnnotation(Object[] pk); +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueCommand.java b/src/org/scada_lts/dao/pointvalues/IPointValueCommand.java new file mode 100644 index 0000000000..0cd050fd52 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueCommand.java @@ -0,0 +1,21 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.model.point.PointValue; +import org.scada_lts.dao.model.point.PointValueAdnnotation; + +import java.util.List; + +public interface IPointValueCommand { + + long deletePointValuesBeforeWithOutLast(int dataPointId, long time); + long deletePointValue(int dataPointId); + long deleteAllPointData(); + long deletePointValuesWithMismatchedType(int dataPointId, int dataType); + void executeBatchUpdateInsert(List params); + void create(PointValue pointValue, PointValueAdnnotation pointValueAdnnotation, int dataType); + Object[] create(int pointId, int dataType, double dvalue, long time); + Object[] create(PointValue entity); + Object[] createNoTransaction(int pointId, int dataType, double dvalue, long time); + long deletePointValuesBeforeWithOutLastTwo(int dataPointId, long time); + long deletePointValuesWithValueLimit(int dataPointId, int limit); +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueDAO.java b/src/org/scada_lts/dao/pointvalues/IPointValueDAO.java new file mode 100644 index 0000000000..414b4127fc --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueDAO.java @@ -0,0 +1,11 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.DAO; + + +public interface IPointValueDAO extends IPointValueQuery, IPointValueCommand, ILimitConst { + + static IPointValueDAO newCommandRespository() { + return new PointValueDAO(DAO.getInstance().getJdbcTemp()); + } +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueDenormalizedDAO.java b/src/org/scada_lts/dao/pointvalues/IPointValueDenormalizedDAO.java new file mode 100644 index 0000000000..0b7f65fda1 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueDenormalizedDAO.java @@ -0,0 +1,10 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.DAO; + +public interface IPointValueDenormalizedDAO extends IPointValueDAO, IPointValueAdnnotationsDAO, ILimitConst { + + static IPointValueDenormalizedDAO newQueryRespository() { + return new PointValueDenormalizedDAO(DAO.query().getJdbcTemp()); + } +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueQuery.java b/src/org/scada_lts/dao/pointvalues/IPointValueQuery.java new file mode 100644 index 0000000000..d37d0954f5 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueQuery.java @@ -0,0 +1,34 @@ +package org.scada_lts.dao.pointvalues; + +import com.serotonin.mango.rt.dataImage.PointValueTime; +import com.serotonin.mango.vo.bean.LongPair; +import com.serotonin.mango.vo.bean.PointHistoryCount; +import org.scada_lts.dao.model.point.PointValue; + +import java.sql.ResultSet; +import java.util.List; + +public interface IPointValueQuery { + + double applyBounds(double value); + long dateRangeCount(int dataPointId, long from, long to); + long getStartTime(List dataPointIds); + long getEndTime(List dataPointIds); + long getMinTs(int dataPointId); + long getMaxTs(int dataPointId); + Long getInceptionDate(int dataPointId); + Long getLatestPointValue(int dataPointId); + //PointValue findById(Object[] pk); + PointValue getPointValueRow(ResultSet rs, int rowNum); + // PointValueTime getPointValue(long id); + LongPair getStartAndEndTime(List dataPointIds); + List getFiledataIds(); + List getTopPointHistoryCounts(); + + List findAll(); + List findAllWithUserName(); + List findByIdAndTs(long id, long ts); + List filtered(String filter, Object[] argsFilter, long limit); + + void createTableForDatapoint(int pointId); +} diff --git a/src/org/scada_lts/dao/pointvalues/IPointValueQuestDbDAO.java b/src/org/scada_lts/dao/pointvalues/IPointValueQuestDbDAO.java new file mode 100644 index 0000000000..8b5d9bbefc --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/IPointValueQuestDbDAO.java @@ -0,0 +1,12 @@ +package org.scada_lts.dao.pointvalues; + +import org.scada_lts.dao.DAO; + +public interface IPointValueQuestDbDAO extends IPointValueDAO, ILimitConst{ + + static IPointValueQuestDbDAO newQueryRespository() { + return new PointValueQuestDbDAO(DAO.query().getJdbcTemp()); + } + + long dropPartition(long time, int dataPointId); +} diff --git a/src/org/scada_lts/dao/pointvalues/PointValueAdnnotationsDAO.java b/src/org/scada_lts/dao/pointvalues/PointValueAdnnotationsDAO.java index 462c46fc36..e66a851a73 100644 --- a/src/org/scada_lts/dao/pointvalues/PointValueAdnnotationsDAO.java +++ b/src/org/scada_lts/dao/pointvalues/PointValueAdnnotationsDAO.java @@ -28,10 +28,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.scada_lts.dao.DAO; -import org.scada_lts.dao.GenericDaoCR; -import org.scada_lts.dao.PointValuesWithAdnotationAboutChangeOwner; import org.scada_lts.dao.model.point.PointValue; import org.scada_lts.dao.model.point.PointValueAdnnotation; +import org.springframework.jdbc.core.JdbcOperations; import org.springframework.jdbc.core.RowMapper; import com.serotonin.mango.rt.dataImage.AnnotatedPointValueTime; @@ -45,12 +44,12 @@ * @author grzegorz bylica Abil'I.T. development team, sdt@abilit.eu * */ -public class PointValueAdnnotationsDAO implements GenericDaoCR, PointValuesWithAdnotationAboutChangeOwner { +public class PointValueAdnnotationsDAO implements IPointValueAdnnotationsDAO { private static final Log LOG = LogFactory.getLog(PointValueAdnnotationsDAO.class); - private static PointValueAdnnotationsDAO instance; - + private static IPointValueAdnnotationsDAO instance; + private final static String COLUMN_NAME_POINT_VALUE_ID = "pointValueId"; private final static String COLUMN_NAME_TEXT_POINT_VALUE_SHORT = "textPointValueShort"; private final static String COLUMN_NAME_TEXT_POINT_VALUE_LONG = "textPointValueLong"; @@ -80,7 +79,7 @@ public class PointValueAdnnotationsDAO implements GenericDaoCR { public PointValueAdnnotation mapRow(ResultSet rs, int rowNum) throws SQLException { @@ -132,26 +149,14 @@ public PointValueAdnnotation mapRow(ResultSet rs, int rowNum) throws SQLExceptio } } - public static PointValueAdnnotationsDAO getInstance() { - if (instance == null) { - instance = new PointValueAdnnotationsDAO(); - } - return instance; - } - - @Override - public List findAllWithAdnotationsAboutChangeOwner() { - return (List) DAO.getInstance().getJdbcTemp().query(POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE, new Object[]{ }, new PointValueAdnnotationRowMapperWithAdnnotationAboutChangeOwner()); - } - @Override - public List findAllWithUserName() { - return null; + public List findAllWithUserNamePointValueAdnnotations() { + return (List) jdbcTemplate.query(POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE, new Object[]{ }, new PointValueAdnnotationRowMapperWithAdnnotationAboutChangeOwner()); } @Override - public List findAll() { - return (List) DAO.getInstance().getJdbcTemp().query(POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE/*POINT_VALUE_ADNNOTATIONS_SELECT*/, new Object[]{ }, new PointValueAdnnotationRowMapper()); + public List findAllPointValueAdnnotations() { + return (List) jdbcTemplate.query(POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE/*POINT_VALUE_ADNNOTATIONS_SELECT*/, new Object[]{ }, new PointValueAdnnotationRowMapper()); } @@ -159,18 +164,17 @@ public List findAll() { * @arguments id - idPointValue * @see org.scada_lts.dao.GenericDaoCR#findById(long) */ - @Override - public PointValueAdnnotation findById(Object[] pk) { - return (PointValueAdnnotation) DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_ADNNOTATIONS_SELECT +" where "+POINT_VALUE_ADNNOTATIONS_FILTER_BASE_ON_POINT_VALUES_ID, pk, new PointValueAdnnotationRowMapper()); + public PointValueAdnnotation findByIdPointValueAdnnotation(Object[] pk) { + return (PointValueAdnnotation) jdbcTemplate.queryForObject(POINT_VALUE_ADNNOTATIONS_SELECT +" where "+POINT_VALUE_ADNNOTATIONS_FILTER_BASE_ON_POINT_VALUES_ID, pk, new PointValueAdnnotationRowMapper()); } @Override - public List filtered(String filter, Object[] argsFilter, long limit) { + public List filteredPointValueAdnnotations(String filter, Object[] argsFilter, long limit) { String myLimit=""; if (limit != NO_LIMIT) { myLimit = LIMIT+limit; } - return (List) DAO.getInstance().getJdbcTemp().query(POINT_VALUE_ADNNOTATIONS_SELECT+" where "+ filter + myLimit, argsFilter, new PointValueAdnnotationRowMapper()); + return (List) jdbcTemplate.query(POINT_VALUE_ADNNOTATIONS_SELECT+" where "+ filter + myLimit, argsFilter, new PointValueAdnnotationRowMapper()); } @@ -180,7 +184,7 @@ public Object[] create(final PointValueAdnnotation entity) { LOG.trace(entity); } - DAO.getInstance().getJdbcTemp().update(POINT_VALUE_ADNNOTATIONS_INSERT, new Object[] { + jdbcTemplate.update(POINT_VALUE_ADNNOTATIONS_INSERT, new Object[] { entity.getPointValueId(), entity.getTextPointValueShort(), entity.getTextPointValueLong(), @@ -193,6 +197,7 @@ public Object[] create(final PointValueAdnnotation entity) { return new Object[] {0}; } + @Override @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) public void update(int userId) { @@ -200,9 +205,10 @@ public void update(int userId) { LOG.trace("update(int userId) userId:" + userId); } - DAO.getInstance().getJdbcTemp().update(POINT_VALUE_ANNOTATIONS_UPDATE, new Object[]{userId}); + jdbcTemplate.update(POINT_VALUE_ANNOTATIONS_UPDATE, new Object[]{userId}); } + @Override public void updateAnnotations(List values) { Map> userIds = new HashMap>(); List alist; diff --git a/src/org/scada_lts/dao/pointvalues/PointValueAmChartDAO.java b/src/org/scada_lts/dao/pointvalues/PointValueAmChartDAO.java index 51778b59db..2f0d8bf595 100644 --- a/src/org/scada_lts/dao/pointvalues/PointValueAmChartDAO.java +++ b/src/org/scada_lts/dao/pointvalues/PointValueAmChartDAO.java @@ -5,13 +5,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.scada_lts.dao.DAO; +import org.springframework.jdbc.core.JdbcOperations; import org.springframework.jdbc.core.RowMapper; import java.sql.ResultSet; import java.sql.SQLException; import java.text.MessageFormat; import java.util.*; -import java.util.concurrent.atomic.AtomicLong; + +import static org.scada_lts.dao.pointvalues.PointValueAmChartUtils.*; /** * Point Value DAO for AmChart Interface @@ -24,10 +26,12 @@ * * @author Radoslaw Jajko */ -public class PointValueAmChartDAO { +public class PointValueAmChartDAO implements IAmChartDAO { private static final Log LOG = LogFactory.getLog(PointValueAmChartDAO.class); + private static final String TABLE_NAME_DENORMALIZED = "pointValuesDenormalized"; + private static final String TABLE_NAME = "pointValues"; private static final String COLUMN_DATAPOINT_ID = "dataPointId"; private static final String COLUMN_TIMESTAMP = "ts"; @@ -38,19 +42,19 @@ public class PointValueAmChartDAO { private static final String SELECT_VALUES = "SELECT " + COLUMN_DATAPOINT_ID + ", " + COLUMN_VALUE + ", " + - COLUMN_TIMESTAMP + " FROM " + TABLE_NAME; + COLUMN_TIMESTAMP + " FROM "; private static final String SELECT_ORDER = " ORDER BY " + COLUMN_TIMESTAMP + " ASC"; private static final String SELECT_LIMIT = " LIMIT ?"; private static final String SELECT_AVG_VALUES_FOR_AGGREGATION = "SELECT " + COLUMN_DATAPOINT_ID + ", " + "avg(" + COLUMN_VALUE + ") AS " + COLUMN_VALUE + ", " + "max(" + COLUMN_TIMESTAMP + ") AS " + COLUMN_TIMESTAMP + " " + - "FROM " + TABLE_NAME; + "FROM "; private static final String SELECT_MAX_VALUES_FOR_AGGREGATION = "SELECT " + COLUMN_DATAPOINT_ID + ", " + "max(" + COLUMN_VALUE + ") AS " + COLUMN_VALUE + ", " + "max(" + COLUMN_TIMESTAMP + ") AS " + COLUMN_TIMESTAMP + " " + - "FROM " + TABLE_NAME; + "FROM "; private static final String CONDITION_TIMESTAMP =" AND " + COLUMN_TIMESTAMP + " >=? AND " + COLUMN_TIMESTAMP + " <=?"; private static final String GROUPBY_DATAPOINT_ID_TIMESTAMP = " GROUP BY " + COLUMN_DATAPOINT_ID + ", floor(" + COLUMN_TIMESTAMP + "/?)"; @@ -58,6 +62,18 @@ public class PointValueAmChartDAO { private static final String CONDITION_DATA_TYPE = " AND " + COLUMN_DATA_TYPE + "=?"; private static final String WHERE_DATA_POINT = " WHERE " + COLUMN_DATAPOINT_ID + "=?"; + private JdbcOperations jdbcTemplate; + private boolean denormalized; + + public PointValueAmChartDAO() { + this.jdbcTemplate = DAO.getInstance().getJdbcTemp(); + } + + public PointValueAmChartDAO(JdbcOperations jdbcTemplate, boolean denormalized) { + this.jdbcTemplate = jdbcTemplate; + this.denormalized = denormalized; + } + /** * Get Point Values from Time Range * @@ -70,27 +86,29 @@ public class PointValueAmChartDAO { * @param endTs Ending Timestamp Range value * @return List with dynamic Map of point values */ + @Override public List> getPointValuesFromRange(int[] pointIds, long startTs, long endTs) { String selectDataPoints = prepareWhereDataPointIdsStatement(pointIds); String selectTimestamp = prepareWhereTimestampStatement(startTs, endTs); - String sqlQuery = SELECT_VALUES + selectDataPoints + selectTimestamp + SELECT_ORDER; + String sqlQuery = SELECT_VALUES + getTableName(denormalized) + selectDataPoints + selectTimestamp + SELECT_ORDER; - List pvcList = DAO.getInstance().getJdbcTemp().query(sqlQuery, new PointValueChartRowMapper()); + List pvcList = jdbcTemplate.query(sqlQuery, new PointValueChartRowMapper()); return convertToAmChartDataObject(pvcList); } + @Override public List> getPointValuesToCompareFromRange(int[] pointIds, long startTs, long endTs) { String selectDataPoints = prepareWhereDataPointIdsStatement(pointIds); String selectTimestamp = prepareWhereTimestampStatement(startTs, endTs); - String sqlQuery = SELECT_VALUES + selectDataPoints + selectTimestamp + SELECT_ORDER; + String sqlQuery = SELECT_VALUES + getTableName(denormalized) + selectDataPoints + selectTimestamp + SELECT_ORDER; - List pvcList = DAO.getInstance().getJdbcTemp().query(sqlQuery, new PointValueChartRowMapper()); + List pvcList = jdbcTemplate.query(sqlQuery, new PointValueChartRowMapper()); return convertToAmChartCompareDataObject(pvcList, pointIds[0]); @@ -107,75 +125,22 @@ public List aggregatePointValues(DataPointVO dataPoint, lo LOG.warn(dataPointInfo(dataPoint)); return Collections.emptyList(); } - QueryArgs aggregationQuery = toAggregationQuery(dataPoint.getId(), dataPoint.getPointLocator().getDataTypeId(), startTs, endTs, intervalMs, limit); - return DAO.getInstance().getJdbcTemp().query(aggregationQuery.getQuery(), - aggregationQuery.getArgs(), new PointValueChartRowMapper()); + QueryArgs aggregationQuery = toAggregationQuery(dataPoint.getId(), dataPoint.getPointLocator().getDataTypeId(), + startTs, endTs, intervalMs, limit, denormalized); + return jdbcTemplate.query(aggregationQuery.getQuery(), aggregationQuery.getArgs(), new PointValueChartRowMapper()); } + @Override public List getPointValuesFromRangeWithLimit(int[] pointIds, long startTs, long endTs, int limit) { String selectDataPoints = prepareWhereDataPointIdsStatement(pointIds); String selectTimestamp = prepareWhereTimestampStatement(startTs, endTs); - String sqlQuery = SELECT_VALUES + selectDataPoints + selectTimestamp + SELECT_ORDER + " LIMIT " + limit; + String sqlQuery = SELECT_VALUES + getTableName(denormalized) + selectDataPoints + selectTimestamp + SELECT_ORDER + " LIMIT " + limit; - return DAO.getInstance().getJdbcTemp().query(sqlQuery, new PointValueChartRowMapper()); + return jdbcTemplate.query(sqlQuery, new PointValueChartRowMapper()); } - public List> convertToAmChartCompareDataObject(List result, int basePointId) { - List> chartData = new ArrayList<>(); - Map entry = new HashMap<>(); - - for (DataPointSimpleValue pvc: result) { - if(pvc.pointId == basePointId) { - entry.put("date", (double) pvc.timestamp); - } else { - entry.put("date2", (double) pvc.timestamp); - } - entry.put(String.valueOf(pvc.pointId), pvc.value); - chartData.add(entry); - entry = new HashMap<>(); - } - return chartData; - } - - /** - * Convert from SQL objects to AmChart Data interface - * - * Convert single SQL rows to match AmChart Data object Interface - * that looks like example below: - * [ - * { - * date: timestamp - * seriesName1: value - * seriesName2: value - * ... - * seriesNameN: value - * } - * ] - * - * @param result SQL result of Statement - * @return List of Java Maps containing values for data points in specific timestamp - */ - public List> convertToAmChartDataObject(List result) { - List> chartData = new ArrayList<>(); - Map entry = new HashMap<>(); - AtomicLong lastTimestamp = new AtomicLong(); - - for (DataPointSimpleValue pvc: result) { - if (lastTimestamp.get() != pvc.timestamp) { - if(!entry.isEmpty()) { - chartData.add(entry); - entry = new HashMap<>(); - } - entry.put("date", (double) pvc.timestamp); - } - entry.put(String.valueOf(pvc.pointId), pvc.value); - lastTimestamp.set(pvc.timestamp); - } - chartData.add(entry); - return chartData; - } private String prepareWhereDataPointIdsStatement(int[] pointIds) { if (pointIds.length == 0) { @@ -202,21 +167,6 @@ private String prepareWhereTimestampStatement(long startTs, long endTs) { return statement.toString(); } - public static class DataPointSimpleValue { - int pointId; - double value; - long timestamp; - - DataPointSimpleValue(int pointId, double value, long timestamp) { - this.pointId = pointId; - this.value = value; - this.timestamp = timestamp; - } - - public long getTimestamp() { - return timestamp; - } - } private static class PointValueChartRowMapper implements RowMapper { @Override @@ -229,9 +179,13 @@ public DataPointSimpleValue mapRow(ResultSet rs, int rowNum) throws SQLException } } + public static QueryArgs toAggregationQuery(int pointId, int dataType, long startTs, long endTs, + long intervalMs, int limit) { + return toAggregationQuery(pointId, dataType, startTs, endTs, intervalMs, limit, false); + } public static QueryArgs toAggregationQuery(int pointId, int dataType, long startTs, long endTs, - long intervalMs, int limit) { + long intervalMs, int limit, boolean denormalized) { String sqlQuery; @@ -245,6 +199,7 @@ public static QueryArgs toAggregationQuery(int pointId, int dataType, long start switch (dataType) { case DataTypes.BINARY: sqlQuery = SELECT_MAX_VALUES_FOR_AGGREGATION + + getTableName(denormalized) + WHERE_DATA_POINT + CONDITION_TIMESTAMP + CONDITION_DATA_TYPE + @@ -254,6 +209,7 @@ public static QueryArgs toAggregationQuery(int pointId, int dataType, long start break; case DataTypes.MULTISTATE: sqlQuery = SELECT_VALUES + + getTableName(denormalized) + WHERE_DATA_POINT + CONDITION_TIMESTAMP + CONDITION_DATA_TYPE + @@ -261,6 +217,7 @@ public static QueryArgs toAggregationQuery(int pointId, int dataType, long start break; case DataTypes.NUMERIC: sqlQuery = SELECT_AVG_VALUES_FOR_AGGREGATION + + getTableName(denormalized) + WHERE_DATA_POINT + CONDITION_TIMESTAMP + CONDITION_DATA_TYPE + @@ -270,6 +227,7 @@ public static QueryArgs toAggregationQuery(int pointId, int dataType, long start break; default: sqlQuery = SELECT_VALUES + + getTableName(denormalized) + WHERE_DATA_POINT + CONDITION_TIMESTAMP + CONDITION_DATA_TYPE + @@ -304,4 +262,8 @@ public Object[] getArgs() { public static String dataPointInfo(DataPointVO dataPoint) { return MessageFormat.format("PointLocator is null for dataPoint: {0} (id: {1}, xid: {2}, dataSource: {3} (id:{4}))", dataPoint.getName(), dataPoint.getId(), dataPoint.getXid(), dataPoint.getDataSourceName(), dataPoint.getDataSourceId()); } + + private static String getTableName(boolean denormalized) { + return (denormalized ? TABLE_NAME_DENORMALIZED : TABLE_NAME); + } } diff --git a/src/org/scada_lts/dao/pointvalues/PointValueAmChartQuestDbDAO.java b/src/org/scada_lts/dao/pointvalues/PointValueAmChartQuestDbDAO.java new file mode 100644 index 0000000000..15173e1071 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/PointValueAmChartQuestDbDAO.java @@ -0,0 +1,249 @@ +package org.scada_lts.dao.pointvalues; + +import com.serotonin.mango.DataTypes; +import com.serotonin.mango.vo.DataPointVO; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.scada_lts.dao.DAO; +import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.MessageFormat; +import java.util.*; +import java.util.stream.Collectors; + +import static org.scada_lts.dao.pointvalues.PointValueAmChartUtils.*; + +/** + * Point Value DAO for AmChart Interface + * + * Improve performance of DataPoint Values access and reduce + * unnecessary steps. Fetch values for all selected data points + * in just one query to reduce database access. Operate only + * on simple objects that does not contains any additional properties + * to reduce the memory usage. Match result with AmChart data interface. + * + * @author Radoslaw Jajko + */ +public class PointValueAmChartQuestDbDAO implements IAmChartDAO{ + + private static final Log LOG = LogFactory.getLog(PointValueAmChartQuestDbDAO.class); + + private static final String COLUMN_DATAPOINT_ID = "dataPointId"; + private static final String COLUMN_TS = "ts"; + private static final String COLUMN_TIMESTAMP = "timestamp"; + private static final String COLUMN_VALUE = "pointValue"; + + private static final String TABLE_POINT_VALUES = "pointValues$dpId"; + + private static final String DATAPOINT_ID = "$dpId"; + + private static final String SELECT_VALUES = "SELECT " + + DATAPOINT_ID + " as " + COLUMN_DATAPOINT_ID + ", " + + COLUMN_VALUE + ", " + + COLUMN_TS + " FROM "; + + private static final String SELECT_ORDER = " ORDER BY " + COLUMN_TIMESTAMP + " ASC"; + private static final String SELECT_ORDER_TS = " ORDER BY " + COLUMN_TS + " ASC"; + private static final String SELECT_LIMIT = " LIMIT ?"; + + private static final String SELECT_AVG_VALUES_FOR_AGGREGATION = "SELECT " + COLUMN_DATAPOINT_ID + ", " + + "tsGrouped, " + + "avg(" + COLUMN_VALUE + ") AS " + COLUMN_VALUE + ", " + + "max(" + COLUMN_TS + ") AS " + COLUMN_TS + " " + + "FROM "; + + private static final String SELECT_MAX_VALUES_FOR_AGGREGATION = "SELECT " + DATAPOINT_ID + " as " + COLUMN_DATAPOINT_ID + ", " + + "tsGrouped, " + + "max(" + COLUMN_VALUE + ") AS " + COLUMN_VALUE + ", " + + "max(" + COLUMN_TS + ") AS " + COLUMN_TS + " " + + "FROM "; + + private static final String CONDITION_TIMESTAMP =" WHERE " + COLUMN_TIMESTAMP + " >= to_timezone($from, 'CET') AND " + COLUMN_TIMESTAMP + " <= to_timezone($to, 'CET')"; + private static final String GROUPBY_DATAPOINT_ID_TIMESTAMP = " GROUP BY " + COLUMN_DATAPOINT_ID + ", tsGrouped "; + + private static final String SELECT_FOR_GROUP_BY = "(SELECT " + + DATAPOINT_ID + " as " + COLUMN_DATAPOINT_ID + ", " + + "cast(" + COLUMN_VALUE + " as double) as " + COLUMN_VALUE + ", " + + " round_down(ts/$divider, 0) as tsGrouped, " + COLUMN_TS + ", " + + COLUMN_TIMESTAMP + " " + + "FROM " + TABLE_POINT_VALUES + ") "; + + + private JdbcOperations jdbcTemplate; + + public PointValueAmChartQuestDbDAO() { + this.jdbcTemplate = DAO.getInstance().getJdbcTemp(); + } + + public PointValueAmChartQuestDbDAO(JdbcOperations jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + /** + * Get Point Values from Time Range + * + * Get Point Values for one or multiple Data Points + * using just one SQL query. Limit result to match the + * time range constraint. + * + * @param pointIds Array of PointIDs + * @param startTs Starting Timestamp Range value + * @param endTs Ending Timestamp Range value + * @return List with dynamic Map of point values + */ + + @Override + public List> getPointValuesFromRange(int[] pointIds, long startTs, long endTs) { + List pvcList = new ArrayList<>(); + + String selectTimestamp = prepareWhereTimestampStatement(startTs*1000, endTs*1000); + + for (int i = 0; i < pointIds.length; i++) { + String dpId = String.valueOf(pointIds[i]); + String sqlQuery = SELECT_VALUES.replace(DATAPOINT_ID, dpId) + TABLE_POINT_VALUES.replace(DATAPOINT_ID, dpId) + selectTimestamp + SELECT_ORDER; + List pvcListPerPoint = jdbcTemplate.query(sqlQuery, new PointValueChartRowMapper()); + pvcList.addAll(pvcListPerPoint); + } + + return convertToAmChartDataObject(pvcList); + } + + @Override + public List> getPointValuesToCompareFromRange(int[] pointIds, long startTs, long endTs) { + + List pvcList = new ArrayList<>(); + + String selectTimestamp = prepareWhereTimestampStatement(startTs*1000, endTs*1000); + + for (int i = 0; i < pointIds.length; i++) { + String dpId = String.valueOf(pointIds[i]); + String sqlQuery = SELECT_VALUES.replace(DATAPOINT_ID, dpId) + TABLE_POINT_VALUES.replace(DATAPOINT_ID, dpId) + selectTimestamp + SELECT_ORDER; + List pvcListPerPoint = jdbcTemplate.query(sqlQuery, new PointValueChartRowMapper()); + pvcList.addAll(pvcListPerPoint); + } + + return convertToAmChartCompareDataObject(pvcList, pointIds[0]); + } + + @Override + public List aggregatePointValues(DataPointVO dataPoint, long startTs, long endTs, + long intervalMs, int limit) { + + if(dataPoint == null) { + LOG.warn("dataPoint is null!"); + return Collections.emptyList(); + } + if(dataPoint.getPointLocator() == null) { + LOG.warn(dataPointInfo(dataPoint)); + return Collections.emptyList(); + } + QueryArgs aggregationQuery = toAggregationQuery(dataPoint.getId(), dataPoint.getPointLocator().getDataTypeId(), startTs, endTs, intervalMs, limit); + return jdbcTemplate.query(aggregationQuery.getQuery(), + aggregationQuery.getArgs(), new PointValueChartRowMapper()); + } + + @Override + public List getPointValuesFromRangeWithLimit(int[] pointIds, long startTs, long endTs, int limit) { + + List pvcList = new ArrayList<>(); + + String selectTimestamp = prepareWhereTimestampStatement(startTs*1000, endTs*1000); + + for (int i = 0; i < pointIds.length; i++) { + String dpId = String.valueOf(pointIds[i]); + String sqlQuery = SELECT_VALUES.replace(DATAPOINT_ID, dpId) + TABLE_POINT_VALUES.replace(DATAPOINT_ID, dpId) + selectTimestamp + SELECT_ORDER; + List pvcListPerPoint = jdbcTemplate.query(sqlQuery, new PointValueChartRowMapper()); + pvcList.addAll(pvcListPerPoint); + } + + return pvcList.stream().limit(limit).collect(Collectors.toList()); + } + + private String prepareWhereTimestampStatement(long startTs, long endTs) { + StringBuilder statement = new StringBuilder(); + statement.append(" WHERE ").append(COLUMN_TIMESTAMP).append(" >= ").append("to_timezone(").append(startTs).append(", 'CET')") + .append(" AND ").append(COLUMN_TIMESTAMP).append(" <= ").append("to_timezone(").append(endTs).append(", 'CET')"); + return statement.toString(); + } + + private static class PointValueChartRowMapper implements RowMapper { + @Override + public DataPointSimpleValue mapRow(ResultSet rs, int rowNum) throws SQLException { + return new DataPointSimpleValue( + rs.getInt(COLUMN_DATAPOINT_ID), + rs.getDouble(COLUMN_VALUE), + rs.getLong(COLUMN_TS) + ); + } + } + + public static QueryArgs toAggregationQuery(int pointId, int dataType, long startTs, long endTs, + long intervalMs, int limit) { + String sqlQuery; + + List args = new ArrayList<>(); + + String conditionTimestamp = CONDITION_TIMESTAMP.replace("$from", String.valueOf(startTs * 1000)) + .replace("$to", String.valueOf(endTs * 1000)); + switch (dataType) { + case DataTypes.BINARY: + sqlQuery = SELECT_MAX_VALUES_FOR_AGGREGATION + + SELECT_FOR_GROUP_BY.replace(DATAPOINT_ID, String.valueOf(pointId)) + .replace("$divider", String.valueOf(intervalMs)) + + conditionTimestamp + + GROUPBY_DATAPOINT_ID_TIMESTAMP + + SELECT_ORDER_TS; + break; + case DataTypes.MULTISTATE: + sqlQuery = SELECT_VALUES.replace(DATAPOINT_ID, String.valueOf(pointId)) + + TABLE_POINT_VALUES.replace(DATAPOINT_ID, String.valueOf(pointId)) + + conditionTimestamp + + SELECT_ORDER_TS; + break; + case DataTypes.NUMERIC: + sqlQuery = SELECT_AVG_VALUES_FOR_AGGREGATION + + SELECT_FOR_GROUP_BY.replace(DATAPOINT_ID, String.valueOf(pointId)) + .replace("$divider", String.valueOf(intervalMs)) + + conditionTimestamp + + GROUPBY_DATAPOINT_ID_TIMESTAMP + + SELECT_ORDER_TS; + break; + default: + sqlQuery = SELECT_VALUES.replace(DATAPOINT_ID, String.valueOf(pointId)) + + TABLE_POINT_VALUES.replace(DATAPOINT_ID, String.valueOf(pointId)) + + conditionTimestamp + + SELECT_ORDER_TS; + } + +// if(limit > 0) { +// sqlQuery += SELECT_LIMIT; +// args.add(limit); +// } + return new QueryArgs(sqlQuery, args.toArray()); + } + + public static class QueryArgs { + private String query; + private Object[] args; + + QueryArgs(String query, Object[] args) { + this.query = query; + this.args = args; + } + + public String getQuery() { + return query; + } + + public Object[] getArgs() { + return args; + } + } + + public static String dataPointInfo(DataPointVO dataPoint) { + return MessageFormat.format("PointLocator is null for dataPoint: {0} (id: {1}, xid: {2}, dataSource: {3} (id:{4}))", dataPoint.getName(), dataPoint.getId(), dataPoint.getXid(), dataPoint.getDataSourceName(), dataPoint.getDataSourceId()); + } +} diff --git a/src/org/scada_lts/dao/pointvalues/PointValueAmChartUtils.java b/src/org/scada_lts/dao/pointvalues/PointValueAmChartUtils.java new file mode 100644 index 0000000000..d104180e44 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/PointValueAmChartUtils.java @@ -0,0 +1,67 @@ +package org.scada_lts.dao.pointvalues; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +public final class PointValueAmChartUtils { + + private PointValueAmChartUtils() {} + + public static List> convertToAmChartCompareDataObject(List result, int basePointId) { + List> chartData = new ArrayList<>(); + Map entry = new HashMap<>(); + + for (DataPointSimpleValue pvc: result) { + if(pvc.pointId == basePointId) { + entry.put("date", (double) pvc.timestamp); + } else { + entry.put("date2", (double) pvc.timestamp); + } + entry.put(String.valueOf(pvc.pointId), pvc.value); + chartData.add(entry); + entry = new HashMap<>(); + } + return chartData; + } + + /** + * Convert from SQL objects to AmChart Data interface + * + * Convert single SQL rows to match AmChart Data object Interface + * that looks like example below: + * [ + * { + * date: timestamp + * seriesName1: value + * seriesName2: value + * ... + * seriesNameN: value + * } + * ] + * + * @param result SQL result of Statement + * @return List of Java Maps containing values for data points in specific timestamp + */ + public static List> convertToAmChartDataObject(List result) { + List> chartData = new ArrayList<>(); + Map entry = new HashMap<>(); + AtomicLong lastTimestamp = new AtomicLong(); + + for (DataPointSimpleValue pvc: result) { + if (lastTimestamp.get() != pvc.timestamp) { + if(!entry.isEmpty()) { + chartData.add(entry); + entry = new HashMap<>(); + } + entry.put("date", (double) pvc.timestamp); + } + entry.put(String.valueOf(pvc.pointId), pvc.value); + lastTimestamp.set(pvc.timestamp); + } + chartData.add(entry); + return chartData; + } +} diff --git a/src/org/scada_lts/dao/pointvalues/PointValueDAO.java b/src/org/scada_lts/dao/pointvalues/PointValueDAO.java index 3716957d7f..8897070833 100644 --- a/src/org/scada_lts/dao/pointvalues/PointValueDAO.java +++ b/src/org/scada_lts/dao/pointvalues/PointValueDAO.java @@ -28,12 +28,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.scada_lts.dao.DAO; -import org.scada_lts.dao.GenericDaoCR; import org.scada_lts.dao.model.point.PointValue; +import org.scada_lts.dao.model.point.PointValueAdnnotation; import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.ArgumentPreparedStatementSetter; -import org.springframework.jdbc.core.PreparedStatementCreator; -import org.springframework.jdbc.core.RowMapper; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.jdbc.core.*; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.support.GeneratedKeyHolder; import org.springframework.jdbc.support.KeyHolder; @@ -61,11 +60,11 @@ * */ @Repository -public class PointValueDAO implements GenericDaoCR { +public class PointValueDAO implements IPointValueDAO { private static final Log LOG = LogFactory.getLog(PointValueDAO.class); - private static PointValueDAO instance; + private static IPointValueDAO instance; private final static String COLUMN_NAME_ID = "id"; private final static String COLUMN_NAME_DATA_TYPE = "dataType"; @@ -161,8 +160,8 @@ public class PointValueDAO implements GenericDaoCR { + " where " + COLUMN_NAME_DATA_POINT_ID + "=? and " + COLUMN_NAME_TIME_STAMP +"=?"; - - + + private static final String POINT_VALUE_INSERT = "" + "insert pointValues (" + COLUMN_NAME_DATA_POINT_ID + "," @@ -246,7 +245,7 @@ public class PointValueDAO implements GenericDaoCR { + "order by id DESC " + "limit 2 " + ") lastId ) and " + COLUMN_NAME_TIME_STAMP + " { + "order by 2 desc "; // @formatter:on - + + private JdbcOperations jdbcTemplate; + + public PointValueDAO() { + this.jdbcTemplate = DAO.getInstance().getJdbcTemp(); + } + + public PointValueDAO(JdbcOperations jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + public static IPointValueDAO getInstance() { + if (instance == null) { + instance = new PointValueDAO(DAO.getInstance().getJdbcTemp()); + } + return instance; + } + //RowMappers private class PointValueRowMapper implements RowMapper { public PointValue mapRow(ResultSet rs, int rowNum) throws SQLException { @@ -322,6 +338,7 @@ public PointValue mapRow(ResultSet rs, int rowNum) throws SQLException { return pv; } } + @Override public PointValue getPointValueRow(ResultSet rs, int rowNum){ try { return new PointValueRowMapperWithUserName().mapRow(rs, rowNum); @@ -431,30 +448,24 @@ MangoValue createMangoValue(ResultSet rs) } return value; } - - public static PointValueDAO getInstance() { - if (instance == null) { - instance = new PointValueDAO(); - } - return instance; - } + @Override public List findAllWithUserName(){ - return (List) DAO.getInstance().getJdbcTemp().query(POINT_VALUE_SELECT_WITH_USERNAME, new Object[]{ }, new PointValueRowMapper()); + return (List) jdbcTemplate.query(POINT_VALUE_SELECT_WITH_USERNAME, new Object[]{ }, new PointValueRowMapper()); } @Override public List findAll() { - return (List) DAO.getInstance().getJdbcTemp().query(POINT_VALUE_SELECT, new Object[]{ }, new PointValueRowMapper()); + return (List) jdbcTemplate.query(POINT_VALUE_SELECT, new Object[]{ }, new PointValueRowMapper()); } - @Override public PointValue findById(Object[] pk) { - return (PointValue) DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_SELECT_ON_BASE_ID, pk, new PointValueRowMapper()); + return (PointValue) jdbcTemplate.queryForObject(POINT_VALUE_SELECT_ON_BASE_ID, pk, new PointValueRowMapper()); } - + + @Override public List findByIdAndTs(long id, long ts) { - return DAO.getInstance().getJdbcTemp().query(POINT_VALUE_SELECT_ON_BASE_ID_TS, new Object[] { id,ts }, new PointValueRowMapper()); + return jdbcTemplate.query(POINT_VALUE_SELECT_ON_BASE_ID_TS, new Object[] { id,ts }, new PointValueRowMapper()); } @Override @@ -465,7 +476,7 @@ public List filtered(String filter, Object[] argsFilter, long limit) myLimit = LIMIT+limit; } List res = - (List) DAO.getInstance().getJdbcTemp().query( + (List) jdbcTemplate.query( "select " + "pv."+COLUMN_NAME_ID + "," + "pv."+COLUMN_NAME_DATA_POINT_ID + ", " @@ -493,7 +504,7 @@ public Object[] create(final PointValue entity) { } KeyHolder keyHolder = new GeneratedKeyHolder(); - DAO.getInstance().getJdbcTemp().update(new PreparedStatementCreator() { + jdbcTemplate.update(new PreparedStatementCreator() { @Override public PreparedStatement createPreparedStatement(Connection connection) throws SQLException { PreparedStatement ps = connection.prepareStatement(POINT_VALUE_INSERT, Statement.RETURN_GENERATED_KEYS); @@ -512,22 +523,39 @@ public PreparedStatement createPreparedStatement(Connection connection) throws S } + @Override @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) public Object[] create(final int pointId,final int dataType,final double dvalue,final long time) { return createNoTransaction(pointId, dataType, dvalue, time); } - - - public Object[] createNoTransaction(final int pointId,final int dataType,final double dvalue,final long time) { + + @Override + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + public void create(PointValue pointValue, PointValueAdnnotation pointValueAdnnotation, int dataType) { + + PointValueTime pointValueTime = pointValue.getPointValue(); + MangoValue mangoValue = pointValueTime.getValue(); + + if (LOG.isTraceEnabled()) { + LOG.trace("pointId:"+pointValue.getDataPointId()+" dataType:"+mangoValue.getDataType()+" dvalue:"+mangoValue.getObjectValue()); + } + + create((int)pointValue.getDataPointId(), dataType, mangoValue.getDoubleValue(), pointValueTime.getTime()); + IPointValueAdnnotationsDAO.newCommandRepository().create(pointValueAdnnotation); + } + + + @Override + public Object[] createNoTransaction(final int pointId, final int dataType, final double dvalue, final long time) { if (LOG.isTraceEnabled()) { LOG.trace("pointId:"+pointId+" dataType:"+dataType+" dvalue:"+time); } KeyHolder keyHolder = new GeneratedKeyHolder(); - DAO.getInstance().getJdbcTemp().update(new PreparedStatementCreator() { + jdbcTemplate.update(new PreparedStatementCreator() { @Override public PreparedStatement createPreparedStatement(Connection connection) throws SQLException { PreparedStatement ps = connection.prepareStatement(POINT_VALUE_INSERT, Statement.RETURN_GENERATED_KEYS); @@ -535,18 +563,26 @@ public PreparedStatement createPreparedStatement(Connection connection) throws S pointId, dataType, dvalue, - time + time, }).setValues(ps); return ps; } }, keyHolder); - - return new Object[] {keyHolder.getKey().longValue()}; + + try { + return new Object[]{keyHolder.getKey().longValue()}; + } catch (InvalidDataAccessApiUsageException ex) { + return new Object[]{Long.parseLong(String.valueOf(keyHolder.getKeys().get("id")))}; + } } - - - + + @Override + public void createTableForDatapoint(int pointId) { + } + + + @Override @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) public void executeBatchUpdateInsert( List params) { if (LOG.isTraceEnabled()) { @@ -557,54 +593,60 @@ public void executeBatchUpdateInsert( List params) { } } - DAO.getInstance().getJdbcTemp().batchUpdate(POINT_VALUE_INSERT,params); + jdbcTemplate.batchUpdate(POINT_VALUE_INSERT,params); } - + + @Override public Long getInceptionDate(int dataPointId) { - return DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_INCEPTION_DATA, new Object[] {dataPointId}, Long.class); + return jdbcTemplate.queryForObject(POINT_VALUE_INCEPTION_DATA, new Object[] {dataPointId}, Long.class); } + @Override public long dateRangeCount(int dataPointId, long from, long to) { - return DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_DATA_RANGE_COUNT, new Object[] {dataPointId, from, to}, Long.class); + return jdbcTemplate.queryForObject(POINT_VALUE_DATA_RANGE_COUNT, new Object[] {dataPointId, from, to}, Long.class); } - + + @Override public LongPair getStartAndEndTime(List dataPointIds) { if (dataPointIds.isEmpty()) return null; MapSqlParameterSource parameters = new MapSqlParameterSource(); parameters.addValue("ids",dataPointIds); - LongPair longPair = DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT,new LongPairRowMapper(),parameters); + LongPair longPair = jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT,new LongPairRowMapper(),parameters); return longPair; } - + + @Override public long getStartTime(List dataPointIds) { if (dataPointIds.isEmpty()) return -1; MapSqlParameterSource parameters = new MapSqlParameterSource(); parameters.addValue("ids",dataPointIds); - Long minTs = DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT,new LongRowMapper(),parameters); + Long minTs = jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT,new LongRowMapper(),parameters); return minTs; } - + + @Override public long getEndTime(List dataPointIds) { if (dataPointIds.isEmpty()) return -1; MapSqlParameterSource parameters = new MapSqlParameterSource(); parameters.addValue("ids",dataPointIds); - Long maxTs = DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT, new LongRowMapper(),parameters); + Long maxTs = jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT, new LongRowMapper(),parameters); return maxTs; } - + + @Override public List getFiledataIds() { //TODO rewrite - return DAO.getInstance().getJdbcTemp().queryForList( + return jdbcTemplate.queryForList( "select distinct id from ( " // + " select id as id from pointValues where dataType=" + DataTypes.IMAGE @@ -615,10 +657,11 @@ public List getFiledataIds() { + DataTypes.IMAGE + ") a order by 1", new Object[] {}, Long.class); } - + + @Override public Long getLatestPointValue(int dataPointId) { try { - return DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_SELECT_MAX_BASE_ON_DATA_POINT_ID, new Object[] {dataPointId}, Long.class); + return jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MAX_BASE_ON_DATA_POINT_ID, new Object[] {dataPointId}, Long.class); } catch (EmptyResultDataAccessException e) { return null; } @@ -641,7 +684,8 @@ private double getValueBaseOnType(int dataType, PointValueTime value) { } // Apply database specific bounds on double values. - public double applyBounds(double value) { + @Override + public double applyBounds(double value) { if (Double.isNaN(value)) return 0; if (value == Double.POSITIVE_INFINITY) @@ -651,47 +695,56 @@ public double applyBounds(double value) { return value; } - + + @Override @Deprecated - public long deletePointValuesBeforeWithOutLast(int dataPointId, long time) { - Long lastId = DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_ID_OF_LAST_VALUE, new Object[] { dataPointId }, Long.class ); - return DAO.getInstance().getJdbcTemp().update(POINT_VALUE_DELETE_BEFORE, new Object[] {dataPointId, time, lastId}); + public long deletePointValuesBeforeWithOutLast(int dataPointId, long time) { + Long lastId = jdbcTemplate.queryForObject(POINT_VALUE_ID_OF_LAST_VALUE, new Object[] { dataPointId }, Long.class ); + return jdbcTemplate.update(POINT_VALUE_DELETE_BEFORE, new Object[] {dataPointId, time, lastId}); } + @Override public long deletePointValuesBeforeWithOutLastTwo(int dataPointId, long time) { - return DAO.getInstance().getJdbcTemp().update(POINT_VALUE_DELETE_BEFORE_WITHOUT_LAST_TWO, new Object[] {dataPointId, dataPointId, time}); + return jdbcTemplate.update(POINT_VALUE_DELETE_BEFORE_WITHOUT_LAST_TWO, new Object[] {dataPointId, dataPointId, time}); } - - public long deletePointValue(int dataPointId) { - return DAO.getInstance().getJdbcTemp().update(POINT_VALUE_DELETE_BASE_ON_POINT_ID, new Object[] {dataPointId}); + + @Override + public long deletePointValue(int dataPointId) { + return jdbcTemplate.update(POINT_VALUE_DELETE_BASE_ON_POINT_ID, new Object[] {dataPointId}); } - public long deleteAllPointData() { - return DAO.getInstance().getJdbcTemp().update(POINT_VALUE_DELETE, new Object[] {}); + @Override + public long deleteAllPointData() { + return jdbcTemplate.update(POINT_VALUE_DELETE, new Object[] {}); } - public long deletePointValuesWithMismatchedType(int dataPointId, int dataType) { - return DAO.getInstance().getJdbcTemp().update(DELETE_POINT_VALUE_WITH_MISMATCHED_TYPE, new Object[] {dataPointId, dataType}); + @Override + public long deletePointValuesWithMismatchedType(int dataPointId, int dataType) { + return jdbcTemplate.update(DELETE_POINT_VALUE_WITH_MISMATCHED_TYPE, new Object[] {dataPointId, dataType}); } + @Override public long deletePointValuesWithValueLimit(int dataPointId, int limit){ - return DAO.getInstance().getJdbcTemp().update(DELETE_POINT_VALUE_BASED_ON_DATAPOINT_WITH_VALUE_LIMIT, new Object[] {dataPointId, dataPointId, limit}); + return jdbcTemplate.update(DELETE_POINT_VALUE_BASED_ON_DATAPOINT_WITH_VALUE_LIMIT, new Object[] {dataPointId, dataPointId, limit}); } - - public PointValueTime getPointValue(long id) { - return ((PointValue) DAO.getInstance().getJdbcTemp().queryForObject(POINT_VALUE_SELECT + " where " + POINT_VALUE_FILTER_BASE_ON_ID, new Object[] {id}, new PointValueRowMapper())).getPointValue(); + + public PointValueTime getPointValue(long id) { + return ((PointValue) jdbcTemplate.queryForObject(POINT_VALUE_SELECT + " where " + POINT_VALUE_FILTER_BASE_ON_ID, new Object[] {id}, new PointValueRowMapper())).getPointValue(); } + @Override public long getMinTs(int dataPointId) { - return DAO.getInstance().getJdbcTemp().queryForObject(SELECT_MIN_TIME_WHERE_DATA_POINT_ID, new Object[] {dataPointId}, Long.TYPE); + return jdbcTemplate.queryForObject(SELECT_MIN_TIME_WHERE_DATA_POINT_ID, new Object[] {dataPointId}, Long.TYPE); } + @Override public long getMaxTs(int dataPointId) { - return DAO.getInstance().getJdbcTemp().queryForObject(SELECT_MAX_TIME_WHERE_DATA_POINT_ID, new Object[] {dataPointId}, Long.TYPE); + return jdbcTemplate.queryForObject(SELECT_MAX_TIME_WHERE_DATA_POINT_ID, new Object[] {dataPointId}, Long.TYPE); } + @Override public List getTopPointHistoryCounts() { - return DAO.getInstance().getJdbcTemp().query(SELECT_DP_ID_AND_COUNT, new RowMapper() { + return jdbcTemplate.query(SELECT_DP_ID_AND_COUNT, new RowMapper() { @Override public PointHistoryCount mapRow(ResultSet rs, int rowNum) throws SQLException { PointHistoryCount phc = new PointHistoryCount(); diff --git a/src/org/scada_lts/dao/pointvalues/PointValueDAO4REST.java b/src/org/scada_lts/dao/pointvalues/PointValueDAO4REST.java index db7db5044b..8024c63ec0 100644 --- a/src/org/scada_lts/dao/pointvalues/PointValueDAO4REST.java +++ b/src/org/scada_lts/dao/pointvalues/PointValueDAO4REST.java @@ -20,9 +20,10 @@ import java.sql.SQLException; import java.util.Date; +import com.serotonin.mango.Common; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.scada_lts.dao.DAO; +import org.scada_lts.dao.model.point.PointValue; import org.scada_lts.dao.model.point.PointValueAdnnotation; import org.scada_lts.dao.model.point.PointValueTypeOfREST; import org.springframework.stereotype.Repository; @@ -77,27 +78,52 @@ public PointValueTime save(String value, int typePointValueOfREST, int dpid) { new RuntimeException("Value not compatible with type (double)"); } } else if (typePointValueOfREST==PointValueTypeOfREST.TYPE_STRING) { - - pvt = new PointValueTime(value, new Date().getTime() ); - Object[] resultPointValue = new PointValueDAO().createNoTransaction(dpid, DataTypes.ALPHANUMERIC, 0, new Date().getTime()); - PointValueAdnnotation pva = new PointValueAdnnotation(); - Long pointValueId = (Long) resultPointValue[0]; - pva.setPointValueId(pointValueId); - pva.setSourceType(DataTypes.ALPHANUMERIC); - - int lengthShortValue = 128; - if (value.length()<=lengthShortValue) { - pva.setTextPointValueShort(value); - } else { - pva.setTextPointValueLong(value); + + boolean dbWriteEnabled = Common.getEnvironmentProfile().getBoolean("db.values.write.enabled", true); + long pointValueId = Common.NEW_ID; + if(dbWriteEnabled) { + pvt = new PointValueTime(value, new Date().getTime()); + Object[] resultPointValue = new PointValueDAO().createNoTransaction(dpid, DataTypes.ALPHANUMERIC, 0, new Date().getTime()); + PointValueAdnnotation pva = new PointValueAdnnotation(); + pointValueId = (Long) resultPointValue[0]; + pva.setPointValueId(pointValueId); + pva.setSourceType(DataTypes.ALPHANUMERIC); + + int lengthShortValue = 128; + if (value.length() <= lengthShortValue) { + pva.setTextPointValueShort(value); + } else { + pva.setTextPointValueLong(value); + } + + new PointValueAdnnotationsDAO().create(pva); + + if (LOG.isTraceEnabled()) { + LOG.trace("save data string:" + dpid); + } } - - new PointValueAdnnotationsDAO().create(pva); - - if (LOG.isTraceEnabled()) { - LOG.trace("save data string:" + dpid); + + boolean dbQueryEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.enabled", false); + if(dbQueryEnabled) { + pvt = new PointValueTime(value, new Date().getTime()); + PointValue pv = new PointValue(); + pv.setPointValue(pvt); + pv.setDataPointId(dpid); + pv.setPointValue(pvt); + + PointValueAdnnotation pva = new PointValueAdnnotation(); + pva.setPointValueId(pointValueId); + pva.setSourceType(DataTypes.ALPHANUMERIC); + + int lengthShortValue = 128; + if (value.length() <= lengthShortValue) { + pva.setTextPointValueShort(value); + } else { + pva.setTextPointValueLong(value); + } + IPointValueDenormalizedDAO.newQueryRespository().create(pv, pva, typePointValueOfREST); } - + } else { new RuntimeException("Unknown value type"); } diff --git a/src/org/scada_lts/dao/pointvalues/PointValueDenormalizedDAO.java b/src/org/scada_lts/dao/pointvalues/PointValueDenormalizedDAO.java new file mode 100644 index 0000000000..4b526952c0 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/PointValueDenormalizedDAO.java @@ -0,0 +1,892 @@ +/* + * (c) 2015 Abil'I.T. http://abilit.eu/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package org.scada_lts.dao.pointvalues; + +import com.serotonin.mango.DataTypes; +import com.serotonin.mango.rt.dataImage.AnnotatedPointValueTime; +import com.serotonin.mango.rt.dataImage.PointValueTime; +import com.serotonin.mango.rt.dataImage.SetPointSource; +import com.serotonin.mango.rt.dataImage.types.*; +import com.serotonin.mango.vo.bean.LongPair; +import com.serotonin.mango.vo.bean.PointHistoryCount; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.scada_lts.dao.DAO; +import org.scada_lts.dao.model.point.PointValue; +import org.scada_lts.dao.model.point.PointValueAdnnotation; +import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.jdbc.core.*; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * + * @author grzegorz bylica Abil'I.T. development team, sdt@abilit.eu + * + */ +@Repository +class PointValueDenormalizedDAO implements IPointValueDenormalizedDAO { + + private static final Log LOG = LogFactory.getLog(PointValueDenormalizedDAO.class); + + private static IPointValueDenormalizedDAO instance; + + private final static String COLUMN_NAME_DATA_TYPE = "dataType"; + private final static String COLUMN_NAME_POINT_VALUE = "pointValue"; + private final static String COLUMN_NAME_TEXT_POINT_VALUE_SHORT = "textPointValueShort"; + private final static String COLUMN_NAME_TEXT_POINT_VALUE_LONG = "textPointValueLong"; + private final static String COLUMN_NAME_SOURCE_TYPE = "sourceType"; + private final static String COLUMN_NAME_TS = "ts"; + private final static String COLUMN_NAME_TIMESTAMP = "timestamp"; + private final static String COLUMN_NAME_SOURCE_ID = "sourceId"; + private final static String COLUMN_NAME_DATA_POINT_ID = "dataPointId"; + private final static String COLUMN_NAME_MIN_TIME_STAMP = "minTs"; + private final static String COLUMN_NAME_MAX_TIME_STAMP = "maxTs"; + private final static String COLUMN_NAME_USERNAME_IN_TABLE_USERS = "username"; + + + // @formatter:off + private static final String POINT_VALUE_SELECT = "" + + "select " + + COLUMN_NAME_DATA_POINT_ID + ", " + + COLUMN_NAME_DATA_TYPE + ", " + + COLUMN_NAME_POINT_VALUE + ", " + + COLUMN_NAME_TS + ", " + + COLUMN_NAME_TEXT_POINT_VALUE_SHORT + ", " + + COLUMN_NAME_TEXT_POINT_VALUE_LONG + ", " + + COLUMN_NAME_SOURCE_TYPE + ", " + + COLUMN_NAME_SOURCE_ID + ", " + + COLUMN_NAME_USERNAME_IN_TABLE_USERS + " " + + "from " + + "pointValuesDenormalized pv "; + + private static final String POINT_VALUE_SELECT_WITH_USERNAME = "" + + "select " + + COLUMN_NAME_DATA_POINT_ID + ", " + + COLUMN_NAME_DATA_TYPE + ", " + + COLUMN_NAME_POINT_VALUE + ", " + + COLUMN_NAME_TS + ", " + + COLUMN_NAME_TEXT_POINT_VALUE_SHORT + ", " + + COLUMN_NAME_TEXT_POINT_VALUE_LONG + ", " + + COLUMN_NAME_SOURCE_TYPE + ", " + + COLUMN_NAME_SOURCE_ID + ", " + + COLUMN_NAME_USERNAME_IN_TABLE_USERS + " " + + "from " + + "pointValuesDenormalized pv "; +/* + private static final String POINT_VALUE_FILTER_BASE_ON_ID = "" + + "pv."+COLUMN_NAME_ID+"=?";*/ + + private static final String POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT = "" + +"select " + +"min(ts) as "+COLUMN_NAME_MIN_TIME_STAMP+", " + +"max(ts) as "+COLUMN_NAME_MAX_TIME_STAMP+" " + +"from " + + "pointValuesDenormalized " + + "where " + COLUMN_NAME_DATA_POINT_ID + " in (:ids)"; + + private static final String POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT = "" + +"select " + +"min(ts) as "+ COLUMN_NAME_MIN_TIME_STAMP + " " + +"from " + + "pointValuesDenormalized " + + "where " + + COLUMN_NAME_DATA_POINT_ID + " in (:ids)"; + + private static final String POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT = "" + +"select " + + "max(ts) as "+ COLUMN_NAME_MAX_TIME_STAMP+" " + +"from " + + "pointValuesDenormalized " + + "where " + + COLUMN_NAME_DATA_POINT_ID + " in (:ids)"; + + private static final String POINT_VALUE_SELECT_MAX_BASE_ON_DATA_POINT_ID = "" + +"select " + +"max(ts) as " + COLUMN_NAME_MAX_TIME_STAMP + " " + +"from " + + "pointValuesDenormalized " + +"where " + + COLUMN_NAME_DATA_POINT_ID + "=?"; + +/* + private static final String POINT_VALUE_SELECT_ON_BASE_ID = "" + + POINT_VALUE_SELECT + + " where " + + COLUMN_NAME_ID + "=?";*/ + + private static final String POINT_VALUE_SELECT_ON_BASE_ID_TS = "" + + POINT_VALUE_SELECT + + " where " + + COLUMN_NAME_DATA_POINT_ID + "=? and " + + COLUMN_NAME_TS +"=?"; + + private static final String POINT_VALUE_DENORMALIZED_INSERT = "" + + "insert into pointValuesDenormalized (" + + ""+COLUMN_NAME_DATA_POINT_ID + ", " + + ""+COLUMN_NAME_DATA_TYPE + ", " + + ""+COLUMN_NAME_POINT_VALUE + ", " + + ""+COLUMN_NAME_TS + ", " + + ""+COLUMN_NAME_TIMESTAMP + ", " + + ""+COLUMN_NAME_TEXT_POINT_VALUE_SHORT + ", " + + ""+COLUMN_NAME_TEXT_POINT_VALUE_LONG + ", " + + ""+COLUMN_NAME_SOURCE_TYPE + ", " + + ""+COLUMN_NAME_SOURCE_ID + ", " + + ""+COLUMN_NAME_USERNAME_IN_TABLE_USERS + +") " + + "values (?,?,?,?,?,?,?,?,?,?)"; + + private static final String POINT_VALUE_INCEPTION_DATA = " " + +"select " + + "min(ts) " + + "from " + + "pointValuesDenormalized " + + "where " + + COLUMN_NAME_DATA_POINT_ID+"=?"; + + private static final String POINT_VALUE_DATA_RANGE_COUNT = " " + +"select " + + "count(*) " + + "from " + + "pointValuesDenormalized " + + "where " + + COLUMN_NAME_DATA_POINT_ID+"=? and "+ COLUMN_NAME_TS +">=? and "+ COLUMN_NAME_TS +"<=?"; + + public static final String POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP = " " + + "pv."+COLUMN_NAME_DATA_POINT_ID+"=? and " + + "pv."+ COLUMN_NAME_TS +" >= ? order by "+ COLUMN_NAME_TS; + + public static final String POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP_FROM_TO = " " + + "pv."+COLUMN_NAME_DATA_POINT_ID+"=? and " + + "pv."+ COLUMN_NAME_TS +">=? and pv."+ COLUMN_NAME_TS +"?"; + + private static final String SELECT_MAX_TIME_WHERE_DATA_POINT_ID = "" + + "select max(" + + COLUMN_NAME_TS + ") " + + "from pointValuesDenormalized where " + + COLUMN_NAME_DATA_POINT_ID + "=? "; + + private static final String SELECT_MIN_TIME_WHERE_DATA_POINT_ID = "" + + "select min(" + + COLUMN_NAME_TS + ") " + + "from pointValuesDenormalized where " + + COLUMN_NAME_DATA_POINT_ID + "=? "; + + private static final String SELECT_DP_ID_AND_COUNT = "" + + "select " + + COLUMN_NAME_DATA_POINT_ID + ", " + + "count(*) from pointValuesDenormalized group by " + + COLUMN_NAME_DATA_POINT_ID + " " + + "order by 2 desc "; + + // @formatter:on + + //RowMappers + private class PointValueRowMapper implements RowMapper { + public PointValue mapRow(ResultSet rs, int rowNum) throws SQLException { + //TODO rewrite MangoValue + MangoValue value = createMangoValue(rs); + long time = rs.getLong(COLUMN_NAME_TS); + + PointValue pv = new PointValue(); + //pv.setId(rs.getLong(COLUMN_NAME_ID)); + pv.setDataPointId(rs.getInt(COLUMN_NAME_DATA_POINT_ID)); + int sourceId = rs.getInt(COLUMN_NAME_SOURCE_ID); + + int sourceType = rs.getInt(COLUMN_NAME_SOURCE_TYPE); + + if (rs.wasNull()) { + pv.setPointValue(new PointValueTime(value, time)); + } else { + pv.setPointValue(new AnnotatedPointValueTime(value, time, sourceType, sourceId)); + } + return pv; + } + } + public PointValue getPointValueRow(ResultSet rs, int rowNum){ + try { + return new PointValueRowMapperWithUserName().mapRow(rs, rowNum); + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + private class PointValueRowMapperWithUserName implements RowMapper { + + public PointValue mapRow(ResultSet rs, int rowNum) throws SQLException { + //TODO rewrite MangoValue + MangoValue value = createMangoValue(rs); + long time = rs.getLong(COLUMN_NAME_TS); + + PointValue pv = new PointValue(); + //pv.setId(rs.getLong(COLUMN_NAME_ID)); + pv.setDataPointId(rs.getInt(COLUMN_NAME_DATA_POINT_ID)); + int sourceId = rs.getInt(COLUMN_NAME_SOURCE_ID); + + int sourceType = rs.getInt(COLUMN_NAME_SOURCE_TYPE); + int username = -1; + String userName = null; + try{ + username = rs.getInt(COLUMN_NAME_USERNAME_IN_TABLE_USERS); + } + catch (Exception e){ + + userName = rs.getString(COLUMN_NAME_USERNAME_IN_TABLE_USERS); + } + + pv.setPointValue( rs.wasNull() + ? createPointValueTime( userName, value, time, username ) + : createAnnotatedPointValueTime( value,time, sourceType, sourceId, userName )); + + return pv; + } + private AnnotatedPointValueTime createAnnotatedPointValueTime(MangoValue value, long time, int sourceType, int sourceId, String userName){ + return ( userName!= null ) + ?new AnnotatedPointValueTime(value, time, sourceType, sourceId, userName) + :new AnnotatedPointValueTime(value, time, sourceType, sourceId); + } + private PointValueTime createPointValueTime(String userName, MangoValue value, long time, int username ){ + + if(userName!= null){ + return new PointValueTime(value, time,userName); + } + else if( username!=0 ){ + return new PointValueTime(value, time,String.valueOf(username)); + } + else { + return new PointValueTime(value, time); + } + + } + } + + private class LongPairRowMapper implements RowMapper { + public LongPair mapRow(ResultSet rs, int index) throws SQLException { + long myLongValue = rs.getLong(COLUMN_NAME_MIN_TIME_STAMP); + if (rs.wasNull()) + return null; + return new LongPair(myLongValue, rs.getLong(COLUMN_NAME_MAX_TIME_STAMP)); + } + } + + private class LongRowMapper implements RowMapper { + public Long mapRow(ResultSet rs, int index) throws SQLException { + return rs.getLong(COLUMN_NAME_MIN_TIME_STAMP); + } + } + + //TODO rewrite for new types + MangoValue createMangoValue(ResultSet rs) + throws SQLException { + + int dataType = rs.getInt(COLUMN_NAME_DATA_TYPE); + MangoValue value = null; + switch (dataType) { + case (DataTypes.NUMERIC): + value = new NumericValue(rs.getDouble(COLUMN_NAME_POINT_VALUE)); + break; + case (DataTypes.BINARY): + value = new BinaryValue(rs.getDouble(COLUMN_NAME_POINT_VALUE)==1); + break; + case (DataTypes.MULTISTATE): + value = new MultistateValue(rs.getInt(COLUMN_NAME_POINT_VALUE)); + break; + case (DataTypes.ALPHANUMERIC): + String s = rs.getString(COLUMN_NAME_TEXT_POINT_VALUE_SHORT); + if (s == null) + s = rs.getString(COLUMN_NAME_TEXT_POINT_VALUE_LONG); + value = new AlphanumericValue(s); + break; + case (DataTypes.IMAGE): { + try { + value = new ImageValue(Integer.parseInt(rs + .getString(COLUMN_NAME_TEXT_POINT_VALUE_SHORT)), + rs.getInt(COLUMN_NAME_TEXT_POINT_VALUE_LONG)); + } catch (NumberFormatException e) { + LOG.info(e.getMessage()); + } + break; + } + default: + value = null; + } + return value; + } + + public static IPointValueDenormalizedDAO getInstance() { + if (instance == null) { + instance = new PointValueDenormalizedDAO(DAO.query().getJdbcTemp()); + } + return instance; + } + + private JdbcOperations jdbcTemplate; + + public PointValueDenormalizedDAO() { + this.jdbcTemplate = DAO.getInstance().getJdbcTemp(); + } + + public PointValueDenormalizedDAO(JdbcOperations jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public List findAllWithUserName(){ + return (List) jdbcTemplate.query(POINT_VALUE_SELECT_WITH_USERNAME, new Object[]{ }, new PointValueRowMapper()); + } + @Override + public List findAll() { + + return (List) jdbcTemplate.query(POINT_VALUE_SELECT, new Object[]{ }, new PointValueRowMapper()); + } +/* + @Override + public PointValue findById(Object[] pk) { + return (PointValue) jdbcTemplate.queryForObject(POINT_VALUE_SELECT_ON_BASE_ID, pk, new PointValueRowMapper()); + }*/ + + public List findByIdAndTs(long id, long ts) { + return jdbcTemplate.query(POINT_VALUE_SELECT_ON_BASE_ID_TS, new Object[] { id,ts }, new PointValueRowMapper()); + } + + @Override + public List filtered(String filter, Object[] argsFilter, long limit) { + String myLimit=""; + if (limit != NO_LIMIT) { + //TODO rewrite limit adding in argsFilter + myLimit = LIMIT+limit; + } + List res = + (List) jdbcTemplate.query( + "select " + + "pv."+COLUMN_NAME_DATA_POINT_ID + ", " + + "pv."+COLUMN_NAME_DATA_TYPE + ", " + + "pv."+COLUMN_NAME_POINT_VALUE + ", " + + "pv."+ COLUMN_NAME_TS + ", " + + "pv."+COLUMN_NAME_TEXT_POINT_VALUE_SHORT + ", " + + "pv."+COLUMN_NAME_TEXT_POINT_VALUE_LONG + ", " + + "pv."+COLUMN_NAME_SOURCE_TYPE + ", " + + "pv."+COLUMN_NAME_SOURCE_ID + ", " + + "pv." +COLUMN_NAME_USERNAME_IN_TABLE_USERS + + " from " + + "pointValuesDenormalized pv where "+ filter + myLimit, argsFilter, new PointValueRowMapperWithUserName()); + return res; + } + + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + @Override + public Object[] create(final PointValue entity) { + + if (LOG.isTraceEnabled()) { + LOG.trace(entity); + } + + KeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.update(new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection connection) throws SQLException { + PreparedStatement ps = connection.prepareStatement(POINT_VALUE_DENORMALIZED_INSERT, Statement.RETURN_GENERATED_KEYS); + new ArgumentPreparedStatementSetter( new Object[] { + entity.getDataPointId(), + entity.getPointValue().getValue().getDataType(), + getValueBaseOnType(entity.getPointValue().getValue().getDataType(), entity.getPointValue()), + entity.getPointValue().getTime(), + new Date(entity.getPointValue().getTime()), + null, + null, + null, + null, + null + }).setValues(ps); + return ps; + } + }, keyHolder); + + return new Object[] {keyHolder.getKey().longValue()}; + + } + + + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + public Object[] create(final int pointId,final int dataType,final double dvalue,final long time) { + + return createNoTransaction(pointId, dataType, dvalue, time); + + } + + @Override + public void create(PointValue pointValue, PointValueAdnnotation pointValueAdnnotation, int dataType) { + if (LOG.isTraceEnabled()) { + LOG.trace("pointId:"+pointValue.getDataPointId()+" dataType:"+pointValue.getPointValue() + .getValue().getDataType()+" time:"+pointValue.getPointValue().getTime()); + } + + PointValueTime pointValueTime = pointValue.getPointValue(); + MangoValue mangoValue = pointValueTime.getValue(); + + jdbcTemplate.update(connection -> { + PreparedStatement ps = connection.prepareStatement(POINT_VALUE_DENORMALIZED_INSERT, Statement.RETURN_GENERATED_KEYS); + new ArgumentPreparedStatementSetter( new Object[] { + pointValue.getDataPointId(), + mangoValue.getDataType(), + getValueBaseOnType(mangoValue.getDataType(), pointValueTime), + pointValueTime.getTime(), + new Date(pointValueTime.getTime()), + pointValueAdnnotation.getTextPointValueShort(), + pointValueAdnnotation.getTextPointValueLong(), + pointValueAdnnotation.getSourceType(), + pointValueAdnnotation.getSourceId(), + pointValueAdnnotation.getChangeOwner() + }).setValues(ps); + return ps; + }); + } + + + public Object[] createNoTransaction(final int pointId,final int dataType,final double dvalue,final long time) { + + if (LOG.isTraceEnabled()) { + LOG.trace("pointId:"+pointId+" dataType:"+dataType+" dvalue:"+time); + } + + KeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.update(new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection connection) throws SQLException { + PreparedStatement ps = connection.prepareStatement(POINT_VALUE_DENORMALIZED_INSERT, Statement.RETURN_GENERATED_KEYS); + new ArgumentPreparedStatementSetter( new Object[] { + pointId, + dataType, + dvalue, + time, + new Date(time), + null, + null, + null, + null, + null + }).setValues(ps); + return ps; + } + }, keyHolder); + + try { + return new Object[]{keyHolder.getKey().longValue()}; + } catch (InvalidDataAccessApiUsageException ex) { + return new Object[]{Long.parseLong(String.valueOf(keyHolder.getKeys().get("id")))}; + } + + } + + @Override + public long deletePointValuesBeforeWithOutLastTwo(int dataPointId, long time) { + return 0; + } + + @Override + public long deletePointValuesWithValueLimit(int dataPointId, int limit) { + return 0; + } + + @Override + public void createTableForDatapoint(int pointId) { + + } + + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + public void executeBatchUpdateInsert( List params) { + + if (LOG.isTraceEnabled()) { + for (Object[] param : params) { + for (Object arg :param) { + LOG.trace("arg:"+arg); + } + } + } + if(params.isEmpty()) { + return; + } + if(params.get(0).length == 4) { + List paramsExt = new ArrayList<>(); + for (Object[] args : params) { + paramsExt.add(new Object[]{args[0], args[1], args[2], args[3], new Date((long)args[3]), null, null, null, null, null}); + } + jdbcTemplate.batchUpdate(POINT_VALUE_DENORMALIZED_INSERT,paramsExt); + } else + jdbcTemplate.batchUpdate(POINT_VALUE_DENORMALIZED_INSERT,params); + + } + + public Long getInceptionDate(int dataPointId) { + return jdbcTemplate.queryForObject(POINT_VALUE_INCEPTION_DATA, new Object[] {dataPointId}, Long.class); + } + + public long dateRangeCount(int dataPointId, long from, long to) { + return jdbcTemplate.queryForObject(POINT_VALUE_DATA_RANGE_COUNT, new Object[] {dataPointId, from, to}, Long.class); + } + + public LongPair getStartAndEndTime(List dataPointIds) { + if (dataPointIds.isEmpty()) return null; + + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("ids",dataPointIds); + + LongPair longPair = jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT,new LongPairRowMapper(),parameters); + + return longPair; + } + + public long getStartTime(List dataPointIds) { + if (dataPointIds.isEmpty()) return -1; + + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("ids",dataPointIds); + + Long minTs = jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT,new LongRowMapper(),parameters); + + return minTs; + } + + public long getEndTime(List dataPointIds) { + if (dataPointIds.isEmpty()) return -1; + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("ids",dataPointIds); + + Long maxTs = jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT, new LongRowMapper(),parameters); + + return maxTs; + + } + + public List getFiledataIds() { + //TODO rewrite + return jdbcTemplate.queryForList( + "select distinct id from ( " // + + " select id as id from pointValues where dataType=" + + DataTypes.IMAGE + + " union" + + " select d.pointValueId as id from reportInstanceData d " + + " join reportInstancePoints p on d.reportInstancePointId=p.id" + + " where p.dataType=" + + DataTypes.IMAGE + + ") a order by 1", new Object[] {}, Long.class); + } + + public Long getLatestPointValue(int dataPointId) { + try { + return jdbcTemplate.queryForObject(POINT_VALUE_SELECT_MAX_BASE_ON_DATA_POINT_ID, new Object[] {dataPointId}, Long.class); + } catch (EmptyResultDataAccessException e) { + return null; + } + } + + /** + * When save value ALPHANUMERIC or IMAGE then data save in adnnotations + * @param dataType + * @param value + * @return + */ + private double getValueBaseOnType(int dataType, PointValueTime value) { + Double avalue = null; + if ( (dataType==DataTypes.ALPHANUMERIC) || (dataType==DataTypes.IMAGE) || value == null) { + avalue = 0.0; + } else { + avalue = value.getDoubleValue(); + } + return avalue; + } + + // Apply database specific bounds on double values. + public double applyBounds(double value) { + if (Double.isNaN(value)) + return 0; + if (value == Double.POSITIVE_INFINITY) + return Double.MAX_VALUE; + if (value == Double.NEGATIVE_INFINITY) + return -Double.MAX_VALUE; + + return value; + } + + public long deletePointValuesBeforeWithOutLast(int dataPointId, long time) { + Long lastId = jdbcTemplate.queryForObject(POINT_VALUE_ID_OF_LAST_VALUE, new Object[] { dataPointId }, Long.class ); + return jdbcTemplate.update(POINT_VALUE_DELETE_BEFORE, new Object[] {dataPointId, time, lastId}); + } + + public long deletePointValue(int dataPointId) { + return jdbcTemplate.update(POINT_VALUE_DELETE_BASE_ON_POINT_ID, new Object[] {dataPointId}); + } + + public long deleteAllPointData() { + return jdbcTemplate.update(POINT_VALUE_DELETE, new Object[] {}); + } + + public long deletePointValuesWithMismatchedType(int dataPointId, int dataType) { + return jdbcTemplate.update(DELETE_POINT_VALUE_WITH_MISMATCHED_TYPE, new Object[] {dataPointId, dataType}); + } + /* + public PointValueTime getPointValue(long id) { + return ((PointValue) jdbcTemplate.queryForObject(POINT_VALUE_SELECT + " where " + POINT_VALUE_FILTER_BASE_ON_ID, new Object[] {id}, new PointValueRowMapper())).getPointValue(); + }*/ + + public long getMinTs(int dataPointId) { + return jdbcTemplate.queryForObject(SELECT_MIN_TIME_WHERE_DATA_POINT_ID, new Object[] {dataPointId}, Long.TYPE); + } + + public long getMaxTs(int dataPointId) { + return jdbcTemplate.queryForObject(SELECT_MAX_TIME_WHERE_DATA_POINT_ID, new Object[] {dataPointId}, Long.TYPE); + } + + public List getTopPointHistoryCounts() { + return jdbcTemplate.query(SELECT_DP_ID_AND_COUNT, new RowMapper() { + @Override + public PointHistoryCount mapRow(ResultSet rs, int rowNum) throws SQLException { + PointHistoryCount phc = new PointHistoryCount(); + phc.setPointId(rs.getInt(1)); + phc.setCount(rs.getInt(2)); + return phc; + } + }); + } + + // @formatter:off + private static final String POINT_VALUE_ADNNOTATIONS_SELECT = "" + + "select " + // +COLUMN_NAME_POINT_VALUE_ID + ", " + +COLUMN_NAME_TEXT_POINT_VALUE_SHORT + ", " + +COLUMN_NAME_TEXT_POINT_VALUE_LONG + ", " + +COLUMN_NAME_SOURCE_TYPE + ", " + +COLUMN_NAME_SOURCE_ID + " " + + "from " + +"pointValuesDenormalized "; + private static final String PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE = "pv."; + private static final String PREFIX_NAME_FOR_USERS_TABLE = "us."; + private static final String POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE = "" + + "select " + //+PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_POINT_VALUE_ID + ", " + +PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_TEXT_POINT_VALUE_SHORT + ", " + +PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_TEXT_POINT_VALUE_LONG + ", " + +PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_SOURCE_TYPE + ", " + +PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_SOURCE_ID + ", " + +PREFIX_NAME_FOR_USERS_TABLE+"username " + + "from " + +"pointValuesDenormalized"; + private static final String POINT_VALUE_ADNNOTATIONS_INSERT = "" + + "insert into pointValuesDenormalized (" + //+ COLUMN_NAME_POINT_VALUE_ID + "," + + COLUMN_NAME_TEXT_POINT_VALUE_SHORT + "," + + COLUMN_NAME_TEXT_POINT_VALUE_LONG + "," + + COLUMN_NAME_SOURCE_TYPE + "," + + COLUMN_NAME_SOURCE_ID + +") " + + "values (?,?,?,?)"; + + private static final String POINT_VALUE_ANNOTATIONS_UPDATE = "" + + "update pointValuesDenormalized set " + + COLUMN_NAME_SOURCE_ID + "=null " + + "where " + + COLUMN_NAME_SOURCE_ID + "=? " + + "and " + + COLUMN_NAME_SOURCE_TYPE + "=" + + SetPointSource.Types.USER; +/* + public static final String POINT_VALUE_ADNNOTATIONS_FILTER_BASE_ON_POINT_VALUES_ID = " " + +COLUMN_NAME_POINT_VALUE_ID+"=? "; +*/ + // @formatter:on + + //RowMapper + private class PointValueAdnnotationRowMapper implements RowMapper { + + public PointValueAdnnotation mapRow(ResultSet rs, int rowNum) throws SQLException { + PointValueAdnnotation pva = new PointValueAdnnotation(); + + //pva.setPointValueId(rs.getLong(COLUMN_NAME_POINT_VALUE_ID)); + pva.setTextPointValueShort(rs.getString(COLUMN_NAME_TEXT_POINT_VALUE_SHORT)); + pva.setTextPointValueLong(rs.getString(COLUMN_NAME_TEXT_POINT_VALUE_LONG)); + pva.setSourceType(rs.getInt(COLUMN_NAME_SOURCE_TYPE)); + pva.setSourceId(rs.getLong(COLUMN_NAME_SOURCE_ID)); + return pva; + } + } + private class PointValueAdnnotationRowMapperWithAdnnotationAboutChangeOwner implements RowMapper { + + public PointValueAdnnotation mapRow(ResultSet rs, int rowNum) throws SQLException { + PointValueAdnnotation pva = new PointValueAdnnotation(); + + //pva.setPointValueId(rs.getLong(PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_POINT_VALUE_ID)); + pva.setTextPointValueShort(rs.getString(PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_TEXT_POINT_VALUE_SHORT)); + pva.setTextPointValueLong(rs.getString(PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_TEXT_POINT_VALUE_LONG)); + pva.setSourceType(rs.getInt(PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_SOURCE_TYPE)); + pva.setSourceId(rs.getLong(PREFIX_NAME_FOR_POINTVALUEADNNOTATION_TABLE+COLUMN_NAME_SOURCE_ID)); + pva.setChangeOwner(rs.getString(PREFIX_NAME_FOR_USERS_TABLE+"username")); + return pva; + } + } + + @Override + public List findAllWithUserNamePointValueAdnnotations() { + return (List) jdbcTemplate.query(POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE, new Object[]{ }, new PointValueAdnnotationRowMapperWithAdnnotationAboutChangeOwner()); + + } + + @Override + public List findAllPointValueAdnnotations() { + return (List) jdbcTemplate.query(POINT_VALUE_ADNNOTATIONS_SELECT_WITH_OWNER_OF_CHANGE/*POINT_VALUE_ADNNOTATIONS_SELECT*/, new Object[]{ }, new PointValueAdnnotationRowMapper()); + } + + /* + * @arguments id - idPointValue + * @see org.scada_lts.dao.GenericDaoCR#findById(long) + */ + /*@Override + public PointValueAdnnotation findByIdPointValueAdnnotation(Object[] pk) { + return (PointValueAdnnotation) jdbcTemplate.queryForObject(POINT_VALUE_ADNNOTATIONS_SELECT +" where "+POINT_VALUE_ADNNOTATIONS_FILTER_BASE_ON_POINT_VALUES_ID, pk, new PointValueAdnnotationRowMapper()); + }*/ + + @Override + public List filteredPointValueAdnnotations(String filter, Object[] argsFilter, long limit) { + String myLimit=""; + if (limit != NO_LIMIT) { + myLimit = LIMIT+limit; + } + return (List) jdbcTemplate.query(POINT_VALUE_ADNNOTATIONS_SELECT+" where "+ filter + myLimit, argsFilter, new PointValueAdnnotationRowMapper()); + } + + + @Override + public Object[] create(final PointValueAdnnotation entity) { + if (LOG.isTraceEnabled()) { + LOG.trace(entity); + } + + jdbcTemplate.update(POINT_VALUE_ADNNOTATIONS_INSERT, new Object[] { + entity.getPointValueId(), + entity.getPointValueId(), + entity.getTextPointValueShort(), + entity.getTextPointValueLong(), + entity.getSourceType(), + entity.getSourceId() + }); + + + // table hav'nt pk //TODO add key + return new Object[] {0}; + } + + @Override + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + public void update(int userId) { + + if (LOG.isTraceEnabled()) { + LOG.trace("update(int userId) userId:" + userId); + } + + jdbcTemplate.update(POINT_VALUE_ANNOTATIONS_UPDATE, new Object[]{userId}); + } + + @Override + public void updateAnnotations(List values) { + Map> userIds = new HashMap>(); + List alist; + + // Look for annotated point values. + AnnotatedPointValueTime apv; + for (PointValue pv : values) { + if (pv.getPointValue() instanceof AnnotatedPointValueTime) { + apv = (AnnotatedPointValueTime) pv.getPointValue(); + if (apv.getSourceType() == SetPointSource.Types.USER) { + alist = userIds.get(apv.getSourceId()); + if (alist == null) { + alist = new ArrayList(); + userIds.put(apv.getSourceId(), alist); + } + alist.add(apv); + } + } + } + } + +} \ No newline at end of file diff --git a/src/org/scada_lts/dao/pointvalues/PointValueQuestDbDAO.java b/src/org/scada_lts/dao/pointvalues/PointValueQuestDbDAO.java new file mode 100644 index 0000000000..406cf08ec2 --- /dev/null +++ b/src/org/scada_lts/dao/pointvalues/PointValueQuestDbDAO.java @@ -0,0 +1,732 @@ +/* + * (c) 2015 Abil'I.T. http://abilit.eu/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package org.scada_lts.dao.pointvalues; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.serotonin.mango.DataTypes; +import com.serotonin.mango.rt.dataImage.AnnotatedPointValueTime; +import com.serotonin.mango.rt.dataImage.PointValueTime; +import com.serotonin.mango.rt.dataImage.types.*; +import com.serotonin.mango.vo.DataPointVO; +import com.serotonin.mango.vo.bean.LongPair; +import com.serotonin.mango.vo.bean.PointHistoryCount; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.json.JSONException; +import org.json.JSONObject; +import org.scada_lts.dao.DAO; +import org.scada_lts.dao.DataPointDAO; +import org.scada_lts.dao.model.point.PointValue; +import org.scada_lts.dao.model.point.PointValueAdnnotation; +import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.jdbc.core.*; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.*; +import java.util.*; +import java.util.stream.Collectors; + + +@Repository +public class PointValueQuestDbDAO implements IPointValueQuestDbDAO { + + private static final Log LOG = LogFactory.getLog(PointValueQuestDbDAO.class); + + private static IPointValueQuestDbDAO instance; + + private final static String COLUMN_NAME_DATA_TYPE = "dataType"; + private final static String COLUMN_NAME_POINT_VALUE = "pointValue"; + private final static String COLUMN_NAME_SOURCE_TYPE = "sourceType"; + private final static String COLUMN_NAME_TS = "ts"; + private final static String COLUMN_NAME_TIMESTAMP = "timestamp"; + private final static String COLUMN_NAME_SOURCE_ID = "sourceId"; + private final static String COLUMN_NAME_MIN_TIME_STAMP = "minTs"; + private final static String COLUMN_NAME_MAX_TIME_STAMP = "maxTs"; + private final static String COLUMN_NAME_USERNAME_IN_TABLE_USERS = "username"; + + private final static String TABLE_POINT_VALUES = "pointValues$dpId"; + private final static String COLUMN_NAME_META_DATA = "metaData"; + + + // @formatter:off + private static final String POINT_VALUE_SELECT = "" + + "select " + + COLUMN_NAME_POINT_VALUE + ", " + + COLUMN_NAME_TS + ", " + + COLUMN_NAME_META_DATA + ", " + + COLUMN_NAME_DATA_TYPE + " " + + "from " + + TABLE_POINT_VALUES + " "; + + private static final String UNION_ALL = " union all "; + + private static final String POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT = "" + +"select " + +"min(ts) as "+COLUMN_NAME_MIN_TIME_STAMP+", " + +"max(ts) as "+COLUMN_NAME_MAX_TIME_STAMP+" " + +"from " + + TABLE_POINT_VALUES + " "; + + private static final String POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT = "" + +"select " + +"min(ts) as "+ COLUMN_NAME_MIN_TIME_STAMP + " " + +"from " + + TABLE_POINT_VALUES + " "; + + private static final String POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT = "" + +"select " + + "max(ts) as "+ COLUMN_NAME_MAX_TIME_STAMP+" " + +"from " + + TABLE_POINT_VALUES + " "; + + private static final String POINT_VALUE_SELECT_MAX_BASE_ON_DATA_POINT_ID = "" + +"select " + +"max(ts) as " + COLUMN_NAME_MAX_TIME_STAMP + " " + +"from " + + TABLE_POINT_VALUES + " "; + + private static final String POINT_VALUE_SELECT_ON_BASE_ID_TS = "" + + POINT_VALUE_SELECT + + " where " + + COLUMN_NAME_TIMESTAMP +"=to_timezone(?, 'CET')"; + + private static final String POINT_VALUE_INSERT = "" + + "insert into " + + TABLE_POINT_VALUES + " (" + + COLUMN_NAME_POINT_VALUE + ", " + + COLUMN_NAME_TS + ", " + + COLUMN_NAME_TIMESTAMP + ", " + + COLUMN_NAME_META_DATA + ", " + + COLUMN_NAME_DATA_TYPE + +") " + + "values (?,?,to_timezone(?, 'CET'),?,?)"; + + private static final String POINT_VALUE_INCEPTION_DATA = " " + +"select " + + "min(ts) " + + "from " + + TABLE_POINT_VALUES + " "; + + private static final String POINT_VALUE_DATA_RANGE_COUNT = " " + +"select " + + "count(*) " + + "from " + + TABLE_POINT_VALUES + " " + + "where " + + COLUMN_NAME_TS +">=to_timezone(?, 'CET') and "+ COLUMN_NAME_TS +"<=to_timezone(?, 'CET')"; + + private static final String SELECT_MAX_TIME_WHERE_DATA_POINT_ID = "" + + "select max(" + + COLUMN_NAME_TS + ") " + + "from " + TABLE_POINT_VALUES + " "; + + private static final String SELECT_MIN_TIME_WHERE_DATA_POINT_ID = "" + + "select min(" + + COLUMN_NAME_TS + ") " + + "from " + TABLE_POINT_VALUES + " "; + + private static final String DROP_PARTITION = "" + + "alter table " + + TABLE_POINT_VALUES + + " drop partition "; + + private static final String WHERE_TIMESTAMP = " WHERE " + COLUMN_NAME_TIMESTAMP + " < "; + + private static final String TO_TIMEZONE = "" + + "to_timezone($timestamp, 'CET')"; + + public static final String POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP = "where " + + "pv."+COLUMN_NAME_TIMESTAMP+" >= to_timezone($from, 'CET') order by "+COLUMN_NAME_TIMESTAMP; + + public static final String POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP_FROM_TO = "where " + + "pv."+COLUMN_NAME_TIMESTAMP+">=to_timezone($from, 'CET') and pv."+COLUMN_NAME_TIMESTAMP+" { + private long dataPointId; + + public PointValueRowMapper(long dataPointId) { + this.dataPointId = dataPointId; + } + + public PointValue mapRow(ResultSet rs, int rowNum) throws SQLException { + //TODO rewrite MangoValue + JSONObject metaData = null; + try { + metaData = new JSONObject(rs.getString(COLUMN_NAME_META_DATA)); + + } catch (JSONException e) { + e.printStackTrace(); + } + + MangoValue value = createMangoValue(rs); + + long time = rs.getLong(COLUMN_NAME_TS); + + PointValue pv = new PointValue(); + pv.setDataPointId(this.dataPointId); + int sourceId = metaData.optInt(COLUMN_NAME_SOURCE_ID); + + int sourceType = metaData.optInt(COLUMN_NAME_SOURCE_TYPE); + + if (sourceType == 0) { + pv.setPointValue(new PointValueTime(value, time)); + } else { + pv.setPointValue(new AnnotatedPointValueTime(value, time, sourceType, sourceId)); + } + return pv; + } + } + + private class PointValueRowMapperWithUserName implements RowMapper { + + private long dataPointId; + + public PointValueRowMapperWithUserName(long dataPointId) { + this.dataPointId = dataPointId; + } + + public PointValue mapRow(ResultSet rs, int rowNum) throws SQLException { + //TODO rewrite MangoValue + JSONObject metaData = null; + try { + metaData = new JSONObject(rs.getString(COLUMN_NAME_META_DATA)); + } catch (JSONException e) { + e.printStackTrace(); + } + + MangoValue value = createMangoValue(rs); + long time = rs.getLong(COLUMN_NAME_TS); + + PointValue pv = new PointValue(); + pv.setDataPointId(this.dataPointId); + int sourceId = metaData.optInt(COLUMN_NAME_SOURCE_ID); + + int sourceType = metaData.optInt(COLUMN_NAME_SOURCE_TYPE); + int username = -1; + String userName = null; + try{ + username = metaData.getInt(COLUMN_NAME_USERNAME_IN_TABLE_USERS); + } + catch (Exception e){ + + userName = metaData.optString(COLUMN_NAME_USERNAME_IN_TABLE_USERS); + } + + pv.setPointValue( sourceType == 0 + ? createPointValueTime( userName, value, time, username ) + : createAnnotatedPointValueTime( value,time, sourceType, sourceId, userName )); + + return pv; + } + private AnnotatedPointValueTime createAnnotatedPointValueTime(MangoValue value, long time, int sourceType, int sourceId, String userName){ + return ( userName!= null ) + ?new AnnotatedPointValueTime(value, time, sourceType, sourceId, userName) + :new AnnotatedPointValueTime(value, time, sourceType, sourceId); + } + private PointValueTime createPointValueTime(String userName, MangoValue value, long time, int username ){ + + if(userName!= null){ + return new PointValueTime(value, time,userName); + } + else if( username!=0 ){ + return new PointValueTime(value, time,String.valueOf(username)); + } + else { + return new PointValueTime(value, time); + } + } + } + + private class LongPairRowMapper implements RowMapper { + public LongPair mapRow(ResultSet rs, int index) throws SQLException { + long myLongValue = rs.getLong(COLUMN_NAME_MIN_TIME_STAMP); + if (rs.wasNull()) + return null; + return new LongPair(myLongValue, rs.getLong(COLUMN_NAME_MAX_TIME_STAMP)); + } + } + + private class LongRowMapper implements RowMapper { + public Long mapRow(ResultSet rs, int index) throws SQLException { + return rs.getLong(COLUMN_NAME_MIN_TIME_STAMP); + } + } + + //TODO rewrite for new types + MangoValue createMangoValue(ResultSet rs) + throws SQLException{ + int dataType = rs.getInt(COLUMN_NAME_DATA_TYPE); + MangoValue value = null; + switch (dataType) { + case (DataTypes.NUMERIC): + value = new NumericValue(rs.getDouble(COLUMN_NAME_POINT_VALUE)); + break; + case (DataTypes.BINARY): + value = new BinaryValue(rs.getDouble(COLUMN_NAME_POINT_VALUE)==1); + break; + case (DataTypes.MULTISTATE): + value = new MultistateValue(rs.getInt(COLUMN_NAME_POINT_VALUE)); + break; + case (DataTypes.ALPHANUMERIC): + value = new AlphanumericValue(rs.getString(COLUMN_NAME_POINT_VALUE)); + break; + case (DataTypes.IMAGE): { + try { + value = new ImageValue(rs.getInt(COLUMN_NAME_POINT_VALUE), + 0); + } catch (NumberFormatException e) { + LOG.info(e.getMessage()); + } + break; + } + default: + value = null; + } + return value; + } + + public static IPointValueQuestDbDAO getInstance() { + if (instance == null) { + instance = new PointValueQuestDbDAO(DAO.query().getJdbcTemp()); + } + return instance; + } + + private JdbcOperations jdbcTemplate; + + public PointValueQuestDbDAO() { + this.jdbcTemplate = DAO.getInstance().getJdbcTemp(); + } + + public PointValueQuestDbDAO(JdbcOperations jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } +/* + @Override + public PointValue findById(Object[] pk) { + return (PointValue) jdbcTemplate.queryForObject(POINT_VALUE_SELECT_ON_BASE_ID, pk, new PointValueRowMapper()); + }*/ + + public List findByIdAndTs(long id, long ts) { + String query = POINT_VALUE_SELECT_ON_BASE_ID_TS.replace(DATAPOINT_ID, String.valueOf(id)); + return jdbcTemplate.query(query, new Object[] { ts }, new PointValueRowMapper(id)); + } + + @Override + public List filtered(String filter, Object[] argsFilter, long limit) { + long dpId = ((Number) argsFilter[0]).longValue(); + String myLimit=""; + boolean reverseOrder = filter.equals(POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID); + if (limit != NO_LIMIT) { + //TODO rewrite limit adding in argsFilter + if (reverseOrder) { + limit *= (-1); + } + myLimit = LIMIT+limit; + } + List res = + (List) jdbcTemplate.query( + "select " + + "pv."+COLUMN_NAME_POINT_VALUE + ", " + + "pv."+ COLUMN_NAME_TS + ", " + + "pv." + COLUMN_NAME_META_DATA + ", " + + "pv."+ COLUMN_NAME_DATA_TYPE + + " from " + + "pointValues" + dpId + " pv "+ filter + myLimit, new PointValueRowMapperWithUserName(dpId)); + if (reverseOrder) + Collections.reverse(res); + return res; + } + + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + @Override + public Object[] create(final PointValue entity) { + + if (LOG.isTraceEnabled()) { + LOG.trace(entity); + } + + KeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.update(new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection connection) throws SQLException { + String query = POINT_VALUE_INSERT.replace(DATAPOINT_ID, String.valueOf(entity.getDataPointId())); + PreparedStatement ps = connection.prepareStatement(query, Statement.RETURN_GENERATED_KEYS); + new ArgumentPreparedStatementSetter( new Object[] { + getValueBaseOnType(entity.getPointValue().getValue().getDataType(), entity.getPointValue()), + entity.getPointValue().getTime(), + entity.getPointValue().getTime()*1000, + prepareMetaData(), + entity.getPointValue().getValue().getDataType() + }).setValues(ps); + return ps; + } + }, keyHolder); + + return new Object[] {keyHolder.getKey().longValue()}; + + } + + private String prepareMetaData() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode metaData = mapper.createObjectNode(); + String nullValue = null; + metaData.put(COLUMN_NAME_SOURCE_ID, nullValue) + .put(COLUMN_NAME_USERNAME_IN_TABLE_USERS, nullValue) + .put(COLUMN_NAME_SOURCE_TYPE, nullValue); + String json = null; + try { + json = mapper.writeValueAsString(metaData); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + json = json.replace("\"", ""); + return json; + } + + private String prepareMetaDataAnnotations(PointValueAdnnotation pointValueAdnnotation) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode metaData = mapper.createObjectNode(); + metaData.put(COLUMN_NAME_SOURCE_ID, pointValueAdnnotation.getSourceId()) + .put(COLUMN_NAME_USERNAME_IN_TABLE_USERS, pointValueAdnnotation.getChangeOwner()) + .put(COLUMN_NAME_SOURCE_TYPE, pointValueAdnnotation.getSourceType()); + String json = null; + try { + json = mapper.writeValueAsString(metaData); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + json = json.replace("\"", ""); + return json; + } + + + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + public Object[] create(final int pointId,final int dataType,final double dvalue,final long time) { + + return createNoTransaction(pointId, dataType, dvalue, time); + + } + + @Override + public void create(PointValue pointValue, PointValueAdnnotation pointValueAdnnotation, int dataType) { + if (LOG.isTraceEnabled()) { + LOG.trace("pointId:"+pointValue.getDataPointId()+" dataType:"+pointValue.getPointValue() + .getValue().getDataType()+" time:"+pointValue.getPointValue().getTime()); + } + + PointValueTime pointValueTime = pointValue.getPointValue(); + + jdbcTemplate.update(connection -> { + String query = POINT_VALUE_INSERT.replace(DATAPOINT_ID, String.valueOf(pointValue.getDataPointId())); + PreparedStatement ps = connection.prepareStatement(query, Statement.RETURN_GENERATED_KEYS); + new ArgumentPreparedStatementSetter( new Object[] { + getValueBaseOnType(dataType, pointValueTime), + pointValueTime.getTime(), + pointValueTime.getTime()*1000, + prepareMetaDataAnnotations(pointValueAdnnotation), + dataType + }).setValues(ps); + return ps; + }); + } + + + public Object[] createNoTransaction(final int pointId,final int dataType,final double dvalue,final long time) { + + if (LOG.isTraceEnabled()) { + LOG.trace("pointId:"+pointId+" dataType:"+dataType+" dvalue:"+time); + } + + KeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.update(new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection connection) throws SQLException { + String query = POINT_VALUE_INSERT.replace(DATAPOINT_ID, String.valueOf(pointId)); + PreparedStatement ps = connection.prepareStatement(query, Statement.RETURN_GENERATED_KEYS); + new ArgumentPreparedStatementSetter( new Object[] { + dvalue, + time, + time*1000, + prepareMetaData(), + dataType + }).setValues(ps); + return ps; + } + }, keyHolder); + + try { + return new Object[]{keyHolder.getKey().longValue()}; + } catch (InvalidDataAccessApiUsageException ex) { + return new Object[]{Long.parseLong(String.valueOf(keyHolder.getKeys().get("id")))}; + } + + } + + @Override + public long deletePointValuesBeforeWithOutLastTwo(int dataPointId, long time) { + String ts = TO_TIMEZONE.replace("$timestamp", String.valueOf(time*1000)); + String query = DROP_PARTITION.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.update(query + WHERE_TIMESTAMP + ts); + } + + @Override + public long deletePointValuesWithValueLimit(int dataPointId, int limit) { + //TODO when delete operation in QuestDB will be supported + return 0; + } + + @Transactional(readOnly = false,propagation= Propagation.REQUIRES_NEW,isolation= Isolation.READ_COMMITTED,rollbackFor=SQLException.class) + public void executeBatchUpdateInsert( List params) { + + Map> groupedParams = params.stream().collect(Collectors.groupingBy(p -> p[0])); + + if (LOG.isTraceEnabled()) { + for (Object[] param : params) { + for (Object arg :param) { + LOG.trace("arg:"+arg); + } + } + } + if(params.isEmpty()) { + return; + } + groupedParams.forEach((key, value) -> { + String query = POINT_VALUE_INSERT.replace(DATAPOINT_ID, String.valueOf(key)); + if(value.get(0).length == 4) { + List paramsExt = new ArrayList<>(); + for (Object[] args : value) { + paramsExt.add(new Object[]{args[2], args[3], (long)args[3]*1000, prepareMetaData(), (int)args[1]}); + } + jdbcTemplate.batchUpdate(query, paramsExt); + } else + jdbcTemplate.batchUpdate(query, value); + }); + } + + public Long getInceptionDate(int dataPointId) { + String query = POINT_VALUE_INCEPTION_DATA.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.queryForObject(query, Long.class); + } + + public long dateRangeCount(int dataPointId, long from, long to) { + String query = POINT_VALUE_DATA_RANGE_COUNT.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.queryForObject(query, new Object[] {from, to}, Long.class); + } + + public LongPair getStartAndEndTime(List dataPointIds) { + if (dataPointIds.isEmpty()) return null; + + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("ids",dataPointIds); + + String query = POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT.replace(DATAPOINT_ID, String.valueOf(dataPointIds.get(0)));; + + for (int i = 1; i < dataPointIds.size(); i++) { + query += UNION_ALL + POINT_VALUE_SELECT_MIN_MAX_TS_BASE_ON_LIST_DATA_POINT.replace(DATAPOINT_ID, String.valueOf(dataPointIds.get(i)));; + } + + LongPair longPair = jdbcTemplate.queryForObject(query,new LongPairRowMapper(),parameters); + + return longPair; + } + + public long getStartTime(List dataPointIds) { + if (dataPointIds.isEmpty()) return -1; + + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("ids",dataPointIds); + + String query = POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT.replace(DATAPOINT_ID, String.valueOf(dataPointIds.get(0)));; + + for (int i = 1; i < dataPointIds.size(); i++) { + query += UNION_ALL + POINT_VALUE_SELECT_MIN_TS_BASE_ON_LIST_DATA_POINT.replace(DATAPOINT_ID, String.valueOf(dataPointIds.get(i)));; + } + + Long minTs = jdbcTemplate.queryForObject(query,new LongRowMapper(),parameters); + + return minTs; + } + + public long getEndTime(List dataPointIds) { + if (dataPointIds.isEmpty()) return -1; + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("ids",dataPointIds); + + String query = POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT.replace(DATAPOINT_ID, String.valueOf(dataPointIds.get(0))); + + for (int i = 1; i < dataPointIds.size(); i++) { + query += UNION_ALL + POINT_VALUE_SELECT_MAX_TS_BASE_ON_LIST_DATA_POINT.replace(DATAPOINT_ID, String.valueOf(dataPointIds.get(i))); + } + + Long maxTs = jdbcTemplate.queryForObject(query, new LongRowMapper(),parameters); + + return maxTs; + + } + + public Long getLatestPointValue(int dataPointId) { + try { + String query = POINT_VALUE_SELECT_MAX_BASE_ON_DATA_POINT_ID.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.queryForObject(query, Long.class); + } catch (EmptyResultDataAccessException e) { + return null; + } + } + + @Override + public PointValue getPointValueRow(ResultSet rs, int rowNum) { + return null; + } + + /** + * When save value ALPHANUMERIC or IMAGE then data save in adnnotations + * @param dataType + * @param value + * @return + */ + private String getValueBaseOnType(int dataType, PointValueTime value) { + if (dataType==DataTypes.ALPHANUMERIC || dataType==DataTypes.IMAGE) + return value.getStringValue(); + if ( value == null) { + return String.valueOf(0.0); + } else { + return String.valueOf(value.getDoubleValue()); + } + } + + // Apply database specific bounds on double values. + public double applyBounds(double value) { + if (Double.isNaN(value)) + return 0; + if (value == Double.POSITIVE_INFINITY) + return Double.MAX_VALUE; + if (value == Double.NEGATIVE_INFINITY) + return -Double.MAX_VALUE; + + return value; + } + + public long deletePointValuesBeforeWithOutLast(int dataPointId, long time) { + String ts = TO_TIMEZONE.replace("$timestamp", String.valueOf(time*1000)); + String query = DROP_PARTITION.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.update(query + WHERE_TIMESTAMP + ts); + } + + public long deletePointValue(int dataPointId) { + String query = DROP_PARTITION.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + String nowTs = "to_timezone(now(), 'CET')"; + return jdbcTemplate.update(query + WHERE_TIMESTAMP + nowTs); + } + + public long deleteAllPointData() { + List dataPoints = new DataPointDAO().getDataPoints(); + for (DataPointVO dataPoint : dataPoints) { + deletePointValue(dataPoint.getId()); + } + return 0; + } + + public long getMinTs(int dataPointId) { + String query = SELECT_MIN_TIME_WHERE_DATA_POINT_ID.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.queryForObject(query, Long.TYPE); + } + + public long getMaxTs(int dataPointId) { + String query = SELECT_MAX_TIME_WHERE_DATA_POINT_ID.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + return jdbcTemplate.queryForObject(query, Long.TYPE); + } + + public void createTableForDatapoint(int dpId) { + String query = CREATE_TABLE_FOR_DATAPOINT.replace(DATAPOINT_ID, String.valueOf(dpId)); + jdbcTemplate.update(query); + } + + @Override + public List getFiledataIds() { + return null; + } + + @Override + public List getTopPointHistoryCounts() { + return null; + } + + @Override + public List findAll() { + return null; + } + + @Override + public List findAllWithUserName() { + return null; + } + + @Override + public long deletePointValuesWithMismatchedType(int dataPointId, int dataType) { + return 0; + } + + public long dropPartition(long time, int dataPointId) { + String query = DROP_PARTITION.replace(DATAPOINT_ID, String.valueOf(dataPointId)); + String timestamp = TO_TIMEZONE.replace("$timestamp", String.valueOf(time)); + return jdbcTemplate.update(query + timestamp); + } +} \ No newline at end of file diff --git a/src/org/scada_lts/mango/adapter/MangoPointValues.java b/src/org/scada_lts/mango/adapter/MangoPointValues.java index d61cbd283a..41c7ec66c8 100644 --- a/src/org/scada_lts/mango/adapter/MangoPointValues.java +++ b/src/org/scada_lts/mango/adapter/MangoPointValues.java @@ -46,7 +46,7 @@ public interface MangoPointValues { long deletePointValuesWithValueLimit(int dataPointId, int limit); - PointValueTime getPointValue(long id); + //PointValueTime getPointValue(long id); void clearUnsavedPointValues(); diff --git a/src/org/scada_lts/mango/service/DataPointService.java b/src/org/scada_lts/mango/service/DataPointService.java index 5fcf01b237..198692254b 100644 --- a/src/org/scada_lts/mango/service/DataPointService.java +++ b/src/org/scada_lts/mango/service/DataPointService.java @@ -53,9 +53,7 @@ import org.scada_lts.dao.pointhierarchy.PointHierarchyDAO; import org.scada_lts.dao.PointLinkDAO; import org.scada_lts.dao.UserCommentDAO; -import org.scada_lts.dao.pointvalues.PointValueAmChartDAO; -import org.scada_lts.dao.pointvalues.PointValueDAO; -import org.scada_lts.dao.pointvalues.PointValueDAO4REST; +import org.scada_lts.dao.pointvalues.*; import org.scada_lts.dao.watchlist.WatchListDAO; import org.scada_lts.mango.adapter.MangoDataPoint; import org.scada_lts.mango.adapter.MangoPointHierarchy; @@ -69,6 +67,7 @@ import org.springframework.stereotype.Service; import static org.scada_lts.utils.AggregateUtils.*; +import static org.scada_lts.dao.pointvalues.PointValueAmChartUtils.*; /** * Service for DataPointDAO @@ -100,7 +99,18 @@ public class DataPointService implements MangoDataPoint { private static final PointHierarchyService pointHierarchyService = new PointHierarchyService(); - private static final PointValueAmChartDAO pointValueAmChartDao = new PointValueAmChartDAO(); + private IPointValueQuestDbDAO pointValueQueryRepository; + + private IAmChartDAO pointValueAmChartCommandRepository; + private IAmChartDAO pointValueAmChartQueryRepository; + private boolean dbQueryEnabled; + + public DataPointService() { + this.pointValueQueryRepository = IPointValueQuestDbDAO.newQueryRespository(); + this.pointValueAmChartCommandRepository = new PointValueAmChartDAO(DAO.getInstance().getJdbcTemp(), false); + this.pointValueAmChartQueryRepository = new PointValueAmChartQuestDbDAO(DAO.query().getJdbcTemp()); + this.dbQueryEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.enabled", false); + } @Override public String generateUniqueXid() { @@ -323,8 +333,10 @@ public void insertDataPoint(final DataPointVO dp) { dp.defaultTextRenderer(); } - dp.setId(dataPointDAO.insert(dp)); + int dpId = dataPointDAO.insert(dp); + dp.setId(dpId); saveEventDetectors(dp); + createTableForDatapoint(dpId); } @Override @@ -647,9 +659,9 @@ public List> getPointValuesFromRange(List point if (pointIds.isEmpty()) return Collections.emptyList(); if (aggregateSettings.isEnabled()) { - return pointValueAmChartDao.convertToAmChartDataObject(aggregateValuesFromRange(startTs, endTs, pointIds, aggregateSettings)); + return convertToAmChartDataObject(aggregateValuesFromRange(startTs, endTs, pointIds, aggregateSettings)); } - return pointValueAmChartDao.getPointValuesFromRange(getPointIds(pointIds), startTs, endTs); + return getPointValueAmChartRepository().getPointValuesFromRange(getPointIds(pointIds), startTs, endTs); } public List> getPointValuesToCompareFromRange(List dataPoints, long startTs, long endTs, @@ -657,9 +669,9 @@ public List> getPointValuesToCompareFromRange(List getDataPoints(Set pointIds) { @@ -688,11 +700,11 @@ public List getDataPointsByXid(Set xids) { return pointIds; } - private List aggregateValuesFromRange(long startTs, long endTs, - List pointIds, - AggregateSettings aggregateSettings) { + private List aggregateValuesFromRange(long startTs, long endTs, + List pointIds, + AggregateSettings aggregateSettings) { int limit = aggregateSettings.getValuesLimit(); - List pvcList = pointValueAmChartDao + List pvcList = getPointValueAmChartRepository() .getPointValuesFromRangeWithLimit(getPointIds(pointIds), startTs, endTs, limit + 1); if (pvcList.size() > limit) { pvcList.clear(); @@ -711,12 +723,12 @@ private int[] getPointIds(List pointIds) { return pointIds.stream().mapToInt(DataPointVO::getId).toArray(); } - private List aggregateSortValues(long startTs, long endTs, + private List aggregateSortValues(long startTs, long endTs, List dataPoints, int limit, long intervalMs) { return dataPoints.stream() - .flatMap(dataPoint -> pointValueAmChartDao.aggregatePointValues(dataPoint, startTs, endTs, intervalMs, limit).stream()) - .sorted(Comparator.comparingLong(PointValueAmChartDAO.DataPointSimpleValue::getTimestamp)) + .flatMap(dataPoint -> getPointValueAmChartRepository().aggregatePointValues(dataPoint, startTs, endTs, intervalMs, limit).stream()) + .sorted(Comparator.comparingLong(DataPointSimpleValue::getTimestamp)) .collect(Collectors.toList()); } @@ -738,4 +750,18 @@ public DataPointVO createDataPoint(DataPointVO dataPoint) { Common.ctx.getRuntimeManager().saveDataPoint(created); return created; } + + private IAmChartDAO getPointValueAmChartRepository() { + boolean readEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.values.read.enabled", true); + if(dbQueryEnabled && readEnabled) { + return pointValueAmChartQueryRepository; + } + return pointValueAmChartCommandRepository; + } + + private void createTableForDatapoint(int dpId) { + if (dbQueryEnabled) { + pointValueQueryRepository.createTableForDatapoint(dpId); + } + } } diff --git a/src/org/scada_lts/mango/service/DataSourceService.java b/src/org/scada_lts/mango/service/DataSourceService.java index 8a5222f426..2779848f0e 100644 --- a/src/org/scada_lts/mango/service/DataSourceService.java +++ b/src/org/scada_lts/mango/service/DataSourceService.java @@ -54,7 +54,7 @@ public class DataSourceService implements MangoDataSource { //TODO spring private static final DataSourceDAO dataSourceDAO = new DataSourceDAO(); - private static final DataPointService dataPointService = new DataPointService(); + private final DataPointService dataPointService = new DataPointService(); @Override public List> getDataSources() { diff --git a/src/org/scada_lts/mango/service/PointValueService.java b/src/org/scada_lts/mango/service/PointValueService.java index 5d15d2b21d..0af377c00c 100644 --- a/src/org/scada_lts/mango/service/PointValueService.java +++ b/src/org/scada_lts/mango/service/PointValueService.java @@ -17,10 +17,17 @@ */ package org.scada_lts.mango.service; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; +import java.net.URI; +import java.net.URL; +import java.net.URLEncoder; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -34,19 +41,23 @@ import com.serotonin.mango.rt.dataSource.meta.MetaDataSourceRT; import com.serotonin.mango.rt.dataSource.meta.MetaPointLocatorRT; import com.serotonin.mango.rt.dataSource.meta.ScriptExecutor; -import com.serotonin.mango.util.LoggingScriptUtils; import com.serotonin.mango.vo.DataPointVO; import com.serotonin.mango.vo.User; import com.serotonin.mango.vo.dataSource.DataSourceVO; import com.serotonin.mango.vo.dataSource.meta.MetaDataSourceVO; import com.serotonin.mango.vo.dataSource.meta.MetaPointLocatorVO; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.multipart.FilePart; +import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; +import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.scada_lts.dao.GenericDaoCR; import org.scada_lts.dao.model.point.PointValue; import org.scada_lts.dao.model.point.PointValueAdnnotation; -import org.scada_lts.dao.pointvalues.PointValueAdnnotationsDAO; -import org.scada_lts.dao.pointvalues.PointValueDAO; +import org.scada_lts.dao.pointvalues.*; import org.scada_lts.mango.adapter.MangoPointValues; import org.scada_lts.mango.adapter.MangoPointValuesWithChangeOwner; import org.springframework.dao.ConcurrencyFailureException; @@ -80,20 +91,33 @@ public class PointValueService implements MangoPointValues, MangoPointValuesWith private static List UNSAVED_POINT_VALUES = new ArrayList(); private static final int POINT_VALUE_INSERT_VALUES_COUNT = 4; - private static PointValueAdnnotationsDAO pointValueAnnotationsDAO = new PointValueAdnnotationsDAO(); private DataPointService dataPointService = new DataPointService(); private DataSourceService dataSourceService = new DataSourceService(); private static final Log LOG = LogFactory.getLog(PointValueService.class); - public PointValueService() { + private IPointValueAdnnotationsDAO pointValueAnnotationsCommandRepository; + private IPointValueDAO pointValueCommandRepository; + private IPointValueQuestDbDAO pointValueQueryRepository; + + private boolean dbQueryEnabled; + private boolean dbWriteEnabled; + public PointValueService() { + pointValueCommandRepository = IPointValueDAO.newCommandRespository(); + pointValueQueryRepository = IPointValueQuestDbDAO.newQueryRespository(); + pointValueAnnotationsCommandRepository = IPointValueAdnnotationsDAO.newCommandRepository(); + dbQueryEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.enabled", + false); + dbWriteEnabled = Common.getEnvironmentProfile().getBoolean("db.values.write.enabled", + true); } @Override public List findAllWithAdnotationsAboutChangeOwner(){ - return pointValueAnnotationsDAO.findAllWithAdnotationsAboutChangeOwner(); + return getPointValueAnnotationsRepository().findAllWithUserNamePointValueAdnnotations(); } + /** * Only the PointValueCache should call this method during runtime. Do not * use. @@ -102,20 +126,20 @@ public PointValueTime savePointValueSync(int pointId, PointValueTime pointValue, SetPointSource source) { long id = savePointValueImpl(pointId, pointValue, source, false); - PointValueTime savedPointValue; + /*PointValueTime savedPointValue; int retries = 5; while (true) { try { - savedPointValue = PointValueDAO.getInstance().findById(new Object[]{id}).getPointValue(); + savedPointValue = getPointValueRepository().findById(new Object[]{id}).getPointValue(); break; } catch (ConcurrencyFailureException e) { if (retries <= 0) throw e; retries--; } - } + }*/ - return savedPointValue; + return pointValue; } /** @@ -227,7 +251,7 @@ public void savePointValue(int pointId, PointValueTime pointValue) { @Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW, isolation = Isolation.READ_COMMITTED, rollbackFor = SQLException.class) long savePointValueInTrasaction(final int pointId, final int dataType, double dvalue, final long time, final String svalue, final SetPointSource source, boolean async) { // Apply database specific bounds on double values. - dvalue = PointValueDAO.getInstance().applyBounds(dvalue); + dvalue = getPointValueRepository().applyBounds(dvalue); if (async) { BatchWriteBehind.add(new BatchWriteBehindEntry(pointId, dataType, @@ -256,7 +280,7 @@ public long savePointValue(final int pointId, final int dataType, double dvalue, final long time, final String svalue, final SetPointSource source, boolean async) { // Apply database specific bounds on double values. - dvalue = PointValueDAO.getInstance().applyBounds(dvalue); + dvalue = getPointValueRepository().applyBounds(dvalue); if (async) { BatchWriteBehind.add(new BatchWriteBehindEntry(pointId, dataType, @@ -283,7 +307,70 @@ public long savePointValue(final int pointId, final int dataType, double dvalue, private long savePointValueImpl(int pointId, int dataType, double dvalue, long time, String svalue, SetPointSource source) { - long id = (Long) PointValueDAO.getInstance().create(pointId, dataType, dvalue, time)[0]; + long id = Common.NEW_ID; + + if(dbWriteEnabled) { + id = createPointValue(pointId, dataType, dvalue, time, svalue, source); + } + + if(dbQueryEnabled) { + createPointValueDenormalized(pointId, dataType, dvalue, time, svalue, source, id); + } + + return id; + } + + private void createPointValueDenormalized(int pointId, int dataType, double dvalue, long time, String svalue, SetPointSource source, long id) { + + PointValue pointValue = new PointValue(); + pointValue.setDataPointId(pointId); + PointValueAdnnotation pointValueAdnnotation = new PointValueAdnnotation(); + if (svalue == null && dataType == DataTypes.IMAGE) { + svalue = Long.toString(id); + } + + // Check if we need to create an annotation. + + Integer sourceType = null, sourceId = null; + if (source != null) { + sourceType = source.getSetPointSourceType(); + sourceId = source.getSetPointSourceId(); + } else { + sourceType = SetPointSource.Types.UNKNOWN; + sourceId = 1; + } + + String shortString = null; + String longString = null; + PointValueTime pointValueTime = null; + if (svalue != null) { + if (svalue.length() > 128) + longString = svalue; + else + shortString = svalue; + pointValueTime = new PointValueTime(svalue, time); + } else + pointValueTime = new PointValueTime(dvalue, time); + + pointValue.setPointValue(pointValueTime); + pointValue.setId(id); + + pointValueAdnnotation.setPointValueId(id); + pointValueAdnnotation.setTextPointValueShort(shortString); + pointValueAdnnotation.setTextPointValueLong(longString); + pointValueAdnnotation.setSourceType(sourceType); + pointValueAdnnotation.setSourceId(sourceId); + + if (source != null && source.getSetPointSourceType() == SetPointSource.Types.USER) { + setUsername(source, pointValueAdnnotation); + } + + pointValueQueryRepository.create(pointValue, pointValueAdnnotation, dataType); + } + + private long createPointValue(int pointId, int dataType, double dvalue, long time, String svalue, SetPointSource source) { + + long id = (Long) pointValueCommandRepository.create(pointId, dataType, dvalue, time)[0]; if (svalue == null && dataType == DataTypes.IMAGE) { @@ -309,14 +396,25 @@ private long savePointValueImpl(int pointId, int dataType, double dvalue, long t else shortString = svalue; } + PointValueAdnnotation pointValueAdnnotation = new PointValueAdnnotation(id, shortString, longString, sourceType, sourceId); - PointValueAdnnotationsDAO.getInstance().create(pointValueAdnnotation); + if(source != null && source.getSetPointSourceType() == SetPointSource.Types.USER) { + setUsername(source, pointValueAdnnotation); + } + pointValueAnnotationsCommandRepository.create(pointValueAdnnotation); } - return id; } + private void setUsername(SetPointSource source, PointValueAdnnotation pointValueAdnnotation) { + UserService userService = new UserService(); + User user = userService.getUser(source.getSetPointSourceId()); + if(user != null) { + pointValueAdnnotation.setChangeOwner(user.getUsername()); + } + } + //TODO rewrite private List getLstPointValueTime(List lstIn) { List lst = new ArrayList(); @@ -327,53 +425,68 @@ private List getLstPointValueTime(List lstIn) { return lst; } + private boolean isReadingFromQueryEnabled() { + boolean readEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.values.read.enabled", true); + return dbQueryEnabled && readEnabled; + } + public List getPointValues(int dataPointId, long since) { - List lst = PointValueDAO.getInstance().filtered( - PointValueDAO.POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP, + List lst = getPointValueRepository().filtered( + isReadingFromQueryEnabled() ? PointValueQuestDbDAO.POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP + .replace("$from", String.valueOf(since*1000)): + PointValueDAO.POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP, new Object[]{dataPointId, since}, GenericDaoCR.NO_LIMIT); return getLstPointValueTime(lst); } public List getPointValuesBetween(int dataPointId, long from, long to) { - List lst = PointValueDAO.getInstance().filtered( - PointValueDAO.POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP_FROM_TO, + List lst = getPointValueRepository().filtered( + isReadingFromQueryEnabled() ? + PointValueQuestDbDAO.POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP_FROM_TO + .replace("$from", String.valueOf(from*1000)) + .replace("$to", String.valueOf(to*1000)) : + PointValueDAO.POINT_VALUE_FILTER_BASE_ON_DATA_POINT_ID_AND_TIME_STAMP_FROM_TO, new Object[]{dataPointId, from, to}, GenericDaoCR.NO_LIMIT); return getLstPointValueTime(lst); } public List getLatestPointValues(int dataPointId, int limit) { - List lst = PointValueDAO.getInstance().filtered( - PointValueDAO.POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID, + List lst = getPointValueRepository().filtered( + isReadingFromQueryEnabled() ? PointValueQuestDbDAO.POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID : + PointValueDAO.POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID, new Object[]{dataPointId}, limit); return getLstPointValueTime(lst); } public List getLatestPointValues(int dataPointId, int limit, long before) { - List lst = PointValueDAO.getInstance().filtered( - PointValueDAO.POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID, + List lst = getPointValueRepository().filtered( + isReadingFromQueryEnabled() ? PointValueQuestDbDAO.POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID : + PointValueDAO.POINT_VALUE_FILTER_LAST_BASE_ON_DATA_POINT_ID, new Object[]{dataPointId, before}, limit); return getLstPointValueTime(lst); } public PointValueTime getLatestPointValue(int dataPointId) { - Long maxTs = PointValueDAO.getInstance().getLatestPointValue(dataPointId); + Long maxTs = getPointValueRepository().getLatestPointValue(dataPointId); if (maxTs == null || maxTs == 0) return null; - List lstValues = PointValueDAO.getInstance().findByIdAndTs(dataPointId, maxTs); + List lstValues = getPointValueRepository().findByIdAndTs(dataPointId, maxTs); - PointValueAdnnotationsDAO.getInstance().updateAnnotations(lstValues); + pointValueAnnotationsCommandRepository.updateAnnotations(lstValues); if (lstValues.size() == 0) return null; return lstValues.get(0).getPointValue(); } public PointValueTime getPointValueBefore(int dataPointId, long time) { - List lst = PointValueDAO.getInstance().filtered( - PointValueDAO.POINT_VALUE_FILTER_BEFORE_TIME_STAMP_BASE_ON_DATA_POINT_ID, + List lst = getPointValueRepository().filtered( + isReadingFromQueryEnabled() ? PointValueQuestDbDAO.POINT_VALUE_FILTER_BEFORE_TIME_STAMP_BASE_ON_DATA_POINT_ID + .replace("$to", String.valueOf(time*1000)): + PointValueDAO.POINT_VALUE_FILTER_BEFORE_TIME_STAMP_BASE_ON_DATA_POINT_ID, new Object[]{dataPointId, time}, 1); if (lst != null && lst.size() > 0) { return lst.get(0).getPointValue(); @@ -383,9 +496,12 @@ public PointValueTime getPointValueBefore(int dataPointId, long time) { } public PointValueTime getPointValueAt(int dataPointId, long time) { - List lst = PointValueDAO.getInstance().filtered( - PointValueDAO.POINT_VALUE_FILTER_AT_TIME_STAMP_BASE_ON_DATA_POINT_ID, - new Object[]{dataPointId, time}, 1); + List lst = getPointValueRepository().filtered( + isReadingFromQueryEnabled() ? PointValueQuestDbDAO.POINT_VALUE_FILTER_AT_TIME_STAMP_BASE_ON_DATA_POINT_ID + .replace("$time", String.valueOf(time*1000)): + PointValueDAO.POINT_VALUE_FILTER_AT_TIME_STAMP_BASE_ON_DATA_POINT_ID, + new Object[]{dataPointId, time}, 1); + if (lst != null && lst.size() > 0) { return lst.get(0).getPointValue(); } else { @@ -395,42 +511,44 @@ public PointValueTime getPointValueAt(int dataPointId, long time) { @Deprecated public long deletePointValuesBeforeWithOutLast(int dataPointId, long time) { - return PointValueDAO.getInstance().deletePointValuesBeforeWithOutLast(dataPointId, time); + if (dbQueryEnabled) + deletePointValuesBeforeForDatapoint(dataPointId, time); + return pointValueCommandRepository.deletePointValuesBeforeWithOutLast(dataPointId, time); } @Override public long deletePointValuesBeforeWithOutLastTwo(int dataPointId, long time) { - return PointValueDAO.getInstance().deletePointValuesBeforeWithOutLastTwo(dataPointId, time); + return getPointValueRepository().deletePointValuesBeforeWithOutLastTwo(dataPointId, time); } @Override public long dateRangeCount(int dataPointId, long from, long to) { - return PointValueDAO.getInstance().dateRangeCount(dataPointId, from, to); + return getPointValueRepository().dateRangeCount(dataPointId, from, to); } @Override public long getInceptionDate(int dataPointId) { - return PointValueDAO.getInstance().getInceptionDate(dataPointId); + return getPointValueRepository().getInceptionDate(dataPointId); } @Override public long getStartTime(List dataPointIds) { - return PointValueDAO.getInstance().getStartTime(dataPointIds); + return getPointValueRepository().getStartTime(dataPointIds); } @Override public long getEndTime(List dataPointIds) { - return PointValueDAO.getInstance().getEndTime(dataPointIds); + return getPointValueRepository().getEndTime(dataPointIds); } @Override public LongPair getStartAndEndTime(List dataPointIds) { - return PointValueDAO.getInstance().getStartAndEndTime(dataPointIds); + return getPointValueRepository().getStartAndEndTime(dataPointIds); } @Override public List getFiledataIds() { - return PointValueDAO.getInstance().getFiledataIds(); + return pointValueCommandRepository.getFiledataIds(); } /** @@ -562,8 +680,14 @@ public void execute() { int retries = 10; while (true) { try { - PointValueDAO.getInstance().executeBatchUpdateInsert(params); + boolean dbWriteEnabled = Common.getEnvironmentProfile().getBoolean("db.values.write.enabled", true); + if(dbWriteEnabled) + IPointValueDAO.newCommandRespository().executeBatchUpdateInsert(params); + + boolean dbQueryEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.enabled", false); + if(dbQueryEnabled) + IPointValueQuestDbDAO.newQueryRespository().executeBatchUpdateInsert(params); break; } catch (ConcurrencyFailureException e) { if (retries <= 0) { @@ -602,36 +726,41 @@ public int getPriority() { return WorkItem.PRIORITY_HIGH; } } - +/* public PointValueTime getPointValue(long id) { - return PointValueDAO.getInstance().getPointValue(id); + return getPointValueRepository().getPointValue(id); } +*/ public List getAllPointValueAnnotations(){ - return pointValueAnnotationsDAO.findAll(); + return getPointValueAnnotationsRepository().findAllPointValueAdnnotations(); } public void updatePointValueAnnotations(int userId) { - pointValueAnnotationsDAO.update(userId); + pointValueAnnotationsCommandRepository.update(userId); } @Override public long deletePointValues(int dataPointId) { - return PointValueDAO.getInstance().deletePointValue(dataPointId); + if (dbQueryEnabled) + pointValueQueryRepository.deletePointValue(dataPointId); + return pointValueCommandRepository.deletePointValue(dataPointId); } @Override public long deleteAllPointValue() { - return PointValueDAO.getInstance().deleteAllPointData(); + if (dbQueryEnabled) + pointValueQueryRepository.deleteAllPointData(); + return pointValueCommandRepository.deleteAllPointData(); } @Override public long deletePointValuesWithMismatchedType(int dataPointId, int dataType) { - return PointValueDAO.getInstance().deletePointValuesWithMismatchedType(dataPointId, dataType); + return pointValueCommandRepository.deletePointValuesWithMismatchedType(dataPointId, dataType); } @Override public long deletePointValuesWithValueLimit(int dataPointId, int limit) { - return PointValueDAO.getInstance().deletePointValuesWithValueLimit(dataPointId, limit); + return pointValueCommandRepository.deletePointValuesWithValueLimit(dataPointId, limit); } @@ -799,6 +928,78 @@ public void updateAllMetaDataPointsByScript() { } + private IPointValueDAO getPointValueRepository() { + boolean readEnabled = Common.getEnvironmentProfile().getBoolean("dbquery.values.read.enabled", + true); + if(dbQueryEnabled && readEnabled) { + return pointValueQueryRepository; + } + return pointValueCommandRepository; + } + + private IPointValueAdnnotationsDAO getPointValueAnnotationsRepository() { + return pointValueAnnotationsCommandRepository; + } + + public void createTableForDatapoint(int dpId) { + if (dbQueryEnabled) { + pointValueQueryRepository.createTableForDatapoint(dpId); + } + } + + public void deletePointValuesBeforeForDatapoint(int dataPointId, long time) { + long questTime = time*1000; + File exported = null; + try { + exported = exportFromQuestDb(dataPointId, questTime); + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + pointValueQueryRepository.deletePointValuesBeforeWithOutLast(dataPointId, time); + importToQuestDb(exported, dataPointId); + try { + Files.delete(exported.toPath()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public File exportFromQuestDb(int dataPointId, long timestamp) throws IOException, InterruptedException { + String query = "select * from pointValues"+dataPointId+" where timestamp < " + + "timestamp_ceil('M', to_timezone(" + timestamp + ",'CET')) " + + " and timestamp > to_timezone(" + timestamp + ", 'CET')"; + String url = "http://localhost:9000/exp?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8); + + java.net.http.HttpClient client = java.net.http.HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).build(); + + HttpResponse response = client.send(request, + HttpResponse.BodyHandlers.ofFileDownload(Path.of(System.getProperty("java.io.tmpdir")), + StandardOpenOption.CREATE, StandardOpenOption.WRITE)); + + Path path = response.body(); + return path.toFile(); + + } + + public static void importToQuestDb(File csv, int dataPointId) { + try { + URL resource = PointValueService.class.getClassLoader().getResource("questdb-schema-data-retention.json"); + File schema = Paths.get(resource.toURI()).toFile(); + PostMethod postMethod = new PostMethod("http://localhost:9000/imp?name=pointValues"+dataPointId+"×tamp=timestamp&partitionBy=MONTH&overwrite=false"); + MultipartRequestEntity entity = new MultipartRequestEntity(new Part[]{new FilePart("schema", schema), new FilePart("data", csv)}, postMethod.getParams()); + postMethod.setRequestEntity(entity); + MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager(); + HttpClient httpClient = new HttpClient(manager); + httpClient.executeMethod(postMethod); + String response = postMethod.getResponseBodyAsString(); + LOG.info("QuestDb import finished: \r\n" + response); + } catch (Exception ex) { + LOG.error(ex.getMessage(), ex); + } + } public void updateAllMetaDataPointsByScript(User user) { List metaDataPoints = new ArrayList<>(); diff --git a/src/org/scada_lts/mango/service/SystemSettingsService.java b/src/org/scada_lts/mango/service/SystemSettingsService.java index 50a899dd81..4f8780db52 100644 --- a/src/org/scada_lts/mango/service/SystemSettingsService.java +++ b/src/org/scada_lts/mango/service/SystemSettingsService.java @@ -19,6 +19,7 @@ import org.scada_lts.serorepl.utils.DirectoryInfo; import org.scada_lts.serorepl.utils.DirectoryUtils; import org.scada_lts.web.mvc.api.AggregateSettings; +import org.scada_lts.web.mvc.api.DbQuerySettings; import org.scada_lts.web.mvc.api.json.*; import org.springframework.stereotype.Service; @@ -346,4 +347,14 @@ public void saveAggregateSettings(AggregateSettings aggregateSettings) { systemSettingsDAO.setValue(SystemSettingsDAO.AGGREGATION_LIMIT_FACTOR, String.valueOf(aggregateSettings.getLimitFactor())); systemSettingsDAO.setValue(SystemSettingsDAO.AGGREGATION_ENABLED, String.valueOf(aggregateSettings.isEnabled())); } + + public DbQuerySettings getDbQuerySettings() { + DbQuerySettings dbQuerySettings = new DbQuerySettings(); + dbQuerySettings.setReadEnabled(SystemSettingsDAO.getBooleanValueOrDefault(SystemSettingsDAO.DB_QUERY_READ_ENABLED)); + return dbQuerySettings; + } + + public void saveDbQuerySettings(DbQuerySettings dbQuerySettings) { + systemSettingsDAO.setValue(SystemSettingsDAO.DB_QUERY_READ_ENABLED, String.valueOf(dbQuerySettings.isReadEnabled())); + } } diff --git a/src/org/scada_lts/utils/QueryUtils.java b/src/org/scada_lts/utils/QueryUtils.java index f6fe61949a..48131d3259 100644 --- a/src/org/scada_lts/utils/QueryUtils.java +++ b/src/org/scada_lts/utils/QueryUtils.java @@ -1,7 +1,13 @@ package org.scada_lts.utils; +import org.apache.commons.logging.LogFactory; +import org.scada_lts.config.ScadaConfig; + public final class QueryUtils { + private static final String DB_QUERY_READ_VALUES_ENABLED_KEY = "dbquery.values.read.enabled"; + private static final org.apache.commons.logging.Log LOG = LogFactory.getLog(QueryUtils.class); + private QueryUtils() {} public static String getArgsIn(int size) { @@ -12,4 +18,13 @@ public static String getArgsIn(int size) { args.delete(args.length() - 1, args.length()); return args.toString(); } + + public static boolean isReadEnabled() { + try { + return ScadaConfig.getInstance().getBoolean(DB_QUERY_READ_VALUES_ENABLED_KEY, false); + } catch (Exception e) { + LOG.error(e.getMessage()); + return false; + } + } } diff --git a/src/org/scada_lts/web/mvc/api/DbQuerySettings.java b/src/org/scada_lts/web/mvc/api/DbQuerySettings.java new file mode 100644 index 0000000000..c8f5be2a73 --- /dev/null +++ b/src/org/scada_lts/web/mvc/api/DbQuerySettings.java @@ -0,0 +1,27 @@ +package org.scada_lts.web.mvc.api; + +import org.scada_lts.utils.QueryUtils; + +public class DbQuerySettings { + + private Boolean readEnabled; + + public DbQuerySettings() { + } + + public DbQuerySettings(Boolean readEnabled) { + this.readEnabled = readEnabled; + } + + public static DbQuerySettings fromEnvProperties() { + return new DbQuerySettings(QueryUtils.isReadEnabled()); + } + + public Boolean isReadEnabled() { + return readEnabled; + } + + public void setReadEnabled(Boolean readEnabled) { + this.readEnabled = readEnabled; + } +} diff --git a/src/org/scada_lts/web/mvc/api/SystemSettingsAPI.java b/src/org/scada_lts/web/mvc/api/SystemSettingsAPI.java index 1131484e09..84aac2b3c5 100644 --- a/src/org/scada_lts/web/mvc/api/SystemSettingsAPI.java +++ b/src/org/scada_lts/web/mvc/api/SystemSettingsAPI.java @@ -595,4 +595,37 @@ public ResponseEntity saveAggregateSettings(HttpServletRequest request, return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); } } + + @GetMapping(value = "/getDbQuerySettings", produces = "application/json") + public ResponseEntity getDbQuerySettings(HttpServletRequest request) { + LOG.info("/api/systemSettings/getDbQueryRead"); + try { + User user = Common.getUser(request); + if (user != null && user.isAdmin()) { + return new ResponseEntity<>(systemSettingsService.getDbQuerySettings(), HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.UNAUTHORIZED); + } + } catch (Exception e) { + LOG.error(e); + return new ResponseEntity<>(HttpStatus.BAD_REQUEST); + } + } + + @PostMapping(value = "/saveDbQuerySettings", consumes = {"text/plain", "application/*"}) + public ResponseEntity saveDbQuerySettings(HttpServletRequest request, @RequestBody DbQuerySettings dbQuerySettings) { + LOG.info("/api/systemSettings/saveDbQueryRead"); + try { + User user = Common.getUser(request); + if (user != null && user.isAdmin()) { + systemSettingsService.saveDbQuerySettings(dbQuerySettings); + return new ResponseEntity<>(SAVED_MSG, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.UNAUTHORIZED); + } + } catch (Exception e) { + LOG.error(e); + return new ResponseEntity<>(HttpStatus.BAD_REQUEST); + } + } } diff --git a/src/org/scada_lts/web/mvc/api/components/AmChartValuesAPI.java b/src/org/scada_lts/web/mvc/api/components/AmChartValuesAPI.java index 2f631a3d24..7c9b83a137 100644 --- a/src/org/scada_lts/web/mvc/api/components/AmChartValuesAPI.java +++ b/src/org/scada_lts/web/mvc/api/components/AmChartValuesAPI.java @@ -29,7 +29,7 @@ public class AmChartValuesAPI { private static final Log LOG = LogFactory.getLog(AmChartValuesAPI.class); - private static final DataPointService dpService = new DataPointService(); + private final DataPointService dpService = new DataPointService(); private static final SystemSettingsService systemSettingsService = new SystemSettingsService(); @GetMapping("/") diff --git a/test/env.properties b/test/env.properties index 8e8369269f..f1c28843d5 100644 --- a/test/env.properties +++ b/test/env.properties @@ -35,6 +35,38 @@ db.type=mysql db.datasource=true db.datasourceName=java:comp/test_env/jdbc/scadalts +db.migration.package=org.scada_lts.dao.migration.mysql +db.values.write.enabled=true +db.table.tocheck=users + +#export +db.values.export.limit=30000000 + +# +dbquery.enabled=true +dbquery.type=questdb +dbquery.datasource=true +dbquery.datasourceName=java:comp/test_env/jdbc/scadaltsquery +dbquery.migration.package=org.scada_lts.dao.migration.query.questdb +dbquery.embedded=true +dbquery.table.tocheck=pointValuesDenormalized + +#import +dbquery.import.overwrite=false +dbquery.values.import.enabled=false + + +#dbquery.enabled=true +#dbquery.type=POSTGRES +#dbquery.datasource=true +#dbquery.datasourceName=java:comp/env/jdbc/scadaltsquery3 +#dbquery.migration.package=org.scada_lts.dao.migration.query.postgres +#dbquery.embedded=false +#dbquery.table.tocheck=pointValuesDenormalized + +#runtime +dbquery.values.read.enabled=true + #db.url=jdbc:mysql://localhost:3306/scadalts #db.username=root #db.password=root diff --git a/test/org/scada_lts/CreateObjectTest.java b/test/org/scada_lts/CreateObjectTest.java new file mode 100644 index 0000000000..003cc5931e --- /dev/null +++ b/test/org/scada_lts/CreateObjectTest.java @@ -0,0 +1,90 @@ +package org.scada_lts; + +import com.serotonin.mango.Common; +import com.serotonin.mango.db.DatabaseAccess; +import com.serotonin.mango.rt.EventManager; +import com.serotonin.mango.web.ContextWrapper; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.scada_lts.dao.IUserDAO; +import org.scada_lts.permissions.service.GetShareUsers; +import org.scada_lts.utils.ApplicationBeans; +import org.springframework.context.ApplicationContext; +import utils.NameClass; + +import javax.sql.DataSource; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static utils.ClassFinderUtils.*; + +@RunWith(Parameterized.class) +public class CreateObjectTest { + + @Parameterized.Parameters(name= "{index}: test constructor: {0}") + public static NameClass[] data() { + String classPackage = "br.org.scadabr"; + String classPackage2 = "cc.radiuino.scadabr"; + String classPackage3 = "com.serotonin"; + String classPackage4 = "org.scada_lts"; + + String classPackage5 = "org.scada_lts.mango.service"; + String classPackage6 = "org.scada_lts.permissions.service"; + String classPackage7 = "org.scada_lts.service"; + String classPackage8 = "org.scada_lts.dao"; + + Set nameClasses = getDaoServiceNoInterfacePublicClasses(classPackage, classPackage2, classPackage3, classPackage4); + nameClasses.addAll(getNoInterfacePublicClasses(classPackage5, classPackage6, classPackage7, classPackage8)); + + return nameClasses.toArray(new NameClass[]{}); + } + + private final NameClass nameClass; + + public CreateObjectTest(NameClass nameClass) { + this.nameClass = nameClass; + } + + @BeforeClass + public static void config() { + ContextWrapper contextWrapper = mock(ContextWrapper.class); + DatabaseAccess databaseAccess = mock(DatabaseAccess.class); + EventManager eventManager = mock(EventManager.class); + when(contextWrapper.getDatabaseAccess()).thenReturn(databaseAccess); + when(contextWrapper.getDatabaseQueryAccess()).thenReturn(databaseAccess); + when(contextWrapper.getEventManager()).thenReturn(eventManager); + DataSource dataSource = mock(DataSource.class); + when(databaseAccess.getDataSource()).thenReturn(dataSource); + Common.ctx = contextWrapper; + + ApplicationContext applicationContext = mock(ApplicationContext.class); + when(applicationContext.getBean("getShareUsers")).thenReturn(mock(GetShareUsers.class)); + when(applicationContext.getBean("userDAO")).thenReturn(mock(IUserDAO.class)); + when(applicationContext.getBean("userDaoWithCache")).thenReturn(mock(IUserDAO.class)); + + ApplicationBeans applicationBeans = new ApplicationBeans(); + applicationBeans.setApplicationContext(applicationContext); + } + + @Test + public void when_create_object_then_no_error() throws InstantiationException, + IllegalAccessException, InvocationTargetException { + Class clazz = nameClass.getClazz(); + + Optional> constructorOpt = getConstructor(clazz); + + if(constructorOpt.isPresent() && constructorOpt.get().getParameters().length > 0) { + Constructor constructor = constructorOpt.get(); + Object[] args = nulls(constructor.getParameters()); + constructor.newInstance(args); + } else { + clazz.newInstance(); + } + } +} diff --git a/test/org/scada_lts/InitObjectTestsSuite.java b/test/org/scada_lts/InitObjectTestsSuite.java new file mode 100644 index 0000000000..098062490d --- /dev/null +++ b/test/org/scada_lts/InitObjectTestsSuite.java @@ -0,0 +1,12 @@ +package org.scada_lts; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + InitStaticFieldTest.class, + CreateObjectTest.class +}) +public class InitObjectTestsSuite { +} diff --git a/test/org/scada_lts/InitStaticFieldTest.java b/test/org/scada_lts/InitStaticFieldTest.java new file mode 100644 index 0000000000..8fa1ce00a6 --- /dev/null +++ b/test/org/scada_lts/InitStaticFieldTest.java @@ -0,0 +1,44 @@ +package org.scada_lts; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import utils.NameClass; + +import java.util.*; + +import static org.mockito.Mockito.mock; +import static utils.ClassFinderUtils.*; + +@RunWith(Parameterized.class) +public class InitStaticFieldTest { + + @Parameterized.Parameters(name= "{index}: class name: {0}") + public static NameClass[] data() { + String classPackage = "br.org.scadabr"; + String classPackage2 = "cc.radiuino.scadabr"; + String classPackage3 = "com.serotonin"; + String classPackage4 = "org.scada_lts"; + + String classPackage5 = "org.scada_lts.mango.service"; + String classPackage6 = "org.scada_lts.permissions.service"; + String classPackage7 = "org.scada_lts.service"; + String classPackage8 = "org.scada_lts.dao"; + + Set nameClasses = getDaoServiceClasses(classPackage, classPackage2, classPackage3, classPackage4); + nameClasses.addAll(getClasses(classPackage5, classPackage6, classPackage7, classPackage8)); + + return nameClasses.toArray(new NameClass[]{}); + } + + private final NameClass nameClass; + + public InitStaticFieldTest(NameClass nameClass) { + this.nameClass = nameClass; + } + + @Test + public void when_init_static_field_then_ok(){ + mock(nameClass.getClazz()); + } +} diff --git a/test/org/scada_lts/dao/PointValueAdnnotationDaoTest.java b/test/org/scada_lts/dao/PointValueAdnnotationDaoTest.java index 8236a3abd8..796d9aec4a 100644 --- a/test/org/scada_lts/dao/PointValueAdnnotationDaoTest.java +++ b/test/org/scada_lts/dao/PointValueAdnnotationDaoTest.java @@ -148,25 +148,25 @@ public void test() { // read // TODO because valueAdnnotations don't have pk error when adnnotation more then one //find (adnnotations for every pointValues in test) - PointValueAdnnotation pointValueAdnnotationNewLongText = pointValuesAdnnotationsDAO.findById(new Object[] {1}); + PointValueAdnnotation pointValueAdnnotationNewLongText = pointValuesAdnnotationsDAO.findByIdPointValueAdnnotation(new Object[] {1}); boolean test1 = pointValueAdnnotationNewLongText.equals(pointValueAdnnotationAlphanumericLongText); assertTrue(test1); - PointValueAdnnotation pointValueAdnnotationNewShortText = pointValuesAdnnotationsDAO.findById(new Object[] {2}); + PointValueAdnnotation pointValueAdnnotationNewShortText = pointValuesAdnnotationsDAO.findByIdPointValueAdnnotation(new Object[] {2}); boolean test2 = pointValueAdnnotationNewShortText.equals(pointValueAdnnotationAlphanumericShortText); assertTrue(test2); - PointValueAdnnotation pointValueAdnnotationNewImage1 = pointValuesAdnnotationsDAO.findById(new Object[] {3}); + PointValueAdnnotation pointValueAdnnotationNewImage1 = pointValuesAdnnotationsDAO.findByIdPointValueAdnnotation(new Object[] {3}); boolean test3 = pointValueAdnnotationNewImage1.equals(pointValueAdnnotationImage1); assertTrue(test3); - PointValueAdnnotation pointValueAdnnotationNewImage2 = pointValuesAdnnotationsDAO.findById(new Object[] {4}); + PointValueAdnnotation pointValueAdnnotationNewImage2 = pointValuesAdnnotationsDAO.findByIdPointValueAdnnotation(new Object[] {4}); boolean test4 = pointValueAdnnotationNewImage2.equals(pointValueAdnnotationImage2); assertTrue(test4); //filtered (values for every type point) - List valuesAdnnotationsFiltered = pointValuesAdnnotationsDAO.filtered( - PointValueAdnnotationsDAO.POINT_VALUE_ADNNOTATIONS_FILTER_BASE_ON_POINT_VALUES_ID, + List valuesAdnnotationsFiltered = pointValuesAdnnotationsDAO.filteredPointValueAdnnotations( + PointValueAdnnotationsDAO.POINT_VALUE_ADNNOTATIONS_FILTER_BASE_ON_POINT_VALUES_ID, new Object[]{1}, 10); boolean testFiltered = (valuesAdnnotationsFiltered.size() == 1); diff --git a/test/org/scada_lts/dao/UserServiceTest.java b/test/org/scada_lts/dao/UserServiceTest.java index 0465f85e71..2d689345ed 100644 --- a/test/org/scada_lts/dao/UserServiceTest.java +++ b/test/org/scada_lts/dao/UserServiceTest.java @@ -48,7 +48,7 @@ public class UserServiceTest extends TestDAO { private static final int SECOND_ALARM_EMAIL = 3; private static final boolean SECOND_AUDIT_EVENT = true; - private static final UserService userService = new UserService(); + private UserService userService = new UserService(); private User user; private User secondUser; diff --git a/test/utils/ClassFinderUtils.java b/test/utils/ClassFinderUtils.java new file mode 100644 index 0000000000..92dc448ed2 --- /dev/null +++ b/test/utils/ClassFinderUtils.java @@ -0,0 +1,167 @@ +package utils; + +import com.serotonin.mango.vo.DataPointVO; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.net.URL; +import java.util.*; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class ClassFinderUtils { + + private static final Log LOG = LogFactory.getLog(ClassFinderUtils.class); + + private static final Predicate> DAO_SERVICE_FILTER = a -> a.getSimpleName().contains("Dao") || a.getSimpleName().contains("DAO") + || a.getSimpleName().contains("service") || a.getSimpleName().contains("Service"); + + private static final Predicate> NO_INTERFACE_PUBLIC_FILTER = a -> !Modifier.isInterface(a.getModifiers()) + && !Modifier.isAbstract(a.getModifiers()) && Modifier.isPublic(a.getModifiers()) + && getConstructor(a).isPresent() && Modifier.isPublic(getConstructor(a).get().getModifiers()); + + private static final Predicate> DAO_SERVICE_NO_INTERFACE_PUBLIC_FILTER = + DAO_SERVICE_FILTER.and(NO_INTERFACE_PUBLIC_FILTER); + + public static Set getDaoServiceClasses(String... packageNames) { + return getClasses(DAO_SERVICE_FILTER, packageNames); + } + + public static Set getNoInterfacePublicClasses(String... packageNames) { + return getClasses(NO_INTERFACE_PUBLIC_FILTER, packageNames); + } + + public static Set getDaoServiceNoInterfacePublicClasses(String... packageNames) { + return getClasses(DAO_SERVICE_NO_INTERFACE_PUBLIC_FILTER, packageNames); + } + + public static Set getClasses(String... packageNames) { + Set set = new HashSet<>(); + for(String packageName: packageNames) { + set.addAll(getClasses(packageName)); + } + return set; + } + + public static Set getClasses(Predicate> filter, String... packageNames) { + Set set = new HashSet<>(); + for(String packageName: packageNames) { + set.addAll(getClasses(packageName, filter)); + } + return set; + } + + public static Set getClasses(String packageName) { + return getClasses(packageName, a -> true); + } + + public static Set getClasses(String packageName, Predicate> filter) { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + assert classLoader != null; + String path = packageName.replace('.', '/'); + Enumeration resources = null; + try { + resources = classLoader.getResources(path); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + List dirs = new ArrayList<>(); + while (resources.hasMoreElements()) { + URL resource = resources.nextElement(); + dirs.add(new File(resource.getFile())); + } + List> classes = new ArrayList<>(); + for (File directory : dirs) { + classes.addAll(findClasses(directory, packageName)); + } + return classes.stream() + .filter(a -> !Modifier.isFinal(a.getModifiers())) + .filter(a -> !a.getSimpleName().endsWith("Test")) + .filter(a -> !a.getSimpleName().isEmpty()) + .filter(filter) + .map(NameClass::new) + .collect(Collectors.toSet()); + } + + public static Object[] nulls(Parameter... parameters) { + List list = new ArrayList<>(); + for(Parameter parameter: parameters) { + Class type = parameter.getType(); + if(type.isPrimitive()) { + if(type.equals(boolean.class)) { + list.add(false); + } else if(type.equals(double.class)) { + list.add(0.0); + } else { + list.add(0); + } + } else { + try { + if(type.equals(DataPointVO.class)) { + list.add(type.getConstructor(int.class).newInstance(1)); + } else { + Optional> constructor = getConstructor(type); + Object object; + if (constructor.isPresent() && constructor.get().getParameters().length > 0) { + object = constructor.get().newInstance(nulls(constructor.get().getParameters())); + } else { + object = type.newInstance(); + } + list.add(object); + } + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + list.add(null); + } + } + } + return list.toArray(); + } + + private static List> findClasses(File directory, String packageName) { + List> classes = new ArrayList<>(); + if (!directory.exists()) { + return classes; + } + File[] files = directory.listFiles(); + for (File file : files) { + if (file.isDirectory()) { + assert !file.getName().contains("."); + classes.addAll(findClasses(file, packageName + "." + file.getName())); + } else if (file.getName().endsWith(".class")) { + try { + LOG.info(file); + classes.add(Class.forName(packageName + '.' + file.getName().substring(0, file.getName().length() - 6))); + } catch (ClassNotFoundException e) { + LOG.error(e.getMessage(), e); + } + } + } + return classes; + } + + public static Optional> getConstructor(Class clazz) { + for(Constructor constructor: clazz.getConstructors()) { + if(constructor.getParameters().length == 0) { + return Optional.of(constructor); + } + } + for(Constructor constructor: clazz.getConstructors()) { + if(constructor.getParameters().length > 0) { + if(constructor.getParameters()[0].getType().isPrimitive()) + return Optional.of(constructor); + } + } + for(Constructor constructor: clazz.getConstructors()) { + if(constructor.getParameters().length > 0) { + return Optional.of(constructor); + } + } + return Optional.empty(); + } +} diff --git a/test/utils/NameClass.java b/test/utils/NameClass.java new file mode 100644 index 0000000000..29e003ec85 --- /dev/null +++ b/test/utils/NameClass.java @@ -0,0 +1,35 @@ +package utils; + +import java.util.Objects; + +import static utils.ClassFinderUtils.getConstructor; + +public class NameClass { + private final Class clazz; + + public NameClass(Class clazz) { + this.clazz = clazz; + } + + public Class getClazz() { + return clazz; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NameClass)) return false; + NameClass nameClass = (NameClass) o; + return Objects.equals(getClazz().getName(), nameClass.getClazz().getName()); + } + + @Override + public int hashCode() { + return Objects.hash(getClazz().getName()); + } + + @Override + public String toString() { + return getConstructor(clazz).isPresent() ? getConstructor(clazz).get().toGenericString() : clazz.getName(); + } +} \ No newline at end of file diff --git a/tomcat/config/context.xml b/tomcat/config/context.xml new file mode 100644 index 0000000000..c5ca29d0cd --- /dev/null +++ b/tomcat/config/context.xml @@ -0,0 +1,50 @@ + + + WEB-INF/web.xml + + + + + \ No newline at end of file diff --git a/tomcat/lib/postgresql-42.2.23.jar b/tomcat/lib/postgresql-42.2.23.jar new file mode 100644 index 0000000000..00ad52ea73 Binary files /dev/null and b/tomcat/lib/postgresql-42.2.23.jar differ diff --git a/webapp-resources/env.properties b/webapp-resources/env.properties index 6d8fa45ffe..226f2e5673 100644 --- a/webapp-resources/env.properties +++ b/webapp-resources/env.properties @@ -33,6 +33,38 @@ db.type=mysql db.datasource=true db.datasourceName=java:comp/env/jdbc/scadalts +db.migration.package=org.scada_lts.dao.migration.mysql +db.values.write.enabled=true +db.table.tocheck=users + +#export +db.values.export.limit=30000000 + +# +dbquery.enabled=true +dbquery.type=questdb +dbquery.datasource=true +dbquery.datasourceName=java:comp/env/jdbc/scadaltsquery +dbquery.migration.package=org.scada_lts.dao.migration.query.questdb +dbquery.embedded=true +dbquery.table.tocheck=pointValuesDenormalized + +#import +dbquery.import.overwrite=true +dbquery.values.import.enabled=true + + +#dbquery.enabled=true +#dbquery.type=POSTGRES +#dbquery.datasource=true +#dbquery.datasourceName=java:comp/env/jdbc/scadaltsquery3 +#dbquery.migration.package=org.scada_lts.dao.migration.query.postgres +#dbquery.embedded=false +#dbquery.table.tocheck=pointValuesDenormalized + +#runtime +dbquery.values.read.enabled=true + #db.url=jdbc:mysql://localhost:3306/scadalts #db.username=root #db.password=root diff --git a/webapp-resources/questdb-schema-data-retention.json b/webapp-resources/questdb-schema-data-retention.json new file mode 100644 index 0000000000..dd3e940fd3 --- /dev/null +++ b/webapp-resources/questdb-schema-data-retention.json @@ -0,0 +1,23 @@ +[ + { + "name":"timestamp", + "type": "TIMESTAMP", + "pattern": "yyyy-MM-ddTHH:mm:ss.SSSUUUZ" + }, + { + "name":"ts", + "type": "LONG" + }, + { + "name":"pointValue", + "type": "STRING" + }, + { + "name":"metaData", + "type": "STRING" + }, + { + "name":"dataType", + "type": "INT" + } +] \ No newline at end of file diff --git a/webapp-resources/questdb-schema.json b/webapp-resources/questdb-schema.json new file mode 100644 index 0000000000..dde6ac1359 --- /dev/null +++ b/webapp-resources/questdb-schema.json @@ -0,0 +1,23 @@ +[ + { + "name":"timestamp", + "type": "TIMESTAMP", + "pattern": "yyyy-MM-ddTHH:mm:ss.SSSZ" + }, + { + "name":"ts", + "type": "LONG" + }, + { + "name":"pointValue", + "type": "STRING" + }, + { + "name":"metaData", + "type": "STRING" + }, + { + "name":"dataType", + "type": "INT" + } +] \ No newline at end of file