From b254866ee0bb219e27e6d94a1a2af210079b2f3d Mon Sep 17 00:00:00 2001 From: Jonathan Roesner Date: Mon, 13 Jan 2025 08:22:21 +0100 Subject: [PATCH 1/2] WIP --- server/build.gradle.kts | 3 ++- server/src/main/kotlin/com/espero/yaade/Main.kt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/build.gradle.kts b/server/build.gradle.kts index df1c40d9..679da289 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -31,8 +31,9 @@ dependencies { implementation("io.vertx:vertx-web-validation") implementation("io.vertx:vertx-auth-oauth2") implementation("io.vertx:vertx-web-client") - + implementation("com.h2database:h2:1.4.200") + implementation("org.xerial:sqlite-jdbc:3.47.2.0") implementation("com.zaxxer:HikariCP:5.0.1") implementation("com.j256.ormlite:ormlite-core:4.48") implementation("com.j256.ormlite:ormlite-jdbc:4.48") diff --git a/server/src/main/kotlin/com/espero/yaade/Main.kt b/server/src/main/kotlin/com/espero/yaade/Main.kt index ef1fac1b..74e44f16 100644 --- a/server/src/main/kotlin/com/espero/yaade/Main.kt +++ b/server/src/main/kotlin/com/espero/yaade/Main.kt @@ -12,7 +12,7 @@ import io.vertx.core.http.HttpServerOptions.DEFAULT_MAX_HEADER_SIZE import io.vertx.ext.web.handler.BodyHandler.DEFAULT_BODY_LIMIT val PORT = System.getenv("YAADE_PORT")?.toInt() ?: 9339 -val JDBC_URL = System.getenv("YAADE_JDBC_URL") ?: "jdbc:h2:file:./app/data/yaade-db" +val JDBC_URL = System.getenv("YAADE_JDBC_URL") ?: "jdbc:sqlite:./app/data/yaade-db-sqlite.db" val JDBC_USR = System.getenv("YAADE_JDBC_USERNAME") ?: "sa" val JDBC_PWD = System.getenv("YAADE_JDBC_PASSWORD") ?: "" val ADMIN_USERNAME: String = System.getenv("YAADE_ADMIN_USERNAME") ?: "" From 4eba9acf6e8fe88bf6204afb7a0b20ed21366f66 Mon Sep 17 00:00:00 2001 From: Jonathan Roesner Date: Fri, 14 Mar 2025 22:55:28 +0100 Subject: [PATCH 2/2] WIP --- .../src/main/kotlin/com/espero/yaade/Main.kt | 3 +- .../kotlin/com/espero/yaade/db/ConfigDao.kt | 2 + .../main/kotlin/com/espero/yaade/init/Init.kt | 150 +++++++++++++++++ .../espero/yaade/server/routes/AdminRoute.kt | 159 +++++++++++++++--- 4 files changed, 287 insertions(+), 27 deletions(-) diff --git a/server/src/main/kotlin/com/espero/yaade/Main.kt b/server/src/main/kotlin/com/espero/yaade/Main.kt index 74e44f16..4fcafb5a 100644 --- a/server/src/main/kotlin/com/espero/yaade/Main.kt +++ b/server/src/main/kotlin/com/espero/yaade/Main.kt @@ -12,7 +12,8 @@ import io.vertx.core.http.HttpServerOptions.DEFAULT_MAX_HEADER_SIZE import io.vertx.ext.web.handler.BodyHandler.DEFAULT_BODY_LIMIT val PORT = System.getenv("YAADE_PORT")?.toInt() ?: 9339 -val JDBC_URL = System.getenv("YAADE_JDBC_URL") ?: "jdbc:sqlite:./app/data/yaade-db-sqlite.db" + +val JDBC_URL = System.getenv("YAADE_JDBC_URL") ?: "jdbc:sqlite:./app/data/yaade-db.sqlite" val JDBC_USR = System.getenv("YAADE_JDBC_USERNAME") ?: "sa" val JDBC_PWD = System.getenv("YAADE_JDBC_PASSWORD") ?: "" val ADMIN_USERNAME: String = System.getenv("YAADE_ADMIN_USERNAME") ?: "" diff --git a/server/src/main/kotlin/com/espero/yaade/db/ConfigDao.kt b/server/src/main/kotlin/com/espero/yaade/db/ConfigDao.kt index 252d4a16..ab9f70ec 100644 --- a/server/src/main/kotlin/com/espero/yaade/db/ConfigDao.kt +++ b/server/src/main/kotlin/com/espero/yaade/db/ConfigDao.kt @@ -6,5 +6,7 @@ import com.j256.ormlite.support.ConnectionSource class ConfigDao(connectionSource: ConnectionSource) : BaseDao(connectionSource, ConfigDb::class.java) { + fun getAll(): List = dao.queryForAll() + fun getByName(name: String): ConfigDb? = dao.queryForEq("name", name).getOrNull(0) } diff --git a/server/src/main/kotlin/com/espero/yaade/init/Init.kt b/server/src/main/kotlin/com/espero/yaade/init/Init.kt index 96542343..ad325e58 100644 --- a/server/src/main/kotlin/com/espero/yaade/init/Init.kt +++ b/server/src/main/kotlin/com/espero/yaade/init/Init.kt @@ -1,9 +1,159 @@ package com.espero.yaade.init import com.espero.yaade.db.DaoManager +import com.espero.yaade.server.Server +import io.vertx.core.impl.logging.LoggerFactory +import java.io.File + +const val defaultH2FilePath = "./app/data/yaade-db" +const val h2Extension = ".mv.db" +const val defaultSqliteFilePath = "./app/data/yaade-db.sqlite" + +private val log = LoggerFactory.getLogger(Server::class.java) fun createDaoManager(jdbcUrl: String, jdbcUser: String, jdbcPwd: String): DaoManager { + checkForMigration(jdbcUrl) val daoManager = DaoManager() daoManager.init(jdbcUrl, jdbcUser, jdbcPwd) return daoManager } + +fun checkForMigration(jdbcUrl: String) { + if (jdbcUrl != "jdbc:sqlite:$defaultSqliteFilePath") { + return + } + if (File(defaultSqliteFilePath).exists()) { + return + } + if (!File("$defaultH2FilePath$h2Extension").exists()) { + return + } + doSqliteMigration(defaultSqliteFilePath, defaultH2FilePath) +} + +fun doSqliteMigration(sqliteFilePath: String, h2FilePath: String) { + log.info("Migrating from H2 to SQLite") + val sqliteManager = DaoManager() + sqliteManager.init("jdbc:sqlite:$sqliteFilePath", "", "") + val h2Manager = DaoManager() + h2Manager.init("jdbc:h2:file:$h2FilePath", "sa", "") + + // Create backup of original H2 database + h2Manager.dataSource.connection.use { conn -> + conn.prepareStatement("BACKUP TO '$h2FilePath$h2Extension.bak'").executeUpdate() + } + + // Create tables in SQLite first (this happens automatically when we access the DAOs) + sqliteManager.accessTokenDao.getAll() + sqliteManager.certificatesDao.getAll() + sqliteManager.collectionDao.getAll() + sqliteManager.requestDao.getById(0) + sqliteManager.configDao.getById(0) + sqliteManager.fileDao.getById(0) + sqliteManager.jobScriptDao.getAll() + sqliteManager.userDao.getById(0) + + // Tables to migrate + val tables = listOf( + "accesstokens", + "certificatedb", + "collections", + "requests", + "config", + "file", + "jobscript", + "users" + ) + + // Get connections for direct SQL execution + h2Manager.dataSource.connection.use { h2Conn -> + sqliteManager.dataSource.connection.use { sqliteConn -> + // Disable auto-commit for batch operations + sqliteConn.autoCommit = false + + try { + // Migrate each table + for (table in tables) { + migrateTable(h2Conn, sqliteConn, table) + } + + // Commit all changes + sqliteConn.commit() + log.info("Migration completed successfully") + } catch (e: Exception) { + // Rollback on error + sqliteConn.rollback() + log.error("Migration failed", e) + throw e + } finally { + // Reset autocommit + sqliteConn.autoCommit = true + } + } + } + + h2Manager.dataSource.close() + sqliteManager.dataSource.close() + log.info("Migration complete") +} + +private fun migrateTable( + h2Conn: java.sql.Connection, + sqliteConn: java.sql.Connection, + tableName: String +) { + log.info("Migrating table: $tableName") + val statement = h2Conn.createStatement() + val resultSet = statement.executeQuery("SELECT * FROM $tableName") + val metaData = resultSet.metaData + val columnCount = metaData.columnCount + + // Generate column names and placeholder string for the prepared statement + val columnNames = ArrayList(columnCount) + val placeholders = StringBuilder() + + for (i in 1..columnCount) { + columnNames.add(metaData.getColumnName(i)) + placeholders.append("?") + if (i < columnCount) { + placeholders.append(",") + } + } + + // Build the SQL insert statement + val insertSQL = + "INSERT INTO $tableName (${columnNames.joinToString(",")}) VALUES ($placeholders)" + val insertStatement = sqliteConn.prepareStatement(insertSQL) + + var count = 0 + // For each row in the result set + while (resultSet.next()) { + // Set parameters for each column + for (i in 1..columnCount) { + when (metaData.getColumnType(i)) { + java.sql.Types.BLOB, java.sql.Types.BINARY, java.sql.Types.VARBINARY, java.sql.Types.LONGVARBINARY -> { + val bytes = resultSet.getBytes(i) + if (bytes == null) { + insertStatement.setNull(i, java.sql.Types.BLOB) + } else { + insertStatement.setBytes(i, bytes) + } + } + + else -> { + val value = resultSet.getObject(i) + insertStatement.setObject(i, value) + } + } + } + + insertStatement.executeUpdate() + count++ + } + + resultSet.close() + statement.close() + insertStatement.close() + + log.info("Migrated $count records from table $tableName") +} diff --git a/server/src/main/kotlin/com/espero/yaade/server/routes/AdminRoute.kt b/server/src/main/kotlin/com/espero/yaade/server/routes/AdminRoute.kt index d739a5da..b65da435 100644 --- a/server/src/main/kotlin/com/espero/yaade/server/routes/AdminRoute.kt +++ b/server/src/main/kotlin/com/espero/yaade/server/routes/AdminRoute.kt @@ -1,24 +1,27 @@ package com.espero.yaade.server.routes +import com.espero.yaade.FILE_STORAGE_PATH import com.espero.yaade.JDBC_PWD import com.espero.yaade.JDBC_URL import com.espero.yaade.JDBC_USR import com.espero.yaade.db.DaoManager +import com.espero.yaade.init.doSqliteMigration import com.espero.yaade.model.db.ConfigDb import com.espero.yaade.model.db.UserDb import com.espero.yaade.server.Server import com.espero.yaade.server.errors.ServerError -import com.espero.yaade.server.utils.awaitBlocking import io.netty.handler.codec.http.HttpResponseStatus import io.vertx.core.Vertx +import io.vertx.core.buffer.Buffer import io.vertx.core.http.HttpHeaders +import io.vertx.core.impl.logging.LoggerFactory import io.vertx.core.json.JsonArray import io.vertx.core.json.JsonObject import io.vertx.ext.web.RoutingContext import io.vertx.kotlin.coroutines.awaitBlocking import io.vertx.kotlin.coroutines.coAwait import net.lingala.zip4j.ZipFile -import org.h2.tools.DeleteDbFiles +import java.io.File import java.util.* import java.util.concurrent.Callable @@ -29,43 +32,147 @@ class AdminRoute( private val server: Server ) { + private val log = LoggerFactory.getLogger(AdminRoute::class.java) + suspend fun exportBackup(ctx: RoutingContext) { - val fileUuid = UUID.randomUUID().toString() - daoManager.dataSource.connection.use { conn -> - conn.prepareStatement("BACKUP TO '/tmp/$fileUuid'").executeUpdate() - } + try { + println("Exporting backup") + val dbFilename = "yaade-db.sqlite" + val filesDirname = "files" + val backupFilename = "yaade-db.backup.zip" + val metadataFilename = "metadata.json" + + val backupFile = File("/tmp/$dbFilename") + if (backupFile.exists()) { + backupFile.delete() + } + + daoManager.dataSource.connection.use { conn -> + val stmt = conn.createStatement() + stmt.executeUpdate("VACUUM INTO '/tmp/$dbFilename'") + } + + val backupMetadata = JsonObject() + .put("version", 1) + .put("createdAt", System.currentTimeMillis()) + .put("db", dbFilename) + .put("files", filesDirname) - ctx.response() - .putHeader("Content-Disposition", "attachment; filename=\"yaade-db.mv.db.zip\"") - .putHeader(HttpHeaders.TRANSFER_ENCODING, "chunked") - .sendFile("/tmp/$fileUuid").coAwait() + awaitBlocking { + vertx.fileSystem() + .writeFileBlocking( + "/tmp/$metadataFilename", + Buffer.buffer(backupMetadata.encode()) + ) - vertx.fileSystem().delete("/tmp/$fileUuid") + val zipFile = ZipFile("/tmp/$backupFilename") + zipFile.addFile("/tmp/$metadataFilename") + zipFile.addFile("/tmp/$dbFilename") + + val filesDir = File("/tmp/$filesDirname") + filesDir.deleteRecursively() + filesDir.mkdir() + + val files = vertx.fileSystem().readDirBlocking(FILE_STORAGE_PATH) + for (file in files) { + val filename = File(file).name + vertx.fileSystem().copyBlocking(file, "/tmp/$filesDirname/$filename") + } + + zipFile.addFolder(filesDir) + } + + ctx.response() + .putHeader( + "Content-Disposition", + "attachment; filename=\"yaade.backup\"" + ) + .putHeader(HttpHeaders.TRANSFER_ENCODING, "chunked") + .sendFile("/tmp/$backupFilename").coAwait() + + vertx.fileSystem().delete("/tmp/$metadataFilename").coAwait() + vertx.fileSystem().delete("/tmp/$dbFilename").coAwait() + vertx.fileSystem().delete("/tmp/$backupFilename").coAwait() + } catch (e: Throwable) { + println("Error exporting backup: ${e.message}") + } } suspend fun importBackup(ctx: RoutingContext) { val f = ctx.fileUploads().iterator().next() - - // create a backup so that data is not really lost... val fileUuid = UUID.randomUUID().toString() - vertx.awaitBlocking { - daoManager.dataSource.connection.use { conn -> - conn.prepareStatement("BACKUP TO './app/data/$fileUuid'").executeUpdate() - } + val backupPath = "./app/data/$fileUuid.sqlite" + if (!JDBC_URL.startsWith("jdbc:sqlite:")) { + throw ServerError( + HttpResponseStatus.BAD_REQUEST.code(), + "Importing backups is only supported for Sqlite databases" + ) } + val dbPath = JDBC_URL.replace("jdbc:sqlite:", "") + // Close the current database connection daoManager.close() - DeleteDbFiles.execute("./app/data", "yaade-db", false) - awaitBlocking { - ZipFile(f.uploadedFileName()).extractAll("./app/data") - } - vertx.fileSystem().delete(f.uploadedFileName()).coAwait() + try { + vertx.fileSystem().copy(dbPath, backupPath).coAwait() + vertx.fileSystem().copyRecursive(FILE_STORAGE_PATH, "/tmp/files", true).coAwait() + vertx.fileSystem().delete(dbPath).coAwait() + vertx.fileSystem().deleteRecursive(FILE_STORAGE_PATH, true).coAwait() - daoManager.init(JDBC_URL, JDBC_USR, JDBC_PWD) + awaitBlocking { + ZipFile(f.uploadedFileName()).extractAll("/tmp/$fileUuid") + val extracted = File("/tmp/$fileUuid") - val response = JsonObject().put(f.fileName(), f.size()) - ctx.response().end(response.encode()).coAwait() - server.restartServer() + val metadataFile = File("/tmp/$fileUuid/metadata.json") + + // if it doesn't exist, it's an old h2 backup + if (!metadataFile.exists()) { + doSqliteMigration(dbPath, extracted.absolutePath + "/yaade-db.mv.db") + return@awaitBlocking + } + + val metadata = JsonObject(metadataFile.readText()) + val dbFilename = metadata.getString("db") + val filesDirname = metadata.getString("files") + + val dbFile = File("/tmp/$fileUuid/$dbFilename") + if (!dbFile.exists()) { + throw ServerError(HttpResponseStatus.BAD_REQUEST.code(), "Invalid backup file") + } + vertx.fileSystem().copyBlocking(dbFile.absolutePath, dbPath) + + val filesDir = File("/tmp/$fileUuid/$filesDirname") + if (!filesDir.exists()) { + throw ServerError(HttpResponseStatus.BAD_REQUEST.code(), "Invalid backup file") + } + + val files = filesDir.listFiles() + for (file in files) { + val filename = file.name + vertx.fileSystem() + .copyBlocking(file.absolutePath, "$FILE_STORAGE_PATH/$filename") + } + + } + + try { + vertx.fileSystem().delete(f.uploadedFileName()).coAwait() + vertx.fileSystem().deleteRecursive("/tmp/$fileUuid", true).coAwait() + vertx.fileSystem().deleteRecursive("/tmp/files", true).coAwait() + } catch (e: Throwable) { + log.warn("Error cleaning up after import: ${e.message}") + } + } catch (e: Throwable) { + log.error("Error importing database backup, reverting: ${e.message}") + vertx.fileSystem().copy(backupPath, dbPath).coAwait() + vertx.fileSystem().copyRecursive("/tmp/files", FILE_STORAGE_PATH, true).coAwait() + } finally { + // Reinitialize the database connection + daoManager.init(JDBC_URL, JDBC_USR, JDBC_PWD) + + val response = JsonObject().put(f.fileName(), f.size()) + ctx.response().end(response.encode()).coAwait() + server.restartServer() + } } suspend fun createUser(ctx: RoutingContext) {