Jr/migrate sqlite

server/build.gradle.kts

@@ -31,8 +31,9 @@ dependencies {
     implementation("io.vertx:vertx-web-validation")
     implementation("io.vertx:vertx-auth-oauth2")
     implementation("io.vertx:vertx-web-client")
-
+    
     implementation("com.h2database:h2:1.4.200")
+    implementation("org.xerial:sqlite-jdbc:3.47.2.0")
     implementation("com.zaxxer:HikariCP:5.0.1")
     implementation("com.j256.ormlite:ormlite-core:4.48")
     implementation("com.j256.ormlite:ormlite-jdbc:4.48")

server/src/main/kotlin/com/espero/yaade/Main.kt

@@ -12,7 +12,8 @@ import io.vertx.core.http.HttpServerOptions.DEFAULT_MAX_HEADER_SIZE
 import io.vertx.ext.web.handler.BodyHandler.DEFAULT_BODY_LIMIT
 
 val PORT = System.getenv("YAADE_PORT")?.toInt() ?: 9339
-val JDBC_URL = System.getenv("YAADE_JDBC_URL") ?: "jdbc:h2:file:./app/data/yaade-db"
+
+val JDBC_URL = System.getenv("YAADE_JDBC_URL") ?: "jdbc:sqlite:./app/data/yaade-db.sqlite"
 val JDBC_USR = System.getenv("YAADE_JDBC_USERNAME") ?: "sa"
 val JDBC_PWD = System.getenv("YAADE_JDBC_PASSWORD") ?: ""
 val ADMIN_USERNAME: String = System.getenv("YAADE_ADMIN_USERNAME") ?: ""

server/src/main/kotlin/com/espero/yaade/db/ConfigDao.kt

@@ -6,5 +6,7 @@ import com.j256.ormlite.support.ConnectionSource
 class ConfigDao(connectionSource: ConnectionSource) :
     BaseDao<ConfigDb>(connectionSource, ConfigDb::class.java) {
 
+    fun getAll(): List<ConfigDb> = dao.queryForAll()
+
     fun getByName(name: String): ConfigDb? = dao.queryForEq("name", name).getOrNull(0)
 }

server/src/main/kotlin/com/espero/yaade/init/Init.kt

@@ -1,9 +1,159 @@
 package com.espero.yaade.init
 
 import com.espero.yaade.db.DaoManager
+import com.espero.yaade.server.Server
+import io.vertx.core.impl.logging.LoggerFactory
+import java.io.File
+
+const val defaultH2FilePath = "./app/data/yaade-db"
+const val h2Extension = ".mv.db"
+const val defaultSqliteFilePath = "./app/data/yaade-db.sqlite"
+
+private val log = LoggerFactory.getLogger(Server::class.java)
 
 fun createDaoManager(jdbcUrl: String, jdbcUser: String, jdbcPwd: String): DaoManager {
+    checkForMigration(jdbcUrl)
     val daoManager = DaoManager()
     daoManager.init(jdbcUrl, jdbcUser, jdbcPwd)
     return daoManager
 }
+
+fun checkForMigration(jdbcUrl: String) {
+    if (jdbcUrl != "jdbc:sqlite:$defaultSqliteFilePath") {
+        return
+    }
+    if (File(defaultSqliteFilePath).exists()) {
+        return
+    }
+    if (!File("$defaultH2FilePath$h2Extension").exists()) {
+        return
+    }
+    doSqliteMigration(defaultSqliteFilePath, defaultH2FilePath)
+}
+
+fun doSqliteMigration(sqliteFilePath: String, h2FilePath: String) {
+    log.info("Migrating from H2 to SQLite")
+    val sqliteManager = DaoManager()
+    sqliteManager.init("jdbc:sqlite:$sqliteFilePath", "", "")
+    val h2Manager = DaoManager()
+    h2Manager.init("jdbc:h2:file:$h2FilePath", "sa", "")
+
+    // Create backup of original H2 database
+    h2Manager.dataSource.connection.use { conn ->
+        conn.prepareStatement("BACKUP TO '$h2FilePath$h2Extension.bak'").executeUpdate()
+    }
+
+    // Create tables in SQLite first (this happens automatically when we access the DAOs)
+    sqliteManager.accessTokenDao.getAll()
+    sqliteManager.certificatesDao.getAll()
+    sqliteManager.collectionDao.getAll()
+    sqliteManager.requestDao.getById(0)
+    sqliteManager.configDao.getById(0)
+    sqliteManager.fileDao.getById(0)
+    sqliteManager.jobScriptDao.getAll()
+    sqliteManager.userDao.getById(0)
+
+    // Tables to migrate
+    val tables = listOf(
+        "accesstokens",
+        "certificatedb",
+        "collections",
+        "requests",
+        "config",
+        "file",
+        "jobscript",
+        "users"
+    )
+
+    // Get connections for direct SQL execution
+    h2Manager.dataSource.connection.use { h2Conn ->
+        sqliteManager.dataSource.connection.use { sqliteConn ->
+            // Disable auto-commit for batch operations
+            sqliteConn.autoCommit = false
+
+            try {
+                // Migrate each table
+                for (table in tables) {
+                    migrateTable(h2Conn, sqliteConn, table)
+                }
+
+                // Commit all changes
+                sqliteConn.commit()
+                log.info("Migration completed successfully")
+            } catch (e: Exception) {
+                // Rollback on error
+                sqliteConn.rollback()
+                log.error("Migration failed", e)
+                throw e
+            } finally {
+                // Reset autocommit
+                sqliteConn.autoCommit = true
+            }
+        }
+    }
+
+    h2Manager.dataSource.close()
+    sqliteManager.dataSource.close()
+    log.info("Migration complete")
+}
+
+private fun migrateTable(
+    h2Conn: java.sql.Connection,
+    sqliteConn: java.sql.Connection,
+    tableName: String
+) {
+    log.info("Migrating table: $tableName")
+    val statement = h2Conn.createStatement()
+    val resultSet = statement.executeQuery("SELECT * FROM $tableName")
+    val metaData = resultSet.metaData
+    val columnCount = metaData.columnCount
+
+    // Generate column names and placeholder string for the prepared statement
+    val columnNames = ArrayList<String>(columnCount)
+    val placeholders = StringBuilder()
+
+    for (i in 1..columnCount) {
+        columnNames.add(metaData.getColumnName(i))
+        placeholders.append("?")
+        if (i < columnCount) {
+            placeholders.append(",")
+        }
+    }
+
+    // Build the SQL insert statement
+    val insertSQL =
+        "INSERT INTO $tableName (${columnNames.joinToString(",")}) VALUES ($placeholders)"
+    val insertStatement = sqliteConn.prepareStatement(insertSQL)
+
+    var count = 0
+    // For each row in the result set
+    while (resultSet.next()) {
+        // Set parameters for each column
+        for (i in 1..columnCount) {
+            when (metaData.getColumnType(i)) {
+                java.sql.Types.BLOB, java.sql.Types.BINARY, java.sql.Types.VARBINARY, java.sql.Types.LONGVARBINARY -> {
+                    val bytes = resultSet.getBytes(i)
+                    if (bytes == null) {
+                        insertStatement.setNull(i, java.sql.Types.BLOB)
+                    } else {
+                        insertStatement.setBytes(i, bytes)
+                    }
+                }
+
+                else -> {
+                    val value = resultSet.getObject(i)
+                    insertStatement.setObject(i, value)
+                }
+            }
+        }
+
+        insertStatement.executeUpdate()
+        count++
+    }
+
+    resultSet.close()
+    statement.close()
+    insertStatement.close()
+
+    log.info("Migrated $count records from table $tableName")
+}

server/src/main/kotlin/com/espero/yaade/server/routes/AdminRoute.kt

@@ -1,24 +1,27 @@
 package com.espero.yaade.server.routes
 
+import com.espero.yaade.FILE_STORAGE_PATH
 import com.espero.yaade.JDBC_PWD
 import com.espero.yaade.JDBC_URL
 import com.espero.yaade.JDBC_USR
 import com.espero.yaade.db.DaoManager
+import com.espero.yaade.init.doSqliteMigration
 import com.espero.yaade.model.db.ConfigDb
 import com.espero.yaade.model.db.UserDb
 import com.espero.yaade.server.Server
 import com.espero.yaade.server.errors.ServerError
-import com.espero.yaade.server.utils.awaitBlocking
 import io.netty.handler.codec.http.HttpResponseStatus
 import io.vertx.core.Vertx
+import io.vertx.core.buffer.Buffer
 import io.vertx.core.http.HttpHeaders
+import io.vertx.core.impl.logging.LoggerFactory
 import io.vertx.core.json.JsonArray
 import io.vertx.core.json.JsonObject
 import io.vertx.ext.web.RoutingContext
 import io.vertx.kotlin.coroutines.awaitBlocking
 import io.vertx.kotlin.coroutines.coAwait
 import net.lingala.zip4j.ZipFile
-import org.h2.tools.DeleteDbFiles
+import java.io.File
 import java.util.*
 import java.util.concurrent.Callable
 
@@ -29,43 +32,147 @@ class AdminRoute(
     private val server: Server
 ) {
 
+    private val log = LoggerFactory.getLogger(AdminRoute::class.java)
+
     suspend fun exportBackup(ctx: RoutingContext) {
-        val fileUuid = UUID.randomUUID().toString()
-        daoManager.dataSource.connection.use { conn ->
-            conn.prepareStatement("BACKUP TO '/tmp/$fileUuid'").executeUpdate()
-        }
+        try {
+            println("Exporting backup")
+            val dbFilename = "yaade-db.sqlite"
+            val filesDirname = "files"
+            val backupFilename = "yaade-db.backup.zip"
+            val metadataFilename = "metadata.json"
+
+            val backupFile = File("/tmp/$dbFilename")
+            if (backupFile.exists()) {
+                backupFile.delete()
+            }
+
+            daoManager.dataSource.connection.use { conn ->
+                val stmt = conn.createStatement()
+                stmt.executeUpdate("VACUUM INTO '/tmp/$dbFilename'")
+            }
+
+            val backupMetadata = JsonObject()
+                .put("version", 1)
+                .put("createdAt", System.currentTimeMillis())
+                .put("db", dbFilename)
+                .put("files", filesDirname)
 
-        ctx.response()
-            .putHeader("Content-Disposition", "attachment; filename=\"yaade-db.mv.db.zip\"")
-            .putHeader(HttpHeaders.TRANSFER_ENCODING, "chunked")
-            .sendFile("/tmp/$fileUuid").coAwait()
+            awaitBlocking {
+                vertx.fileSystem()
+                    .writeFileBlocking(
+                        "/tmp/$metadataFilename",
+                        Buffer.buffer(backupMetadata.encode())
+                    )
 
-        vertx.fileSystem().delete("/tmp/$fileUuid")
+                val zipFile = ZipFile("/tmp/$backupFilename")
+                zipFile.addFile("/tmp/$metadataFilename")
+                zipFile.addFile("/tmp/$dbFilename")
+
+                val filesDir = File("/tmp/$filesDirname")
+                filesDir.deleteRecursively()
+                filesDir.mkdir()
+
+                val files = vertx.fileSystem().readDirBlocking(FILE_STORAGE_PATH)
+                for (file in files) {
+                    val filename = File(file).name
+                    vertx.fileSystem().copyBlocking(file, "/tmp/$filesDirname/$filename")
+                }
+
+                zipFile.addFolder(filesDir)
+            }
+
+            ctx.response()
+                .putHeader(
+                    "Content-Disposition",
+                    "attachment; filename=\"yaade.backup\""
+                )
+                .putHeader(HttpHeaders.TRANSFER_ENCODING, "chunked")
+                .sendFile("/tmp/$backupFilename").coAwait()
+
+            vertx.fileSystem().delete("/tmp/$metadataFilename").coAwait()
+            vertx.fileSystem().delete("/tmp/$dbFilename").coAwait()
+            vertx.fileSystem().delete("/tmp/$backupFilename").coAwait()
+        } catch (e: Throwable) {
+            println("Error exporting backup: ${e.message}")
+        }
     }
 
     suspend fun importBackup(ctx: RoutingContext) {
         val f = ctx.fileUploads().iterator().next()
-
-        // create a backup so that data is not really lost...
         val fileUuid = UUID.randomUUID().toString()
-        vertx.awaitBlocking {
-            daoManager.dataSource.connection.use { conn ->
-                conn.prepareStatement("BACKUP TO './app/data/$fileUuid'").executeUpdate()
-            }
+        val backupPath = "./app/data/$fileUuid.sqlite"
+        if (!JDBC_URL.startsWith("jdbc:sqlite:")) {
+            throw ServerError(
+                HttpResponseStatus.BAD_REQUEST.code(),
+                "Importing backups is only supported for Sqlite databases"
+            )
         }
+        val dbPath = JDBC_URL.replace("jdbc:sqlite:", "")
+        // Close the current database connection
         daoManager.close()
 
-        DeleteDbFiles.execute("./app/data", "yaade-db", false)
-        awaitBlocking {
-            ZipFile(f.uploadedFileName()).extractAll("./app/data")
-        }
-        vertx.fileSystem().delete(f.uploadedFileName()).coAwait()
+        try {
+            vertx.fileSystem().copy(dbPath, backupPath).coAwait()
+            vertx.fileSystem().copyRecursive(FILE_STORAGE_PATH, "/tmp/files", true).coAwait()
+            vertx.fileSystem().delete(dbPath).coAwait()
+            vertx.fileSystem().deleteRecursive(FILE_STORAGE_PATH, true).coAwait()
 
-        daoManager.init(JDBC_URL, JDBC_USR, JDBC_PWD)
+            awaitBlocking {
+                ZipFile(f.uploadedFileName()).extractAll("/tmp/$fileUuid")
+                val extracted = File("/tmp/$fileUuid")
 
-        val response = JsonObject().put(f.fileName(), f.size())
-        ctx.response().end(response.encode()).coAwait()
-        server.restartServer()
+                val metadataFile = File("/tmp/$fileUuid/metadata.json")
+
+                // if it doesn't exist, it's an old h2 backup
+                if (!metadataFile.exists()) {
+                    doSqliteMigration(dbPath, extracted.absolutePath + "/yaade-db.mv.db")
+                    return@awaitBlocking
+                }
+
+                val metadata = JsonObject(metadataFile.readText())
+                val dbFilename = metadata.getString("db")
+                val filesDirname = metadata.getString("files")
+
+                val dbFile = File("/tmp/$fileUuid/$dbFilename")
+                if (!dbFile.exists()) {
+                    throw ServerError(HttpResponseStatus.BAD_REQUEST.code(), "Invalid backup file")
+                }
+                vertx.fileSystem().copyBlocking(dbFile.absolutePath, dbPath)
+
+                val filesDir = File("/tmp/$fileUuid/$filesDirname")
+                if (!filesDir.exists()) {
+                    throw ServerError(HttpResponseStatus.BAD_REQUEST.code(), "Invalid backup file")
+                }
+
+                val files = filesDir.listFiles()
+                for (file in files) {
+                    val filename = file.name
+                    vertx.fileSystem()
+                        .copyBlocking(file.absolutePath, "$FILE_STORAGE_PATH/$filename")
+                }
+
+            }
+
+            try {
+                vertx.fileSystem().delete(f.uploadedFileName()).coAwait()
+                vertx.fileSystem().deleteRecursive("/tmp/$fileUuid", true).coAwait()
+                vertx.fileSystem().deleteRecursive("/tmp/files", true).coAwait()
+            } catch (e: Throwable) {
+                log.warn("Error cleaning up after import: ${e.message}")
+            }
+        } catch (e: Throwable) {
+            log.error("Error importing database backup, reverting: ${e.message}")
+            vertx.fileSystem().copy(backupPath, dbPath).coAwait()
+            vertx.fileSystem().copyRecursive("/tmp/files", FILE_STORAGE_PATH, true).coAwait()
+        } finally {
+            // Reinitialize the database connection
+            daoManager.init(JDBC_URL, JDBC_USR, JDBC_PWD)
+
+            val response = JsonObject().put(f.fileName(), f.size())
+            ctx.response().end(response.encode()).coAwait()
+            server.restartServer()
+        }
     }
 
     suspend fun createUser(ctx: RoutingContext) {