refactor: migrate database encoding to utf8mb4

config.php

@@ -127,7 +127,7 @@
 define('AJAX_ENCODING', 'UTF-8');
 
 /* SQL Character Set. */
-define('SQL_CHARACTER_SET', 'utf8');
+define('SQL_CHARACTER_SET', 'utf8mb4');
 
 /* Insert BOM in the beginning of CSV file */
 /* This is UTF-8 BOM, EF BB BF for UTF-8 */

lib/InstallationTests.php

@@ -402,11 +402,11 @@ public static function checkMySQL($host, $user, $pass, $name)
         );
 
         /* Check CREATE TABLE permissions. */
-        $queryResult = @mysqli_query($db, 'CREATE TABLE `testtable` (`id` int(11) NOT NULL default \'0\') ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;');
+        $queryResult = @mysqli_query($db, 'CREATE TABLE `testtable` (`id` int(11) NOT NULL default \'0\') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;');
         if (!$queryResult)
         {
             mysqli_query($db, 'DROP TABLE testtable');
-            $queryResult = @mysqli_query($db, 'CREATE TABLE `testtable` (`id` int(11) NOT NULL default \'0\') ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;');
+            $queryResult = @mysqli_query($db, 'CREATE TABLE `testtable` (`id` int(11) NOT NULL default \'0\') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;');
         }
         if (!$queryResult)
         {

modules/install/Schema.php

@@ -1532,6 +1532,27 @@ public static function get()
                     ADD COLUMN `time_zone_iana` varchar(64) NOT NULL DEFAULT \'UTC\'
                     AFTER `time_zone`", true);
             ',
+            '382' => 'PHP:
+                $quoteIdentifier = function($identifier)
+                {
+                    return \'`\' . str_replace(\'`\', \'``\', $identifier) . \'`\';
+                };
+                $db->query(\'ALTER DATABASE \' . $quoteIdentifier(DATABASE_NAME) . \' CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci\');
+                $tables = $db->getAllAssoc(
+                    "SELECT TABLE_NAME AS table_name
+                     FROM information_schema.TABLES
+                     WHERE TABLE_SCHEMA = DATABASE()
+                       AND TABLE_TYPE = \'BASE TABLE\'"
+                );
+                foreach ($tables as $row)
+                {
+                    $tableName = $row[\'table_name\'];
+                    $db->query(
+                        \'ALTER TABLE \' . $quoteIdentifier($tableName)
+                        . \' CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci\'
+                    );
+                }
+            ',
 
         );
     }

modules/install/ajax/ui.php

@@ -1131,6 +1131,7 @@ function MySQLConnect()
         return false;
     }
 
+    mysqli_set_charset($mySQLConnection, SQL_CHARACTER_SET);
 
     /* Create an array of all tables in the database. */
     $tables = array();

src/OpenCATS/Tests/IntegrationTests/DatabaseConnectionTest.php

@@ -126,6 +126,38 @@ function testMakeQueryIntegerOrNULL()
         }
     }
 
+    function testFourByteUnicodeRoundTrip()
+    {
+        $db = DatabaseConnection::getInstance();
+
+        $db->query(
+            'CREATE TEMPORARY TABLE `utf8mb4_regression_test` ('
+            . '`id` int(11) NOT NULL AUTO_INCREMENT,'
+            . '`value` text NOT NULL,'
+            . 'PRIMARY KEY (`id`)'
+            . ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci'
+        );
+
+        $fourByteString = "OpenCATS \u{1F600}";
+
+        $db->query(
+            'INSERT INTO `utf8mb4_regression_test` (`value`) VALUES ('
+            . $db->makeQueryString($fourByteString)
+            . ')'
+        );
+
+        $rows = $db->getAllAssoc('SELECT `value` FROM `utf8mb4_regression_test` LIMIT 1');
+
+        $db->query('DROP TEMPORARY TABLE `utf8mb4_regression_test`');
+
+        $this->assertCount(1, $rows, 'Expected one row back from utf8mb4 regression table');
+        $this->assertSame(
+            $fourByteString,
+            $rows[0]['value'],
+            'Four-byte Unicode string must survive a database round-trip unchanged'
+        );
+    }
+
     function testMakeQueryDouble()
     {
         $db = DatabaseConnection::getInstance();

test/config.php

@@ -127,7 +127,7 @@
 define('AJAX_ENCODING', 'UTF-8');
 
 /* SQL Character Set. */
-define('SQL_CHARACTER_SET', 'utf8');
+define('SQL_CHARACTER_SET', 'utf8mb4');
 
 /* Insert BOM in the beginning of CSV file */
 /* This is UTF-8 BOM, EF BB BF for UTF-8 */