rendered paste bodyIndex: D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage.class===================================================================--- D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage.class (revision 14982)+++ D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage.class (working copy)@@ -1334,9 +1334,10 @@ * * @param string $value the value * @param int $size (as a STORAGE_SIZE_XXX constant)+ * @param bool $lengthInBytes (optional) whether to interpret the size in bytes or in characters * @return the truncated string */- function _truncateString($value, $size) {+ function _truncateString($value, $size, $lengthInBytes=false) { switch ($size) { case STORAGE_SIZE_SMALL: $size = 32;@@ -1350,7 +1351,11 @@ $size = 255; break; }- return GalleryUtilities::utf8Substring($value, 0, $size);+ if ($lengthInBytes) {+ return GalleryUtilities::utf8Strcut($value, 0, $size);+ } else {+ return GalleryUtilities::utf8Substring($value, 0, $size);+ } } } @@ -1707,5 +1712,15 @@ return $extras->_getUniqueIdWithConnection($db); }++ /**+ * Truncate UTF-8 strings either to given character or to byte length depending on the MySQL+ * version.+ * @see GalleryStorage::_truncateString+ */+ function _truncateString($value, $size, $lengthInBytes=false) {+ $lengthInBytes = $this->_serverInfo && version_compare($this->_serverInfo, '4.1.0', '<');+ return parent::_truncateString($value, $size, $lengthInBytes);+ } } ?>Index: D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage/Db2Storage.class===================================================================--- D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage/Db2Storage.class (revision 14982)+++ D:/eclipse_workspace/gallery2/modules/core/classes/GalleryStorage/Db2Storage.class (working copy)@@ -278,5 +278,14 @@ return 'CALL ADMIN_CMD (\'RUNSTATS ON TABLE ' . $this->_username . '.%s WITH DISTRIBUTION ON ALL COLUMNS AND INDEXES ALL\')'; }++ /**+ * Truncate UTF-8 strings to given byte length rather than character length since DB2 interprets+ * string lengths in bytes.+ * @see GalleryStorage::_truncateString+ */+ function _truncateString($value, $size, $lengthInBytes=false) {+ return parent::_truncateString($value, $size, true);+ } } ?>Index: D:/eclipse_workspace/gallery2/modules/core/classes/GalleryUtilities.class===================================================================--- D:/eclipse_workspace/gallery2/modules/core/classes/GalleryUtilities.class (revision 14982)+++ D:/eclipse_workspace/gallery2/modules/core/classes/GalleryUtilities.class (working copy)@@ -815,6 +815,80 @@ } /**+ * A multibyte safe version of mb_strcut for UTF-8.+ * @param string $string the input string containing raw UTF-8+ * @param int $start the byte-offset of the start position+ * @param int $length the length in bytes (not in characters)+ * @return string a multibyte safe substring of input value+ */+ function utf8Strcut($string, $start, $length) {+ static $hasMbStrCut;+ if (!isset($hasMbStrCut)) {+ $hasMbStrCut = function_exists('mb_strcut');+ }++ if ($hasMbStrCut) {+ return (string) mb_strcut($string, $start, $length, 'UTF-8');+ } else {+ return GalleryUtilities::_phpUtf8Strcut($string, $start, $length);+ }+ }++ /**+ * A multibyte safe PHP implementation of mb_strcut for UTF-8.+ *+ * This is going to be a whole lot slower than mb_substr, so we should use that instead,+ * whenever we can.+ *+ * @see utf8Strcut+ */+ function _phpUtf8Strcut($string, $start, $length) {+ $strLength = strlen($string);+ $start = $start < 0 ? $strLength + $start : $start;+ $start = $start < 0 ? 0 : $start;+ $length = $length < 0 ? $strLength - $start + $length : $length;+ $length = $length < 0 ? 0 : $length;+ $length = ($start + $length) > $strLength ? $strLength - $start : $length;++ if ($start >= ($strLength - 1) || $strLength == 0 || $length == 0) {+ return '';+ }++ /* 1. Jump to the $start byte and check if it's the start of a character. */+ if ($start != 0 && (ord($string[$start]) & 0xC0) == 0x80) {+ /* It's the 2nd to 4th byte of a multibyte character, go back to the 1st byte of it */+ $j = 0;+ /* Go back a maximum of 3 bytes and stop at the beginning of the string */+ while ($start > 0 && $j++ < 3 && (ord($string[$start]) & 0xC0) == 0x80) {+ $start--;+ }+ }++ /* 2. Exract the multibyte unsafe substring starting at the modified $start */+ $result = substr($string, $start, $length);+ /* 3. Keep the resulting string up to the last complete character */+ $end = $start + $length - 1;+ if ($end >= ($strLength -1)) {+ return $result;+ } else if ((ord($string[$end + 1]) & 0xC0) != 0x80) {+ /* The byte at $end is the last one in the character */+ return $result;+ } else if (ord($string[$end]) & 0x80) { /* Is it a single-byte character? */+ /* It's an incomplete multibyte character, go back to the last character */+ $j = 0;+ /* Go back a maximum of 3 bytes */+ while ($end > $start && $j++ < 3 && (ord($string[$end]) & 0xC0) == 0x80) {+ $end--;+ }+ $end = $end > $start ? $end-- : $start;++ return substr($string, $start, $end - $start); + }++ return $result;+ }++ /** * Convert a numerical unicode value to a multibyte UTF-8 string. Adapted from code found here: * http://us2.php.net/utf8_encode *Index: D:/eclipse_workspace/gallery2/modules/core/test/phpunit/StorageTest.class===================================================================--- D:/eclipse_workspace/gallery2/modules/core/test/phpunit/StorageTest.class (revision 14982)+++ D:/eclipse_workspace/gallery2/modules/core/test/phpunit/StorageTest.class (working copy)@@ -40,6 +40,7 @@ $storage =& $gallery->getStorage(); $this->_db = $storage->_db;+ $this->_serverInfo = isset($storage->_serverInfo) ? $storage->_serverInfo : null; $this->_isEmptyAllowedForNotNullColumn = $storage->_isEmptyAllowedForNotNullColumn; $storage->_db = new StorageTestDB($this); }@@ -49,6 +50,7 @@ $storage =& $gallery->getStorage(); $storage->_db = $this->_db;+ $storage->_serverInfo = $this->_serverInfo; $storage->_isEmptyAllowedForNotNullColumn = $this->_isEmptyAllowedForNotNullColumn; parent::tearDown();@@ -121,10 +123,6 @@ global $gallery; $storage =& $gallery->getStorage(); - /*- * We really should use a fake map, but it's hard to insert a fake map definition into- * GalleryStorage at the moment.- */ $ret = $storage->addMapEntry( 'ExternalIdMap', array('externalId' => str_repeat('x', 200),@@ -772,6 +770,59 @@ return $this->failWithStatus($ret); } }++ function testTruncateString() {+ global $gallery;+ $storage =& $gallery->getStorage();+ /* Use a real SQL connection for this test */+ $storage->_db = $this->_db;+ $storage->_serverInfo = $this->_serverInfo;++ /* Test UTF-8 string with 5 characters / 10 bytes */+ $testString = "\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba";+ /* Get a STRING-LARGE version of that string */+ $originalStringLarge = str_repeat($testString, 51);+ $originalStringTooLarge = str_repeat($testString, 52);+ $key = 'test' . rand();+ $itemId = -7;+ $ret = GalleryCoreApi::addMapEntry('GalleryDerivativePreferencesMap',+ array('itemId' => $itemId,+ 'order' => 1,+ 'derivativeType' => 1,+ 'derivativeOperations' => $originalStringLarge));+ if ($ret) {+ return $this->failWithStatus($ret);+ }++ list ($ret, $results) = GalleryCoreApi::getMapEntry('GalleryDerivativePreferencesMap',+ array('derivativeOperations'),+ array('itemId' => $itemId));+ if ($ret) {+ return $this->failWithStatus($ret);+ }++ /* Cleanup */+ $ret = GalleryCoreApi::removeMapEntry('GalleryDerivativePreferencesMap',+ array('itemId' => $itemId));+ if ($ret) {+ return $this->failWithStatus($ret);+ }+ /* Commit our DB transaction to release locks from the PluginParameterMap table */+ $ret = $storage->checkPoint();+ if ($ret) {+ return $this->failWithStatus($ret);+ }++ $expected = $originalStringLarge;+ if ($storage->getType() == 'db2'+ || ($storage->getType() == 'mysql' && $this->_serverInfo+ && version_compare($this->_serverInfo, '4.1.0', '<'))) {+ $expected = GalleryUtilities::utf8Strcut($originalStringLarge, 0, 255);+ }++ $this->assertEquals(1, $results->resultCount(), 'no / wrong rows selected');+ $this->assertEquals(array($expected), $results->nextResult(), 'inserted string');+ } } class StorageTestDB {Index: D:/eclipse_workspace/gallery2/modules/core/test/phpunit/UtilitiesTest.class===================================================================--- D:/eclipse_workspace/gallery2/modules/core/test/phpunit/UtilitiesTest.class (revision 14982)+++ D:/eclipse_workspace/gallery2/modules/core/test/phpunit/UtilitiesTest.class (working copy)@@ -943,6 +943,58 @@ 'php version'); } + function testAssertMbFunctionOverloadingDisabled() {+ /* Assert that we're not accidentaly using mb_ functions in our fallback */+ $mbFuncOverload = (int) ini_get('mbstring.func_overload');+ $this->assert(!($mbFuncOverload & 2), 'mb function overload str functions. ' .+ 'Please disable mbstring.func_overload to test PHP fallback.'); + }++ function testUtf8Strcut() {+ /* Russian */+ $utf8 = ("\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba\xd0" .+ "\xb8\xd0\xb9\xd0\x97\xd0\xb4\xd1\x80\xd0\xb0" .+ "\xd0\xb2\xd1\x81\xd1\x82\xd0\xb2\xd1\x83");+ $this->assertEquals("\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba",+ $result = GalleryUtilities::utf8Strcut($utf8, 0, 10),+ 'standard version');+ $this->assertEquals(10, strlen($result), 'length with standard version');+ $this->assertEquals("\xd0\xa0\xd1\x83\xd1\x81\xd1\x81\xd0\xba",+ GalleryUtilities::_phpUtf8Strcut($utf8, 0, 10),+ 'PHP version');++ /* With byte offset */+ $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2",+ GalleryUtilities::utf8Strcut($utf8, 24, 7),+ 'standard version with offset');+ $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2",+ GalleryUtilities::_phpUtf8Strcut($utf8, 24, 7),+ 'PHP version with offset');++ /* With byte offset and length matching total length */+ $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2\xd1\x83",+ GalleryUtilities::utf8Strcut($utf8, 24, 8),+ 'standard version with offset up to end');+ $this->assertEquals("\xd1\x81\xd1\x82\xd0\xb2\xd1\x83",+ GalleryUtilities::_phpUtf8Strcut($utf8, 24, 8),+ 'PHP version with offset up to end');++ /* Start position larger than string */+ $this->assertEquals('', GalleryUtilities::utf8Strcut("short", 20, 5),+ 'standard version with too large start position');+ $this->assertEquals('', GalleryUtilities::_phpUtf8Strcut("short", 20, 5),+ 'PHP version with too large start position');++ /* Test some special characters */+ $utf8 = 'apo\'strophe.jpg "quotes"';+ $this->assertEquals('apo\'strophe.jpg "quote',+ GalleryUtilities::utf8Substring($utf8, 0, 22),+ 'standard version special characters');+ $this->assertEquals('apo\'strophe.jpg "quote',+ GalleryUtilities::_phpUtf8Substring($utf8, 0, 22),+ 'php version special characters');+ }+ function testIsEmbedded() { $key = 'G2_EMBED'; $backupValue = null;