1 /* 2 Copyright (c) 2010, 2021, Oracle and/or its affiliates. 3 4 This program is free software; you can redistribute it and/or modify 5 it under the terms of the GNU General Public License, version 2.0, 6 as published by the Free Software Foundation. 7 8 This program is also distributed with certain software (including 9 but not limited to OpenSSL) that is licensed under separate terms, 10 as designated in a particular file or component or in included license 11 documentation. The authors of MySQL hereby grant you an additional 12 permission to link the program and your derivative works with the 13 separately licensed software that they have included with MySQL. 14 15 This program is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU General Public License, version 2.0, for more details. 19 20 You should have received a copy of the GNU General Public License 21 along with this program; if not, write to the Free Software 22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 23 */ 24 25 package testsuite.clusterj; 26 27 import java.nio.ByteBuffer; 28 import java.nio.CharBuffer; 29 import java.nio.charset.Charset; 30 import java.nio.charset.CharsetEncoder; 31 import java.nio.charset.CoderResult; 32 33 import java.sql.PreparedStatement; 34 import java.sql.ResultSet; 35 import java.sql.SQLException; 36 37 import java.util.ArrayList; 38 import java.util.List; 39 import java.util.Properties; 40 41 import testsuite.clusterj.model.CharsetLatin1; 42 import testsuite.clusterj.model.CharsetBig5; 43 import testsuite.clusterj.model.CharsetModel; 44 import testsuite.clusterj.model.CharsetSjis; 45 import testsuite.clusterj.model.CharsetUtf8; 46 47 /** Test that all characters in supported character sets can be read and written. 48 49 * 1. Identify which character sets to test. 50 * 2. For each character set, create a table with an id column and three VARCHAR columns 51 * (one with length < 256 another with length > 256, and a third with length > 8000) 52 * with the test character set. 53 * 3. For each table, write a persistent interface that maps the table. 54 * 4. For each persistent interface: 55 * a) create an empty list of String 56 * b) create a CharBuffer containing all mappable characters for the character set from the range 0:65535 57 * c) map the CharBuffer to a ByteBuffer of length equal to the size of the VARCHAR column 58 * d) create a String from the characters in the CharBuffer that could fit into the column 59 * e) add the String to the list of String 60 * f) continue from c) until all characters have been represented in the list of String 61 * g) remove all rows of the table 62 * h) use JDBC or clusterj to write a row in the database for each String in the list 63 * i) use JDBC or clusterj to read all rows and compare the String to the list of Strings 64 * 65 */ 66 public class CharsetTest extends AbstractClusterJModelTest { 67 68 @Override localSetUp()69 public void localSetUp() { 70 createSessionFactory(); 71 session = sessionFactory.getSession(); 72 setAutoCommit(connection, false); 73 } 74 testLatin1()75 public void testLatin1() { 76 writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 77 writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 78 writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 79 80 writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 81 writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 82 writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 83 84 writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 85 writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 86 writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 87 88 writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 89 writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 90 writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 91 92 failOnError(); 93 } 94 testUtf8()95 public void testUtf8() { 96 writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 97 writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 98 writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 99 100 writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 101 writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 102 writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 103 104 writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 105 writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 106 writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 107 108 writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 109 writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 110 writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 111 112 failOnError(); 113 } 114 testSjis()115 public void testSjis() { 116 /* These tests are excluded due to a JDBC error: 117 * java.sql.SQLException: 118 * Failed to insert charsetsjis at instance 0 errant string: [... 165 167 168... ] 119 * Incorrect string value: '\xC2\xA5\xC2\xA7\xC2\xA8...' for column 'smallcolumn' at row 1 120 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1055) 121 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:956) 122 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3558) 123 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3490) 124 at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1959) 125 at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2109) 126 at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2648) 127 at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2077) 128 at com.mysql.jdbc.PreparedStatement.execute(PreparedStatement.java:1356) 129 at testsuite.clusterj.CharsetTest.writeToJDBC(CharsetTest.java:317) 130 writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL); 131 writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM); 132 writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE); 133 */ 134 135 writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL); 136 writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM); 137 writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE); 138 139 writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL); 140 writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM); 141 writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE); 142 143 failOnError(); 144 } 145 testBig5()146 public void testBig5() { 147 writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 148 writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 149 writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 150 151 writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 152 writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 153 writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 154 155 writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 156 writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 157 writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 158 159 writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 160 writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 161 writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 162 163 failOnError(); 164 } 165 writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)166 protected void writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 167 ColumnDescriptor columnDescriptor) { 168 removeAll(modelClass); 169 List<String> result = null; 170 List<String> strings = generateStrings(columnDescriptor, charsetName); 171 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 172 writeToJDBC(columnDescriptor, tableName, instances); 173 result = readFromJDBC(columnDescriptor, tableName); 174 if (debug) System.out.println("Returned results of size " + result.size()); 175 verify("writeJDBCreadJDBC ", strings, result, columnDescriptor); 176 } 177 writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)178 protected void writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 179 ColumnDescriptor columnDescriptor) { 180 removeAll(modelClass); 181 List<String> result = null; 182 List<String> strings = generateStrings(columnDescriptor, charsetName); 183 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 184 writeToJDBC(columnDescriptor, tableName, instances); 185 result = readFromNDB(columnDescriptor, modelClass); 186 if (debug) System.out.println("Returned results of size " + result.size()); 187 verify("writeJDBCreadNDB ", strings, result, columnDescriptor); 188 } 189 writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)190 protected void writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 191 ColumnDescriptor columnDescriptor) { 192 removeAll(modelClass); 193 List<String> result = null; 194 List<String> strings = generateStrings(columnDescriptor, charsetName); 195 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 196 writeToNDB(columnDescriptor, instances); 197 result = readFromJDBC(columnDescriptor, tableName); 198 if (debug) System.out.println("Returned results of size " + result.size()); 199 verify("writeNDBreadJDBC ", strings, result, columnDescriptor); 200 } 201 writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)202 protected void writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 203 ColumnDescriptor columnDescriptor) { 204 removeAll(modelClass); 205 List<String> result = null; 206 List<String> strings = generateStrings(columnDescriptor, charsetName); 207 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 208 writeToNDB(columnDescriptor, instances); 209 result = readFromNDB(columnDescriptor, modelClass); 210 if (debug) System.out.println("Returned results of size " + result.size()); 211 verify("writeNDBreadNDB ", strings, result, columnDescriptor); 212 } 213 verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor)214 private void verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor) { 215 int maxErrors = 10; 216 for (int i = 0; i < expecteds.size(); ++i) { 217 String expected = expecteds.get(i); 218 String actual = actuals.get(i); 219 if (actual == null) { 220 error(where + columnDescriptor.columnName + " actual column " + i + " was null."); 221 } else { 222 int expectedLength = expected.length(); 223 int actualLength = actual.length(); 224 errorIfNotEqual(where + "got failure on size of column data for column width " + columnDescriptor.columnWidth + " at row " + i, expectedLength, actualLength); 225 if (expectedLength != actualLength) 226 continue; 227 for (int j = 0; j < expected.length(); ++j) { 228 if (--maxErrors > 0) { 229 errorIfNotEqual("Failure to match column data for column width " + columnDescriptor.columnWidth + " at row " + i + " column " + j, 230 expected.codePointAt(j), actual.codePointAt(j)); 231 } 232 } 233 } 234 } 235 } 236 generateStrings(ColumnDescriptor columnDescriptor, String charsetName)237 protected List<String> generateStrings(ColumnDescriptor columnDescriptor, 238 String charsetName) { 239 List<String> result = new ArrayList<String>(); 240 Charset charset = Charset.forName(charsetName); 241 CharBuffer allChars = CharBuffer.allocate(65536); 242 CharsetEncoder encoder = charset.newEncoder(); 243 // add all encodable characters to the buffer 244 int count = 0; 245 for (int i = 0; i < 65536; ++i) { 246 Character ch = (char)i; 247 if (encoder.canEncode(ch)) { 248 allChars.append(ch); 249 ++count; 250 } 251 } 252 if (debug) System.out.print(charsetName + " has " + count + " encodable characters"); 253 allChars.flip(); 254 255 int width = columnDescriptor.getColumnWidth(); 256 // encode all the characters that fit into the output byte buffer 257 boolean done = false; 258 byte[] bytes = new byte[width]; 259 while (!done) { 260 int begin = allChars.position(); 261 allChars.mark(); 262 ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); 263 CoderResult coderResult = encoder.encode(allChars, byteBuffer, false); 264 int end = allChars.position(); 265 int length = end - begin; 266 if (length == 0) { 267 done = true; 268 continue; 269 } 270 char[] chars = new char[length]; 271 allChars.reset(); 272 allChars.get(chars, 0, length); 273 String encodable = String.copyValueOf(chars); 274 result.add(encodable); 275 if (coderResult.isUnderflow()) { 276 done = true; 277 } 278 } 279 if (debug) System.out.println(" in " + result.size() + " row(s) of size " + columnDescriptor.columnWidth); 280 return result; 281 } 282 generateInstances(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass, List<String> strings)283 protected List<CharsetModel> generateInstances(ColumnDescriptor columnDescriptor, 284 Class<? extends CharsetModel> modelClass, List<String> strings) { 285 List<CharsetModel> result = new ArrayList<CharsetModel>(); 286 for (int i = 0; i < strings.size(); ++i) { 287 CharsetModel instance = session.newInstance(modelClass); 288 instance.setId(i); 289 columnDescriptor.set(instance, strings.get(i)); 290 result.add(instance); 291 } 292 if (debug) System.out.println("Created " + result.size() + " instances of " + modelClass.getName()); 293 return result; 294 } 295 writeToJDBC(ColumnDescriptor columnDescriptor, String tableName, List<CharsetModel> instances)296 protected void writeToJDBC(ColumnDescriptor columnDescriptor, 297 String tableName, List<CharsetModel> instances) { 298 StringBuffer buffer = new StringBuffer("INSERT INTO "); 299 buffer.append(tableName); 300 buffer.append(" (id, "); 301 buffer.append(columnDescriptor.getColumnName()); 302 buffer.append(") VALUES (?, ?)"); 303 String statement = buffer.toString(); 304 if (debug) System.out.println(statement); 305 PreparedStatement preparedStatement = null; 306 int i = 0; 307 String value = ""; 308 try { 309 Properties extraProperties = new Properties(); 310 extraProperties.put("characterEncoding", "utf8"); 311 getConnection(extraProperties); 312 setAutoCommit(connection, false); 313 preparedStatement = connection.prepareStatement(statement); 314 if (debug) System.out.println(preparedStatement.toString()); 315 for (i = 0; i < instances.size(); ++i) { 316 CharsetModel instance = instances.get(i); 317 preparedStatement.setInt(1, instance.getId()); 318 value = columnDescriptor.get(instance); 319 preparedStatement.setString(2, value); 320 // if (debug) System.out.println("Value set to column is size " + value.length()); 321 // if (debug) System.out.println(" value " + value); 322 preparedStatement.execute(); 323 } 324 connection.commit(); 325 } catch (SQLException e) { 326 throw new RuntimeException("Failed to insert " + tableName + " at instance " + i + " errant string: " + dump(value), e); 327 } 328 } 329 writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances)330 protected void writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances) { 331 session.currentTransaction().begin(); 332 for (CharsetModel instance: instances) { 333 session.makePersistent(instance); 334 } 335 session.currentTransaction().commit(); 336 } 337 readFromNDB(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass)338 protected List<String> readFromNDB(ColumnDescriptor columnDescriptor, 339 Class<? extends CharsetModel> modelClass) { 340 List<String> result = new ArrayList<String>(); 341 session.currentTransaction().begin(); 342 int i = 0; 343 boolean done = false; 344 while (!done) { 345 CharsetModel instance = session.find(modelClass, i++); 346 if (instance != null) { 347 result.add(columnDescriptor.get(instance)); 348 } else { 349 done = true; 350 } 351 } 352 session.currentTransaction().commit(); 353 return result; 354 } 355 readFromJDBC(ColumnDescriptor columnDescriptor, String tableName)356 protected List<String> readFromJDBC(ColumnDescriptor columnDescriptor, 357 String tableName) { 358 List<String> result = new ArrayList<String>(); 359 StringBuffer buffer = new StringBuffer("SELECT id, "); 360 buffer.append(columnDescriptor.getColumnName()); 361 buffer.append(" FROM "); 362 buffer.append(tableName); 363 buffer.append(" ORDER BY ID"); 364 String statement = buffer.toString(); 365 if (debug) System.out.println(statement); 366 PreparedStatement preparedStatement = null; 367 int i = 0; 368 try { 369 preparedStatement = connection.prepareStatement(statement); 370 ResultSet rs = preparedStatement.executeQuery(); 371 while (rs.next()) { 372 String columnData = rs.getString(2); 373 result.add(columnData); 374 ++i; 375 } 376 connection.commit(); 377 } catch (SQLException e) { 378 throw new RuntimeException("Failed to read " + tableName + " at instance " + i, e); 379 } 380 return result; 381 } 382 383 protected enum ColumnDescriptor { 384 SMALL(200, "smallcolumn", new InstanceHandler() { 385 public void set(CharsetModel instance, String value) { 386 instance.setSmallColumn(value); 387 } 388 public String get(CharsetModel instance) { 389 return instance.getSmallColumn(); 390 } 391 }), 392 MEDIUM(500, "mediumcolumn", new InstanceHandler() { 393 public void set(CharsetModel instance, String value) { 394 instance.setMediumColumn(value); 395 } 396 public String get(CharsetModel instance) { 397 return instance.getMediumColumn(); 398 } 399 }), 400 LARGE(10000, "largecolumn", new InstanceHandler() { 401 public void set(CharsetModel instance, String value) { 402 instance.setLargeColumn(value); 403 } 404 public String get(CharsetModel instance) { 405 return instance.getLargeColumn(); 406 } 407 }); 408 409 private int columnWidth; 410 411 private String columnName; 412 413 private InstanceHandler instanceHandler; 414 getColumnName()415 public String getColumnName() { 416 return columnName; 417 } 418 get(CharsetModel instance)419 public String get(CharsetModel instance) { 420 return instanceHandler.get(instance); 421 } 422 set(CharsetModel instance, String string)423 public void set(CharsetModel instance, String string) { 424 this.instanceHandler.set(instance, string); 425 } 426 getColumnWidth()427 public int getColumnWidth() { 428 return columnWidth; 429 } 430 ColumnDescriptor(int width, String name, InstanceHandler instanceHandler)431 private ColumnDescriptor(int width, String name, InstanceHandler instanceHandler) { 432 this.columnWidth = width; 433 this.columnName = name; 434 this.instanceHandler = instanceHandler; 435 } 436 437 private interface InstanceHandler { set(CharsetModel instance, String value)438 void set(CharsetModel instance, String value); get(CharsetModel instance)439 String get(CharsetModel instance); 440 } 441 442 } 443 444 /** The instances for testing. */ 445 protected List<CharsetModel> charsetTypes = new ArrayList<CharsetModel>(); 446 447 } 448