1 /* 2 Copyright 2010 Sun Microsystems, Inc. 3 All rights reserved. Use is subject to license terms. 4 5 This program is free software; you can redistribute it and/or modify 6 it under the terms of the GNU General Public License, version 2.0, 7 as published by the Free Software Foundation. 8 9 This program is also distributed with certain software (including 10 but not limited to OpenSSL) that is licensed under separate terms, 11 as designated in a particular file or component or in included license 12 documentation. The authors of MySQL hereby grant you an additional 13 permission to link the program and your derivative works with the 14 separately licensed software that they have included with MySQL. 15 16 This program is distributed in the hope that it will be useful, 17 but WITHOUT ANY WARRANTY; without even the implied warranty of 18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 GNU General Public License, version 2.0, for more details. 20 21 You should have received a copy of the GNU General Public License 22 along with this program; if not, write to the Free Software 23 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 24 */ 25 26 package testsuite.clusterj; 27 28 import java.nio.ByteBuffer; 29 import java.nio.CharBuffer; 30 import java.nio.charset.Charset; 31 import java.nio.charset.CharsetEncoder; 32 import java.nio.charset.CoderResult; 33 34 import java.sql.PreparedStatement; 35 import java.sql.ResultSet; 36 import java.sql.SQLException; 37 38 import java.util.ArrayList; 39 import java.util.List; 40 import java.util.Properties; 41 42 import testsuite.clusterj.model.CharsetLatin1; 43 import testsuite.clusterj.model.CharsetBig5; 44 import testsuite.clusterj.model.CharsetModel; 45 import testsuite.clusterj.model.CharsetSjis; 46 import testsuite.clusterj.model.CharsetUtf8; 47 48 /** Test that all characters in supported character sets can be read and written. 49 50 * 1. Identify which character sets to test. 51 * 2. For each character set, create a table with an id column and three VARCHAR columns 52 * (one with length < 256 another with length > 256, and a third with length > 8000) 53 * with the test character set. 54 * 3. For each table, write a persistent interface that maps the table. 55 * 4. For each persistent interface: 56 * a) create an empty list of String 57 * b) create a CharBuffer containing all mappable characters for the character set from the range 0:65535 58 * c) map the CharBuffer to a ByteBuffer of length equal to the size of the VARCHAR column 59 * d) create a String from the characters in the CharBuffer that could fit into the column 60 * e) add the String to the list of String 61 * f) continue from c) until all characters have been represented in the list of String 62 * g) remove all rows of the table 63 * h) use JDBC or clusterj to write a row in the database for each String in the list 64 * i) use JDBC or clusterj to read all rows and compare the String to the list of Strings 65 * 66 */ 67 public class CharsetTest extends AbstractClusterJModelTest { 68 69 @Override localSetUp()70 public void localSetUp() { 71 createSessionFactory(); 72 session = sessionFactory.getSession(); 73 setAutoCommit(connection, false); 74 } 75 testLatin1()76 public void testLatin1() { 77 writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 78 writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 79 writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 80 81 writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 82 writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 83 writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 84 85 writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 86 writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 87 writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 88 89 writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL); 90 writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM); 91 writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE); 92 93 failOnError(); 94 } 95 testUtf8()96 public void testUtf8() { 97 writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 98 writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 99 writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 100 101 writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 102 writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 103 writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 104 105 writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 106 writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 107 writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 108 109 writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL); 110 writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM); 111 writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE); 112 113 failOnError(); 114 } 115 testSjis()116 public void testSjis() { 117 /* These tests are excluded due to a JDBC error: 118 * java.sql.SQLException: 119 * Failed to insert charsetsjis at instance 0 errant string: [... 165 167 168... ] 120 * Incorrect string value: '\xC2\xA5\xC2\xA7\xC2\xA8...' for column 'smallcolumn' at row 1 121 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1055) 122 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:956) 123 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3558) 124 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3490) 125 at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1959) 126 at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2109) 127 at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2648) 128 at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2077) 129 at com.mysql.jdbc.PreparedStatement.execute(PreparedStatement.java:1356) 130 at testsuite.clusterj.CharsetTest.writeToJDBC(CharsetTest.java:317) 131 writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL); 132 writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM); 133 writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE); 134 */ 135 136 writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL); 137 writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM); 138 writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE); 139 140 writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL); 141 writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM); 142 writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE); 143 144 failOnError(); 145 } 146 testBig5()147 public void testBig5() { 148 writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 149 writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 150 writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 151 152 writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 153 writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 154 writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 155 156 writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 157 writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 158 writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 159 160 writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL); 161 writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM); 162 writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE); 163 164 failOnError(); 165 } 166 writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)167 protected void writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 168 ColumnDescriptor columnDescriptor) { 169 removeAll(modelClass); 170 List<String> result = null; 171 List<String> strings = generateStrings(columnDescriptor, charsetName); 172 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 173 writeToJDBC(columnDescriptor, tableName, instances); 174 result = readFromJDBC(columnDescriptor, tableName); 175 if (debug) System.out.println("Returned results of size " + result.size()); 176 // if (debug) System.out.println("Results:\n" + dump(result)); 177 verify("writeJDBCreadJDBC", strings, result, columnDescriptor); 178 } 179 writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)180 protected void writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 181 ColumnDescriptor columnDescriptor) { 182 removeAll(modelClass); 183 List<String> result = null; 184 List<String> strings = generateStrings(columnDescriptor, charsetName); 185 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 186 writeToJDBC(columnDescriptor, tableName, instances); 187 result = readFromNDB(columnDescriptor, modelClass); 188 if (debug) System.out.println("Returned results of size " + result.size()); 189 // if (debug) System.out.println("Results: " + dump(result)); 190 verify("writeJDBCreadNDB", strings, result, columnDescriptor); 191 } 192 writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)193 protected void writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 194 ColumnDescriptor columnDescriptor) { 195 removeAll(modelClass); 196 List<String> result = null; 197 List<String> strings = generateStrings(columnDescriptor, charsetName); 198 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 199 writeToNDB(columnDescriptor, instances); 200 result = readFromJDBC(columnDescriptor, tableName); 201 if (debug) System.out.println("Returned results of size " + result.size()); 202 // if (debug) System.out.println("Results: " + dump(result)); 203 verify("writeNDBreadJDBC", strings, result, columnDescriptor); 204 } 205 writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)206 protected void writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, 207 ColumnDescriptor columnDescriptor) { 208 removeAll(modelClass); 209 List<String> result = null; 210 List<String> strings = generateStrings(columnDescriptor, charsetName); 211 List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings); 212 writeToNDB(columnDescriptor, instances); 213 result = readFromNDB(columnDescriptor, modelClass); 214 if (debug) System.out.println("Returned results of size " + result.size()); 215 // if (debug) System.out.println("Results: " + dump(result)); 216 verify("writeNDBreadNDB", strings, result, columnDescriptor); 217 } 218 verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor)219 private void verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor) { 220 int maxErrors = 10; 221 for (int i = 0; i < expecteds.size(); ++i) { 222 String expected = expecteds.get(i); 223 String actual = actuals.get(i); 224 int expectedLength = expected.length(); 225 int actualLength = actual.length(); 226 errorIfNotEqual(where + " got failure on size of column data for column width " + columnDescriptor.columnWidth + " at row " + i, expectedLength, actualLength); 227 if (expectedLength != actualLength) 228 continue; 229 for (int j = 0; j < expected.length(); ++j) { 230 if (--maxErrors > 0) { 231 errorIfNotEqual("Failure to match column data for column width " + columnDescriptor.columnWidth + " at row " + i + " column " + j, 232 expected.codePointAt(j), actual.codePointAt(j)); 233 } 234 } 235 } 236 } 237 generateStrings(ColumnDescriptor columnDescriptor, String charsetName)238 protected List<String> generateStrings(ColumnDescriptor columnDescriptor, 239 String charsetName) { 240 List<String> result = new ArrayList<String>(); 241 Charset charset = Charset.forName(charsetName); 242 CharBuffer allChars = CharBuffer.allocate(65536); 243 CharsetEncoder encoder = charset.newEncoder(); 244 // add all encodable characters to the buffer 245 int count = 0; 246 // for (int i = 0; i < 65536; ++i) { 247 for (int i = 0; i < 65536; ++i) { 248 Character ch = (char)i; 249 if (encoder.canEncode(ch)) { 250 allChars.append(ch); 251 ++count; 252 } 253 } 254 if (debug) System.out.print(charsetName + " has " + count + " encodable characters"); 255 allChars.flip(); 256 257 int width = columnDescriptor.getColumnWidth(); 258 // encode all the characters that fit into the output byte buffer 259 boolean done = false; 260 byte[] bytes = new byte[width]; 261 while (!done) { 262 int begin = allChars.position(); 263 allChars.mark(); 264 ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); 265 CoderResult coderResult = encoder.encode(allChars, byteBuffer, false); 266 int end = allChars.position(); 267 int length = end - begin; 268 if (length == 0) { 269 done = true; 270 continue; 271 } 272 char[] chars = new char[length]; 273 allChars.reset(); 274 allChars.get(chars, 0, length); 275 String encodable = String.copyValueOf(chars); 276 result.add(encodable); 277 if (coderResult.isUnderflow()) { 278 done = true; 279 } 280 } 281 if (debug) System.out.println(" in " + result.size() + " row(s) of size " + columnDescriptor.columnWidth); 282 return result; 283 } 284 generateInstances(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass, List<String> strings)285 protected List<CharsetModel> generateInstances(ColumnDescriptor columnDescriptor, 286 Class<? extends CharsetModel> modelClass, List<String> strings) { 287 List<CharsetModel> result = new ArrayList<CharsetModel>(); 288 for (int i = 0; i < strings.size(); ++i) { 289 CharsetModel instance = session.newInstance(modelClass); 290 instance.setId(i); 291 columnDescriptor.set(instance, strings.get(i)); 292 result.add(instance); 293 } 294 if (debug) System.out.println("Created " + result.size() + " instances of " + modelClass.getName()); 295 return result; 296 } 297 writeToJDBC(ColumnDescriptor columnDescriptor, String tableName, List<CharsetModel> instances)298 protected void writeToJDBC(ColumnDescriptor columnDescriptor, 299 String tableName, List<CharsetModel> instances) { 300 StringBuffer buffer = new StringBuffer("INSERT INTO "); 301 buffer.append(tableName); 302 buffer.append(" (id, "); 303 buffer.append(columnDescriptor.getColumnName()); 304 buffer.append(") VALUES (?, ?)"); 305 String statement = buffer.toString(); 306 if (debug) System.out.println(statement); 307 PreparedStatement preparedStatement = null; 308 int i = 0; 309 String value = ""; 310 try { 311 Properties extraProperties = new Properties(); 312 extraProperties.put("characterEncoding", "utf8"); 313 getConnection(extraProperties); 314 setAutoCommit(connection, false); 315 preparedStatement = connection.prepareStatement(statement); 316 if (debug) System.out.println(preparedStatement.toString()); 317 for (i = 0; i < instances.size(); ++i) { 318 CharsetModel instance = instances.get(i); 319 preparedStatement.setInt(1, instance.getId()); 320 value = columnDescriptor.get(instance); 321 preparedStatement.setString(2, value); 322 // if (debug) System.out.println("Value set to column is size " + value.length()); 323 // if (debug) System.out.println(" value " + value); 324 preparedStatement.execute(); 325 } 326 connection.commit(); 327 } catch (SQLException e) { 328 throw new RuntimeException("Failed to insert " + tableName + " at instance " + i + " errant string: " + dump(value), e); 329 } 330 } 331 writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances)332 protected void writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances) { 333 session.currentTransaction().begin(); 334 for (CharsetModel instance: instances) { 335 session.makePersistent(instance); 336 } 337 session.currentTransaction().commit(); 338 } 339 readFromNDB(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass)340 protected List<String> readFromNDB(ColumnDescriptor columnDescriptor, 341 Class<? extends CharsetModel> modelClass) { 342 List<String> result = new ArrayList<String>(); 343 session.currentTransaction().begin(); 344 int i = 0; 345 boolean done = false; 346 while (!done) { 347 CharsetModel instance = session.find(modelClass, i++); 348 if (instance != null) { 349 result.add(columnDescriptor.get(instance)); 350 } else { 351 done = true; 352 } 353 } 354 session.currentTransaction().commit(); 355 return result; 356 } 357 readFromJDBC(ColumnDescriptor columnDescriptor, String tableName)358 protected List<String> readFromJDBC(ColumnDescriptor columnDescriptor, 359 String tableName) { 360 List<String> result = new ArrayList<String>(); 361 StringBuffer buffer = new StringBuffer("SELECT id, "); 362 buffer.append(columnDescriptor.getColumnName()); 363 buffer.append(" FROM "); 364 buffer.append(tableName); 365 buffer.append(" ORDER BY ID"); 366 String statement = buffer.toString(); 367 if (debug) System.out.println(statement); 368 PreparedStatement preparedStatement = null; 369 int i = 0; 370 try { 371 preparedStatement = connection.prepareStatement(statement); 372 ResultSet rs = preparedStatement.executeQuery(); 373 while (rs.next()) { 374 String columnData = rs.getString(2); 375 result.add(columnData); 376 ++i; 377 } 378 connection.commit(); 379 } catch (SQLException e) { 380 throw new RuntimeException("Failed to read " + tableName + " at instance " + i, e); 381 } 382 return result; 383 } 384 385 protected enum ColumnDescriptor { 386 SMALL(200, "smallcolumn", new InstanceHandler() { 387 public void set(CharsetModel instance, String value) { 388 instance.setSmallColumn(value); 389 } 390 public String get(CharsetModel instance) { 391 return instance.getSmallColumn(); 392 } 393 }), 394 MEDIUM(500, "mediumcolumn", new InstanceHandler() { 395 public void set(CharsetModel instance, String value) { 396 instance.setMediumColumn(value); 397 } 398 public String get(CharsetModel instance) { 399 return instance.getMediumColumn(); 400 } 401 }), 402 LARGE(10000, "largecolumn", new InstanceHandler() { 403 public void set(CharsetModel instance, String value) { 404 instance.setLargeColumn(value); 405 } 406 public String get(CharsetModel instance) { 407 return instance.getLargeColumn(); 408 } 409 }); 410 411 private int columnWidth; 412 413 private String columnName; 414 415 private InstanceHandler instanceHandler; 416 getColumnName()417 public String getColumnName() { 418 return columnName; 419 } 420 get(CharsetModel instance)421 public String get(CharsetModel instance) { 422 return instanceHandler.get(instance); 423 } 424 set(CharsetModel instance, String string)425 public void set(CharsetModel instance, String string) { 426 this.instanceHandler.set(instance, string); 427 } 428 getColumnWidth()429 public int getColumnWidth() { 430 return columnWidth; 431 } 432 ColumnDescriptor(int width, String name, InstanceHandler instanceHandler)433 private ColumnDescriptor(int width, String name, InstanceHandler instanceHandler) { 434 this.columnWidth = width; 435 this.columnName = name; 436 this.instanceHandler = instanceHandler; 437 } 438 439 private interface InstanceHandler { set(CharsetModel instance, String value)440 void set(CharsetModel instance, String value); get(CharsetModel instance)441 String get(CharsetModel instance); 442 } 443 444 } 445 446 /** The instances for testing. */ 447 protected List<CharsetModel> charsetTypes = new ArrayList<CharsetModel>(); 448 449 } 450