1 /*
2    Copyright 2010 Sun Microsystems, Inc.
3    All rights reserved. Use is subject to license terms.
4 
5    This program is free software; you can redistribute it and/or modify
6    it under the terms of the GNU General Public License, version 2.0,
7    as published by the Free Software Foundation.
8 
9    This program is also distributed with certain software (including
10    but not limited to OpenSSL) that is licensed under separate terms,
11    as designated in a particular file or component or in included license
12    documentation.  The authors of MySQL hereby grant you an additional
13    permission to link the program and your derivative works with the
14    separately licensed software that they have included with MySQL.
15 
16    This program is distributed in the hope that it will be useful,
17    but WITHOUT ANY WARRANTY; without even the implied warranty of
18    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19    GNU General Public License, version 2.0, for more details.
20 
21    You should have received a copy of the GNU General Public License
22    along with this program; if not, write to the Free Software
23    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
24 */
25 
26 package testsuite.clusterj;
27 
28 import java.nio.ByteBuffer;
29 import java.nio.CharBuffer;
30 import java.nio.charset.Charset;
31 import java.nio.charset.CharsetEncoder;
32 import java.nio.charset.CoderResult;
33 
34 import java.sql.PreparedStatement;
35 import java.sql.ResultSet;
36 import java.sql.SQLException;
37 
38 import java.util.ArrayList;
39 import java.util.List;
40 import java.util.Properties;
41 
42 import testsuite.clusterj.model.CharsetLatin1;
43 import testsuite.clusterj.model.CharsetBig5;
44 import testsuite.clusterj.model.CharsetModel;
45 import testsuite.clusterj.model.CharsetSjis;
46 import testsuite.clusterj.model.CharsetUtf8;
47 
48 /** Test that all characters in supported character sets can be read and written.
49 
50  * 1. Identify which character sets to test.
51  * 2. For each character set, create a table with an id column and three VARCHAR columns
52  *    (one with length < 256 another with length > 256, and a third with length > 8000)
53  *    with the test character set.
54  * 3. For each table, write a persistent interface that maps the table.
55  * 4. For each persistent interface:
56  *   a) create an empty list of String
57  *   b) create a CharBuffer containing all mappable characters for the character set from the range 0:65535
58  *   c) map the CharBuffer to a ByteBuffer of length equal to the size of the VARCHAR column
59  *   d) create a String from the characters in the CharBuffer that could fit into the column
60  *   e) add the String to the list of String
61  *   f) continue from c) until all characters have been represented in the list of String
62  *   g) remove all rows of the table
63  *   h) use JDBC or clusterj to write a row in the database for each String in the list
64  *   i) use JDBC or clusterj to read all rows and compare the String to the list of Strings
65  *
66  */
67 public class CharsetTest extends AbstractClusterJModelTest {
68 
69     @Override
localSetUp()70     public void localSetUp() {
71         createSessionFactory();
72         session = sessionFactory.getSession();
73         setAutoCommit(connection, false);
74     }
75 
testLatin1()76     public void testLatin1() {
77         writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
78         writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
79         writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
80 
81         writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
82         writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
83         writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
84 
85         writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
86         writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
87         writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
88 
89         writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
90         writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
91         writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
92 
93         failOnError();
94     }
95 
testUtf8()96     public void testUtf8() {
97         writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
98         writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
99         writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
100 
101         writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
102         writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
103         writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
104 
105         writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
106         writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
107         writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
108 
109         writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
110         writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
111         writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
112 
113         failOnError();
114     }
115 
testSjis()116     public void testSjis() {
117         /* These tests are excluded due to a JDBC error:
118          * java.sql.SQLException:
119          * Failed to insert charsetsjis at instance 0 errant string: [... 165 167 168... ]
120          * Incorrect string value: '\xC2\xA5\xC2\xA7\xC2\xA8...' for column 'smallcolumn' at row 1
121                                 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1055)
122                                 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:956)
123                                 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3558)
124                                 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3490)
125                                 at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1959)
126                                 at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2109)
127                                 at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2648)
128                                 at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2077)
129                                 at com.mysql.jdbc.PreparedStatement.execute(PreparedStatement.java:1356)
130                                 at testsuite.clusterj.CharsetTest.writeToJDBC(CharsetTest.java:317)
131         writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL);
132         writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM);
133         writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE);
134          */
135 
136         writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL);
137         writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM);
138         writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE);
139 
140         writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL);
141         writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM);
142         writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE);
143 
144         failOnError();
145     }
146 
testBig5()147     public void testBig5() {
148         writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
149         writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
150         writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
151 
152         writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
153         writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
154         writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
155 
156         writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
157         writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
158         writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
159 
160         writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
161         writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
162         writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
163 
164         failOnError();
165     }
166 
writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)167     protected void writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
168             ColumnDescriptor columnDescriptor) {
169         removeAll(modelClass);
170         List<String> result = null;
171         List<String> strings = generateStrings(columnDescriptor, charsetName);
172         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
173         writeToJDBC(columnDescriptor, tableName, instances);
174         result = readFromJDBC(columnDescriptor, tableName);
175         if (debug) System.out.println("Returned results of size " + result.size());
176 //        if (debug) System.out.println("Results:\n" + dump(result));
177         verify("writeJDBCreadJDBC", strings, result, columnDescriptor);
178     }
179 
writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)180     protected void writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
181             ColumnDescriptor columnDescriptor) {
182         removeAll(modelClass);
183         List<String> result = null;
184         List<String> strings = generateStrings(columnDescriptor, charsetName);
185         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
186         writeToJDBC(columnDescriptor, tableName, instances);
187         result = readFromNDB(columnDescriptor, modelClass);
188         if (debug) System.out.println("Returned results of size " + result.size());
189 //        if (debug) System.out.println("Results: " + dump(result));
190         verify("writeJDBCreadNDB", strings, result, columnDescriptor);
191     }
192 
writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)193     protected void writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
194             ColumnDescriptor columnDescriptor) {
195         removeAll(modelClass);
196         List<String> result = null;
197         List<String> strings = generateStrings(columnDescriptor, charsetName);
198         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
199         writeToNDB(columnDescriptor, instances);
200         result = readFromJDBC(columnDescriptor, tableName);
201         if (debug) System.out.println("Returned results of size " + result.size());
202 //        if (debug) System.out.println("Results: " + dump(result));
203         verify("writeNDBreadJDBC", strings, result, columnDescriptor);
204     }
205 
writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)206     protected void writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
207             ColumnDescriptor columnDescriptor) {
208         removeAll(modelClass);
209         List<String> result = null;
210         List<String> strings = generateStrings(columnDescriptor, charsetName);
211         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
212         writeToNDB(columnDescriptor, instances);
213         result = readFromNDB(columnDescriptor, modelClass);
214         if (debug) System.out.println("Returned results of size " + result.size());
215 //        if (debug) System.out.println("Results: " + dump(result));
216         verify("writeNDBreadNDB", strings, result, columnDescriptor);
217     }
218 
verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor)219     private void verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor) {
220         int maxErrors = 10;
221         for (int i = 0; i < expecteds.size(); ++i) {
222             String expected = expecteds.get(i);
223             String actual = actuals.get(i);
224             int expectedLength = expected.length();
225             int actualLength = actual.length();
226             errorIfNotEqual(where + " got failure on size of column data for column width " + columnDescriptor.columnWidth + " at row " + i, expectedLength, actualLength);
227             if (expectedLength != actualLength)
228                 continue;
229             for (int j = 0; j < expected.length(); ++j) {
230                 if (--maxErrors > 0) {
231                     errorIfNotEqual("Failure to match column data for column width " + columnDescriptor.columnWidth + " at row " + i + " column " + j,
232                             expected.codePointAt(j), actual.codePointAt(j));
233                 }
234             }
235         }
236     }
237 
generateStrings(ColumnDescriptor columnDescriptor, String charsetName)238     protected List<String> generateStrings(ColumnDescriptor columnDescriptor,
239             String charsetName) {
240         List<String> result = new ArrayList<String>();
241         Charset charset = Charset.forName(charsetName);
242         CharBuffer allChars = CharBuffer.allocate(65536);
243         CharsetEncoder encoder = charset.newEncoder();
244        // add all encodable characters to the buffer
245         int count = 0;
246 //        for (int i = 0; i < 65536; ++i) {
247         for (int i = 0; i < 65536; ++i) {
248             Character ch = (char)i;
249             if (encoder.canEncode(ch)) {
250                 allChars.append(ch);
251                 ++count;
252             }
253         }
254         if (debug) System.out.print(charsetName + " has " + count + " encodable characters");
255         allChars.flip();
256 
257         int width = columnDescriptor.getColumnWidth();
258         // encode all the characters that fit into the output byte buffer
259         boolean done = false;
260         byte[] bytes = new byte[width];
261         while (!done) {
262             int begin = allChars.position();
263             allChars.mark();
264             ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
265             CoderResult coderResult = encoder.encode(allChars, byteBuffer, false);
266             int end = allChars.position();
267             int length = end - begin;
268             if (length == 0) {
269                 done = true;
270                 continue;
271             }
272             char[] chars = new char[length];
273             allChars.reset();
274             allChars.get(chars, 0, length);
275             String encodable = String.copyValueOf(chars);
276             result.add(encodable);
277             if (coderResult.isUnderflow()) {
278                 done = true;
279             }
280         }
281         if (debug) System.out.println(" in " + result.size() + " row(s) of size " + columnDescriptor.columnWidth);
282         return result;
283     }
284 
generateInstances(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass, List<String> strings)285     protected List<CharsetModel> generateInstances(ColumnDescriptor columnDescriptor,
286             Class<? extends CharsetModel> modelClass, List<String> strings) {
287         List<CharsetModel> result = new ArrayList<CharsetModel>();
288         for (int i = 0; i < strings.size(); ++i) {
289             CharsetModel instance = session.newInstance(modelClass);
290             instance.setId(i);
291             columnDescriptor.set(instance, strings.get(i));
292             result.add(instance);
293         }
294         if (debug) System.out.println("Created " + result.size() + " instances of " + modelClass.getName());
295         return result;
296     }
297 
writeToJDBC(ColumnDescriptor columnDescriptor, String tableName, List<CharsetModel> instances)298     protected void writeToJDBC(ColumnDescriptor columnDescriptor,
299             String tableName, List<CharsetModel> instances) {
300         StringBuffer buffer = new StringBuffer("INSERT INTO ");
301         buffer.append(tableName);
302         buffer.append(" (id, ");
303         buffer.append(columnDescriptor.getColumnName());
304         buffer.append(") VALUES (?, ?)");
305         String statement = buffer.toString();
306         if (debug) System.out.println(statement);
307         PreparedStatement preparedStatement = null;
308         int i = 0;
309         String value = "";
310         try {
311             Properties extraProperties = new Properties();
312             extraProperties.put("characterEncoding", "utf8");
313             getConnection(extraProperties);
314             setAutoCommit(connection, false);
315             preparedStatement = connection.prepareStatement(statement);
316             if (debug) System.out.println(preparedStatement.toString());
317             for (i = 0; i < instances.size(); ++i) {
318                 CharsetModel instance = instances.get(i);
319                 preparedStatement.setInt(1, instance.getId());
320                 value = columnDescriptor.get(instance);
321                 preparedStatement.setString(2, value);
322 //                if (debug) System.out.println("Value set to column is size " + value.length());
323 //                if (debug) System.out.println(" value " + value);
324                 preparedStatement.execute();
325             }
326             connection.commit();
327         } catch (SQLException e) {
328             throw new RuntimeException("Failed to insert " + tableName + " at instance " + i + " errant string: " + dump(value), e);
329         }
330     }
331 
writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances)332     protected void writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances) {
333         session.currentTransaction().begin();
334         for (CharsetModel instance: instances) {
335             session.makePersistent(instance);
336         }
337         session.currentTransaction().commit();
338     }
339 
readFromNDB(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass)340     protected List<String> readFromNDB(ColumnDescriptor columnDescriptor,
341             Class<? extends CharsetModel> modelClass) {
342         List<String> result = new ArrayList<String>();
343         session.currentTransaction().begin();
344         int i = 0;
345         boolean done = false;
346         while (!done) {
347             CharsetModel instance = session.find(modelClass, i++);
348             if (instance != null) {
349                 result.add(columnDescriptor.get(instance));
350             } else {
351                 done = true;
352             }
353         }
354         session.currentTransaction().commit();
355         return result;
356     }
357 
readFromJDBC(ColumnDescriptor columnDescriptor, String tableName)358     protected List<String> readFromJDBC(ColumnDescriptor columnDescriptor,
359             String tableName) {
360         List<String> result = new ArrayList<String>();
361         StringBuffer buffer = new StringBuffer("SELECT id, ");
362         buffer.append(columnDescriptor.getColumnName());
363         buffer.append(" FROM ");
364         buffer.append(tableName);
365         buffer.append(" ORDER BY ID");
366         String statement = buffer.toString();
367         if (debug) System.out.println(statement);
368         PreparedStatement preparedStatement = null;
369         int i = 0;
370         try {
371             preparedStatement = connection.prepareStatement(statement);
372             ResultSet rs = preparedStatement.executeQuery();
373             while (rs.next()) {
374                 String columnData = rs.getString(2);
375                 result.add(columnData);
376                 ++i;
377             }
378             connection.commit();
379         } catch (SQLException e) {
380             throw new RuntimeException("Failed to read " + tableName + " at instance " + i, e);
381         }
382         return result;
383     }
384 
385     protected enum ColumnDescriptor {
386         SMALL(200, "smallcolumn", new InstanceHandler() {
387             public void set(CharsetModel instance, String value) {
388                 instance.setSmallColumn(value);
389             }
390             public String get(CharsetModel instance) {
391                 return instance.getSmallColumn();
392             }
393         }),
394         MEDIUM(500, "mediumcolumn", new InstanceHandler() {
395             public void set(CharsetModel instance, String value) {
396                 instance.setMediumColumn(value);
397             }
398             public String get(CharsetModel instance) {
399                 return instance.getMediumColumn();
400             }
401         }),
402         LARGE(10000, "largecolumn", new InstanceHandler() {
403             public void set(CharsetModel instance, String value) {
404                 instance.setLargeColumn(value);
405             }
406             public String get(CharsetModel instance) {
407                 return instance.getLargeColumn();
408             }
409         });
410 
411         private int columnWidth;
412 
413         private String columnName;
414 
415         private InstanceHandler instanceHandler;
416 
getColumnName()417         public String getColumnName() {
418             return columnName;
419         }
420 
get(CharsetModel instance)421         public String get(CharsetModel instance) {
422             return instanceHandler.get(instance);
423         }
424 
set(CharsetModel instance, String string)425         public void set(CharsetModel instance, String string) {
426             this.instanceHandler.set(instance, string);
427         }
428 
getColumnWidth()429         public int getColumnWidth() {
430             return columnWidth;
431         }
432 
ColumnDescriptor(int width, String name, InstanceHandler instanceHandler)433         private ColumnDescriptor(int width, String name, InstanceHandler instanceHandler) {
434             this.columnWidth = width;
435             this.columnName = name;
436             this.instanceHandler = instanceHandler;
437         }
438 
439         private interface InstanceHandler {
set(CharsetModel instance, String value)440             void set(CharsetModel instance, String value);
get(CharsetModel instance)441             String get(CharsetModel instance);
442             }
443 
444     }
445 
446     /** The instances for testing. */
447     protected List<CharsetModel> charsetTypes = new ArrayList<CharsetModel>();
448 
449 }
450