1 /*
2    Copyright (c) 2010, 2021, Oracle and/or its affiliates.
3 
4    This program is free software; you can redistribute it and/or modify
5    it under the terms of the GNU General Public License, version 2.0,
6    as published by the Free Software Foundation.
7 
8    This program is also distributed with certain software (including
9    but not limited to OpenSSL) that is licensed under separate terms,
10    as designated in a particular file or component or in included license
11    documentation.  The authors of MySQL hereby grant you an additional
12    permission to link the program and your derivative works with the
13    separately licensed software that they have included with MySQL.
14 
15    This program is distributed in the hope that it will be useful,
16    but WITHOUT ANY WARRANTY; without even the implied warranty of
17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18    GNU General Public License, version 2.0, for more details.
19 
20    You should have received a copy of the GNU General Public License
21    along with this program; if not, write to the Free Software
22    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
23 */
24 
25 package testsuite.clusterj;
26 
27 import java.nio.ByteBuffer;
28 import java.nio.CharBuffer;
29 import java.nio.charset.Charset;
30 import java.nio.charset.CharsetEncoder;
31 import java.nio.charset.CoderResult;
32 
33 import java.sql.PreparedStatement;
34 import java.sql.ResultSet;
35 import java.sql.SQLException;
36 
37 import java.util.ArrayList;
38 import java.util.List;
39 import java.util.Properties;
40 
41 import testsuite.clusterj.model.CharsetLatin1;
42 import testsuite.clusterj.model.CharsetBig5;
43 import testsuite.clusterj.model.CharsetModel;
44 import testsuite.clusterj.model.CharsetSjis;
45 import testsuite.clusterj.model.CharsetUtf8;
46 
47 /** Test that all characters in supported character sets can be read and written.
48 
49  * 1. Identify which character sets to test.
50  * 2. For each character set, create a table with an id column and three VARCHAR columns
51  *    (one with length < 256 another with length > 256, and a third with length > 8000)
52  *    with the test character set.
53  * 3. For each table, write a persistent interface that maps the table.
54  * 4. For each persistent interface:
55  *   a) create an empty list of String
56  *   b) create a CharBuffer containing all mappable characters for the character set from the range 0:65535
57  *   c) map the CharBuffer to a ByteBuffer of length equal to the size of the VARCHAR column
58  *   d) create a String from the characters in the CharBuffer that could fit into the column
59  *   e) add the String to the list of String
60  *   f) continue from c) until all characters have been represented in the list of String
61  *   g) remove all rows of the table
62  *   h) use JDBC or clusterj to write a row in the database for each String in the list
63  *   i) use JDBC or clusterj to read all rows and compare the String to the list of Strings
64  *
65  */
66 public class CharsetTest extends AbstractClusterJModelTest {
67 
68     @Override
localSetUp()69     public void localSetUp() {
70         createSessionFactory();
71         session = sessionFactory.getSession();
72         setAutoCommit(connection, false);
73     }
74 
testLatin1()75     public void testLatin1() {
76         writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
77         writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
78         writeJDBCreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
79 
80         writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
81         writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
82         writeJDBCreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
83 
84         writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
85         writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
86         writeNDBreadJDBC("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
87 
88         writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.SMALL);
89         writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.MEDIUM);
90         writeNDBreadNDB("windows-1252", "charsetlatin1", CharsetLatin1.class, ColumnDescriptor.LARGE);
91 
92         failOnError();
93     }
94 
testUtf8()95     public void testUtf8() {
96         writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
97         writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
98         writeJDBCreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
99 
100         writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
101         writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
102         writeJDBCreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
103 
104         writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
105         writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
106         writeNDBreadJDBC("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
107 
108         writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.SMALL);
109         writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.MEDIUM);
110         writeNDBreadNDB("UTF-8", "charsetutf8", CharsetUtf8.class, ColumnDescriptor.LARGE);
111 
112         failOnError();
113     }
114 
testSjis()115     public void testSjis() {
116         /* These tests are excluded due to a JDBC error:
117          * java.sql.SQLException:
118          * Failed to insert charsetsjis at instance 0 errant string: [... 165 167 168... ]
119          * Incorrect string value: '\xC2\xA5\xC2\xA7\xC2\xA8...' for column 'smallcolumn' at row 1
120                                 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1055)
121                                 at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:956)
122                                 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3558)
123                                 at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3490)
124                                 at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1959)
125                                 at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2109)
126                                 at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2648)
127                                 at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2077)
128                                 at com.mysql.jdbc.PreparedStatement.execute(PreparedStatement.java:1356)
129                                 at testsuite.clusterj.CharsetTest.writeToJDBC(CharsetTest.java:317)
130         writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL);
131         writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM);
132         writeJDBCreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE);
133          */
134 
135         writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL);
136         writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM);
137         writeNDBreadJDBC("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE);
138 
139         writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.SMALL);
140         writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.MEDIUM);
141         writeNDBreadNDB("SJIS", "charsetsjis", CharsetSjis.class, ColumnDescriptor.LARGE);
142 
143         failOnError();
144     }
145 
testBig5()146     public void testBig5() {
147         writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
148         writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
149         writeJDBCreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
150 
151         writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
152         writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
153         writeJDBCreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
154 
155         writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
156         writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
157         writeNDBreadJDBC("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
158 
159         writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.SMALL);
160         writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.MEDIUM);
161         writeNDBreadNDB("big5", "charsetbig5", CharsetBig5.class, ColumnDescriptor.LARGE);
162 
163         failOnError();
164     }
165 
writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)166     protected void writeJDBCreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
167             ColumnDescriptor columnDescriptor) {
168         removeAll(modelClass);
169         List<String> result = null;
170         List<String> strings = generateStrings(columnDescriptor, charsetName);
171         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
172         writeToJDBC(columnDescriptor, tableName, instances);
173         result = readFromJDBC(columnDescriptor, tableName);
174         if (debug) System.out.println("Returned results of size " + result.size());
175         verify("writeJDBCreadJDBC ", strings, result, columnDescriptor);
176     }
177 
writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)178     protected void writeJDBCreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
179             ColumnDescriptor columnDescriptor) {
180         removeAll(modelClass);
181         List<String> result = null;
182         List<String> strings = generateStrings(columnDescriptor, charsetName);
183         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
184         writeToJDBC(columnDescriptor, tableName, instances);
185         result = readFromNDB(columnDescriptor, modelClass);
186         if (debug) System.out.println("Returned results of size " + result.size());
187         verify("writeJDBCreadNDB ", strings, result, columnDescriptor);
188     }
189 
writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)190     protected void writeNDBreadJDBC(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
191             ColumnDescriptor columnDescriptor) {
192         removeAll(modelClass);
193         List<String> result = null;
194         List<String> strings = generateStrings(columnDescriptor, charsetName);
195         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
196         writeToNDB(columnDescriptor, instances);
197         result = readFromJDBC(columnDescriptor, tableName);
198         if (debug) System.out.println("Returned results of size " + result.size());
199         verify("writeNDBreadJDBC ", strings, result, columnDescriptor);
200     }
201 
writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass, ColumnDescriptor columnDescriptor)202     protected void writeNDBreadNDB(String charsetName, String tableName, Class<? extends CharsetModel> modelClass,
203             ColumnDescriptor columnDescriptor) {
204         removeAll(modelClass);
205         List<String> result = null;
206         List<String> strings = generateStrings(columnDescriptor, charsetName);
207         List<CharsetModel> instances = generateInstances(columnDescriptor, modelClass, strings);
208         writeToNDB(columnDescriptor, instances);
209         result = readFromNDB(columnDescriptor, modelClass);
210         if (debug) System.out.println("Returned results of size " + result.size());
211         verify("writeNDBreadNDB ", strings, result, columnDescriptor);
212     }
213 
verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor)214     private void verify(String where, List<String> expecteds, List<String> actuals, ColumnDescriptor columnDescriptor) {
215         int maxErrors = 10;
216         for (int i = 0; i < expecteds.size(); ++i) {
217             String expected = expecteds.get(i);
218             String actual = actuals.get(i);
219             if (actual == null) {
220                 error(where + columnDescriptor.columnName + " actual column " + i + " was null.");
221             } else {
222                 int expectedLength = expected.length();
223                 int actualLength = actual.length();
224                 errorIfNotEqual(where + "got failure on size of column data for column width " + columnDescriptor.columnWidth + " at row " + i, expectedLength, actualLength);
225                 if (expectedLength != actualLength)
226                     continue;
227                 for (int j = 0; j < expected.length(); ++j) {
228                     if (--maxErrors > 0) {
229                         errorIfNotEqual("Failure to match column data for column width " + columnDescriptor.columnWidth + " at row " + i + " column " + j,
230                                 expected.codePointAt(j), actual.codePointAt(j));
231                     }
232                 }
233             }
234         }
235     }
236 
generateStrings(ColumnDescriptor columnDescriptor, String charsetName)237     protected List<String> generateStrings(ColumnDescriptor columnDescriptor,
238             String charsetName) {
239         List<String> result = new ArrayList<String>();
240         Charset charset = Charset.forName(charsetName);
241         CharBuffer allChars = CharBuffer.allocate(65536);
242         CharsetEncoder encoder = charset.newEncoder();
243        // add all encodable characters to the buffer
244         int count = 0;
245         for (int i = 0; i < 65536; ++i) {
246             Character ch = (char)i;
247             if (encoder.canEncode(ch)) {
248                 allChars.append(ch);
249                 ++count;
250             }
251         }
252         if (debug) System.out.print(charsetName + " has " + count + " encodable characters");
253         allChars.flip();
254 
255         int width = columnDescriptor.getColumnWidth();
256         // encode all the characters that fit into the output byte buffer
257         boolean done = false;
258         byte[] bytes = new byte[width];
259         while (!done) {
260             int begin = allChars.position();
261             allChars.mark();
262             ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
263             CoderResult coderResult = encoder.encode(allChars, byteBuffer, false);
264             int end = allChars.position();
265             int length = end - begin;
266             if (length == 0) {
267                 done = true;
268                 continue;
269             }
270             char[] chars = new char[length];
271             allChars.reset();
272             allChars.get(chars, 0, length);
273             String encodable = String.copyValueOf(chars);
274             result.add(encodable);
275             if (coderResult.isUnderflow()) {
276                 done = true;
277             }
278         }
279         if (debug) System.out.println(" in " + result.size() + " row(s) of size " + columnDescriptor.columnWidth);
280         return result;
281     }
282 
generateInstances(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass, List<String> strings)283     protected List<CharsetModel> generateInstances(ColumnDescriptor columnDescriptor,
284             Class<? extends CharsetModel> modelClass, List<String> strings) {
285         List<CharsetModel> result = new ArrayList<CharsetModel>();
286         for (int i = 0; i < strings.size(); ++i) {
287             CharsetModel instance = session.newInstance(modelClass);
288             instance.setId(i);
289             columnDescriptor.set(instance, strings.get(i));
290             result.add(instance);
291         }
292         if (debug) System.out.println("Created " + result.size() + " instances of " + modelClass.getName());
293         return result;
294     }
295 
writeToJDBC(ColumnDescriptor columnDescriptor, String tableName, List<CharsetModel> instances)296     protected void writeToJDBC(ColumnDescriptor columnDescriptor,
297             String tableName, List<CharsetModel> instances) {
298         StringBuffer buffer = new StringBuffer("INSERT INTO ");
299         buffer.append(tableName);
300         buffer.append(" (id, ");
301         buffer.append(columnDescriptor.getColumnName());
302         buffer.append(") VALUES (?, ?)");
303         String statement = buffer.toString();
304         if (debug) System.out.println(statement);
305         PreparedStatement preparedStatement = null;
306         int i = 0;
307         String value = "";
308         try {
309             Properties extraProperties = new Properties();
310             extraProperties.put("characterEncoding", "utf8");
311             getConnection(extraProperties);
312             setAutoCommit(connection, false);
313             preparedStatement = connection.prepareStatement(statement);
314             if (debug) System.out.println(preparedStatement.toString());
315             for (i = 0; i < instances.size(); ++i) {
316                 CharsetModel instance = instances.get(i);
317                 preparedStatement.setInt(1, instance.getId());
318                 value = columnDescriptor.get(instance);
319                 preparedStatement.setString(2, value);
320 //                if (debug) System.out.println("Value set to column is size " + value.length());
321 //                if (debug) System.out.println(" value " + value);
322                 preparedStatement.execute();
323             }
324             connection.commit();
325         } catch (SQLException e) {
326             throw new RuntimeException("Failed to insert " + tableName + " at instance " + i + " errant string: " + dump(value), e);
327         }
328     }
329 
writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances)330     protected void writeToNDB(ColumnDescriptor columnDescriptor, List<CharsetModel> instances) {
331         session.currentTransaction().begin();
332         for (CharsetModel instance: instances) {
333             session.makePersistent(instance);
334         }
335         session.currentTransaction().commit();
336     }
337 
readFromNDB(ColumnDescriptor columnDescriptor, Class<? extends CharsetModel> modelClass)338     protected List<String> readFromNDB(ColumnDescriptor columnDescriptor,
339             Class<? extends CharsetModel> modelClass) {
340         List<String> result = new ArrayList<String>();
341         session.currentTransaction().begin();
342         int i = 0;
343         boolean done = false;
344         while (!done) {
345             CharsetModel instance = session.find(modelClass, i++);
346             if (instance != null) {
347                 result.add(columnDescriptor.get(instance));
348             } else {
349                 done = true;
350             }
351         }
352         session.currentTransaction().commit();
353         return result;
354     }
355 
readFromJDBC(ColumnDescriptor columnDescriptor, String tableName)356     protected List<String> readFromJDBC(ColumnDescriptor columnDescriptor,
357             String tableName) {
358         List<String> result = new ArrayList<String>();
359         StringBuffer buffer = new StringBuffer("SELECT id, ");
360         buffer.append(columnDescriptor.getColumnName());
361         buffer.append(" FROM ");
362         buffer.append(tableName);
363         buffer.append(" ORDER BY ID");
364         String statement = buffer.toString();
365         if (debug) System.out.println(statement);
366         PreparedStatement preparedStatement = null;
367         int i = 0;
368         try {
369             preparedStatement = connection.prepareStatement(statement);
370             ResultSet rs = preparedStatement.executeQuery();
371             while (rs.next()) {
372                 String columnData = rs.getString(2);
373                 result.add(columnData);
374                 ++i;
375             }
376             connection.commit();
377         } catch (SQLException e) {
378             throw new RuntimeException("Failed to read " + tableName + " at instance " + i, e);
379         }
380         return result;
381     }
382 
383     protected enum ColumnDescriptor {
384         SMALL(200, "smallcolumn", new InstanceHandler() {
385             public void set(CharsetModel instance, String value) {
386                 instance.setSmallColumn(value);
387             }
388             public String get(CharsetModel instance) {
389                 return instance.getSmallColumn();
390             }
391         }),
392         MEDIUM(500, "mediumcolumn", new InstanceHandler() {
393             public void set(CharsetModel instance, String value) {
394                 instance.setMediumColumn(value);
395             }
396             public String get(CharsetModel instance) {
397                 return instance.getMediumColumn();
398             }
399         }),
400         LARGE(10000, "largecolumn", new InstanceHandler() {
401             public void set(CharsetModel instance, String value) {
402                 instance.setLargeColumn(value);
403             }
404             public String get(CharsetModel instance) {
405                 return instance.getLargeColumn();
406             }
407         });
408 
409         private int columnWidth;
410 
411         private String columnName;
412 
413         private InstanceHandler instanceHandler;
414 
getColumnName()415         public String getColumnName() {
416             return columnName;
417         }
418 
get(CharsetModel instance)419         public String get(CharsetModel instance) {
420             return instanceHandler.get(instance);
421         }
422 
set(CharsetModel instance, String string)423         public void set(CharsetModel instance, String string) {
424             this.instanceHandler.set(instance, string);
425         }
426 
getColumnWidth()427         public int getColumnWidth() {
428             return columnWidth;
429         }
430 
ColumnDescriptor(int width, String name, InstanceHandler instanceHandler)431         private ColumnDescriptor(int width, String name, InstanceHandler instanceHandler) {
432             this.columnWidth = width;
433             this.columnName = name;
434             this.instanceHandler = instanceHandler;
435         }
436 
437         private interface InstanceHandler {
set(CharsetModel instance, String value)438             void set(CharsetModel instance, String value);
get(CharsetModel instance)439             String get(CharsetModel instance);
440             }
441 
442     }
443 
444     /** The instances for testing. */
445     protected List<CharsetModel> charsetTypes = new ArrayList<CharsetModel>();
446 
447 }
448