1 ///////////////////////////////////////////////////////////////////////
2 // File: unicharset_extractor.cpp
3 // Description: Unicode character/ligature set extractor.
4 // Author: Thomas Kielbus
5 //
6 // (C) Copyright 2006, Google Inc.
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
17 ///////////////////////////////////////////////////////////////////////
18
19 // Given a list of box files or text files on the command line, this program
20 // normalizes the text according to command-line options and generates
21 // a unicharset.
22
23 #include <cstdlib>
24 #include "boxread.h"
25 #include "commandlineflags.h"
26 #include "commontraining.h" // CheckSharedLibraryVersion
27 #include "lang_model_helpers.h"
28 #include "normstrngs.h"
29 #include "unicharset.h"
30 #include "unicharset_training_utils.h"
31
32 using namespace tesseract;
33
34 static STRING_PARAM_FLAG(output_unicharset, "unicharset", "Output file path");
35 static INT_PARAM_FLAG(norm_mode, 1,
36 "Normalization mode: 1=Combine graphemes, "
37 "2=Split graphemes, 3=Pure unicode");
38
39 namespace tesseract {
40
41 // Helper normalizes and segments the given strings according to norm_mode, and
42 // adds the segmented parts to unicharset.
AddStringsToUnicharset(const std::vector<std::string> & strings,int norm_mode,UNICHARSET * unicharset)43 static void AddStringsToUnicharset(const std::vector<std::string> &strings, int norm_mode,
44 UNICHARSET *unicharset) {
45 for (const auto &string : strings) {
46 std::vector<std::string> normalized;
47 if (NormalizeCleanAndSegmentUTF8(UnicodeNormMode::kNFC, OCRNorm::kNone,
48 static_cast<GraphemeNormMode>(norm_mode),
49 /*report_errors*/ true, string.c_str(), &normalized)) {
50 for (const std::string &normed : normalized) {
51 // normed is a UTF-8 encoded string
52 if (normed.empty() || IsUTF8Whitespace(normed.c_str())) {
53 continue;
54 }
55 unicharset->unichar_insert(normed.c_str());
56 }
57 } else {
58 tprintf("Normalization failed for string '%s'\n", string.c_str());
59 }
60 }
61 }
62
Main(int argc,char ** argv)63 static int Main(int argc, char **argv) {
64 UNICHARSET unicharset;
65 // Load input files
66 for (int arg = 1; arg < argc; ++arg) {
67 std::string file_data = tesseract::ReadFile(argv[arg]);
68 if (file_data.empty()) {
69 continue;
70 }
71 std::vector<std::string> texts;
72 if (ReadMemBoxes(-1, /*skip_blanks*/ true, &file_data[0],
73 /*continue_on_failure*/ false, /*boxes*/ nullptr, &texts,
74 /*box_texts*/ nullptr, /*pages*/ nullptr)) {
75 tprintf("Extracting unicharset from box file %s\n", argv[arg]);
76 } else {
77 tprintf("Extracting unicharset from plain text file %s\n", argv[arg]);
78 texts.clear();
79 texts = split(file_data, '\n');
80 }
81 AddStringsToUnicharset(texts, FLAGS_norm_mode, &unicharset);
82 }
83 SetupBasicProperties(/*report_errors*/ true, /*decompose*/ false, &unicharset);
84 // Write unicharset file.
85 if (unicharset.save_to_file(FLAGS_output_unicharset.c_str())) {
86 tprintf("Wrote unicharset file %s\n", FLAGS_output_unicharset.c_str());
87 } else {
88 tprintf("Cannot save unicharset file %s\n", FLAGS_output_unicharset.c_str());
89 return EXIT_FAILURE;
90 }
91 return EXIT_SUCCESS;
92 }
93
94 } // namespace tesseract
95
main(int argc,char ** argv)96 int main(int argc, char **argv) {
97 tesseract::CheckSharedLibraryVersion();
98 if (argc > 1) {
99 tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
100 }
101 if (argc < 2) {
102 tprintf(
103 "Usage: %s [--output_unicharset filename] [--norm_mode mode]"
104 " box_or_text_file [...]\n",
105 argv[0]);
106 tprintf("Where mode means:\n");
107 tprintf(" 1=combine graphemes (use for Latin and other simple scripts)\n");
108 tprintf(" 2=split graphemes (use for Indic/Khmer/Myanmar)\n");
109 tprintf(" 3=pure unicode (use for Arabic/Hebrew/Thai/Tibetan)\n");
110 tprintf("Reads box or plain text files to extract the unicharset.\n");
111 return EXIT_FAILURE;
112 }
113 return tesseract::Main(argc, argv);
114 }
115