1 // Copyright 2013 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // 16 // Author: dsites@google.com (Dick Sites) 17 // 18 19 #ifndef I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_ 20 #define I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_ 21 22 #include <stdio.h> 23 #include "integral_types.h" // for uint8 etc 24 25 namespace CLD2 { 26 27 28 // Take a set of <key, score> pairs and tote them up. 29 // Key is an 8-bit per-script language 30 // After explicitly sorting, retrieve top key, score pairs 31 // Normal use is key=per-script language 32 // The main data structure is an array of 256 uint16 counts. We normally 33 // expect this to be initialized, added-to about 60 times, then the top three 34 // items found. The reduce the initial and final time, we also keep a bit vector 35 // of unused (and uninitialized) parts, each of 64 bits covering four keys. 36 class Tote { 37 public: 38 Tote(); 39 ~Tote(); 40 void Reinit(); 41 void AddScoreCount(); 42 void Add(uint8 ikey, int idelta); AddBytes(int ibytes)43 void AddBytes(int ibytes) {byte_count_ += ibytes;} 44 void CurrentTopThreeKeys(int* key3) const; GetScoreCount()45 int GetScoreCount() const {return score_count_;} GetByteCount()46 int GetByteCount() const {return byte_count_;} GetScore(int i)47 int GetScore(int i) const {return score_[i];} SetScoreCount(uint16 v)48 void SetScoreCount(uint16 v) {score_count_ = v;} SetScore(int i,int v)49 void SetScore(int i, int v) {score_[i] = v;} 50 51 private: 52 uint64 in_use_mask_; // 64 bits, one for each group of 4 scores. 53 // 0 = not initialized,not used 54 int byte_count_; // Bytes of text scored 55 int score_count_; // Number of quadgrams/etc. scored 56 union { 57 uint64 gscore_[64]; // For alignment and clearing quickly 58 uint16 score_[256]; // Probability score sum 59 }; 60 61 }; 62 63 64 // Take a set of <key, score, reliability> triples and tote them up. 65 // Key is a 16-bit full language 66 // After explicitly sorting, retrieve top key, score, reliability triples 67 class DocTote { 68 public: 69 DocTote(); 70 ~DocTote(); 71 void Reinit(); 72 void Add(uint16 ikey, int ibytes, int score, int ireliability); 73 int Find(uint16 ikey); AddClosePair(int subscr,int val)74 void AddClosePair(int subscr, int val) {closepair_[subscr] += val;} 75 int CurrentTopKey(); RunningScore()76 Tote* RunningScore() {return &runningscore_;} 77 void Sort(int n); 78 void Dump(FILE* f); 79 GetIncrCount()80 int GetIncrCount() const {return incr_count_;} GetClosePair(int subscr)81 int GetClosePair(int subscr) const {return closepair_[subscr];} MaxSize()82 int MaxSize() const {return kMaxSize_;} Key(int i)83 uint16 Key(int i) const {return key_[i];} Value(int i)84 int Value(int i) const {return value_[i];} // byte count Score(int i)85 int Score(int i) const {return score_[i];} // sum lg prob Reliability(int i)86 int Reliability(int i) const {return reliability_[i];} SetKey(int i,int v)87 void SetKey(int i, int v) {key_[i] = v;} SetValue(int i,int v)88 void SetValue(int i, int v) {value_[i] = v;} SetScore(int i,int v)89 void SetScore(int i, int v) {score_[i] = v;} SetReliability(int i,int v)90 void SetReliability(int i, int v) {reliability_[i] = v;} 91 92 static const uint16 kUnusedKey = 0xFFFF; 93 94 private: 95 static const int kMaxSize_ = 24; 96 static const int kMaxClosePairSize_ = 8; 97 98 int incr_count_; // Number of Add calls 99 int sorted_; // Contents have been sorted, cannot Add 100 Tote runningscore_; // Top lang scores across entire doc, for 101 // helping resolve close pairs 102 // Align at multiple of 8 bytes 103 int closepair_[kMaxClosePairSize_]; 104 uint16 key_[kMaxSize_]; // Lang unassigned = 0xFFFF, valid = 1..1023 105 int value_[kMaxSize_]; // Bytecount this lang 106 int score_[kMaxSize_]; // Probability score sum 107 int reliability_[kMaxSize_]; // Percentage 0..100 108 }; 109 110 } // End namespace CLD2 111 112 #endif // I18N_ENCODINGS_CLD2_INTERNAL_TOTE_H_ 113