1 #ifndef FASTREAD_TOKENIZER_H_ 2 #define FASTREAD_TOKENIZER_H_ 3 4 #include "cpp11/R.hpp" 5 #include "cpp11/list.hpp" 6 #include "cpp11/protect.hpp" 7 8 #include "Warnings.h" 9 #include <memory> 10 11 class Token; 12 13 typedef const char* SourceIterator; 14 typedef std::pair<SourceIterator, SourceIterator> SourceIterators; 15 typedef void (*UnescapeFun)(SourceIterator, SourceIterator, std::string*); 16 17 class Tokenizer; 18 typedef std::shared_ptr<Tokenizer> TokenizerPtr; 19 20 class Tokenizer { 21 Warnings* pWarnings_; 22 23 public: Tokenizer()24 Tokenizer() : pWarnings_(NULL) {} ~Tokenizer()25 virtual ~Tokenizer() {} 26 27 virtual void tokenize(SourceIterator begin, SourceIterator end) = 0; 28 virtual Token nextToken() = 0; 29 // Percentage & bytes 30 virtual std::pair<double, size_t> progress() = 0; 31 32 virtual void unescape(SourceIterator begin,SourceIterator end,std::string * pOut)33 unescape(SourceIterator begin, SourceIterator end, std::string* pOut) { 34 pOut->reserve(end - begin); 35 for (SourceIterator cur = begin; cur != end; ++cur) 36 pOut->push_back(*cur); 37 } 38 setWarnings(Warnings * pWarnings)39 void setWarnings(Warnings* pWarnings) { pWarnings_ = pWarnings; } 40 41 inline void warn( 42 int row, 43 int col, 44 const std::string& expected, 45 const std::string& actual = "") { 46 if (pWarnings_ == NULL) { 47 cpp11::warning( 48 "[%i, %i]: expected %s", row + 1, col + 1, expected.c_str()); 49 return; 50 } 51 pWarnings_->addWarning(row, col, expected, actual); 52 } 53 54 static TokenizerPtr create(const cpp11::list& spec); 55 }; 56 57 // ----------------------------------------------------------------------------- 58 // Helper class for parsers - ensures iterator always advanced no matter 59 // how loop is exited 60 61 class Advance { 62 SourceIterator* pIter_; 63 64 public: Advance(SourceIterator * pIter)65 Advance(SourceIterator* pIter) : pIter_(pIter) {} 66 Advance(const Advance&) = delete; 67 Advance& operator=(const Advance&) = delete; ~Advance()68 ~Advance() { (*pIter_)++; } 69 }; 70 71 #endif 72