1 #ifndef FASTREAD_TOKENIZER_H_
2 #define FASTREAD_TOKENIZER_H_
3 
4 #include "cpp11/R.hpp"
5 #include "cpp11/list.hpp"
6 #include "cpp11/protect.hpp"
7 
8 #include "Warnings.h"
9 #include <memory>
10 
11 class Token;
12 
13 typedef const char* SourceIterator;
14 typedef std::pair<SourceIterator, SourceIterator> SourceIterators;
15 typedef void (*UnescapeFun)(SourceIterator, SourceIterator, std::string*);
16 
17 class Tokenizer;
18 typedef std::shared_ptr<Tokenizer> TokenizerPtr;
19 
20 class Tokenizer {
21   Warnings* pWarnings_;
22 
23 public:
Tokenizer()24   Tokenizer() : pWarnings_(NULL) {}
~Tokenizer()25   virtual ~Tokenizer() {}
26 
27   virtual void tokenize(SourceIterator begin, SourceIterator end) = 0;
28   virtual Token nextToken() = 0;
29   // Percentage & bytes
30   virtual std::pair<double, size_t> progress() = 0;
31 
32   virtual void
unescape(SourceIterator begin,SourceIterator end,std::string * pOut)33   unescape(SourceIterator begin, SourceIterator end, std::string* pOut) {
34     pOut->reserve(end - begin);
35     for (SourceIterator cur = begin; cur != end; ++cur)
36       pOut->push_back(*cur);
37   }
38 
setWarnings(Warnings * pWarnings)39   void setWarnings(Warnings* pWarnings) { pWarnings_ = pWarnings; }
40 
41   inline void warn(
42       int row,
43       int col,
44       const std::string& expected,
45       const std::string& actual = "") {
46     if (pWarnings_ == NULL) {
47       cpp11::warning(
48           "[%i, %i]: expected %s", row + 1, col + 1, expected.c_str());
49       return;
50     }
51     pWarnings_->addWarning(row, col, expected, actual);
52   }
53 
54   static TokenizerPtr create(const cpp11::list& spec);
55 };
56 
57 // -----------------------------------------------------------------------------
58 // Helper class for parsers - ensures iterator always advanced no matter
59 // how loop is exited
60 
61 class Advance {
62   SourceIterator* pIter_;
63 
64 public:
Advance(SourceIterator * pIter)65   Advance(SourceIterator* pIter) : pIter_(pIter) {}
66   Advance(const Advance&) = delete;
67   Advance& operator=(const Advance&) = delete;
~Advance()68   ~Advance() { (*pIter_)++; }
69 };
70 
71 #endif
72