1 // Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_
16 #define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
17 
18 // A buffer class capable of holding a contiguous sequence of characters
19 // and a partitioning thereof into preprocessing tokens, along with their
20 // associated provenances.
21 
22 #include "char-block.h"
23 #include "provenance.h"
24 #include <cstddef>
25 #include <cstring>
26 #include <ostream>
27 #include <string>
28 #include <utility>
29 #include <vector>
30 
31 namespace Fortran::parser {
32 
33 // Buffers a contiguous sequence of characters that has been partitioned into
34 // a sequence of preprocessing tokens with provenances.
35 class TokenSequence {
36 public:
TokenSequence()37   TokenSequence() {}
TokenSequence(const TokenSequence & that)38   TokenSequence(const TokenSequence &that) { Put(that); }
39   TokenSequence(
40       const TokenSequence &that, std::size_t at, std::size_t count = 1) {
41     Put(that, at, count);
42   }
TokenSequence(TokenSequence && that)43   TokenSequence(TokenSequence &&that)
44     : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
45       char_{std::move(that.char_)}, provenances_{std::move(that.provenances_)} {
46   }
TokenSequence(const std::string & s,Provenance p)47   TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
48 
49   TokenSequence &operator=(const TokenSequence &that) {
50     clear();
51     Put(that);
52     return *this;
53   }
54   TokenSequence &operator=(TokenSequence &&that);
empty()55   bool empty() const { return start_.empty(); }
56   void clear();
57   void pop_back();
58   void shrink_to_fit();
59   void swap(TokenSequence &);
60 
SizeInTokens()61   std::size_t SizeInTokens() const { return start_.size(); }
SizeInChars()62   std::size_t SizeInChars() const { return char_.size(); }
63 
ToCharBlock()64   CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; }
ToString()65   std::string ToString() const { return ToCharBlock().ToString(); }
66 
TokenAt(std::size_t token)67   CharBlock TokenAt(std::size_t token) const {
68     return {&char_[start_.at(token)], TokenBytes(token)};
69   }
CharAt(std::size_t j)70   char CharAt(std::size_t j) const { return char_.at(j); }
CurrentOpenToken()71   CharBlock CurrentOpenToken() const {
72     return {&char_[nextStart_], char_.size() - nextStart_};
73   }
74 
75   std::size_t SkipBlanks(std::size_t) const;
76 
PutNextTokenChar(char ch,Provenance provenance)77   void PutNextTokenChar(char ch, Provenance provenance) {
78     char_.emplace_back(ch);
79     provenances_.Put({provenance, 1});
80   }
81 
CloseToken()82   void CloseToken() {
83     start_.emplace_back(nextStart_);
84     nextStart_ = char_.size();
85   }
86 
ReopenLastToken()87   void ReopenLastToken() {
88     nextStart_ = start_.back();
89     start_.pop_back();
90   }
91 
92   void RemoveLastToken();
93 
94   void Put(const TokenSequence &);
95   void Put(const TokenSequence &, ProvenanceRange);
96   void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
97   void Put(const char *, std::size_t, Provenance);
98   void Put(const CharBlock &, Provenance);
99   void Put(const std::string &, Provenance);
100   void Put(const std::stringstream &, Provenance);
101 
102   Provenance GetTokenProvenance(
103       std::size_t token, std::size_t offset = 0) const;
104   ProvenanceRange GetTokenProvenanceRange(
105       std::size_t token, std::size_t offset = 0) const;
106   ProvenanceRange GetIntervalProvenanceRange(
107       std::size_t token, std::size_t tokens = 1) const;
108   ProvenanceRange GetProvenanceRange() const;
109 
GetMutableCharData()110   char *GetMutableCharData() { return &char_[0]; }
111   TokenSequence &ToLowerCase();
112   bool HasBlanks(std::size_t firstChar = 0) const;
113   bool HasRedundantBlanks(std::size_t firstChar = 0) const;
114   TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
115   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
116   TokenSequence &ClipComment(bool skipFirst = false);
117   void Emit(CookedSource &) const;
118   void Dump(std::ostream &) const;
119 
120 private:
TokenBytes(std::size_t token)121   std::size_t TokenBytes(std::size_t token) const {
122     return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
123         start_[token];
124   }
125 
126   std::vector<std::size_t> start_;
127   std::size_t nextStart_{0};
128   std::vector<char> char_;
129   OffsetToProvenanceMappings provenances_;
130 };
131 }
132 #endif  // FORTRAN_PARSER_TOKEN_SEQUENCE_H_
133