1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // https://developers.google.com/protocol-buffers/ 4 // 5 // Redistribution and use in source and binary forms, with or without 6 // modification, are permitted provided that the following conditions are 7 // met: 8 // 9 // * Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // * Redistributions in binary form must reproduce the above 12 // copyright notice, this list of conditions and the following disclaimer 13 // in the documentation and/or other materials provided with the 14 // distribution. 15 // * Neither the name of Google Inc. nor the names of its 16 // contributors may be used to endorse or promote products derived from 17 // this software without specific prior written permission. 18 // 19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 #ifndef GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__ 32 #define GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__ 33 34 #include <cstdint> 35 #include <type_traits> 36 37 #include <google/protobuf/parse_context.h> 38 #include <google/protobuf/extension_set.h> 39 #include <google/protobuf/generated_message_tctable_decl.h> 40 #include <google/protobuf/message_lite.h> 41 #include <google/protobuf/metadata_lite.h> 42 #include <google/protobuf/port.h> 43 #include <google/protobuf/wire_format_lite.h> 44 45 // Must come last: 46 #include <google/protobuf/port_def.inc> 47 48 namespace google { 49 namespace protobuf { 50 51 class Message; 52 class UnknownFieldSet; 53 54 namespace internal { 55 56 // PROTOBUF_TC_PARAM_DECL are the parameters for tailcall functions. 57 // 58 // Note that this is performance sensitive: changing the parameters will change 59 // the registers used by the ABI calling convention, which subsequently affects 60 // register selection logic inside the function. 61 #define PROTOBUF_TC_PARAM_DECL \ 62 ::google::protobuf::MessageLite *msg, const char *ptr, \ 63 ::google::protobuf::internal::ParseContext *ctx, \ 64 const ::google::protobuf::internal::TailCallParseTableBase *table, \ 65 uint64_t hasbits, ::google::protobuf::internal::TcFieldData data 66 67 // PROTOBUF_TC_PARAM_PASS passes values to match PROTOBUF_TC_PARAM_DECL. 68 #define PROTOBUF_TC_PARAM_PASS msg, ptr, ctx, table, hasbits, data 69 70 class TcParserBase { 71 public: 72 static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL); 73 static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL); 74 75 template <typename FieldType, typename TagType> SingularParseMessage(PROTOBUF_TC_PARAM_DECL)76 PROTOBUF_NOINLINE static const char* SingularParseMessage( 77 PROTOBUF_TC_PARAM_DECL) { 78 if (PROTOBUF_PREDICT_FALSE(static_cast<TagType>(data.coded_tag()) != 0)) { 79 return table->fallback(PROTOBUF_TC_PARAM_PASS); 80 } 81 ptr += sizeof(TagType); 82 hasbits |= (uint64_t{1} << data.hasbit_idx()); 83 auto& field = RefAt<FieldType*>(msg, data.offset()); 84 if (field == nullptr) { 85 auto arena = ctx->data().arena; 86 if (Arena::is_arena_constructable<FieldType>::value) { 87 field = Arena::CreateMessage<FieldType>(arena); 88 } else { 89 field = Arena::Create<FieldType>(arena); 90 } 91 } 92 SyncHasbits(msg, hasbits, table); 93 return ctx->ParseMessage(field, ptr); 94 } 95 96 template <typename FieldType, typename TagType> RepeatedParseMessage(PROTOBUF_TC_PARAM_DECL)97 PROTOBUF_NOINLINE static const char* RepeatedParseMessage( 98 PROTOBUF_TC_PARAM_DECL) { 99 if (PROTOBUF_PREDICT_FALSE(static_cast<TagType>(data.coded_tag()) != 0)) { 100 return table->fallback(PROTOBUF_TC_PARAM_PASS); 101 } 102 ptr += sizeof(TagType); 103 auto& field = RefAt<RepeatedPtrField<FieldType>>(msg, data.offset()); 104 SyncHasbits(msg, hasbits, table); 105 ptr = ctx->ParseMessage(field.Add(), ptr); 106 return ptr; 107 } 108 109 template <typename LayoutType, typename TagType> 110 static const char* RepeatedFixed(PROTOBUF_TC_PARAM_DECL); 111 template <typename LayoutType, typename TagType> 112 static const char* PackedFixed(PROTOBUF_TC_PARAM_DECL); 113 114 enum VarintDecode { kNoConversion = 0, kZigZag = 1 }; 115 template <typename FieldType, typename TagType, VarintDecode zigzag> 116 static const char* RepeatedVarint(PROTOBUF_TC_PARAM_DECL); 117 template <typename FieldType, typename TagType, VarintDecode zigzag> 118 static const char* PackedVarint(PROTOBUF_TC_PARAM_DECL); 119 120 enum Utf8Type { kNoUtf8 = 0, kUtf8 = 1, kUtf8ValidateOnly = 2 }; 121 template <typename TagType, Utf8Type utf8> 122 static const char* SingularString(PROTOBUF_TC_PARAM_DECL); 123 template <typename TagType, Utf8Type utf8> 124 static const char* RepeatedString(PROTOBUF_TC_PARAM_DECL); 125 126 protected: 127 template <typename T> RefAt(void * x,size_t offset)128 static T& RefAt(void* x, size_t offset) { 129 T* target = reinterpret_cast<T*>(static_cast<char*>(x) + offset); 130 GOOGLE_DCHECK_EQ(0, reinterpret_cast<uintptr_t>(target) % alignof(T)); 131 return *target; 132 } 133 SyncHasbits(MessageLite * msg,uint64_t hasbits,const TailCallParseTableBase * table)134 static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits( 135 MessageLite* msg, uint64_t hasbits, const TailCallParseTableBase* table) { 136 const uint32_t has_bits_offset = table->has_bits_offset; 137 if (has_bits_offset) { 138 // Only the first 32 has-bits are updated. Nothing above those is stored, 139 // but e.g. messages without has-bits update the upper bits. 140 RefAt<uint32_t>(msg, has_bits_offset) = static_cast<uint32_t>(hasbits); 141 } 142 } 143 Return(PROTOBUF_TC_PARAM_DECL)144 static inline PROTOBUF_ALWAYS_INLINE const char* Return( 145 PROTOBUF_TC_PARAM_DECL) { 146 SyncHasbits(msg, hasbits, table); 147 return ptr; 148 } 149 Error(PROTOBUF_TC_PARAM_DECL)150 static inline PROTOBUF_ALWAYS_INLINE const char* Error( 151 PROTOBUF_TC_PARAM_DECL) { 152 SyncHasbits(msg, hasbits, table); 153 return nullptr; 154 } 155 156 class ScopedArenaSwap final { 157 public: ScopedArenaSwap(MessageLite * msg,ParseContext * ctx)158 ScopedArenaSwap(MessageLite* msg, ParseContext* ctx) 159 : ctx_(ctx), saved_(ctx->data().arena) { 160 ctx_->data().arena = msg->GetArenaForAllocation(); 161 } 162 ScopedArenaSwap(const ScopedArenaSwap&) = delete; ~ScopedArenaSwap()163 ~ScopedArenaSwap() { ctx_->data().arena = saved_; } 164 165 private: 166 ParseContext* const ctx_; 167 Arena* const saved_; 168 }; 169 170 template <class MessageBaseT, class UnknownFieldsT> GenericFallbackImpl(PROTOBUF_TC_PARAM_DECL)171 static const char* GenericFallbackImpl(PROTOBUF_TC_PARAM_DECL) { 172 #define CHK_(x) \ 173 if (PROTOBUF_PREDICT_FALSE(!(x))) return nullptr /* NOLINT */ 174 175 SyncHasbits(msg, hasbits, table); 176 uint32_t tag; 177 ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); 178 CHK_(ptr); 179 if ((tag & 7) == WireFormatLite::WIRETYPE_END_GROUP || tag == 0) { 180 ctx->SetLastTag(tag); 181 return ptr; 182 } 183 uint32_t num = tag >> 3; 184 if (table->extension_range_low <= num && 185 num <= table->extension_range_high) { 186 return RefAt<ExtensionSet>(msg, table->extension_offset) 187 .ParseField(tag, ptr, 188 static_cast<const MessageBaseT*>(table->default_instance), 189 &msg->_internal_metadata_, ctx); 190 } 191 return UnknownFieldParse( 192 tag, msg->_internal_metadata_.mutable_unknown_fields<UnknownFieldsT>(), 193 ptr, ctx); 194 #undef CHK_ 195 } 196 }; 197 198 // TcParser implements most of the parsing logic for tailcall tables. 199 // 200 // This is templated on lg2(table size), since dispatching depends upon the size 201 // of the table. The template parameter avoids runtime overhead for computing 202 // the table entry index. 203 template <uint32_t kPowerOf2> 204 struct TcParser final : TcParserBase { 205 // Dispatch to the designated parse function TagDispatchfinal206 inline PROTOBUF_ALWAYS_INLINE static const char* TagDispatch( 207 PROTOBUF_TC_PARAM_DECL) { 208 const auto coded_tag = UnalignedLoad<uint16_t>(ptr); 209 constexpr size_t kIdxMask = ((1 << (kPowerOf2)) - 1); 210 const size_t idx = (coded_tag >> 3) & kIdxMask; 211 data = table->table()[idx].bits; 212 data.data ^= coded_tag; 213 PROTOBUF_MUSTTAIL return table->table()[idx].target(PROTOBUF_TC_PARAM_PASS); 214 } 215 216 // We can only safely call from field to next field if the call is optimized 217 // to a proper tail call. Otherwise we blow through stack. Clang and gcc 218 // reliably do this optimization in opt mode, but do not perform this in debug 219 // mode. Luckily the structure of the algorithm is such that it's always 220 // possible to just return and use the enclosing parse loop as a trampoline. TailCallfinal221 static const char* TailCall(PROTOBUF_TC_PARAM_DECL) { 222 constexpr bool always_return = !PROTOBUF_TAILCALL; 223 if (always_return || !ctx->DataAvailable(ptr)) { 224 PROTOBUF_MUSTTAIL return Return(PROTOBUF_TC_PARAM_PASS); 225 } 226 PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_PASS); 227 } 228 ParseLoopfinal229 static const char* ParseLoop(MessageLite* msg, const char* ptr, 230 ParseContext* ctx, 231 const TailCallParseTableBase* table) { 232 ScopedArenaSwap saved(msg, ctx); 233 const uint32_t has_bits_offset = table->has_bits_offset; 234 while (!ctx->Done(&ptr)) { 235 uint64_t hasbits = 0; 236 if (has_bits_offset) hasbits = RefAt<uint32_t>(msg, has_bits_offset); 237 ptr = TagDispatch(msg, ptr, ctx, table, hasbits, {}); 238 if (ptr == nullptr) break; 239 if (ctx->LastTag() != 1) break; // Ended on terminating tag 240 } 241 return ptr; 242 } 243 244 template <typename LayoutType, typename TagType> 245 static const char* SingularFixed(PROTOBUF_TC_PARAM_DECL); 246 247 template <typename FieldType, typename TagType, VarintDecode zigzag> 248 static const char* SingularVarint(PROTOBUF_TC_PARAM_DECL); 249 }; 250 251 // Declare helper functions: 252 #include <google/protobuf/generated_message_tctable_impl.inc> 253 254 } // namespace internal 255 } // namespace protobuf 256 } // namespace google 257 258 #include <google/protobuf/port_undef.inc> 259 260 #endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__ 261