1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "src/trace_processor/util/descriptors.h"
18 #include "perfetto/ext/base/string_view.h"
19 #include "perfetto/protozero/field.h"
20 #include "perfetto/protozero/scattered_heap_buffer.h"
21 #include "protos/perfetto/common/descriptor.pbzero.h"
22 #include "protos/perfetto/trace_processor/trace_processor.pbzero.h"
23 
24 namespace perfetto {
25 namespace trace_processor {
26 
CreateFieldFromDecoder(const protos::pbzero::FieldDescriptorProto::Decoder & f_decoder,bool is_extension)27 FieldDescriptor CreateFieldFromDecoder(
28     const protos::pbzero::FieldDescriptorProto::Decoder& f_decoder,
29     bool is_extension) {
30   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
31   std::string type_name =
32       f_decoder.has_type_name()
33           ? base::StringView(f_decoder.type_name()).ToStdString()
34           : "";
35   // TODO(lalitm): add support for enums here.
36   uint32_t type =
37       f_decoder.has_type()
38           ? static_cast<uint32_t>(f_decoder.type())
39           : static_cast<uint32_t>(FieldDescriptorProto::TYPE_MESSAGE);
40   return FieldDescriptor(
41       base::StringView(f_decoder.name()).ToStdString(),
42       static_cast<uint32_t>(f_decoder.number()), type, std::move(type_name),
43       f_decoder.label() == FieldDescriptorProto::LABEL_REPEATED, is_extension);
44 }
45 
ResolveShortType(const std::string & parent_path,const std::string & short_type)46 base::Optional<uint32_t> DescriptorPool::ResolveShortType(
47     const std::string& parent_path,
48     const std::string& short_type) {
49   PERFETTO_DCHECK(!short_type.empty());
50 
51   std::string search_path = short_type[0] == '.'
52                                 ? parent_path + short_type
53                                 : parent_path + '.' + short_type;
54   auto opt_idx = FindDescriptorIdx(search_path);
55   if (opt_idx)
56     return opt_idx;
57 
58   if (parent_path.empty())
59     return base::nullopt;
60 
61   auto parent_dot_idx = parent_path.rfind('.');
62   auto parent_substr = parent_dot_idx == std::string::npos
63                            ? ""
64                            : parent_path.substr(0, parent_dot_idx);
65   return ResolveShortType(parent_substr, short_type);
66 }
67 
AddExtensionField(const std::string & package_name,protozero::ConstBytes field_desc_proto)68 util::Status DescriptorPool::AddExtensionField(
69     const std::string& package_name,
70     protozero::ConstBytes field_desc_proto) {
71   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
72   FieldDescriptorProto::Decoder f_decoder(field_desc_proto);
73   auto field = CreateFieldFromDecoder(f_decoder, true);
74 
75   auto extendee_name = base::StringView(f_decoder.extendee()).ToStdString();
76   PERFETTO_CHECK(!extendee_name.empty());
77   if (extendee_name[0] != '.') {
78     // Only prepend if the extendee is not fully qualified
79     extendee_name = package_name + "." + extendee_name;
80   }
81   auto extendee = FindDescriptorIdx(extendee_name);
82   if (!extendee.has_value()) {
83     return util::ErrStatus("Extendee does not exist %s", extendee_name.c_str());
84   }
85   descriptors_[extendee.value()].AddField(field);
86   return util::OkStatus();
87 }
88 
CheckPreviousDefinition(const std::string & file_name,const std::string & descriptor_name)89 void DescriptorPool::CheckPreviousDefinition(
90     const std::string& file_name,
91     const std::string& descriptor_name) {
92   auto prev_idx = FindDescriptorIdx(descriptor_name);
93   if (prev_idx.has_value()) {
94     auto prev_file = descriptors_[*prev_idx].file_name();
95     // We should already make sure we process each file once, so if we're
96     // hitting this path, it means the same message was defined in multiple
97     // files.
98     PERFETTO_FATAL("%s: %s was already defined in file %s", file_name.c_str(),
99                    descriptor_name.c_str(), prev_file.c_str());
100   }
101 }
102 
AddNestedProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto,std::vector<ExtensionInfo> * extensions)103 void DescriptorPool::AddNestedProtoDescriptors(
104     const std::string& file_name,
105     const std::string& package_name,
106     base::Optional<uint32_t> parent_idx,
107     protozero::ConstBytes descriptor_proto,
108     std::vector<ExtensionInfo>* extensions) {
109   protos::pbzero::DescriptorProto::Decoder decoder(descriptor_proto);
110 
111   auto parent_name =
112       parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
113   auto full_name =
114       parent_name + "." + base::StringView(decoder.name()).ToStdString();
115 
116   CheckPreviousDefinition(file_name, full_name);
117 
118   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
119   ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
120                                    ProtoDescriptor::Type::kMessage, parent_idx);
121   for (auto it = decoder.field(); it; ++it) {
122     FieldDescriptorProto::Decoder f_decoder(*it);
123     proto_descriptor.AddField(CreateFieldFromDecoder(f_decoder, false));
124   }
125   descriptors_.emplace_back(std::move(proto_descriptor));
126 
127   auto idx = static_cast<uint32_t>(descriptors_.size()) - 1;
128   for (auto it = decoder.enum_type(); it; ++it) {
129     AddEnumProtoDescriptors(file_name, package_name, idx, *it);
130   }
131   for (auto it = decoder.nested_type(); it; ++it) {
132     AddNestedProtoDescriptors(file_name, package_name, idx, *it, extensions);
133   }
134   for (auto ext_it = decoder.extension(); ext_it; ++ext_it) {
135     extensions->emplace_back(package_name, *ext_it);
136   }
137 }
138 
AddEnumProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto)139 void DescriptorPool::AddEnumProtoDescriptors(
140     const std::string& file_name,
141     const std::string& package_name,
142     base::Optional<uint32_t> parent_idx,
143     protozero::ConstBytes descriptor_proto) {
144   protos::pbzero::EnumDescriptorProto::Decoder decoder(descriptor_proto);
145 
146   auto parent_name =
147       parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
148   auto full_name =
149       parent_name + "." + base::StringView(decoder.name()).ToStdString();
150 
151   CheckPreviousDefinition(file_name, full_name);
152 
153   ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
154                                    ProtoDescriptor::Type::kEnum, base::nullopt);
155   for (auto it = decoder.value(); it; ++it) {
156     protos::pbzero::EnumValueDescriptorProto::Decoder enum_value(it->data(),
157                                                                  it->size());
158     proto_descriptor.AddEnumValue(enum_value.number(),
159                                   enum_value.name().ToStdString());
160   }
161   descriptors_.emplace_back(std::move(proto_descriptor));
162 }
163 
AddFromFileDescriptorSet(const uint8_t * file_descriptor_set_proto,size_t size)164 util::Status DescriptorPool::AddFromFileDescriptorSet(
165     const uint8_t* file_descriptor_set_proto,
166     size_t size) {
167   // First pass: extract all the message descriptors from the file and add them
168   // to the pool.
169   protos::pbzero::FileDescriptorSet::Decoder proto(file_descriptor_set_proto,
170                                                    size);
171   std::vector<ExtensionInfo> extensions;
172   for (auto it = proto.file(); it; ++it) {
173     protos::pbzero::FileDescriptorProto::Decoder file(*it);
174     std::string file_name = file.name().ToStdString();
175     if (processed_files_.find(file_name) != processed_files_.end()) {
176       // This file has been loaded once already. Skip.
177       continue;
178     }
179     processed_files_.insert(file_name);
180     std::string package = "." + base::StringView(file.package()).ToStdString();
181     for (auto message_it = file.message_type(); message_it; ++message_it) {
182       AddNestedProtoDescriptors(file_name, package, base::nullopt, *message_it,
183                                 &extensions);
184     }
185     for (auto enum_it = file.enum_type(); enum_it; ++enum_it) {
186       AddEnumProtoDescriptors(file_name, package, base::nullopt, *enum_it);
187     }
188     for (auto ext_it = file.extension(); ext_it; ++ext_it) {
189       extensions.emplace_back(package, *ext_it);
190     }
191   }
192 
193   // Second pass: Add extension fields to the real protos.
194   for (const auto& extension : extensions) {
195     auto status = AddExtensionField(extension.first, extension.second);
196     if (!status.ok())
197       return status;
198   }
199 
200   // Third pass: resolve the types of all the fields to the correct indiices.
201   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
202   for (auto& descriptor : descriptors_) {
203     for (auto& field : *descriptor.mutable_fields()) {
204       if (!field.resolved_type_name().empty())
205         continue;
206 
207       if (field.type() == FieldDescriptorProto::TYPE_MESSAGE ||
208           field.type() == FieldDescriptorProto::TYPE_ENUM) {
209         auto opt_desc =
210             ResolveShortType(descriptor.full_name(), field.raw_type_name());
211         if (!opt_desc.has_value()) {
212           return util::ErrStatus(
213               "Unable to find short type %s in field inside message %s",
214               field.raw_type_name().c_str(), descriptor.full_name().c_str());
215         }
216         field.set_resolved_type_name(
217             descriptors_[opt_desc.value()].full_name());
218       }
219     }
220   }
221   return util::OkStatus();
222 }
223 
FindDescriptorIdx(const std::string & full_name) const224 base::Optional<uint32_t> DescriptorPool::FindDescriptorIdx(
225     const std::string& full_name) const {
226   auto it = std::find_if(descriptors_.begin(), descriptors_.end(),
227                          [full_name](const ProtoDescriptor& desc) {
228                            return desc.full_name() == full_name;
229                          });
230   auto idx = static_cast<uint32_t>(std::distance(descriptors_.begin(), it));
231   return idx < descriptors_.size() ? base::Optional<uint32_t>(idx)
232                                    : base::nullopt;
233 }
234 
SerializeAsDescriptorSet()235 std::vector<uint8_t> DescriptorPool::SerializeAsDescriptorSet() {
236   protozero::HeapBuffered<protos::pbzero::DescriptorSet> descs;
237   for (auto& desc : descriptors()) {
238     protos::pbzero::DescriptorProto* proto_descriptor =
239         descs->add_descriptors();
240     proto_descriptor->set_name(desc.full_name());
241     for (auto& field : desc.fields()) {
242       protos::pbzero::FieldDescriptorProto* field_descriptor =
243           proto_descriptor->add_field();
244       field_descriptor->set_name(field.name());
245       field_descriptor->set_number(static_cast<int32_t>(field.number()));
246       // We do not support required fields. They will show up as optional
247       // after serialization.
248       field_descriptor->set_label(
249           field.is_repeated()
250               ? protos::pbzero::FieldDescriptorProto::LABEL_REPEATED
251               : protos::pbzero::FieldDescriptorProto::LABEL_OPTIONAL);
252       field_descriptor->set_type_name(field.resolved_type_name());
253       field_descriptor->set_type(
254           static_cast<protos::pbzero::FieldDescriptorProto_Type>(field.type()));
255     }
256   }
257   return descs.SerializeAsArray();
258 }
259 
ProtoDescriptor(std::string file_name,std::string package_name,std::string full_name,Type type,base::Optional<uint32_t> parent_id)260 ProtoDescriptor::ProtoDescriptor(std::string file_name,
261                                  std::string package_name,
262                                  std::string full_name,
263                                  Type type,
264                                  base::Optional<uint32_t> parent_id)
265     : file_name_(std::move(file_name)),
266       package_name_(std::move(package_name)),
267       full_name_(std::move(full_name)),
268       type_(type),
269       parent_id_(parent_id) {}
270 
FieldDescriptor(std::string name,uint32_t number,uint32_t type,std::string raw_type_name,bool is_repeated,bool is_extension)271 FieldDescriptor::FieldDescriptor(std::string name,
272                                  uint32_t number,
273                                  uint32_t type,
274                                  std::string raw_type_name,
275                                  bool is_repeated,
276                                  bool is_extension)
277     : name_(std::move(name)),
278       number_(number),
279       type_(type),
280       raw_type_name_(std::move(raw_type_name)),
281       is_repeated_(is_repeated),
282       is_extension_(is_extension) {}
283 
284 }  // namespace trace_processor
285 }  // namespace perfetto
286