1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/descriptors.h"
18 #include "perfetto/ext/base/string_view.h"
19 #include "perfetto/protozero/field.h"
20 #include "perfetto/protozero/scattered_heap_buffer.h"
21 #include "protos/perfetto/common/descriptor.pbzero.h"
22 #include "protos/perfetto/trace_processor/trace_processor.pbzero.h"
23
24 namespace perfetto {
25 namespace trace_processor {
26
CreateFieldFromDecoder(const protos::pbzero::FieldDescriptorProto::Decoder & f_decoder,bool is_extension)27 FieldDescriptor CreateFieldFromDecoder(
28 const protos::pbzero::FieldDescriptorProto::Decoder& f_decoder,
29 bool is_extension) {
30 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
31 std::string type_name =
32 f_decoder.has_type_name()
33 ? base::StringView(f_decoder.type_name()).ToStdString()
34 : "";
35 // TODO(lalitm): add support for enums here.
36 uint32_t type =
37 f_decoder.has_type()
38 ? static_cast<uint32_t>(f_decoder.type())
39 : static_cast<uint32_t>(FieldDescriptorProto::TYPE_MESSAGE);
40 return FieldDescriptor(
41 base::StringView(f_decoder.name()).ToStdString(),
42 static_cast<uint32_t>(f_decoder.number()), type, std::move(type_name),
43 f_decoder.label() == FieldDescriptorProto::LABEL_REPEATED, is_extension);
44 }
45
ResolveShortType(const std::string & parent_path,const std::string & short_type)46 base::Optional<uint32_t> DescriptorPool::ResolveShortType(
47 const std::string& parent_path,
48 const std::string& short_type) {
49 PERFETTO_DCHECK(!short_type.empty());
50
51 std::string search_path = short_type[0] == '.'
52 ? parent_path + short_type
53 : parent_path + '.' + short_type;
54 auto opt_idx = FindDescriptorIdx(search_path);
55 if (opt_idx)
56 return opt_idx;
57
58 if (parent_path.empty())
59 return base::nullopt;
60
61 auto parent_dot_idx = parent_path.rfind('.');
62 auto parent_substr = parent_dot_idx == std::string::npos
63 ? ""
64 : parent_path.substr(0, parent_dot_idx);
65 return ResolveShortType(parent_substr, short_type);
66 }
67
AddExtensionField(const std::string & package_name,protozero::ConstBytes field_desc_proto)68 util::Status DescriptorPool::AddExtensionField(
69 const std::string& package_name,
70 protozero::ConstBytes field_desc_proto) {
71 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
72 FieldDescriptorProto::Decoder f_decoder(field_desc_proto);
73 auto field = CreateFieldFromDecoder(f_decoder, true);
74
75 auto extendee_name = base::StringView(f_decoder.extendee()).ToStdString();
76 PERFETTO_CHECK(!extendee_name.empty());
77 if (extendee_name[0] != '.') {
78 // Only prepend if the extendee is not fully qualified
79 extendee_name = package_name + "." + extendee_name;
80 }
81 auto extendee = FindDescriptorIdx(extendee_name);
82 if (!extendee.has_value()) {
83 return util::ErrStatus("Extendee does not exist %s", extendee_name.c_str());
84 }
85 descriptors_[extendee.value()].AddField(field);
86 return util::OkStatus();
87 }
88
CheckPreviousDefinition(const std::string & file_name,const std::string & descriptor_name)89 void DescriptorPool::CheckPreviousDefinition(
90 const std::string& file_name,
91 const std::string& descriptor_name) {
92 auto prev_idx = FindDescriptorIdx(descriptor_name);
93 if (prev_idx.has_value()) {
94 auto prev_file = descriptors_[*prev_idx].file_name();
95 // We should already make sure we process each file once, so if we're
96 // hitting this path, it means the same message was defined in multiple
97 // files.
98 PERFETTO_FATAL("%s: %s was already defined in file %s", file_name.c_str(),
99 descriptor_name.c_str(), prev_file.c_str());
100 }
101 }
102
AddNestedProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto,std::vector<ExtensionInfo> * extensions)103 void DescriptorPool::AddNestedProtoDescriptors(
104 const std::string& file_name,
105 const std::string& package_name,
106 base::Optional<uint32_t> parent_idx,
107 protozero::ConstBytes descriptor_proto,
108 std::vector<ExtensionInfo>* extensions) {
109 protos::pbzero::DescriptorProto::Decoder decoder(descriptor_proto);
110
111 auto parent_name =
112 parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
113 auto full_name =
114 parent_name + "." + base::StringView(decoder.name()).ToStdString();
115
116 CheckPreviousDefinition(file_name, full_name);
117
118 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
119 ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
120 ProtoDescriptor::Type::kMessage, parent_idx);
121 for (auto it = decoder.field(); it; ++it) {
122 FieldDescriptorProto::Decoder f_decoder(*it);
123 proto_descriptor.AddField(CreateFieldFromDecoder(f_decoder, false));
124 }
125 descriptors_.emplace_back(std::move(proto_descriptor));
126
127 auto idx = static_cast<uint32_t>(descriptors_.size()) - 1;
128 for (auto it = decoder.enum_type(); it; ++it) {
129 AddEnumProtoDescriptors(file_name, package_name, idx, *it);
130 }
131 for (auto it = decoder.nested_type(); it; ++it) {
132 AddNestedProtoDescriptors(file_name, package_name, idx, *it, extensions);
133 }
134 for (auto ext_it = decoder.extension(); ext_it; ++ext_it) {
135 extensions->emplace_back(package_name, *ext_it);
136 }
137 }
138
AddEnumProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto)139 void DescriptorPool::AddEnumProtoDescriptors(
140 const std::string& file_name,
141 const std::string& package_name,
142 base::Optional<uint32_t> parent_idx,
143 protozero::ConstBytes descriptor_proto) {
144 protos::pbzero::EnumDescriptorProto::Decoder decoder(descriptor_proto);
145
146 auto parent_name =
147 parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
148 auto full_name =
149 parent_name + "." + base::StringView(decoder.name()).ToStdString();
150
151 CheckPreviousDefinition(file_name, full_name);
152
153 ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
154 ProtoDescriptor::Type::kEnum, base::nullopt);
155 for (auto it = decoder.value(); it; ++it) {
156 protos::pbzero::EnumValueDescriptorProto::Decoder enum_value(it->data(),
157 it->size());
158 proto_descriptor.AddEnumValue(enum_value.number(),
159 enum_value.name().ToStdString());
160 }
161 descriptors_.emplace_back(std::move(proto_descriptor));
162 }
163
AddFromFileDescriptorSet(const uint8_t * file_descriptor_set_proto,size_t size)164 util::Status DescriptorPool::AddFromFileDescriptorSet(
165 const uint8_t* file_descriptor_set_proto,
166 size_t size) {
167 // First pass: extract all the message descriptors from the file and add them
168 // to the pool.
169 protos::pbzero::FileDescriptorSet::Decoder proto(file_descriptor_set_proto,
170 size);
171 std::vector<ExtensionInfo> extensions;
172 for (auto it = proto.file(); it; ++it) {
173 protos::pbzero::FileDescriptorProto::Decoder file(*it);
174 std::string file_name = file.name().ToStdString();
175 if (processed_files_.find(file_name) != processed_files_.end()) {
176 // This file has been loaded once already. Skip.
177 continue;
178 }
179 processed_files_.insert(file_name);
180 std::string package = "." + base::StringView(file.package()).ToStdString();
181 for (auto message_it = file.message_type(); message_it; ++message_it) {
182 AddNestedProtoDescriptors(file_name, package, base::nullopt, *message_it,
183 &extensions);
184 }
185 for (auto enum_it = file.enum_type(); enum_it; ++enum_it) {
186 AddEnumProtoDescriptors(file_name, package, base::nullopt, *enum_it);
187 }
188 for (auto ext_it = file.extension(); ext_it; ++ext_it) {
189 extensions.emplace_back(package, *ext_it);
190 }
191 }
192
193 // Second pass: Add extension fields to the real protos.
194 for (const auto& extension : extensions) {
195 auto status = AddExtensionField(extension.first, extension.second);
196 if (!status.ok())
197 return status;
198 }
199
200 // Third pass: resolve the types of all the fields to the correct indiices.
201 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
202 for (auto& descriptor : descriptors_) {
203 for (auto& field : *descriptor.mutable_fields()) {
204 if (!field.resolved_type_name().empty())
205 continue;
206
207 if (field.type() == FieldDescriptorProto::TYPE_MESSAGE ||
208 field.type() == FieldDescriptorProto::TYPE_ENUM) {
209 auto opt_desc =
210 ResolveShortType(descriptor.full_name(), field.raw_type_name());
211 if (!opt_desc.has_value()) {
212 return util::ErrStatus(
213 "Unable to find short type %s in field inside message %s",
214 field.raw_type_name().c_str(), descriptor.full_name().c_str());
215 }
216 field.set_resolved_type_name(
217 descriptors_[opt_desc.value()].full_name());
218 }
219 }
220 }
221 return util::OkStatus();
222 }
223
FindDescriptorIdx(const std::string & full_name) const224 base::Optional<uint32_t> DescriptorPool::FindDescriptorIdx(
225 const std::string& full_name) const {
226 auto it = std::find_if(descriptors_.begin(), descriptors_.end(),
227 [full_name](const ProtoDescriptor& desc) {
228 return desc.full_name() == full_name;
229 });
230 auto idx = static_cast<uint32_t>(std::distance(descriptors_.begin(), it));
231 return idx < descriptors_.size() ? base::Optional<uint32_t>(idx)
232 : base::nullopt;
233 }
234
SerializeAsDescriptorSet()235 std::vector<uint8_t> DescriptorPool::SerializeAsDescriptorSet() {
236 protozero::HeapBuffered<protos::pbzero::DescriptorSet> descs;
237 for (auto& desc : descriptors()) {
238 protos::pbzero::DescriptorProto* proto_descriptor =
239 descs->add_descriptors();
240 proto_descriptor->set_name(desc.full_name());
241 for (auto& field : desc.fields()) {
242 protos::pbzero::FieldDescriptorProto* field_descriptor =
243 proto_descriptor->add_field();
244 field_descriptor->set_name(field.name());
245 field_descriptor->set_number(static_cast<int32_t>(field.number()));
246 // We do not support required fields. They will show up as optional
247 // after serialization.
248 field_descriptor->set_label(
249 field.is_repeated()
250 ? protos::pbzero::FieldDescriptorProto::LABEL_REPEATED
251 : protos::pbzero::FieldDescriptorProto::LABEL_OPTIONAL);
252 field_descriptor->set_type_name(field.resolved_type_name());
253 field_descriptor->set_type(
254 static_cast<protos::pbzero::FieldDescriptorProto_Type>(field.type()));
255 }
256 }
257 return descs.SerializeAsArray();
258 }
259
ProtoDescriptor(std::string file_name,std::string package_name,std::string full_name,Type type,base::Optional<uint32_t> parent_id)260 ProtoDescriptor::ProtoDescriptor(std::string file_name,
261 std::string package_name,
262 std::string full_name,
263 Type type,
264 base::Optional<uint32_t> parent_id)
265 : file_name_(std::move(file_name)),
266 package_name_(std::move(package_name)),
267 full_name_(std::move(full_name)),
268 type_(type),
269 parent_id_(parent_id) {}
270
FieldDescriptor(std::string name,uint32_t number,uint32_t type,std::string raw_type_name,bool is_repeated,bool is_extension)271 FieldDescriptor::FieldDescriptor(std::string name,
272 uint32_t number,
273 uint32_t type,
274 std::string raw_type_name,
275 bool is_repeated,
276 bool is_extension)
277 : name_(std::move(name)),
278 number_(number),
279 type_(type),
280 raw_type_name_(std::move(raw_type_name)),
281 is_repeated_(is_repeated),
282 is_extension_(is_extension) {}
283
284 } // namespace trace_processor
285 } // namespace perfetto
286