1 /** @file
2
3 A brief file description
4
5 @section license License
6
7 Licensed to the Apache Software Foundation (ASF) under one
8 or more contributor license agreements. See the NOTICE file
9 distributed with this work for additional information
10 regarding copyright ownership. The ASF licenses this file
11 to you under the Apache License, Version 2.0 (the
12 "License"); you may not use this file except in compliance
13 with the License. You may obtain a copy of the License at
14
15 http://www.apache.org/licenses/LICENSE-2.0
16
17 Unless required by applicable law or agreed to in writing, software
18 distributed under the License is distributed on an "AS IS" BASIS,
19 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 See the License for the specific language governing permissions and
21 limitations under the License.
22 */
23
24 /****************************************************************************
25
26 HdrHeap.h
27
28 Description:
29
30
31 ****************************************************************************/
32
33 #pragma once
34
35 #include "tscore/Ptr.h"
36 #include "tscore/ink_assert.h"
37 #include "tscore/Scalar.h"
38 #include "HdrToken.h"
39
40 // Objects in the heap must currently be aligned to 8 byte boundaries,
41 // so their (address & HDR_PTR_ALIGNMENT_MASK) == 0
42
43 static constexpr size_t HDR_PTR_SIZE = sizeof(uint64_t);
44 static constexpr size_t HDR_PTR_ALIGNMENT_MASK = HDR_PTR_SIZE - 1L;
45 using HdrHeapMarshalBlocks = ts::Scalar<HDR_PTR_SIZE>;
46
47 // A many of the operations regarding read-only str
48 // heaps are hand unrolled in the code. Changing
49 // this value requires a full pass through HdrBuf.cc
50 // to fix the unrolled operations
51 static constexpr unsigned HDR_BUF_RONLY_HEAPS = 3;
52
53 class CoreUtils;
54 class IOBufferBlock;
55
56 enum {
57 HDR_HEAP_OBJ_EMPTY = 0,
58 HDR_HEAP_OBJ_RAW = 1,
59 HDR_HEAP_OBJ_URL = 2,
60 HDR_HEAP_OBJ_HTTP_HEADER = 3,
61 HDR_HEAP_OBJ_MIME_HEADER = 4,
62 HDR_HEAP_OBJ_FIELD_BLOCK = 5,
63 HDR_HEAP_OBJ_FIELD_STANDALONE = 6, // not a type that lives in HdrHeaps
64 HDR_HEAP_OBJ_FIELD_SDK_HANDLE = 7, // not a type that lives in HdrHeaps
65
66 HDR_HEAP_OBJ_MAGIC = 0x0FEEB1E0
67 };
68
69 struct HdrHeapObjImpl {
70 uint32_t m_type : 8;
71 uint32_t m_length : 20;
72 uint32_t m_obj_flags : 4;
73 };
74
75 /*-------------------------------------------------------------------------
76 -------------------------------------------------------------------------*/
77
78 extern void obj_describe(HdrHeapObjImpl *obj, bool recurse);
79
80 inline int
obj_is_aligned(HdrHeapObjImpl * obj)81 obj_is_aligned(HdrHeapObjImpl *obj)
82 {
83 return (((((uintptr_t)obj) & HDR_PTR_ALIGNMENT_MASK) == 0) && ((obj->m_length & HDR_PTR_ALIGNMENT_MASK) == 0));
84 }
85
86 inline void
obj_clear_data(HdrHeapObjImpl * obj)87 obj_clear_data(HdrHeapObjImpl *obj)
88 {
89 char *ptr = (char *)obj;
90 int hdr_length = sizeof(HdrHeapObjImpl);
91 memset(ptr + hdr_length, '\0', obj->m_length - hdr_length);
92 }
93
94 inline void
obj_copy_data(HdrHeapObjImpl * s_obj,HdrHeapObjImpl * d_obj)95 obj_copy_data(HdrHeapObjImpl *s_obj, HdrHeapObjImpl *d_obj)
96 {
97 char *src, *dst;
98
99 ink_assert((s_obj->m_length == d_obj->m_length) && (s_obj->m_type == d_obj->m_type));
100
101 int hdr_length = sizeof(HdrHeapObjImpl);
102 src = (char *)s_obj + hdr_length;
103 dst = (char *)d_obj + hdr_length;
104 memcpy(dst, src, d_obj->m_length - hdr_length);
105 }
106
107 inline void
obj_copy(HdrHeapObjImpl * s_obj,char * d_addr)108 obj_copy(HdrHeapObjImpl *s_obj, char *d_addr)
109 {
110 memcpy(d_addr, (char *)s_obj, s_obj->m_length);
111 }
112
113 inline void
obj_init_header(HdrHeapObjImpl * obj,uint32_t type,uint32_t nbytes,uint32_t obj_flags)114 obj_init_header(HdrHeapObjImpl *obj, uint32_t type, uint32_t nbytes, uint32_t obj_flags)
115 {
116 obj->m_type = type;
117 obj->m_length = nbytes;
118 obj->m_obj_flags = obj_flags;
119 }
120
121 /*-------------------------------------------------------------------------
122 -------------------------------------------------------------------------*/
123
124 enum {
125 HDR_BUF_MAGIC_ALIVE = 0xabcdfeed,
126 HDR_BUF_MAGIC_MARSHALED = 0xdcbafeed,
127 HDR_BUF_MAGIC_DEAD = 0xabcddead,
128 HDR_BUF_MAGIC_CORRUPT = 0xbadbadcc
129 };
130
131 class HdrStrHeap : public RefCountObj
132 {
133 public:
134 static constexpr int DEFAULT_SIZE = 2048;
135
136 void free() override;
137
138 char *allocate(int nbytes);
139 char *expand(char *ptr, int old_size, int new_size);
140 int space_avail();
141
142 uint32_t m_heap_size;
143 char *m_free_start;
144 uint32_t m_free_size;
145
146 bool contains(const char *str) const;
147 };
148
149 inline bool
contains(const char * str)150 HdrStrHeap::contains(const char *str) const
151 {
152 return reinterpret_cast<char const *>(this + 1) <= str && str < reinterpret_cast<char const *>(this) + m_heap_size;
153 }
154
155 struct StrHeapDesc {
156 StrHeapDesc() = default;
157
158 Ptr<RefCountObj> m_ref_count_ptr;
159 char const *m_heap_start = nullptr;
160 int32_t m_heap_len = 0;
161 bool m_locked = false;
162
163 bool
containsStrHeapDesc164 contains(const char *str) const
165 {
166 return (str >= m_heap_start && str < (m_heap_start + m_heap_len));
167 }
168 };
169
170 class HdrHeap
171 {
172 friend class CoreUtils;
173
174 public:
175 static constexpr int DEFAULT_SIZE = 2048;
176
177 void init();
178 inkcoreapi void destroy();
179
180 // PtrHeap allocation
181 HdrHeapObjImpl *allocate_obj(int nbytes, int type);
182 void deallocate_obj(HdrHeapObjImpl *obj);
183
184 // StrHeap allocation
185 char *allocate_str(int nbytes);
186 char *expand_str(const char *old_str, int old_len, int new_len);
187 char *duplicate_str(const char *str, int nbytes);
188 void free_string(const char *s, int len);
189
190 // Marshalling
191 inkcoreapi int marshal_length();
192 inkcoreapi int marshal(char *buf, int length);
193 int unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, RefCountObj *block_ref);
194 /// Computes the valid data size of an unmarshalled instance.
195 /// Callers should round up to HDR_PTR_SIZE to get the actual footprint.
196 int unmarshal_size() const; // TBD - change this name, it's confusing.
197 // One option - overload marshal_length to return this value if @a magic is HDR_BUF_MAGIC_MARSHALED.
198
199 void inherit_string_heaps(const HdrHeap *inherit_from);
200 int attach_block(IOBufferBlock *b, const char *use_start);
201 void set_ronly_str_heap_end(int slot, const char *end);
202
203 // Lock read only str heaps so that can't be moved around
204 // by a heap consolidation. Does NOT lock for Multi-Threaded
205 // access!
206 void
lock_ronly_str_heap(unsigned i)207 lock_ronly_str_heap(unsigned i)
208 {
209 m_ronly_heap[i].m_locked = true;
210 }
211
212 void
unlock_ronly_str_heap(unsigned i)213 unlock_ronly_str_heap(unsigned i)
214 {
215 m_ronly_heap[i].m_locked = false;
216 // INKqa11238
217 // Move slot i to the first available slot in m_ronly_heap[].
218 // The move is necessary because the rest of the code assumes
219 // heaps are always allocated in order.
220 for (unsigned j = 0; j < i; j++) {
221 if (m_ronly_heap[j].m_heap_start == nullptr) {
222 // move slot i to slot j
223 m_ronly_heap[j].m_ref_count_ptr = m_ronly_heap[i].m_ref_count_ptr;
224 m_ronly_heap[j].m_heap_start = m_ronly_heap[i].m_heap_start;
225 m_ronly_heap[j].m_heap_len = m_ronly_heap[i].m_heap_len;
226 m_ronly_heap[j].m_locked = m_ronly_heap[i].m_locked;
227 m_ronly_heap[i].m_ref_count_ptr = nullptr;
228 m_ronly_heap[i].m_heap_start = nullptr;
229 m_ronly_heap[i].m_heap_len = 0;
230 m_ronly_heap[i].m_locked = false;
231 break; // Did the move, time to go.
232 }
233 }
234 }
235
236 // Working function to copy strings into a new heap
237 // Unlike the HDR_MOVE_STR macro, this function will call
238 // allocate_str which will update the new_heap to create more space
239 // if there is not originally sufficient space
240 inline std::string_view
localize(const std::string_view & string)241 localize(const std::string_view &string)
242 {
243 auto length = string.length();
244 if (length > 0) {
245 char *new_str = this->allocate_str(length);
246 if (new_str) {
247 memcpy(new_str, string.data(), length);
248 } else {
249 length = 0;
250 }
251 return {new_str, length};
252 }
253 return {nullptr, 0};
254 }
255
256 // Sanity Check Functions
257 void sanity_check_strs();
258 bool check_marshalled(uint32_t buf_length);
259
260 // Debugging functions
261 void dump_heap(int len = -1);
262
263 uint32_t m_magic;
264 char *m_free_start;
265 char *m_data_start;
266 uint32_t m_size;
267
268 bool m_writeable;
269
270 // Overflow block ptr
271 // Overflow blocks are necessary because we can
272 // run out of space in the header heap and the
273 // heap is not rellocatable
274 // Overflow blocks have the HdrHeap full structure
275 // header on them, although only first block can
276 // point to string heaps
277 HdrHeap *m_next;
278
279 // HdrBuf heap pointers
280 uint32_t m_free_size;
281
282 int demote_rw_str_heap();
283 void coalesce_str_heaps(int incoming_size = 0);
284 void evacuate_from_str_heaps(HdrStrHeap *new_heap);
285 size_t required_space_for_evacuation();
286 bool attach_str_heap(char const *h_start, int h_len, RefCountObj *h_ref_obj, int *index);
287
288 uint64_t total_used_size() const;
289
290 /** Struct to prevent garbage collection on heaps.
291 This bumps the reference count to the heap containing the pointer
292 while the instance of this class exists. When it goes out of scope
293 the reference is dropped. This is useful inside a method or block
294 to keep the required heap data around until leaving the scope.
295 */
296 struct HeapGuard {
297 /// Construct the protection.
HeapGuardHeapGuard298 HeapGuard(HdrHeap *heap, const char *str)
299 {
300 if (heap->m_read_write_heap && heap->m_read_write_heap->contains(str)) {
301 m_ptr = heap->m_read_write_heap.get();
302 } else {
303 for (auto &i : heap->m_ronly_heap) {
304 if (i.contains(str)) {
305 m_ptr = i.m_ref_count_ptr;
306 break;
307 }
308 }
309 }
310 }
311
312 // There's no need to have a destructor here, the default dtor will take care of
313 // releasing the (potentially) locked heap.
314
315 /// The heap we protect (if any)
316 Ptr<RefCountObj> m_ptr;
317 };
318
319 // String Heap access
320 Ptr<HdrStrHeap> m_read_write_heap;
321 StrHeapDesc m_ronly_heap[HDR_BUF_RONLY_HEAPS];
322 int m_lost_string_space;
323 };
324
325 static constexpr HdrHeapMarshalBlocks HDR_HEAP_HDR_SIZE{ts::round_up(sizeof(HdrHeap))};
326 static constexpr size_t HDR_MAX_ALLOC_SIZE = HdrHeap::DEFAULT_SIZE - HDR_HEAP_HDR_SIZE;
327
328 inline void
free_string(const char * s,int len)329 HdrHeap::free_string(const char *s, int len)
330 {
331 if (s && len > 0) {
332 m_lost_string_space += len;
333 }
334 }
335
336 inline int
unmarshal_size()337 HdrHeap::unmarshal_size() const
338 {
339 return m_size + m_ronly_heap[0].m_heap_len;
340 }
341
342 //
343 struct MarshalXlate {
344 char const *start = nullptr;
345 char const *end = nullptr;
346 char const *offset = nullptr;
MarshalXlateMarshalXlate347 MarshalXlate() {}
348 };
349
350 struct HeapCheck {
351 char const *start;
352 char const *end;
353 };
354
355 // Nasty macro to do string marshalling
356 #define HDR_MARSHAL_STR(ptr, table, nentries) \
357 if (ptr) { \
358 int found = 0; \
359 for (int i = 0; i < nentries; i++) { \
360 if (ptr >= table[i].start && ptr <= table[i].end) { \
361 ptr = (((char *)ptr) - (uintptr_t)table[i].offset); \
362 found = 1; \
363 break; \
364 } \
365 } \
366 ink_assert(found); \
367 if (found == 0) { \
368 return -1; \
369 } \
370 }
371
372 // Nasty macro to do string marshalling
373 #define HDR_MARSHAL_STR_1(ptr, table) \
374 if (ptr) { \
375 int found = 0; \
376 if (ptr >= table[0].start && ptr <= table[0].end) { \
377 ptr = (((char *)ptr) - (uintptr_t)table[0].offset); \
378 found = 1; \
379 } \
380 ink_assert(found); \
381 if (found == 0) { \
382 return -1; \
383 } \
384 }
385
386 #define HDR_MARSHAL_PTR(ptr, type, table, nentries) \
387 if (ptr) { \
388 int found = 0; \
389 for (int i = 0; i < nentries; i++) { \
390 if ((char *)ptr >= table[i].start && (char *)ptr <= table[i].end) { \
391 ptr = (type *)(((char *)ptr) - (uintptr_t)table[i].offset); \
392 found = 1; \
393 break; \
394 } \
395 } \
396 ink_assert(found); \
397 if (found == 0) { \
398 return -1; \
399 } \
400 }
401
402 #define HDR_MARSHAL_PTR_1(ptr, type, table) \
403 if (ptr) { \
404 int found = 0; \
405 if ((char *)ptr >= table[0].start && (char *)ptr <= table[0].end) { \
406 ptr = (type *)(((char *)ptr) - (uintptr_t)table[0].offset); \
407 found = 1; \
408 } \
409 ink_assert(found); \
410 if (found == 0) { \
411 return -1; \
412 } \
413 }
414
415 #define HDR_UNMARSHAL_STR(ptr, offset) \
416 if (ptr) { \
417 ptr = ((char *)ptr) + offset; \
418 }
419
420 #define HDR_UNMARSHAL_PTR(ptr, type, offset) \
421 if (ptr) { \
422 ptr = (type *)(((char *)ptr) + offset); \
423 }
424
425 // Nasty macro to do string evacuation. Assumes
426 // new heap = new_heap
427 #define HDR_MOVE_STR(str, len) \
428 { \
429 if (str) { \
430 char *new_str = new_heap->allocate(len); \
431 if (new_str) \
432 memcpy(new_str, str, len); \
433 str = new_str; \
434 } \
435 }
436
437 // Nasty macro to do verify all strings it
438 // in attached heaps
439 #define CHECK_STR(str, len, _heaps, _num_heaps) \
440 { \
441 if (str) { \
442 int found = 0; \
443 for (int i = 0; i < _num_heaps; i++) { \
444 if (str >= _heaps[i].start && str + len <= heaps[i].end) { \
445 found = 1; \
446 } \
447 } \
448 ink_release_assert(found); \
449 } \
450 }
451
452 // struct HdrHeapSDKHandle()
453 //
454 // Handle to a HdrHeap.
455 //
456 // Intended to be subclassed and contain a
457 // object pointer that points into the heap
458 //
459 struct HdrHeapSDKHandle {
460 public:
HdrHeapSDKHandleHdrHeapSDKHandle461 HdrHeapSDKHandle() {}
~HdrHeapSDKHandleHdrHeapSDKHandle462 ~HdrHeapSDKHandle() { clear(); }
463 // clear() only deallocates chained SDK return values
464 // The underlying MBuffer is left untouched
465 void clear();
466
467 // destroy() frees the underlying MBuffer and deallocates all chained
468 // SDK return values
469 void destroy();
470
471 void set(const HdrHeapSDKHandle *from);
472 const char *make_sdk_string(const char *raw_str, int raw_str_len);
473
474 HdrHeap *m_heap = nullptr;
475
476 // In order to prevent gratitous refcounting,
477 // automatic C++ copies are disabled!
478 HdrHeapSDKHandle(const HdrHeapSDKHandle &r) = delete;
479 HdrHeapSDKHandle &operator=(const HdrHeapSDKHandle &r) = delete;
480 };
481
482 inline void
destroy()483 HdrHeapSDKHandle::destroy()
484 {
485 if (m_heap) {
486 m_heap->destroy();
487 }
488 clear();
489 }
490
491 inline void
clear()492 HdrHeapSDKHandle::clear()
493 {
494 m_heap = nullptr;
495 }
496
497 inline void
set(const HdrHeapSDKHandle * from)498 HdrHeapSDKHandle::set(const HdrHeapSDKHandle *from)
499 {
500 clear();
501 m_heap = from->m_heap;
502 }
503
504 HdrStrHeap *new_HdrStrHeap(int requested_size);
505 inkcoreapi HdrHeap *new_HdrHeap(int size = HdrHeap::DEFAULT_SIZE);
506
507 void hdr_heap_test();
508