30 static bool queue_stack_map_is_empty(struct bpf_queue_stack *qs)  in queue_stack_map_is_empty()  argument
32 	return qs->head == qs->tail;  in queue_stack_map_is_empty()
35 static bool queue_stack_map_is_full(struct bpf_queue_stack *qs)  in queue_stack_map_is_full()  argument
37 	u32 head = qs->head + 1;  in queue_stack_map_is_full()
39 	if (unlikely(head >= qs->size))  in queue_stack_map_is_full()
42 	return head == qs->tail;  in queue_stack_map_is_full()
70 	struct bpf_queue_stack *qs;  in queue_stack_map_alloc()  local
74 	queue_size = sizeof(*qs) + size * attr->value_size;  in queue_stack_map_alloc()
76 	qs = bpf_map_area_alloc(queue_size, numa_node);  in queue_stack_map_alloc()
77 	if (!qs)  in queue_stack_map_alloc()
80 	memset(qs, 0, sizeof(*qs));  in queue_stack_map_alloc()
82 	bpf_map_init_from_attr(&qs->map, attr);  in queue_stack_map_alloc()
84 	qs->size = size;  in queue_stack_map_alloc()
86 	raw_spin_lock_init(&qs->lock);  in queue_stack_map_alloc()
88 	return &qs->map;  in queue_stack_map_alloc()
94 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in queue_stack_map_free()  local
96 	bpf_map_area_free(qs);  in queue_stack_map_free()
101 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in __queue_map_get()  local
106 	raw_spin_lock_irqsave(&qs->lock, flags);  in __queue_map_get()
108 	if (queue_stack_map_is_empty(qs)) {  in __queue_map_get()
109 		memset(value, 0, qs->map.value_size);  in __queue_map_get()
114 	ptr = &qs->elements[qs->tail * qs->map.value_size];  in __queue_map_get()
115 	memcpy(value, ptr, qs->map.value_size);  in __queue_map_get()
118 		if (unlikely(++qs->tail >= qs->size))  in __queue_map_get()
119 			qs->tail = 0;  in __queue_map_get()
123 	raw_spin_unlock_irqrestore(&qs->lock, flags);  in __queue_map_get()
130 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in __stack_map_get()  local
136 	raw_spin_lock_irqsave(&qs->lock, flags);  in __stack_map_get()
138 	if (queue_stack_map_is_empty(qs)) {  in __stack_map_get()
139 		memset(value, 0, qs->map.value_size);  in __stack_map_get()
144 	index = qs->head - 1;  in __stack_map_get()
145 	if (unlikely(index >= qs->size))  in __stack_map_get()
146 		index = qs->size - 1;  in __stack_map_get()
148 	ptr = &qs->elements[index * qs->map.value_size];  in __stack_map_get()
149 	memcpy(value, ptr, qs->map.value_size);  in __stack_map_get()
152 		qs->head = index;  in __stack_map_get()
155 	raw_spin_unlock_irqrestore(&qs->lock, flags);  in __stack_map_get()
187 	struct bpf_queue_stack *qs = bpf_queue_stack(map);  in queue_stack_map_push_elem()  local
201 	raw_spin_lock_irqsave(&qs->lock, irq_flags);  in queue_stack_map_push_elem()
203 	if (queue_stack_map_is_full(qs)) {  in queue_stack_map_push_elem()
209 		if (unlikely(++qs->tail >= qs->size))  in queue_stack_map_push_elem()
210 			qs->tail = 0;  in queue_stack_map_push_elem()
213 	dst = &qs->elements[qs->head * qs->map.value_size];  in queue_stack_map_push_elem()
214 	memcpy(dst, value, qs->map.value_size);  in queue_stack_map_push_elem()
216 	if (unlikely(++qs->head >= qs->size))  in queue_stack_map_push_elem()
217 		qs->head = 0;  in queue_stack_map_push_elem()
220 	raw_spin_unlock_irqrestore(&qs->lock, irq_flags);  in queue_stack_map_push_elem()