1package ebpf
2
3import (
4	"errors"
5	"fmt"
6	"os"
7	"unsafe"
8
9	"github.com/cilium/ebpf/internal"
10	"github.com/cilium/ebpf/internal/btf"
11	"github.com/cilium/ebpf/internal/unix"
12)
13
14// Generic errors returned by BPF syscalls.
15var (
16	ErrNotExist = errors.New("requested object does not exist")
17)
18
19// bpfObjName is a null-terminated string made up of
20// 'A-Za-z0-9_' characters.
21type bpfObjName [unix.BPF_OBJ_NAME_LEN]byte
22
23// newBPFObjName truncates the result if it is too long.
24func newBPFObjName(name string) bpfObjName {
25	var result bpfObjName
26	copy(result[:unix.BPF_OBJ_NAME_LEN-1], name)
27	return result
28}
29
30func invalidBPFObjNameChar(char rune) bool {
31	dotAllowed := objNameAllowsDot() == nil
32
33	switch {
34	case char >= 'A' && char <= 'Z':
35		fallthrough
36	case char >= 'a' && char <= 'z':
37		fallthrough
38	case char >= '0' && char <= '9':
39		fallthrough
40	case dotAllowed && char == '.':
41		fallthrough
42	case char == '_':
43		return false
44	default:
45		return true
46	}
47}
48
49type bpfMapCreateAttr struct {
50	mapType        MapType
51	keySize        uint32
52	valueSize      uint32
53	maxEntries     uint32
54	flags          uint32
55	innerMapFd     uint32     // since 4.12 56f668dfe00d
56	numaNode       uint32     // since 4.14 96eabe7a40aa
57	mapName        bpfObjName // since 4.15 ad5b177bd73f
58	mapIfIndex     uint32
59	btfFd          uint32
60	btfKeyTypeID   btf.TypeID
61	btfValueTypeID btf.TypeID
62}
63
64type bpfMapOpAttr struct {
65	mapFd   uint32
66	padding uint32
67	key     internal.Pointer
68	value   internal.Pointer
69	flags   uint64
70}
71
72type bpfMapInfo struct {
73	mapType    uint32
74	id         uint32
75	keySize    uint32
76	valueSize  uint32
77	maxEntries uint32
78	flags      uint32
79	mapName    bpfObjName // since 4.15 ad5b177bd73f
80}
81
82type bpfProgLoadAttr struct {
83	progType           ProgramType
84	insCount           uint32
85	instructions       internal.Pointer
86	license            internal.Pointer
87	logLevel           uint32
88	logSize            uint32
89	logBuf             internal.Pointer
90	kernelVersion      uint32     // since 4.1  2541517c32be
91	progFlags          uint32     // since 4.11 e07b98d9bffe
92	progName           bpfObjName // since 4.15 067cae47771c
93	progIfIndex        uint32     // since 4.15 1f6f4cb7ba21
94	expectedAttachType AttachType // since 4.17 5e43f899b03a
95	progBTFFd          uint32
96	funcInfoRecSize    uint32
97	funcInfo           internal.Pointer
98	funcInfoCnt        uint32
99	lineInfoRecSize    uint32
100	lineInfo           internal.Pointer
101	lineInfoCnt        uint32
102	attachBTFID        btf.TypeID
103	attachProgFd       uint32
104}
105
106type bpfProgInfo struct {
107	progType     uint32
108	id           uint32
109	tag          [unix.BPF_TAG_SIZE]byte
110	jitedLen     uint32
111	xlatedLen    uint32
112	jited        internal.Pointer
113	xlated       internal.Pointer
114	loadTime     uint64 // since 4.15 cb4d2b3f03d8
115	createdByUID uint32
116	nrMapIDs     uint32
117	mapIds       internal.Pointer
118	name         bpfObjName
119}
120
121type bpfProgTestRunAttr struct {
122	fd          uint32
123	retval      uint32
124	dataSizeIn  uint32
125	dataSizeOut uint32
126	dataIn      internal.Pointer
127	dataOut     internal.Pointer
128	repeat      uint32
129	duration    uint32
130}
131
132type bpfGetFDByIDAttr struct {
133	id   uint32
134	next uint32
135}
136
137type bpfMapFreezeAttr struct {
138	mapFd uint32
139}
140
141type bpfObjGetNextIDAttr struct {
142	startID   uint32
143	nextID    uint32
144	openFlags uint32
145}
146
147func bpfProgLoad(attr *bpfProgLoadAttr) (*internal.FD, error) {
148	for {
149		fd, err := internal.BPF(internal.BPF_PROG_LOAD, unsafe.Pointer(attr), unsafe.Sizeof(*attr))
150		// As of ~4.20 the verifier can be interrupted by a signal,
151		// and returns EAGAIN in that case.
152		if err == unix.EAGAIN {
153			continue
154		}
155
156		if err != nil {
157			return nil, err
158		}
159
160		return internal.NewFD(uint32(fd)), nil
161	}
162}
163
164func bpfProgTestRun(attr *bpfProgTestRunAttr) error {
165	_, err := internal.BPF(internal.BPF_PROG_TEST_RUN, unsafe.Pointer(attr), unsafe.Sizeof(*attr))
166	return err
167}
168
169func bpfMapCreate(attr *bpfMapCreateAttr) (*internal.FD, error) {
170	fd, err := internal.BPF(internal.BPF_MAP_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr))
171	if errors.Is(err, os.ErrPermission) {
172		return nil, errors.New("permission denied or insufficient rlimit to lock memory for map")
173	}
174
175	if err != nil {
176		return nil, err
177	}
178
179	return internal.NewFD(uint32(fd)), nil
180}
181
182var haveNestedMaps = internal.FeatureTest("nested maps", "4.12", func() (bool, error) {
183	inner, err := bpfMapCreate(&bpfMapCreateAttr{
184		mapType:    Array,
185		keySize:    4,
186		valueSize:  4,
187		maxEntries: 1,
188	})
189	if err != nil {
190		return false, err
191	}
192	defer inner.Close()
193
194	innerFd, _ := inner.Value()
195	nested, err := bpfMapCreate(&bpfMapCreateAttr{
196		mapType:    ArrayOfMaps,
197		keySize:    4,
198		valueSize:  4,
199		maxEntries: 1,
200		innerMapFd: innerFd,
201	})
202	if err != nil {
203		return false, nil
204	}
205
206	_ = nested.Close()
207	return true, nil
208})
209
210var haveMapMutabilityModifiers = internal.FeatureTest("read- and write-only maps", "5.2", func() (bool, error) {
211	// This checks BPF_F_RDONLY_PROG and BPF_F_WRONLY_PROG. Since
212	// BPF_MAP_FREEZE appeared in 5.2 as well we don't do a separate check.
213	m, err := bpfMapCreate(&bpfMapCreateAttr{
214		mapType:    Array,
215		keySize:    4,
216		valueSize:  4,
217		maxEntries: 1,
218		flags:      unix.BPF_F_RDONLY_PROG,
219	})
220	if err != nil {
221		return false, nil
222	}
223	_ = m.Close()
224	return true, nil
225})
226
227func bpfMapLookupElem(m *internal.FD, key, valueOut internal.Pointer) error {
228	fd, err := m.Value()
229	if err != nil {
230		return err
231	}
232
233	attr := bpfMapOpAttr{
234		mapFd: fd,
235		key:   key,
236		value: valueOut,
237	}
238	_, err = internal.BPF(internal.BPF_MAP_LOOKUP_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
239	return wrapMapError(err)
240}
241
242func bpfMapLookupAndDelete(m *internal.FD, key, valueOut internal.Pointer) error {
243	fd, err := m.Value()
244	if err != nil {
245		return err
246	}
247
248	attr := bpfMapOpAttr{
249		mapFd: fd,
250		key:   key,
251		value: valueOut,
252	}
253	_, err = internal.BPF(internal.BPF_MAP_LOOKUP_AND_DELETE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
254	return wrapMapError(err)
255}
256
257func bpfMapUpdateElem(m *internal.FD, key, valueOut internal.Pointer, flags uint64) error {
258	fd, err := m.Value()
259	if err != nil {
260		return err
261	}
262
263	attr := bpfMapOpAttr{
264		mapFd: fd,
265		key:   key,
266		value: valueOut,
267		flags: flags,
268	}
269	_, err = internal.BPF(internal.BPF_MAP_UPDATE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
270	return wrapMapError(err)
271}
272
273func bpfMapDeleteElem(m *internal.FD, key internal.Pointer) error {
274	fd, err := m.Value()
275	if err != nil {
276		return err
277	}
278
279	attr := bpfMapOpAttr{
280		mapFd: fd,
281		key:   key,
282	}
283	_, err = internal.BPF(internal.BPF_MAP_DELETE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
284	return wrapMapError(err)
285}
286
287func bpfMapGetNextKey(m *internal.FD, key, nextKeyOut internal.Pointer) error {
288	fd, err := m.Value()
289	if err != nil {
290		return err
291	}
292
293	attr := bpfMapOpAttr{
294		mapFd: fd,
295		key:   key,
296		value: nextKeyOut,
297	}
298	_, err = internal.BPF(internal.BPF_MAP_GET_NEXT_KEY, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
299	return wrapMapError(err)
300}
301
302func objGetNextID(cmd internal.BPFCmd, start uint32) (uint32, error) {
303	attr := bpfObjGetNextIDAttr{
304		startID: start,
305	}
306	_, err := internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
307	return attr.nextID, wrapObjError(err)
308}
309
310func wrapObjError(err error) error {
311	if err == nil {
312		return nil
313	}
314	if errors.Is(err, unix.ENOENT) {
315		return fmt.Errorf("%w", ErrNotExist)
316	}
317
318	return errors.New(err.Error())
319}
320
321func wrapMapError(err error) error {
322	if err == nil {
323		return nil
324	}
325
326	if errors.Is(err, unix.ENOENT) {
327		return ErrKeyNotExist
328	}
329
330	if errors.Is(err, unix.EEXIST) {
331		return ErrKeyExist
332	}
333
334	return errors.New(err.Error())
335}
336
337func bpfMapFreeze(m *internal.FD) error {
338	fd, err := m.Value()
339	if err != nil {
340		return err
341	}
342
343	attr := bpfMapFreezeAttr{
344		mapFd: fd,
345	}
346	_, err = internal.BPF(internal.BPF_MAP_FREEZE, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
347	return err
348}
349
350func bpfGetProgInfoByFD(fd *internal.FD) (*bpfProgInfo, error) {
351	var info bpfProgInfo
352	if err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)); err != nil {
353		return nil, fmt.Errorf("can't get program info: %w", err)
354	}
355	return &info, nil
356}
357
358func bpfGetMapInfoByFD(fd *internal.FD) (*bpfMapInfo, error) {
359	var info bpfMapInfo
360	err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info))
361	if err != nil {
362		return nil, fmt.Errorf("can't get map info: %w", err)
363	}
364	return &info, nil
365}
366
367var haveObjName = internal.FeatureTest("object names", "4.15", func() (bool, error) {
368	attr := bpfMapCreateAttr{
369		mapType:    Array,
370		keySize:    4,
371		valueSize:  4,
372		maxEntries: 1,
373		mapName:    newBPFObjName("feature_test"),
374	}
375
376	fd, err := bpfMapCreate(&attr)
377	if err != nil {
378		return false, nil
379	}
380
381	_ = fd.Close()
382	return true, nil
383})
384
385var objNameAllowsDot = internal.FeatureTest("dot in object names", "5.2", func() (bool, error) {
386	if err := haveObjName(); err != nil {
387		return false, err
388	}
389
390	attr := bpfMapCreateAttr{
391		mapType:    Array,
392		keySize:    4,
393		valueSize:  4,
394		maxEntries: 1,
395		mapName:    newBPFObjName(".test"),
396	}
397
398	fd, err := bpfMapCreate(&attr)
399	if err != nil {
400		return false, nil
401	}
402
403	_ = fd.Close()
404	return true, nil
405})
406
407func bpfObjGetFDByID(cmd internal.BPFCmd, id uint32) (*internal.FD, error) {
408	attr := bpfGetFDByIDAttr{
409		id: id,
410	}
411	ptr, err := internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
412	return internal.NewFD(uint32(ptr)), wrapObjError(err)
413}
414