1package ebpf 2 3import ( 4 "errors" 5 "fmt" 6 "os" 7 "unsafe" 8 9 "github.com/cilium/ebpf/internal" 10 "github.com/cilium/ebpf/internal/btf" 11 "github.com/cilium/ebpf/internal/unix" 12) 13 14// Generic errors returned by BPF syscalls. 15var ( 16 ErrNotExist = errors.New("requested object does not exist") 17) 18 19// bpfObjName is a null-terminated string made up of 20// 'A-Za-z0-9_' characters. 21type bpfObjName [unix.BPF_OBJ_NAME_LEN]byte 22 23// newBPFObjName truncates the result if it is too long. 24func newBPFObjName(name string) bpfObjName { 25 var result bpfObjName 26 copy(result[:unix.BPF_OBJ_NAME_LEN-1], name) 27 return result 28} 29 30func invalidBPFObjNameChar(char rune) bool { 31 dotAllowed := objNameAllowsDot() == nil 32 33 switch { 34 case char >= 'A' && char <= 'Z': 35 fallthrough 36 case char >= 'a' && char <= 'z': 37 fallthrough 38 case char >= '0' && char <= '9': 39 fallthrough 40 case dotAllowed && char == '.': 41 fallthrough 42 case char == '_': 43 return false 44 default: 45 return true 46 } 47} 48 49type bpfMapCreateAttr struct { 50 mapType MapType 51 keySize uint32 52 valueSize uint32 53 maxEntries uint32 54 flags uint32 55 innerMapFd uint32 // since 4.12 56f668dfe00d 56 numaNode uint32 // since 4.14 96eabe7a40aa 57 mapName bpfObjName // since 4.15 ad5b177bd73f 58 mapIfIndex uint32 59 btfFd uint32 60 btfKeyTypeID btf.TypeID 61 btfValueTypeID btf.TypeID 62} 63 64type bpfMapOpAttr struct { 65 mapFd uint32 66 padding uint32 67 key internal.Pointer 68 value internal.Pointer 69 flags uint64 70} 71 72type bpfMapInfo struct { 73 mapType uint32 74 id uint32 75 keySize uint32 76 valueSize uint32 77 maxEntries uint32 78 flags uint32 79 mapName bpfObjName // since 4.15 ad5b177bd73f 80} 81 82type bpfProgLoadAttr struct { 83 progType ProgramType 84 insCount uint32 85 instructions internal.Pointer 86 license internal.Pointer 87 logLevel uint32 88 logSize uint32 89 logBuf internal.Pointer 90 kernelVersion uint32 // since 4.1 2541517c32be 91 progFlags uint32 // since 4.11 e07b98d9bffe 92 progName bpfObjName // since 4.15 067cae47771c 93 progIfIndex uint32 // since 4.15 1f6f4cb7ba21 94 expectedAttachType AttachType // since 4.17 5e43f899b03a 95 progBTFFd uint32 96 funcInfoRecSize uint32 97 funcInfo internal.Pointer 98 funcInfoCnt uint32 99 lineInfoRecSize uint32 100 lineInfo internal.Pointer 101 lineInfoCnt uint32 102 attachBTFID btf.TypeID 103 attachProgFd uint32 104} 105 106type bpfProgInfo struct { 107 progType uint32 108 id uint32 109 tag [unix.BPF_TAG_SIZE]byte 110 jitedLen uint32 111 xlatedLen uint32 112 jited internal.Pointer 113 xlated internal.Pointer 114 loadTime uint64 // since 4.15 cb4d2b3f03d8 115 createdByUID uint32 116 nrMapIDs uint32 117 mapIds internal.Pointer 118 name bpfObjName 119} 120 121type bpfProgTestRunAttr struct { 122 fd uint32 123 retval uint32 124 dataSizeIn uint32 125 dataSizeOut uint32 126 dataIn internal.Pointer 127 dataOut internal.Pointer 128 repeat uint32 129 duration uint32 130} 131 132type bpfGetFDByIDAttr struct { 133 id uint32 134 next uint32 135} 136 137type bpfMapFreezeAttr struct { 138 mapFd uint32 139} 140 141type bpfObjGetNextIDAttr struct { 142 startID uint32 143 nextID uint32 144 openFlags uint32 145} 146 147func bpfProgLoad(attr *bpfProgLoadAttr) (*internal.FD, error) { 148 for { 149 fd, err := internal.BPF(internal.BPF_PROG_LOAD, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) 150 // As of ~4.20 the verifier can be interrupted by a signal, 151 // and returns EAGAIN in that case. 152 if err == unix.EAGAIN { 153 continue 154 } 155 156 if err != nil { 157 return nil, err 158 } 159 160 return internal.NewFD(uint32(fd)), nil 161 } 162} 163 164func bpfProgTestRun(attr *bpfProgTestRunAttr) error { 165 _, err := internal.BPF(internal.BPF_PROG_TEST_RUN, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) 166 return err 167} 168 169func bpfMapCreate(attr *bpfMapCreateAttr) (*internal.FD, error) { 170 fd, err := internal.BPF(internal.BPF_MAP_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) 171 if errors.Is(err, os.ErrPermission) { 172 return nil, errors.New("permission denied or insufficient rlimit to lock memory for map") 173 } 174 175 if err != nil { 176 return nil, err 177 } 178 179 return internal.NewFD(uint32(fd)), nil 180} 181 182var haveNestedMaps = internal.FeatureTest("nested maps", "4.12", func() (bool, error) { 183 inner, err := bpfMapCreate(&bpfMapCreateAttr{ 184 mapType: Array, 185 keySize: 4, 186 valueSize: 4, 187 maxEntries: 1, 188 }) 189 if err != nil { 190 return false, err 191 } 192 defer inner.Close() 193 194 innerFd, _ := inner.Value() 195 nested, err := bpfMapCreate(&bpfMapCreateAttr{ 196 mapType: ArrayOfMaps, 197 keySize: 4, 198 valueSize: 4, 199 maxEntries: 1, 200 innerMapFd: innerFd, 201 }) 202 if err != nil { 203 return false, nil 204 } 205 206 _ = nested.Close() 207 return true, nil 208}) 209 210var haveMapMutabilityModifiers = internal.FeatureTest("read- and write-only maps", "5.2", func() (bool, error) { 211 // This checks BPF_F_RDONLY_PROG and BPF_F_WRONLY_PROG. Since 212 // BPF_MAP_FREEZE appeared in 5.2 as well we don't do a separate check. 213 m, err := bpfMapCreate(&bpfMapCreateAttr{ 214 mapType: Array, 215 keySize: 4, 216 valueSize: 4, 217 maxEntries: 1, 218 flags: unix.BPF_F_RDONLY_PROG, 219 }) 220 if err != nil { 221 return false, nil 222 } 223 _ = m.Close() 224 return true, nil 225}) 226 227func bpfMapLookupElem(m *internal.FD, key, valueOut internal.Pointer) error { 228 fd, err := m.Value() 229 if err != nil { 230 return err 231 } 232 233 attr := bpfMapOpAttr{ 234 mapFd: fd, 235 key: key, 236 value: valueOut, 237 } 238 _, err = internal.BPF(internal.BPF_MAP_LOOKUP_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 239 return wrapMapError(err) 240} 241 242func bpfMapLookupAndDelete(m *internal.FD, key, valueOut internal.Pointer) error { 243 fd, err := m.Value() 244 if err != nil { 245 return err 246 } 247 248 attr := bpfMapOpAttr{ 249 mapFd: fd, 250 key: key, 251 value: valueOut, 252 } 253 _, err = internal.BPF(internal.BPF_MAP_LOOKUP_AND_DELETE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 254 return wrapMapError(err) 255} 256 257func bpfMapUpdateElem(m *internal.FD, key, valueOut internal.Pointer, flags uint64) error { 258 fd, err := m.Value() 259 if err != nil { 260 return err 261 } 262 263 attr := bpfMapOpAttr{ 264 mapFd: fd, 265 key: key, 266 value: valueOut, 267 flags: flags, 268 } 269 _, err = internal.BPF(internal.BPF_MAP_UPDATE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 270 return wrapMapError(err) 271} 272 273func bpfMapDeleteElem(m *internal.FD, key internal.Pointer) error { 274 fd, err := m.Value() 275 if err != nil { 276 return err 277 } 278 279 attr := bpfMapOpAttr{ 280 mapFd: fd, 281 key: key, 282 } 283 _, err = internal.BPF(internal.BPF_MAP_DELETE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 284 return wrapMapError(err) 285} 286 287func bpfMapGetNextKey(m *internal.FD, key, nextKeyOut internal.Pointer) error { 288 fd, err := m.Value() 289 if err != nil { 290 return err 291 } 292 293 attr := bpfMapOpAttr{ 294 mapFd: fd, 295 key: key, 296 value: nextKeyOut, 297 } 298 _, err = internal.BPF(internal.BPF_MAP_GET_NEXT_KEY, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 299 return wrapMapError(err) 300} 301 302func objGetNextID(cmd internal.BPFCmd, start uint32) (uint32, error) { 303 attr := bpfObjGetNextIDAttr{ 304 startID: start, 305 } 306 _, err := internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 307 return attr.nextID, wrapObjError(err) 308} 309 310func wrapObjError(err error) error { 311 if err == nil { 312 return nil 313 } 314 if errors.Is(err, unix.ENOENT) { 315 return fmt.Errorf("%w", ErrNotExist) 316 } 317 318 return errors.New(err.Error()) 319} 320 321func wrapMapError(err error) error { 322 if err == nil { 323 return nil 324 } 325 326 if errors.Is(err, unix.ENOENT) { 327 return ErrKeyNotExist 328 } 329 330 if errors.Is(err, unix.EEXIST) { 331 return ErrKeyExist 332 } 333 334 return errors.New(err.Error()) 335} 336 337func bpfMapFreeze(m *internal.FD) error { 338 fd, err := m.Value() 339 if err != nil { 340 return err 341 } 342 343 attr := bpfMapFreezeAttr{ 344 mapFd: fd, 345 } 346 _, err = internal.BPF(internal.BPF_MAP_FREEZE, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 347 return err 348} 349 350func bpfGetProgInfoByFD(fd *internal.FD) (*bpfProgInfo, error) { 351 var info bpfProgInfo 352 if err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)); err != nil { 353 return nil, fmt.Errorf("can't get program info: %w", err) 354 } 355 return &info, nil 356} 357 358func bpfGetMapInfoByFD(fd *internal.FD) (*bpfMapInfo, error) { 359 var info bpfMapInfo 360 err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)) 361 if err != nil { 362 return nil, fmt.Errorf("can't get map info: %w", err) 363 } 364 return &info, nil 365} 366 367var haveObjName = internal.FeatureTest("object names", "4.15", func() (bool, error) { 368 attr := bpfMapCreateAttr{ 369 mapType: Array, 370 keySize: 4, 371 valueSize: 4, 372 maxEntries: 1, 373 mapName: newBPFObjName("feature_test"), 374 } 375 376 fd, err := bpfMapCreate(&attr) 377 if err != nil { 378 return false, nil 379 } 380 381 _ = fd.Close() 382 return true, nil 383}) 384 385var objNameAllowsDot = internal.FeatureTest("dot in object names", "5.2", func() (bool, error) { 386 if err := haveObjName(); err != nil { 387 return false, err 388 } 389 390 attr := bpfMapCreateAttr{ 391 mapType: Array, 392 keySize: 4, 393 valueSize: 4, 394 maxEntries: 1, 395 mapName: newBPFObjName(".test"), 396 } 397 398 fd, err := bpfMapCreate(&attr) 399 if err != nil { 400 return false, nil 401 } 402 403 _ = fd.Close() 404 return true, nil 405}) 406 407func bpfObjGetFDByID(cmd internal.BPFCmd, id uint32) (*internal.FD, error) { 408 attr := bpfGetFDByIDAttr{ 409 id: id, 410 } 411 ptr, err := internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) 412 return internal.NewFD(uint32(ptr)), wrapObjError(err) 413} 414