465 lines
11 KiB
Go
465 lines
11 KiB
Go
package ebpf
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"unsafe"
|
|
|
|
"github.com/cilium/ebpf/asm"
|
|
"github.com/cilium/ebpf/internal"
|
|
"github.com/cilium/ebpf/internal/unix"
|
|
)
|
|
|
|
// ErrNotExist is returned when loading a non-existing map or program.
|
|
//
|
|
// Deprecated: use os.ErrNotExist instead.
|
|
var ErrNotExist = os.ErrNotExist
|
|
|
|
// invalidBPFObjNameChar returns true if char may not appear in
|
|
// a BPF object name.
|
|
func invalidBPFObjNameChar(char rune) bool {
|
|
dotAllowed := objNameAllowsDot() == nil
|
|
|
|
switch {
|
|
case char >= 'A' && char <= 'Z':
|
|
return false
|
|
case char >= 'a' && char <= 'z':
|
|
return false
|
|
case char >= '0' && char <= '9':
|
|
return false
|
|
case dotAllowed && char == '.':
|
|
return false
|
|
case char == '_':
|
|
return false
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
|
|
type bpfMapOpAttr struct {
|
|
mapFd uint32
|
|
padding uint32
|
|
key internal.Pointer
|
|
value internal.Pointer
|
|
flags uint64
|
|
}
|
|
|
|
type bpfBatchMapOpAttr struct {
|
|
inBatch internal.Pointer
|
|
outBatch internal.Pointer
|
|
keys internal.Pointer
|
|
values internal.Pointer
|
|
count uint32
|
|
mapFd uint32
|
|
elemFlags uint64
|
|
flags uint64
|
|
}
|
|
|
|
type bpfMapInfo struct {
|
|
map_type uint32 // since 4.12 1e2709769086
|
|
id uint32
|
|
key_size uint32
|
|
value_size uint32
|
|
max_entries uint32
|
|
map_flags uint32
|
|
name internal.BPFObjName // since 4.15 ad5b177bd73f
|
|
ifindex uint32 // since 4.16 52775b33bb50
|
|
btf_vmlinux_value_type_id uint32 // since 5.6 85d33df357b6
|
|
netns_dev uint64 // since 4.16 52775b33bb50
|
|
netns_ino uint64
|
|
btf_id uint32 // since 4.18 78958fca7ead
|
|
btf_key_type_id uint32 // since 4.18 9b2cf328b2ec
|
|
btf_value_type_id uint32
|
|
}
|
|
|
|
type bpfProgInfo struct {
|
|
prog_type uint32
|
|
id uint32
|
|
tag [unix.BPF_TAG_SIZE]byte
|
|
jited_prog_len uint32
|
|
xlated_prog_len uint32
|
|
jited_prog_insns internal.Pointer
|
|
xlated_prog_insns internal.Pointer
|
|
load_time uint64 // since 4.15 cb4d2b3f03d8
|
|
created_by_uid uint32
|
|
nr_map_ids uint32 // since 4.15 cb4d2b3f03d8
|
|
map_ids internal.Pointer
|
|
name internal.BPFObjName // since 4.15 067cae47771c
|
|
ifindex uint32
|
|
gpl_compatible uint32
|
|
netns_dev uint64
|
|
netns_ino uint64
|
|
nr_jited_ksyms uint32
|
|
nr_jited_func_lens uint32
|
|
jited_ksyms internal.Pointer
|
|
jited_func_lens internal.Pointer
|
|
btf_id uint32
|
|
func_info_rec_size uint32
|
|
func_info internal.Pointer
|
|
nr_func_info uint32
|
|
nr_line_info uint32
|
|
line_info internal.Pointer
|
|
jited_line_info internal.Pointer
|
|
nr_jited_line_info uint32
|
|
line_info_rec_size uint32
|
|
jited_line_info_rec_size uint32
|
|
nr_prog_tags uint32
|
|
prog_tags internal.Pointer
|
|
run_time_ns uint64
|
|
run_cnt uint64
|
|
}
|
|
|
|
type bpfProgTestRunAttr struct {
|
|
fd uint32
|
|
retval uint32
|
|
dataSizeIn uint32
|
|
dataSizeOut uint32
|
|
dataIn internal.Pointer
|
|
dataOut internal.Pointer
|
|
repeat uint32
|
|
duration uint32
|
|
}
|
|
|
|
type bpfMapFreezeAttr struct {
|
|
mapFd uint32
|
|
}
|
|
|
|
type bpfObjGetNextIDAttr struct {
|
|
startID uint32
|
|
nextID uint32
|
|
openFlags uint32
|
|
}
|
|
|
|
func bpfProgTestRun(attr *bpfProgTestRunAttr) error {
|
|
_, err := internal.BPF(internal.BPF_PROG_TEST_RUN, unsafe.Pointer(attr), unsafe.Sizeof(*attr))
|
|
return err
|
|
}
|
|
|
|
var haveNestedMaps = internal.FeatureTest("nested maps", "4.12", func() error {
|
|
_, err := internal.BPFMapCreate(&internal.BPFMapCreateAttr{
|
|
MapType: uint32(ArrayOfMaps),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: 1,
|
|
// Invalid file descriptor.
|
|
InnerMapFd: ^uint32(0),
|
|
})
|
|
if errors.Is(err, unix.EINVAL) {
|
|
return internal.ErrNotSupported
|
|
}
|
|
if errors.Is(err, unix.EBADF) {
|
|
return nil
|
|
}
|
|
return err
|
|
})
|
|
|
|
var haveMapMutabilityModifiers = internal.FeatureTest("read- and write-only maps", "5.2", func() error {
|
|
// This checks BPF_F_RDONLY_PROG and BPF_F_WRONLY_PROG. Since
|
|
// BPF_MAP_FREEZE appeared in 5.2 as well we don't do a separate check.
|
|
m, err := internal.BPFMapCreate(&internal.BPFMapCreateAttr{
|
|
MapType: uint32(Array),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: 1,
|
|
Flags: unix.BPF_F_RDONLY_PROG,
|
|
})
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
_ = m.Close()
|
|
return nil
|
|
})
|
|
|
|
var haveMmapableMaps = internal.FeatureTest("mmapable maps", "5.5", func() error {
|
|
// This checks BPF_F_MMAPABLE, which appeared in 5.5 for array maps.
|
|
m, err := internal.BPFMapCreate(&internal.BPFMapCreateAttr{
|
|
MapType: uint32(Array),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: 1,
|
|
Flags: unix.BPF_F_MMAPABLE,
|
|
})
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
_ = m.Close()
|
|
return nil
|
|
})
|
|
|
|
var haveInnerMaps = internal.FeatureTest("inner maps", "5.10", func() error {
|
|
// This checks BPF_F_INNER_MAP, which appeared in 5.10.
|
|
m, err := internal.BPFMapCreate(&internal.BPFMapCreateAttr{
|
|
MapType: uint32(Array),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: 1,
|
|
Flags: unix.BPF_F_INNER_MAP,
|
|
})
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
_ = m.Close()
|
|
return nil
|
|
})
|
|
|
|
func bpfMapLookupElem(m *internal.FD, key, valueOut internal.Pointer) error {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := bpfMapOpAttr{
|
|
mapFd: fd,
|
|
key: key,
|
|
value: valueOut,
|
|
}
|
|
_, err = internal.BPF(internal.BPF_MAP_LOOKUP_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return wrapMapError(err)
|
|
}
|
|
|
|
func bpfMapLookupAndDelete(m *internal.FD, key, valueOut internal.Pointer) error {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := bpfMapOpAttr{
|
|
mapFd: fd,
|
|
key: key,
|
|
value: valueOut,
|
|
}
|
|
_, err = internal.BPF(internal.BPF_MAP_LOOKUP_AND_DELETE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return wrapMapError(err)
|
|
}
|
|
|
|
func bpfMapUpdateElem(m *internal.FD, key, valueOut internal.Pointer, flags uint64) error {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := bpfMapOpAttr{
|
|
mapFd: fd,
|
|
key: key,
|
|
value: valueOut,
|
|
flags: flags,
|
|
}
|
|
_, err = internal.BPF(internal.BPF_MAP_UPDATE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return wrapMapError(err)
|
|
}
|
|
|
|
func bpfMapDeleteElem(m *internal.FD, key internal.Pointer) error {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := bpfMapOpAttr{
|
|
mapFd: fd,
|
|
key: key,
|
|
}
|
|
_, err = internal.BPF(internal.BPF_MAP_DELETE_ELEM, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return wrapMapError(err)
|
|
}
|
|
|
|
func bpfMapGetNextKey(m *internal.FD, key, nextKeyOut internal.Pointer) error {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := bpfMapOpAttr{
|
|
mapFd: fd,
|
|
key: key,
|
|
value: nextKeyOut,
|
|
}
|
|
_, err = internal.BPF(internal.BPF_MAP_GET_NEXT_KEY, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return wrapMapError(err)
|
|
}
|
|
|
|
func objGetNextID(cmd internal.BPFCmd, start uint32) (uint32, error) {
|
|
attr := bpfObjGetNextIDAttr{
|
|
startID: start,
|
|
}
|
|
_, err := internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return attr.nextID, err
|
|
}
|
|
|
|
func bpfMapBatch(cmd internal.BPFCmd, m *internal.FD, inBatch, outBatch, keys, values internal.Pointer, count uint32, opts *BatchOptions) (uint32, error) {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
attr := bpfBatchMapOpAttr{
|
|
inBatch: inBatch,
|
|
outBatch: outBatch,
|
|
keys: keys,
|
|
values: values,
|
|
count: count,
|
|
mapFd: fd,
|
|
}
|
|
if opts != nil {
|
|
attr.elemFlags = opts.ElemFlags
|
|
attr.flags = opts.Flags
|
|
}
|
|
_, err = internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
// always return count even on an error, as things like update might partially be fulfilled.
|
|
return attr.count, wrapMapError(err)
|
|
}
|
|
|
|
func wrapMapError(err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
if errors.Is(err, unix.ENOENT) {
|
|
return internal.SyscallError(ErrKeyNotExist, unix.ENOENT)
|
|
}
|
|
|
|
if errors.Is(err, unix.EEXIST) {
|
|
return internal.SyscallError(ErrKeyExist, unix.EEXIST)
|
|
}
|
|
|
|
if errors.Is(err, unix.ENOTSUPP) {
|
|
return internal.SyscallError(ErrNotSupported, unix.ENOTSUPP)
|
|
}
|
|
|
|
if errors.Is(err, unix.E2BIG) {
|
|
return fmt.Errorf("key too big for map: %w", err)
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func bpfMapFreeze(m *internal.FD) error {
|
|
fd, err := m.Value()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := bpfMapFreezeAttr{
|
|
mapFd: fd,
|
|
}
|
|
_, err = internal.BPF(internal.BPF_MAP_FREEZE, unsafe.Pointer(&attr), unsafe.Sizeof(attr))
|
|
return err
|
|
}
|
|
|
|
func bpfGetProgInfoByFD(fd *internal.FD, ids []MapID) (*bpfProgInfo, error) {
|
|
var info bpfProgInfo
|
|
if len(ids) > 0 {
|
|
info.nr_map_ids = uint32(len(ids))
|
|
info.map_ids = internal.NewPointer(unsafe.Pointer(&ids[0]))
|
|
}
|
|
|
|
if err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)); err != nil {
|
|
return nil, fmt.Errorf("can't get program info: %w", err)
|
|
}
|
|
return &info, nil
|
|
}
|
|
|
|
func bpfGetMapInfoByFD(fd *internal.FD) (*bpfMapInfo, error) {
|
|
var info bpfMapInfo
|
|
err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("can't get map info: %w", err)
|
|
}
|
|
return &info, nil
|
|
}
|
|
|
|
var haveObjName = internal.FeatureTest("object names", "4.15", func() error {
|
|
attr := internal.BPFMapCreateAttr{
|
|
MapType: uint32(Array),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: 1,
|
|
MapName: internal.NewBPFObjName("feature_test"),
|
|
}
|
|
|
|
fd, err := internal.BPFMapCreate(&attr)
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
|
|
_ = fd.Close()
|
|
return nil
|
|
})
|
|
|
|
var objNameAllowsDot = internal.FeatureTest("dot in object names", "5.2", func() error {
|
|
if err := haveObjName(); err != nil {
|
|
return err
|
|
}
|
|
|
|
attr := internal.BPFMapCreateAttr{
|
|
MapType: uint32(Array),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: 1,
|
|
MapName: internal.NewBPFObjName(".test"),
|
|
}
|
|
|
|
fd, err := internal.BPFMapCreate(&attr)
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
|
|
_ = fd.Close()
|
|
return nil
|
|
})
|
|
|
|
var haveBatchAPI = internal.FeatureTest("map batch api", "5.6", func() error {
|
|
var maxEntries uint32 = 2
|
|
attr := internal.BPFMapCreateAttr{
|
|
MapType: uint32(Hash),
|
|
KeySize: 4,
|
|
ValueSize: 4,
|
|
MaxEntries: maxEntries,
|
|
}
|
|
|
|
fd, err := internal.BPFMapCreate(&attr)
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
defer fd.Close()
|
|
keys := []uint32{1, 2}
|
|
values := []uint32{3, 4}
|
|
kp, _ := marshalPtr(keys, 8)
|
|
vp, _ := marshalPtr(values, 8)
|
|
nilPtr := internal.NewPointer(nil)
|
|
_, err = bpfMapBatch(internal.BPF_MAP_UPDATE_BATCH, fd, nilPtr, nilPtr, kp, vp, maxEntries, nil)
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
return nil
|
|
})
|
|
|
|
var haveProbeReadKernel = internal.FeatureTest("bpf_probe_read_kernel", "5.5", func() error {
|
|
insns := asm.Instructions{
|
|
asm.Mov.Reg(asm.R1, asm.R10),
|
|
asm.Add.Imm(asm.R1, -8),
|
|
asm.Mov.Imm(asm.R2, 8),
|
|
asm.Mov.Imm(asm.R3, 0),
|
|
asm.FnProbeReadKernel.Call(),
|
|
asm.Return(),
|
|
}
|
|
buf := bytes.NewBuffer(make([]byte, 0, len(insns)*asm.InstructionSize))
|
|
if err := insns.Marshal(buf, internal.NativeEndian); err != nil {
|
|
return err
|
|
}
|
|
bytecode := buf.Bytes()
|
|
|
|
fd, err := internal.BPFProgLoad(&internal.BPFProgLoadAttr{
|
|
ProgType: uint32(Kprobe),
|
|
License: internal.NewStringPointer("GPL"),
|
|
Instructions: internal.NewSlicePointer(bytecode),
|
|
InsCount: uint32(len(bytecode) / asm.InstructionSize),
|
|
})
|
|
if err != nil {
|
|
return internal.ErrNotSupported
|
|
}
|
|
_ = fd.Close()
|
|
return nil
|
|
})
|