eBPF Talk: 谁动了我的 bpf map?

使用 bpf 跟踪 bpf map 的更新、删除函数，以此确认是谁动了 bpf map。

TL;DR 已实现对 bpf map 的更新、删除函数的跟踪，未实现批量操作 bpf map 函数的跟踪；源代码：github.com/Asphaltt/mad。

实现效果

跟踪 pwru 的配置：

1
2
3
4
5
6
7


# ./mad
2024/11/17 07:06:47 mad is running ..
0: map(5755:.rodata:Array) is updated by process(76052:pwru)
key: 00000000 [4 bytes]

0: map(5755:.rodata:Array) is updated by process(76052:pwru)
value: {".rodata": [{"CFG": {"netns": 0, "mark": 0, "ifindex": 0, "output_meta": 0x1, "output_tuple": 0x1, "output_skb": 0x0, "output_shinfo": 0x1, "output_stack": 0x0, "output_caller": 0x0, "output_unused": 0x0, "is_set": 0x1, "track_skb": 0x0, "track_skb_by_stackid": 0x0, "track_xdp": 0x1, "unused": 0x0, "skb_btf_id": 1875, "shinfo_btf_id": 21388}}, {"TRUE": true}, {"ZERO": 0}, {"BPF_PROG_ADDR": 0}]}

即使是字符串，也能识别出来：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


3: map(5766:retsnoop.data:Array) is updated by process(76100:retsnoop)
key: 00000000 [4 bytes]

3: map(5766:retsnoop.data:Array) is updated by process(76100:retsnoop)
value: {".data": [{"FMT_SUCC_VOID": "    EXIT  %s%s [VOID]     "}, {"FMT_SUCC_TRUE": "    EXIT  %s%s [true]     "}, {"FMT_SUCC_FALSE": "    EXIT  %s%s [false]    "}, {"FMT_FAIL_NULL": "[!] EXIT  %s%s [NULL]     "}, {"FMT_FAIL_PTR": "[!] EXIT  %s%s [%d]       "}, {"FMT_SUCC_PTR": "    EXIT  %s%s [0x%lx]    "}, {"FMT_FAIL_LONG": "[!] EXIT  %s%s [%ld]      "}, {"FMT_SUCC_LONG": "    EXIT  %s%s [%ld]      "}, {"FMT_FAIL_INT": "[!] EXIT  %s%s [%d]       "}, {"FMT_SUCC_INT": "    EXIT  %s%s [%d]       "}, {"FMT_SUCC_VOID_COMPAT": "    EXIT  [%d] %s [VOID]  "}, {"FMT_SUCC_TRUE_COMPAT": "    EXIT  [%d] %s [true]  "}, {"FMT_SUCC_FALSE_COMPAT": "    EXIT  [%d] %s [false] "}, {"FMT_FAIL_NULL_COMPAT": "[!] EXIT  [%d] %s [NULL]  "}, {"FMT_FAIL_PTR_COMPAT": "[!] EXIT  [%d] %s [%d]    "}, {"FMT_SUCC_PTR_COMPAT": "    EXIT  [%d] %s [0x%lx] "}, {"FMT_FAIL_LONG_COMPAT": "[!] EXIT  [%d] %s [%ld]   "}, {"FMT_SUCC_LONG_COMPAT": "    EXIT  [%d] %s [%ld]   "}, {"FMT_FAIL_INT_COMPAT": "[!] EXIT  [%d] %s [%d]    "}, {"FMT_SUCC_INT_COMPAT": "    EXIT  [%d] %s [%d]    "}, {"push_call_stack.___fmt": "=== STARTING TRACING %s [COMM %s PID %d] === "}, {"push_call_stack.___fmt.1": "    ENTER %s%s [...] "}, {"push_call_stack.___fmt.2": "=== STARTING TRACING %s [PID %d] === "}, {"push_call_stack.___fmt.3": "=== ...      TRACING [PID %d COMM %s] === "}, {"push_call_stack.___fmt.4": "    ENTER [%d] %s [...] "}, {"save_stitch_stack.___fmt": "SHOULDN'T HAPPEN DEPTH %ld LEN %ld
 "}, {"save_stitch_stack.___fmt.5": "CURRENT DEPTH %d..%d "}, {"save_stitch_stack.___fmt.6": "SAVED DEPTH %d..%d "}, {"save_stitch_stack.___fmt.7": "STITCHED STACK %d..%d to ..%d
 "}, {"save_stitch_stack.___fmt.8": "EMIT PARTIAL STACK DEPTH %d..%d
 "}, {"save_stitch_stack.___fmt.9": "RESETTING SAVED ERR STACK %d..%d to %d..
 "}, {"pop_call_stack.___fmt": "POP(0) UNEXPECTED PID %d DEPTH %d MAX DEPTH %d "}, {"pop_call_stack.___fmt.10": "POP(1) UNEXPECTED GOT  ID %d ADDR %lx NAME %s "}, {"pop_call_stack.___fmt.11": "POP(2) UNEXPECTED WANT ID %u ADDR %lx NAME %s "}, {"pop_call_stack.___fmt.12": "EMIT ERROR STACK DEPTH %d (SAVED ..%d)
 "}, {"pop_call_stack.___fmt.13": "EMIT SUCCESS STACK DEPTH %d (SAVED ..%d)
 "}]}

实现细节之识别目标函数

在 mad 里，并没有写死所有要跟踪的内核函数，而是通过 BTF 去识别这些函数：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36


// https://github.com/Asphaltt/mad/blob/main/bpf_btf.go

func isTgtFunc(typ btf.Type) (string, bool) {
    fn, ok := typ.(*btf.Func)
    if !ok {
        return "", false
    }

    fnName := fn.Name
    fnProto := fn.Type.(*btf.FuncProto)

    if strings.HasSuffix(fnName, "_map_update_elem") && len(fnProto.Params) == 4 {
        if isBpfMap(fnProto.Params[0].Type) && isVoid(fnProto.Params[1].Type) && isVoid(fnProto.Params[2].Type) {
            return fnName, true
        }
    } else if strings.HasSuffix(fnName, "_map_delete_elem") && len(fnProto.Params) == 2 {
        if isBpfMap(fnProto.Params[0].Type) && isVoid(fnProto.Params[1].Type) {
            return fnName, true
        }
    }

    return "", false
}

func retrieveHooks(spec *btf.Spec) ([]string, error) {
    var hooks []string

    iter := spec.Iterate()
    for iter.Next() {
        if name, ok := isTgtFunc(iter.Type); ok {
            hooks = append(hooks, name)
        }
    }

    return hooks, nil
}

如上代码片段的处理逻辑：

遍历整个 BTF spec。
判断遍历的 BTF 类型是不是函数。
判断是不是更新 bpf map 的函数：函数名以 _map_update_elem 结尾且函数参数符合预期。
判断是不是删除 bpf map 的函数：函数名以 _map_delete_elem 结尾且函数参数符合预期。

实现细节之根据 BTF 信息解析 kv 数据

使用 bpftool 命令查看 BPF map 的 key 和 value 的数据结构：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34


# bpftool m d i 5672
[{
        "value": {
            ".rodata": [{
                    "CFG": {
                        "netns": 0,
                        "mark": 0,
                        "ifindex": 0,
                        "output_meta": 0x1,
                        "output_tuple": 0x1,
                        "output_skb": 0x0,
                        "output_shinfo": 0x1,
                        "output_stack": 0x0,
                        "output_caller": 0x0,
                        "output_unused": 0x0,
                        "is_set": 0x1,
                        "track_skb": 0x0,
                        "track_skb_by_stackid": 0x0,
                        "track_xdp": 0x1,
                        "unused": 0x0,
                        "skb_btf_id": 1875,
                        "shinfo_btf_id": 21388
                    }
                },{
                    "TRUE": true
                },{
                    "ZERO": 0
                },{
                    "BPF_PROG_ADDR": 0
                }
            ]
        }
    }
]

在 mad 里，能否实现同样的效果呢？

是个苦力活，基本上抄一下 bpftool 的实现就好：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51


// https://github.com/Asphaltt/mybtf/blob/main/dump.go

func (dd *dataDumper) dumpData(typ btf.Type, bits bitsInfo, data []byte) error {
    switch v := typ.(type) {
    case *btf.Int:
        dd.dumpInt(v, bits, data)

    case *btf.Struct:
        dd.dumpStruct(v, data)

    case *btf.Union:
        dd.dumpUnion(v, data)

    case *btf.Array:
        dd.dumpArray(v, data)

    case *btf.Enum:
        dd.dumpEnum(v, data)

    case *btf.Pointer:
        dd.dumpPointer(v, data)

    case *btf.Fwd:
        dd.print("(fwd-kind-invalid)")
        return fmt.Errorf("fwd kind invalid")

    case *btf.Typedef:
        return dd.dumpData(v.Type, bits, data)

    case *btf.Volatile:
        return dd.dumpData(v.Type, bits, data)

    case *btf.Const:
        return dd.dumpData(v.Type, bits, data)

    case *btf.Restrict:
        return dd.dumpData(v.Type, bits, data)

    case *btf.Var:
        dd.dumpVar(v, bits, data)

    case *btf.Datasec:
        return dd.dumpDataSec(v, data)

    default:
        dd.print("(unsupported-kind)")
        return fmt.Errorf("unsupported kind %T", v)
    }

    return nil
}

以上，便实现了根据 BTF 信息解析数据的功能。

总结

通过 BTF 信息，可以清楚地看出更新、删除 bpf map 的 kv 细节，这对于调试和排查问题是非常有帮助的。

eBPF Talk: 谁动了我的 bpf map?

文章目录

实现效果

实现细节之识别目标函数

实现细节之根据 BTF 信息解析 kv 数据

总结

知识星球

星球里的专栏：

《XDP 进阶手册》