在 eBPF Talk: 自制查看 bpf prog 反汇编的工具 里提到的 bpftool BUG Wrong callq address displayed,花了两三天时间将它给修了。
TL;DR [PATCH bpf v3] bpf, bpftool: Fix incorrect disasm pc 已合入 bpf 仓库。
BUG 复现
使用 bpftool 查看 bpf prog 的反汇编:
1
2
3
4
5
6
7
8
9
|
# bpftool prog dump jited name kprobe_skb_1
...
; event->cpu_id = bpf_get_smp_processor_id();
c70: call 0xffffffffd2702ab4
...
# echo "0xffffffffd2702ab4 is wrong for bpf_get_smp_processor_id"
# grep ffffffffd2702ab4 /proc/kallsyms
# grep bpf_get_smp_processor_id /proc/kallsyms
ffffffff92b0b490 T bpf_get_smp_processor_id
|
翻看了一下 commit 历史,确认所有版本的 bpftool 都存在这个问题,因为从一开始就是错的。
BUG 修复之尝试篇
翻看一下 bpftool prog dump jited
的源代码,找到反汇编的核心代码逻辑:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
// ${KERNEL}/tools/bpf/bpftool/jit_disasm.c
#ifdef HAVE_LLVM_SUPPORT
static int
disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc)
{
char buf[256];
int count;
count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, pc,
buf, sizeof(buf));
if (json_output)
printf_json(buf);
else
printf("%s", buf);
return count;
}
#endif /* HAVE_LLVM_SUPPORT */
#ifdef HAVE_LIBBFD_SUPPORT
static int
disassemble_insn(disasm_ctx_t *ctx, __maybe_unused unsigned char *image,
__maybe_unused ssize_t len, int pc)
{
return ctx->disassemble(pc, ctx->info);
}
#endif /* HAVE_LIBBPFD_SUPPORT */
int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
const char *arch, const char *disassembler_options,
const struct btf *btf,
const struct bpf_prog_linfo *prog_linfo,
__u64 func_ksym, unsigned int func_idx,
bool linum)
{
const struct bpf_line_info *linfo = NULL;
unsigned int nr_skip = 0;
int count, i, pc = 0;
disasm_ctx_t ctx;
if (!len)
return -1;
if (init_context(&ctx, arch, disassembler_options, image, len))
return -1;
// ...
do {
// ...
count = disassemble_insn(&ctx, image, len, pc);
// ...
pc += count;
} while (count > 0 && pc < len);
// ...
destroy_context(&ctx);
return 0;
}
|
找到 LLVMDisasmInstruction() 的文档:
1
2
3
4
5
6
7
8
9
10
11
12
13
|
/**
* Disassemble a single instruction using the disassembler context specified in
* the parameter DC. The bytes of the instruction are specified in the
* parameter Bytes, and contains at least BytesSize number of bytes. The
* instruction is at the address specified by the PC parameter. If a valid
* instruction can be disassembled, its string is returned indirectly in
* OutString whose size is specified in the parameter OutStringSize. This
* function returns the number of bytes in the instruction or zero if there was
* no valid instruction.
*/
size_t LLVMDisasmInstruction(LLVMDisasmContextRef DC, uint8_t *Bytes,
uint64_t BytesSize, uint64_t PC,
char *OutString, size_t OutStringSize);
|
这儿写明了 PC
参数需要是 insn 的地址。
所以,将 disassemble_insn(&ctx, image, len, pc);
里的 pc
参数改成 func_ksym + pc
试试:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
(gdb) run p d j i 378 linum
Starting program: /root/bpftool/src/bpftool p d j i 378 linum
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
int kprobe_skb_1(struct pt_regs * ctx):
bpf_prog_1e1ae32f79e6c86e_kprobe_skb_1:
; PWRU_ADD_KPROBE(1) [file:bpf/kprobe_pwru.c line_num:530 line_col:0]
0:bptool disas, func ksym=ffffffffc03fffc8, pc=ffffffffc03fffc8(0)
Program received signal SIGSEGV, Segmentation fault.
llvm::support::endian::read<unsigned char, 1ul> (memory=0x555517997578, endian=llvm::endianness::little) at /root/llvm-project/llvm/include/llvm/Support/Endian.h:61
61 memcpy(&ret,
(gdb) bt
#0 llvm::support::endian::read<unsigned char, 1ul> (memory=0x555517997578, endian=llvm::endianness::little) at /root/llvm-project/llvm/include/llvm/Support/Endian.h:61
#1 0x000055555575456b in consume<unsigned char> (insn=0x7fffffffdb70, ptr=@0x7fffffffdacb: 0 '\000') at /root/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp:203
#2 0x000055555574628e in readPrefixes (insn=0x7fffffffdb70) at /root/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp:231
#3 0x000055555574ad72 in (anonymous namespace)::X86GenericDisassembler::getInstruction (this=0x5555575a1190, Instr=..., Size=@0x7fffffffdca8: 93824994447723, Bytes=..., Address=18446744072640004040, CStream=...)
at /root/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp:1879
#4 0x0000555555ff5d95 in LLVMDisasmInstruction (DCR=0x5555575a1280, Bytes=0x555517997578 <error: Cannot access memory at address 0x555517997578>, BytesSize=1069547794, PC=18446744072640004040, OutString=0x7fffffffdf60 "0",
OutStringSize=256) at /root/llvm-project/llvm/lib/MC/MCDisassembler/Disassembler.cpp:267
#5 0x00005555556f56c6 in disassemble_insn (image=image@entry=0x5555575975b0 "\017\037D", len=len@entry=218, pc=pc@entry=-1069547576, ctx=<optimized out>) at jit_disasm.c:117
#6 0x00005555556f5927 in disasm_print_insn (image=image@entry=0x5555575975b0 "\017\037D", len=218, opcodes=opcodes@entry=0, arch=arch@entry=0x0, disassembler_options=<optimized out>, btf=btf@entry=0x555557593070,
prog_linfo=0x55555759c560, func_ksym=18446744072640004040, func_idx=0, linum=true) at jit_disasm.c:364
#7 0x0000555555701f48 in prog_dump (linum=<optimized out>, visual=<optimized out>, opcodes=<optimized out>, filepath=<optimized out>, mode=DUMP_JITED, info=0x7fffffffe1e0) at prog.c:825
#8 do_dump (argc=<optimized out>, argv=<optimized out>) at prog.c:986
#9 0x00005555556e6c9a in main (argc=<optimized out>, argv=<optimized out>) at main.c:539
|
能看到所有 debug info,是编译 LLVM 和 bpftool 的时候都带上 -g
选项:
1
2
3
4
5
6
7
8
|
# echo "Build and install llvm, -DCMAKE_BUILD_TYPE=Debug is the key option"
mkdir llvm_build
cmake -S llvm-project/llvm -B llvm_build -DCMAKE_BUILD_TYPE=Debug -DLLVM_ENABLE_PROJECTS="clang;lldb" -DLLVM_TARGETS_TO_BUILD="X86"
make -j64 -C llvm_build llvm-config llvm-libraries clang lldb
sudo make install -j64 -C llvm_build
# echo "Build bpftool with -g"
EXTRA_CFLAGS=-g make -j64 -C src
|
gdb
里已提示 Bytes=0x555517997578 <error: Cannot access memory at address 0x555517997578>
,即提供的 insn buffer 有误。
接着,梳理一下整个函数调用流程,发现不能直接将 pc
调整为 func_ksym + pc
;需要通过以下方式才能修复。
BUG 修复之 LLVM 篇
bpftool 支持使用 LLVM 和 libbfd 作为反汇编的后端;不过,都存在这问题。
其中出问题的地方在于:
1
2
3
4
5
6
7
8
9
10
11
12
|
static int
disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc)
{
char buf[256];
int count;
count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, pc,
buf, sizeof(buf));
// ...
return count;
}
|
提供给 LLVMDisasmInstruction()
的 pc
参数是有错误的。
在 disasm_print_insn()
里,pc
变量指的是当前 bpf prog 里 insn 的索引。
修复方式如下:
1
2
3
4
5
6
7
8
9
10
|
-disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc)
+disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
+ __u64 func_ksym)
{
char buf[256];
int count;
- count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, pc,
+ count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, func_ksym + pc,
buf, sizeof(buf));
|
func_ksym + pc
才是 insn 对应的真实地址。
注:func_ksym
是当前 bpf prog 的 image 的起始地址。
BUG 修复之 libbfd 篇
使用 libbfd 作为反汇编的后端,也存在这个问题:
使用 libbfd 反汇编时没处理 relative address 的问题;比如 call
, je
等指令使用 relative address 的时候,反汇编出来的地址是错误的。
注:这个 7 年前的 commit,使用 libbfd 作为反汇编的后端,但却忽略了 relative address 的处理。
直接翻看 libbfd 的源代码:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
|
// https://github.com/bminor/binutils-gdb/blob/master/opcodes/disassemble.c
disassembler_ftype
disassembler (enum bfd_architecture a,
bool big ATTRIBUTE_UNUSED,
unsigned long mach ATTRIBUTE_UNUSED,
bfd *abfd ATTRIBUTE_UNUSED)
{
disassembler_ftype disassemble;
// ...
#ifdef ARCH_ia64
case bfd_arch_ia64:
disassemble = print_insn_ia64;
break;
#endif
// ...
}
// https://github.com/bminor/binutils-gdb/blob/master/opcodes/ia64-dis.c
int
print_insn_ia64 (bfd_vma memaddr, struct disassemble_info *info)
{
// ...
case IA64_OPND_CLASS_REL:
(*info->print_address_func) (memaddr + value, info);
break;
// ...
}
|
啊哈,当看到 info->print_address_func
用来处理 relative address 时,bpftool 的问题就有解了:提供自己的 print_address_func
,然后给传过来的 memaddr
加上 func_ksym
。
不过,先看看 print_address_func
初始化:
1
2
3
4
5
6
7
8
9
10
11
12
13
|
// https://github.com/bminor/binutils-gdb/blob/master/opcodes/dis-init.c
void
init_disassemble_info (struct disassemble_info *info, void *stream,
fprintf_ftype fprintf_func,
fprintf_styled_ftype fprintf_styled_func)
{
memset (info, 0, sizeof (*info));
// ...
info->print_address_func = generic_print_address;
// ...
}
|
所以,解决办法如下:
1
2
3
4
5
6
7
8
9
10
11
12
|
struct disasm_info {
struct disassemble_info info;
__u64 func_ksym;
};
static void disasm_print_addr(bfd_vma addr, struct disassemble_info *info)
{
struct disasm_info *dinfo = container_of(info, struct disasm_info, info);
addr += dinfo->func_ksym;
generic_print_address(addr, info);
}
|
总结
Gray 大佬一句感叹:这些自古以来的 bug 无人在意会感到孤单北半球。