前段时间，老板让我 手撕 分析 一个陌生的 XDP 程序。

责任重过山头，莫敢推辞，只能硬着头皮上了。

陌生的 XDP 程序

没有源代码，且发现该 XDP 程序还没带上 debug 信息；分析的难度急剧飙升。

不过，根据 XDP 程序开发的套路，还是能够从 bpf 汇编分析一下 XDP 程序大概的处理逻辑。以下为部分 bpf 汇编：

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100

# bpftool p d x i 21712
   0: (bf) r3 = r1
   1: (b7) r0 = 1
   2: (79) r1 = *(u64 *)(r3 +8)
   3: (79) r4 = *(u64 *)(r3 +0)
   4: (bf) r2 = r4
   5: (07) r2 += 14
   6: (2d) if r2 > r1 goto pc+144
   7: (71) r6 = *(u8 *)(r4 +13)
   8: (67) r6 <<= 8
   9: (71) r2 = *(u8 *)(r4 +12)
  10: (4f) r6 |= r2
  11: (15) if r6 == 0xa888 goto pc+3
  12: (b7) r2 = 14
  13: (b7) r5 = 0
  14: (55) if r6 != 0x81 goto pc+7
  15: (bf) r2 = r4
  16: (07) r2 += 18
  17: (b7) r0 = 1
  18: (2d) if r2 > r1 goto pc+132
  19: (b7) r2 = 18
  20: (b7) r5 = 4
  21: (69) r6 = *(u16 *)(r4 +16)
  22: (bf) r0 = r6
  23: (57) r0 &= 65535
  24: (15) if r0 == 0x81 goto pc+2
  25: (15) if r0 == 0xa888 goto pc+1
  26: (05) goto pc+12
  27: (bf) r7 = r2
  28: (07) r7 += 4
  29: (bf) r6 = r4
  30: (0f) r6 += r7
  31: (b7) r0 = 1
  32: (2d) if r6 > r1 goto pc+118
  33: (bf) r0 = r4
  34: (07) r0 += 2
  35: (0f) r0 += r2
  36: (07) r5 += 4
  37: (69) r6 = *(u16 *)(r0 +0)
  38: (bf) r2 = r7
  39: (57) r6 &= 65535
  40: (b7) r0 = 2
  41: (15) if r6 == 0x8 goto pc+1
  42: (05) goto pc+108
  43: (bf) r7 = r4
  44: (0f) r7 += r2
  45: (bf) r6 = r7
  46: (07) r6 += 20
  47: (b7) r0 = 1
  48: (2d) if r6 > r1 goto pc+102
  49: (71) r2 = *(u8 *)(r7 +9)
  50: (b7) r0 = 2
  51: (55) if r2 != 0x6 goto pc+99
  52: (bf) r2 = r6
  53: (07) r2 += 20
  54: (b7) r0 = 1
  55: (2d) if r2 > r1 goto pc+95
  56: (69) r1 = *(u16 *)(r6 +2)
  57: (b7) r0 = 2
  58: (55) if r1 != 0xfcff goto pc+92
  59: (69) r1 = *(u16 *)(r6 +12)
  60: (57) r1 &= 512
  61: (b7) r0 = 2
  62: (15) if r1 == 0x0 goto pc+88
  63: ...
 131: (b7) r8 = 0
 132: (79) r1 = *(u64 *)(r10 -48)
 133: (05) goto pc+29
 134: (18) r1 = 0xa726f72726520
 136: (7b) *(u64 *)(r10 -8) = r1
 137: (18) r1 = 0x67616c665f706374
 139: (7b) *(u64 *)(r10 -16) = r1
 140: (18) r1 = 0x206563616c706572
 142: (7b) *(u64 *)(r10 -24) = r1
 143: (18) r1 = 0x5f6d7573635f346c
 145: (7b) *(u64 *)(r10 -32) = r1
 146: (bf) r1 = r10
 147: (07) r1 += -32
 148: (b7) r2 = 32
 149: (85) call bpf_trace_printk#-55408
 150: (b7) r0 = 1
 151: (95) exit
 152: ...
 196: (b7) r1 = 0
 197: (73) *(u8 *)(r10 -2) = r1
 198: (b7) r1 = 2674
 199: (6b) *(u16 *)(r10 -4) = r1
 200: (b7) r1 = 1869771365
 201: (63) *(u32 *)(r10 -8) = r1
 202: (18) r1 = 0x207165735f6b6361
 204: (7b) *(u64 *)(r10 -16) = r1
 205: (18) r1 = 0x206563616c706572
 207: (7b) *(u64 *)(r10 -24) = r1
 208: (18) r1 = 0x5f6d7573635f346c
 210: (7b) *(u64 *)(r10 -32) = r1
 211: (bf) r1 = r10
 212: (07) r1 += -32
 213: (b7) r2 = 31
 214: (05) goto pc-66
 215: ...

参考

1
2
3
4
5
6
7

enum xdp_action {
    XDP_ABORTED = 0,
    XDP_DROP,
    XDP_PASS,
    XDP_TX,
    XDP_REDIRECT,
};

给 bpf 汇编加上一些注释：

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100

# bpftool p d x i 21712
   0: (bf) r3 = r1
   1: (b7) r0 = 1                       // XDP_DROP
   2: (79) r1 = *(u64 *)(r3 +8)
   3: (79) r4 = *(u64 *)(r3 +0)
   4: (bf) r2 = r4
   5: (07) r2 += 14
   6: (2d) if r2 > r1 goto pc+144
   7: (71) r6 = *(u8 *)(r4 +13)
   8: (67) r6 <<= 8
   9: (71) r2 = *(u8 *)(r4 +12)
  10: (4f) r6 |= r2
  11: (15) if r6 == 0xa888 goto pc+3    // VLAN 8021AD
  12: (b7) r2 = 14
  13: (b7) r5 = 0
  14: (55) if r6 != 0x81 goto pc+7      // VLAN 8021Q
  15: (bf) r2 = r4
  16: (07) r2 += 18
  17: (b7) r0 = 1                       // XDP_DROP
  18: (2d) if r2 > r1 goto pc+132
  19: (b7) r2 = 18
  20: (b7) r5 = 4
  21: (69) r6 = *(u16 *)(r4 +16)
  22: (bf) r0 = r6
  23: (57) r0 &= 65535
  24: (15) if r0 == 0x81 goto pc+2      // VLAN 8021Q
  25: (15) if r0 == 0xa888 goto pc+1    // VLAN 8021AD
  26: (05) goto pc+12
  27: (bf) r7 = r2
  28: (07) r7 += 4
  29: (bf) r6 = r4
  30: (0f) r6 += r7
  31: (b7) r0 = 1                       // XDP_DROP
  32: (2d) if r6 > r1 goto pc+118
  33: (bf) r0 = r4
  34: (07) r0 += 2
  35: (0f) r0 += r2
  36: (07) r5 += 4
  37: (69) r6 = *(u16 *)(r0 +0)
  38: (bf) r2 = r7
  39: (57) r6 &= 65535
  40: (b7) r0 = 2                       // XDP_PASS
  41: (15) if r6 == 0x8 goto pc+1       // ETH_P_IP
  42: (05) goto pc+108
  43: (bf) r7 = r4
  44: (0f) r7 += r2
  45: (bf) r6 = r7
  46: (07) r6 += 20                     // iphdr
  47: (b7) r0 = 1                       // XDP_DROP
  48: (2d) if r6 > r1 goto pc+102
  49: (71) r2 = *(u8 *)(r7 +9)
  50: (b7) r0 = 2                       // XDP_PASS
  51: (55) if r2 != 0x6 goto pc+99      // IPPROTO_TCP
  52: (bf) r2 = r6
  53: (07) r2 += 20                     // tcphdr
  54: (b7) r0 = 1                       // XDP_DROP
  55: (2d) if r2 > r1 goto pc+95
  56: (69) r1 = *(u16 *)(r6 +2)
  57: (b7) r0 = 2                       // XDP_PASS
  58: (55) if r1 != 0xfcff goto pc+92   // dport 65532
  59: (69) r1 = *(u16 *)(r6 +12)
  60: (57) r1 &= 512
  61: (b7) r0 = 2                       // XDP_PASS
  62: (15) if r1 == 0x0 goto pc+88      // tcp flags, not SYN
  63: ...
 131: (b7) r8 = 0
 132: (79) r1 = *(u64 *)(r10 -48)
 133: (05) goto pc+29
 134: (18) r1 = 0xa726f72726520
 136: (7b) *(u64 *)(r10 -8) = r1
 137: (18) r1 = 0x67616c665f706374
 139: (7b) *(u64 *)(r10 -16) = r1
 140: (18) r1 = 0x206563616c706572
 142: (7b) *(u64 *)(r10 -24) = r1
 143: (18) r1 = 0x5f6d7573635f346c
 145: (7b) *(u64 *)(r10 -32) = r1
 146: (bf) r1 = r10
 147: (07) r1 += -32
 148: (b7) r2 = 32
 149: (85) call bpf_trace_printk#-55408     // print "l4_csum_replace tcp_flag error\n"
 150: (b7) r0 = 1                           // XDP_DROP
 151: (95) exit
 152: ...
 196: (b7) r1 = 0
 197: (73) *(u8 *)(r10 -2) = r1
 198: (b7) r1 = 2674
 199: (6b) *(u16 *)(r10 -4) = r1
 200: (b7) r1 = 1869771365
 201: (63) *(u32 *)(r10 -8) = r1
 202: (18) r1 = 0x207165735f6b6361
 204: (7b) *(u64 *)(r10 -16) = r1
 205: (18) r1 = 0x206563616c706572
 207: (7b) *(u64 *)(r10 -24) = r1
 208: (18) r1 = 0x5f6d7573635f346c          // l4_csum_replace ack_seq erro
 210: (7b) *(u64 *)(r10 -32) = r1
 211: (bf) r1 = r10
 212: (07) r1 += -32
 213: (b7) r2 = 31
 214: (05) goto pc-66
 215: ...

结合两个 bpf_trace_printk() 可知，这个 XDP 程序对目的端口为 65532 的 TCP SYN 包进行了网络包修改。

尽管分析的结果还不够透彻，但已满足老板的预期，就不再继续分析了。

分析过程

XDP 套路一：判断协议

在 XDP 程序里处理网络包，首先需要判断的就是当前网络包是不是预期的那个；如果不是，则 XDP_PASS。

而在，判断的过程中，如果发现是畸形包，则直接 XDP_DROP 。

上面的 XDP 程序就判断了 ETH_P_8021Q、ETH_P_8021AD、ETH_P_IP 和 IPPROTO_TCP 等协议。

比如：

1
2
3

eth = (typeof(eth)) xdp->data;
if (eth->h_proto == ETH_P_8021Q || eth->h_proto == ETH_P_8021AD)
    eth = (void *) (eth + 1) + sizeof(struct vlan_hdr);

XDP 套路二：判断范围

在读取网络包内容前，必须先判断读取的内存有没有超出网络包内容的范围了。

上面的 XDP 程序里，如果超出网络包内容的范围，就 XDP_PASS。

比如：

1
2
3

eth = (typeof(eth)) xdp->data;
if ((void *) (eth + 1) > xdp->data_end)
    return XDP_PASS;

XDP 套路三：bpf\_trace\_printk() 的使用

其实，这是大部分 bpf 程序的套路了。

这是因为 bpf_trace_printk() 所使用的字符串都保存到栈上，如：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

 134: (18) r1 = 0xa726f72726520
 136: (7b) *(u64 *)(r10 -8) = r1
 137: (18) r1 = 0x67616c665f706374
 139: (7b) *(u64 *)(r10 -16) = r1
 140: (18) r1 = 0x206563616c706572
 142: (7b) *(u64 *)(r10 -24) = r1
 143: (18) r1 = 0x5f6d7573635f346c
 145: (7b) *(u64 *)(r10 -32) = r1
 146: (bf) r1 = r10
 147: (07) r1 += -32
 148: (b7) r2 = 32
 149: (85) call bpf_trace_printk#-55408     // print "l4_csum_replace tcp_flag error\n"

通过 r1 = 0xa726f72726520; *(u64 *)(r10 -8) = r1 的方式，将字符串保存到栈上，然后准备好 bpf_trace_printk() 的参数 R1 和 R2，最后 call bpf_trace_printk#-55408；这里，R1 是指向字符串保存的栈空间的地址的指针，R2 是字符串大小 32 字节。

所以，可以将保存到栈上的字符串给翻译出来：

1
2
3
4

s := []uint64{0x5f6d7573635f346c, 0x206563616c706572, 0x67616c665f706374, 0xa726f72726520}
b := (*byte)(unsafe.Pointer(&s[0]))
fmt.Println(unsafe.String(b, len(s)*8))
// l4_csum_replace tcp_flag error

这里就是 bpf_printk("l4_csum_replace tcp_flag error\n") 了。

小结

只要掌握了那些 XDP（bpf）程序的开发套路，分析陌生的 XDP 程序就不在话下。