tracepoint介绍

Brendan Gregg大神博客对tracepoint做了说明，同时看到taobao技术博客对文章进行了翻译，学习整理如下。

测试系统版本

$ uname -r
5.4.0-72-generic

利用bcc的tplist工具查看当前版本支持的tracepoint点：

$ sudo /usr/share/bcc/tools/tplist -v 'tcp:*'
tcp:tcp_retransmit_skb
   const void * skbaddr;
   const void * skaddr;
   int state;
   __u16 sport;
   __u16 dport;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];
tcp:tcp_send_reset
   const void * skbaddr;
   const void * skaddr;
   int state;
   __u16 sport;
   __u16 dport;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];
tcp:tcp_receive_reset
   const void * skaddr;
   __u16 sport;
   __u16 dport;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];
   __u64 sock_cookie;
tcp:tcp_destroy_sock
   const void * skaddr;
   __u16 sport;
   __u16 dport;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];
   __u64 sock_cookie;
tcp:tcp_rcv_space_adjust
   const void * skaddr;
   __u16 sport;
   __u16 dport;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];
   __u64 sock_cookie;
tcp:tcp_retransmit_synack
   const void * skaddr;
   const void * req;
   __u16 sport;
   __u16 dport;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];
tcp:tcp_probe
   __u8 saddr[sizeof(struct sockaddr_in6)];
   __u8 daddr[sizeof(struct sockaddr_in6)];
   __u16 sport;
   __u16 dport;
   __u32 mark;
   __u16 data_len;
   __u32 snd_nxt;
   __u32 snd_una;
   __u32 snd_cwnd;
   __u32 ssthresh;
   __u32 snd_wnd;
   __u32 srtt;
   __u32 rcv_wnd;
   __u64 sock_cookie;

$ sudo /usr/share/bcc/tools/tplist -v 'sock:*'
sock:sock_rcvqueue_full
   int rmem_alloc;
   unsigned int truesize;
   int sk_rcvbuf;
sock:sock_exceed_buf_limit
   char name[32];
   long * sysctl_mem;
   long allocated;
   int sysctl_rmem;
   int rmem_alloc;
   int sysctl_wmem;
   int wmem_alloc;
   int wmem_queued;
   int kind;
sock:inet_sock_set_state
   const void * skaddr;
   int oldstate;
   int newstate;
   __u16 sport;
   __u16 dport;
   __u16 family;
   __u8 protocol;
   __u8 saddr[4];
   __u8 daddr[4];
   __u8 saddr_v6[16];
   __u8 daddr_v6[16];

对上述tracepoint的说明如下：

• tcp:tcp_retransmit_skb: 跟踪重传。对于理解包括拥塞在内的网络问题很有用。将会在笔者的 tcpretrans 工具中替换 kprobes。
• tcp:tcp_retransmit_synack: 跟踪 SYN 和 SYN/ACK 重传。将它们剥离出来很有趣，是因为它们可以表明服务器的饱和度（listen backlog 丢包）而不是网络拥塞。它对应着 LINUX_MIB_TCPSYNRETRANS。
• tcp:tcp_destroy_sock: 对于需要统计汇总 TCP 会话的内存详情的程序是需要的，它可以通过 sock 地址来作为主键索引。这个探测点可以得知会话是否已经结束，因此接下来sock 地址将会被复用，任何截止到现在的统计信息都应该被使用然后删除。
• tcp:tcp_send_reset: 这个会跟踪一个有效 socket 下的 RST 发送，用以诊断相关类型的问题。
• tcp:tcp_receive_reset: 跟踪 RST 接受。
• tcp:tcp_probe: 用以跟踪 TCP 拥塞窗口，这也让一个更老的 TCP probe 模块废弃并移除。这个是 Masami Hiramatsu 提交并在 4.16 合入。
• sock:inet_sock_set_state: 可以用来做很多事情。tcplife 工具就是其中一个，并且笔者的 tcpconnect 和 tcpaccept bcc 工具也可以转换为使用这个 tracepoint。我们可以添加单独的 tcp:tcp_connect 和 tcp:tcp_accept tracepoints (或者 tcp:tcp_active_open 和 tcp:tcp_passive_open), 但是可以直接使用 sock:inet_sock_set_state。

使用示例

编写ply程序，利用sock:inet_sock_set_state实现连接状态变化跟踪：

sudo ply 'tracepoint:sock/inet_sock_set_state { printf("saddr: %v sport: %v -> daddr: %v dport: %v, old_state: %v, new_state: %v\n", data->saddr, data->sport, data->daddr, data->dport, data->oldstate, data->newstate);}'

输出如下：

$  sudo ply 'tracepoint:sock/inet_sock_set_state { printf("saddr: %v sport: %v -> daddr: %v dport: %v, old_state: %v, new_state: %v\n", data->saddr, data->sport, data->daddr, data->dport, data->oldstate, data->newstate);}'
ply: active
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 47600 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [13, 225, 93, 17] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 48316 -> daddr: [13, 225, 93, 17] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [99, 84, 203, 13] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 42292 -> daddr: [99, 84, 203, 13] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 90, 87] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 47610 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 47612 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 47606 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 0 -> daddr: [104, 193, 90, 87] dport: 443, old_state: 7, new_state: 2
saddr: [192, 168, 136, 163] sport: 44286 -> daddr: [104, 193, 90, 87] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 47618 -> daddr: [104, 193, 88, 77] dport: 443, old_state: 2, new_state: 1
saddr: [192, 168, 136, 163] sport: 44292 -> daddr: [104, 193, 90, 87] dport: 443, old_state: 2, new_state: 1

其中关于连接状态的枚举值，可以参考Linux内核源码：

# /include/net/tcp_states.h
enum {
   TCP_ESTABLISHED = 1,
   TCP_SYN_SENT,
   TCP_SYN_RECV,
   TCP_FIN_WAIT1,
   TCP_FIN_WAIT2,
   TCP_TIME_WAIT,
   TCP_CLOSE,
   TCP_CLOSE_WAIT,
   TCP_LAST_ACK,
   TCP_LISTEN,
   TCP_CLOSING,	/* Now a valid state */
   TCP_NEW_SYN_RECV,

   TCP_MAX_STATES	/* Leave at the end! */
};

参考

http://www.brendangregg.com/blog/2018-03-22/tcp-tracepoints.html
https://kernel.taobao.org/2019/10/TCP-Tracepoints/