Master Crash: Analyzing vmcore and Inspecting sk_buff Structures in Linux

Crash tool is used to parse vmcore captured by kdump, which contains a memory snapshot of the running system. This article demonstrates how to use Crash to inspect the sk_buff structure as an example.

Basic Usage

In Crash, the struct command displays a structure: [struct] <struct_name> <virtual_address>. The struct keyword is optional.

Example

Start Crash

crash vmlinux vmcore

Find sk_buff address

Use bt to view the current stack and registers.

crash> bt
PID: 27528 TASK: ffff88108e1d00c0 CPU: 6 COMMAND: "ZMSSMediaProces"
#0 [ffff88106f035740] machine_kexec at ffffff8103237b
#1 [ffff88106f0357a0] crash_kexec at ffffff810ba552
#2 [ffff88106f035870] oops_end at ffffff814fc6c0
#3 [ffff88106f0358a0] die at ffffff8100f31f
#4 [ffff88106f0358d0] do_general_protection at ffffff814fc242
#5 [ffff88106f035900] general_protection at ffffff814fba15
#6 [ffff88106f0359c0] skb_release_data at ffffff8142c77f
#7 [ffff88106f0359e0] __kfree_skb at ffffff8142c2fe
#8 [ffff88106f0359a00] kfree_skb at ffffff8142c442
#9 [ffff88106f0359a30] __ip_flush_pending_frames at ffffff814706e3
#10 [ffff88106f0359a50] ip_flush_pending_frames at ffffff8147071c
#11 [ffff88106f0359a60] udp_flush_pending_frames at ffffff81495be0
#12 [ffff88106f0359a70] udp_sendmsg at ffffff814960f6
#13 [ffff88106f0359b70] inet_sendmsg at ffffff8149e34a
#14 [ffff88106f0359bb0] sock_sendmsg at ffffff81424b0a
#15 [ffff88106f0359d60] __sys_sendmsg at ffffff81424fbf
#16 [ffff88106f0359f10] sys_sendmsg at ffffff81425c79
#17 [ffff88106f035f80] system_call_fastpath at ffffff8100b0f2

Disassemble to confirm the address of skb_release_data:

crash> dis -l skb_release_data
/usr/src/debug/kernel-2.6.32-220.el6/.../net/core/skbuff.c:340
0xffffffff8142c700 <skb_release_data>: push %rbp
0xffffffff8142c701 <skb_release_data+1>: mov %rsp,%rbp
0xffffffff8142c704 <skb_release_data+4>: push %r12
0xffffffff8142c706 <skb_release_data+6>: push %rbx
0xffffffff8142c707 <skb_release_data+7>: nopl 0x0(%rax,%rax,1)
...

The address of skb is stored in register r12, which holds ffff8810dd32f280.

Print the structure at that address:

crash> sk_buff ffff8810dd32f280
struct sk_buff {
  next = 0x0,
  prev = 0x0,
  sk = 0xffff88114b4f0b40,
  tstamp = { tv64 = 0 },
  dev = 0x0,
  _skb_dst = 0,
  sp = 0x0,
  cb = "\000...",
  len = 1500,
  data_len = 0,
  mac_len = 0,
  hdr_len = 0,
  head = 0xffff881161aa4800,
  data = 0xffff881161aa4810,
  truesize = 1768,
  users = { counter = 1 }
}

Examine head and data pointers:

crash> x/16xg 0xffff881161aa4800
0xffff881161aa4800: 0x0000000000000000 0x0008000000000000
0xffff881161aa4810: 0x0000000000000000 0x884509ff00000000
0xffff881161aa4820: 0x113d000001006005 0x007fe200f20a27ed
0xffff881161aa4830: 0x4c0508a930310900 0xb1d2745121900000
0xffff881161aa4840: 0x585a547439439fff 0x0000004b02e30400
0xffff881161aa4850: 0x4002030400585a00 0x017306e8a8ed53c9
0xffff881161aa4860: 0xca21900000a40559 0x0000004b5f2b1275
0xffff881161aa4870: 0x4002830400585a00 0x017306e8a8ed53c9

Calculate the address of skb_shared_info (end + 1536) and display it:

crash> skb_shared_info 0xffff881161aa4e00
struct skb_shared_info {
  dataref = { counter = 10749386 },
  nr_frags = 36864,
  gso_size = 51745,
  gso_segs = 4753,
  gso_type = 33323,
  ip6_frag_id = 203,
  tx_flags = { flags = 0 },
  frag_list = 0x17306e8a8ed53c9,
  hwtstamps = { hwtstamp = { tv64 = -3881663074131507766 },
                syststamp = { tv64 = 650841363090 } },
  ...
}

The example shows corrupted skb_shared_info data, indicating memory corruption.

Inspect the associated sock structure via the sk pointer:

crash> sock 0xffff88114b4f0b40
struct sock {
  __sk_common = { skc_node = { next = 0xffff8810debddb80, pprev = 0xffff881204a84ec0 },
                  skc_nulls_node = { next = 0xffff8810debddb80, pprev = 0xffff881204a84ec0 },
                  skc_refcnt = { counter = 2 },
                  skc_hash = 33728,
                  skc_family = 2,
                  skc_state = 1,
                  skc_reuse = 1,
                  ... },
  sk_write_queue = { next = 0xffff8810d3f26780, prev = 0xffff8810d3f26780, qlen = 1, lock = { raw_lock = { slock = 0 } } },
  ...
}

List all sk_buff objects in the socket’s write queue:

crash> sk_buff 0xffff8810d3f26780
struct sk_buff {
  next = 0xffff88114b4f0c08,
  prev = 0xffff88114b4f0c08,
  sk = 0xffff88114b4f0b40,
  len = 1500,
  data_len = 1432,
  mac_len = 0,
  hdr_len = 0,
  ...
}