Sungju's Slow Life

The real merit of the vmcore is that you can trace the code with the current value each variable holds. Here you can find one example that traces the filesystem which ended up with the corrupted data entry somehow.

http://pagead2.googlesyndication.com/pagead/show_ads.js

crash> bt
PID: 6326   TASK: ffff810402165820  CPU: 1   COMMAND: "fuser"
 #0 [ffff8103b54efa80] crash_kexec at ffffffff800b099c
 #1 [ffff8103b54efb40] __die at ffffffff80065137
 #2 [ffff8103b54efb80] do_page_fault at ffffffff80067484
 #3 [ffff8103b54efc70] error_exit at ffffffff8005dde9
    [exception RIP: force_reval_path+0xf]
    RIP: ffffffff800eb856  RSP: ffff8103b54efd28  RFLAGS: 00010246
    RAX: 0000000000000000  RBX: ffff8104b033d228  RCX: ffff8104b033d228
    RDX: 000000005094fed3  RSI: 0000000000000000  RDI: ffff8103b54efe48
    RBP: 0000000000000000   R8: ffff810402a0eb28   R9: ffff8108f731e100
    R10: ffff8103f97ec000  R11: ffffffff8005bc0b  R12: 0000000000000000
    R13: 0000000000000000  R14: ffff8103d10943d8  R15: 0000000000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #4 [ffff8103b54efd30] __link_path_walk at ffffffff8000a5c0
 #5 [ffff8103b54efda0] link_path_walk at ffffffff8000eb23
 #6 [ffff8103b54efdd0] do_path_lookup at ffffffff8000cdf6
 #7 [ffff8103b54efe10] __user_walk_fd at ffffffff800239f8
 #8 [ffff8103b54efe40] vfs_stat_fd at ffffffff80028973
 #9 [ffff8103b54efef0] sys_newstat at ffffffff80023728
#10 [ffff8103b54eff80] tracesys at ffffffff8005d28d (via system_call)
    RIP: 0000003be5cc4485  RSP: 00007fffd25d7e18  RFLAGS: 00000246
    RAX: ffffffffffffffda  RBX: ffffffff8005d28d  RCX: ffffffffffffffff
    RDX: 00007fffd25d7ee0  RSI: 00007fffd25d7ee0  RDI: 00000000025bd610
    RBP: 00000000025bd6e0   R8: 0000000000000000   R9: 0000003be5d18f40
    R10: 0000000000000000  R11: 0000000000000246  R12: 0000000000000000
    R13: 00000000025bd610  R14: 0000000000000004  R15: 0000000000000000
    ORIG_RAX: 0000000000000004  CS: 0033  SS: 002b
 
crash> dis -lr force_reval_path+0xf
/usr/src/debug/kernel-2.6.18/linux-2.6.18-308.8.1.el5.x86_64/fs/namei.c: 435
0xffffffff800eb847 :  push   %rbx
/usr/src/debug/kernel-2.6.18/linux-2.6.18-308.8.1.el5.x86_64/fs/namei.c: 437
0xffffffff800eb848 :      mov    (%rdi),%rbx
/usr/src/debug/kernel-2.6.18/linux-2.6.18-308.8.1.el5.x86_64/fs/namei.c: 444
0xffffffff800eb84b :      mov    0x90(%rbx),%rax
0xffffffff800eb852 :      mov    0x30(%rax),%rax
0xffffffff800eb856 :      testb  $0x40,0x9(%rax)
 
crash> struct dentry -o
struct dentry {
...
  [0x90] struct super_block *d_sb;
...
}
SIZE: 0xd8
 
crash> struct super_block -o
struct super_block {
...
   [0x30] struct file_system_type *s_type;
...
}
 
crash> struct file_system_type -o
struct file_system_type {
   [0x0] const char *name;
   [0x8] int fs_flags;             d_sb->s_type->fs_flags & FS_REVAL_DOT))
 
 
crash> struct nameidata ffff8103b54efe48
struct nameidata {
  dentry = 0xffff8104b033d228, 
  mnt = 0xffff8104bf587bc0, 
...
}
 
 
crash> struct dentry 0xffff8104b033d228
..
 
  d_sb = 0xffff8108feaf0400, 
 
crash> struct super_block 0xffff8108feaf0400
struct super_block {
  s_dev = 0x0, 
  s_blocksize = 0x0, 
  s_blocksize_bits = 0x0, 
  s_dirt = 0x0, 
  s_maxbytes = 0x0, 
  s_type = 0x0, 
  s_op = 0xffffffff8891d260, 
  dq_op = 0x0, 
  s_qcop = 0x0, 
  s_export_op = 0x0, 
 
 
crash> struct vfsmount 0xffff8104bf587bc0
  mnt_devname = 0xffff8104bf6fcd40 "/dev/root", 
 
So, s_type has a value 0 at the time of this access. All data in super_block is 0 and s_op is also has 0 for all fields.
 
crash> struct super_operations 0xffffffff8891d260
struct super_operations {
  alloc_inode = 0, 
  destroy_inode = 0, 
  read_inode = 0, 
  dirty_inode = 0, 
  write_inode = 0, 
  put_inode = 0, 
  drop_inode = 0, 
  delete_inode = 0, 
  put_super = 0, 
  write_super = 0, 
  sync_fs = 0, 
  write_super_lockfs = 0, 
  unlockfs = 0, 
  statfs = 0, 
  remount_fs = 0, 
  clear_inode = 0, 
  umount_begin = 0, 
  show_options = 0, 
  show_stats = 0, 
  quota_read = 0, 
  quota_write = 0, 
  freeze_fs = 0, 
  unfreeze_fs = 0, 
  bdev_try_to_free_page = 0
}