Hi Dave,
Before you responded I noticed that a simple "make modules" didn't work
because my kernel wasn't exporting the symbol. Rather than do anything risky/complex
which might risk mucking up the troubleshooting process, I just rebuilt the kernel. It
built just fine and now I can load crash and I see "DUMPFILE: /dev/crash" when I
load up crash. Let me try walking through the steps that you had me do previously, this
time using /dev/crash instead of /dev/mem and /dev/kmem
From my limited understanding of what's going on here, it would
appear that the dump file is missing some data, or else crash is looking in the wrong
place for it.
-Kevin
---Live system---
KERNEL: vmlinux-devcrash
DUMPFILE: /dev/crash
CPUS: 2
DATE: Tue Oct 14 16:08:28 2008
UPTIME: 00:02:07
LOAD AVERAGE: 0.17, 0.08, 0.03
TASKS: 97
NODENAME: test-machine
RELEASE: 2.6.20-17.39-custom2
VERSION: #1 SMP Tue Oct 14 13:45:17 PDT 2008
MACHINE: i686 (2200 Mhz)
MEMORY: 5 GB
PID: 5628
COMMAND: "crash"
TASK: 5d4c2560 [THREAD_INFO: f3de6000]
CPU: 1
STATE: TASK_RUNNING (ACTIVE)
crash> p modules
modules = $2 = {
next = 0xf8a3ea04,
prev = 0xf8842104
}
crash> module 0xf8a3ea00
struct module {
state = MODULE_STATE_LIVE,
list = {
next = 0xf8d10484,
prev = 0x403c63a4
},
name = "crash\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000",
mkobj = {
kobj = {
k_name = 0xf8a3ea4c "crash",
name = "crash\000\000\000\000\000\000\000\000\000\000\000\000\000\000",
kref = {
refcount = {
counter = 3
}
},
entry = {
next = 0x403c6068,
prev = 0xf8d104e4
},
parent = 0x403c6074
...
crash> vtop 0xf8a3ea00
VIRTUAL PHYSICAL
f8a3ea00 116017a00
PAGE DIRECTORY: 4044b000
PGD: 4044b018 => 6001
PMD: 6e28 => 1d51a067
PTE: 1d51a1f0 => 116017163
PAGE: 116017000
PTE PHYSICAL FLAGS
116017163 116017000 (PRESENT|RW|ACCESSED|DIRTY|GLOBAL)
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
472c02e0 116017000 0 229173 1 80000000
crash> rd -p 116017000 30
116017000: 53e58955 d089c389 4d8bca89 74c98508 U..S.......M...t
116017010: 01e9831f b85b0d74 ffffffea ffffba5d ....t.[.....]...
116017020: 03c3ffff 53132043 26b48d24 00000000 ....C .S$..&....
116017030: 89204389 5d5b2453 26b48dc3 00000000 .C .S$[]...&....
116017040: 83e58955 55892cec 08558be4 89f45d89 U....,.U..U..]..
116017050: 7d89f875 ffeabffc 4d89ffff 8b028be0 u..}.......M....
116017060: c3890452 ac0fd689 45890cf3 0ceec1ec R..........E....
116017070: 5589c889 89d231f0 ...U.1..
crash>
---Using dump file---
please wait... (gathering module symbol data)
WARNING: cannot access vmalloc'd module memory
KERNEL: vmlinux-devcrash
DUMPFILE: /var/crash/vmcore
CPUS: 2
DATE: Tue Oct 14 16:09:32 2008
UPTIME: 00:03:12
LOAD AVERAGE: 0.09, 0.08, 0.02
TASKS: 97
NODENAME: test-machine
RELEASE: 2.6.20-17.39-custom2
VERSION: #1 SMP Tue Oct 14 13:45:17 PDT 2008
MACHINE: i686 (2200 Mhz)
MEMORY: 5 GB
PANIC: "[ 192.148000] SysRq : Trigger a crashdump"
PID: 0
COMMAND: "swapper"
TASK: 403c0440 (1 of 2) [THREAD_INFO: 403f2000]
CPU: 0
STATE: TASK_RUNNING (SYSRQ)
crash> p modules
modules = $2 = {
next = 0xf8a3ea04,
prev = 0xf8842104
}
crash> module 0xf8a3ea00
struct module {
state = MODULE_STATE_LIVE,
list = {
next = 0x0,
prev = 0x0
},
name = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\0
00\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\0
00\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\0
00\000",
mkobj = {
kobj = {
k_name = 0x0,
name = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\0
00\000\000",
kref = {
refcount = {
counter = 0
}
},
entry = {
next = 0x0,
prev = 0x0
...
crash> vtop 0xf8a3ea00
VIRTUAL PHYSICAL
f8a3ea00 116017a00
PAGE DIRECTORY: 4044b000
PGD: 4044b018 => 6001
PMD: 6e28 => 1d51a067
PTE: 1d51a1f0 => 116017163
PAGE: 116017000
PTE PHYSICAL FLAGS
116017163 116017000 (PRESENT|RW|ACCESSED|DIRTY|GLOBAL)
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
472c02e0 116017000 0 229173 1 80000000
crash> rd -p 116017000 30
116017000: 00000000 00000000 00000000 00000000 ................
116017010: 00000000 00000000 00000000 00000000 ................
116017020: 00000000 00000000 00000000 00000000 ................
116017030: 00000000 00000000 00000000 00000000 ................
116017040: 00000000 00000000 00000000 00000000 ................
116017050: 00000000 00000000 00000000 00000000 ................
116017060: 00000000 00000000 00000000 00000000 ................
116017070: 00000000 00000000 ........
crash>