5:24 a.m.
Hello,
I'd like to introduce a small patch for the crash tool (I described an idea behind it
https://www.redhat.com/archives/crash-utility/2013-March/msg00164.html).
* crash version 6.1.6
* arch x86_64
You can find it here https://github.com/hziSot/crash-stack-parser
So, here are some results from a real system:
=======> CPU #0
bin> bt
PID: 0 TASK: ffffffff81a8d020 CPU: 0 COMMAND: "swapper"
#0 [ffff880045603920] machine_kexec at ffffffff8103284b
#1 [ffff880045603980] crash_kexec at ffffffff810ba972
#2 [ffff880045603a50] oops_end at ffffffff81501860
#3 [ffff880045603a80] no_context at ffffffff81043bfb
#4 [ffff880045603ad0] __bad_area_nosemaphore at ffffffff81043e85
#5 [ffff880045603b20] bad_area_nosemaphore at ffffffff81043f53
#6 [ffff880045603b30] __do_page_fault at ffffffff810446b1
#7 [ffff880045603c50] do_page_fault at ffffffff8150383e
#8 [ffff880045603c80] page_fault at ffffffff81500bf5
#9 [ffff880045603d60] activate_task at ffffffff81053953
#10 [ffff880045603d70] try_to_wake_up at ffffffff810600c0
#11 [ffff880045603de0] default_wake_function at ffffffff810602c2
#12 [ffff880045603df0] __wake_up_common at ffffffff8104e369
#13 [ffff880045603e40] complete at ffffffff81053347
#14 [ffff880045603e70] qla24xx_msix_default at ffffffffa00a8f74 [qla2xxx]
#15 [ffff880045603ed0] handle_IRQ_event at ffffffff810dbb10
#16 [ffff880045603f20] handle_edge_irq at ffffffff810de29e
#17 [ffff880045603f60] handle_irq at ffffffff8100df09
#18 [ffff880045603f80] do_IRQ at ffffffff815060bc
--- <IRQ stack> ---
#19 [ffffffff81a01da8] ret_from_intr at ffffffff8100ba53
[exception RIP: intel_idle+222]
RIP: ffffffff812cdc0e RSP: ffffffff81a01e58 RFLAGS: 00000206
RAX: 0000000000000000 RBX: ffffffff81a01ec8 RCX: 0000000000000000
RDX: 000000000000116c RSI: 0000000000000000 RDI: 0000000000441165
RBP: ffffffff8100ba4e R8: 0000000000000005 R9: 000000000000006d
R10: 000000c26ede6d76 R11: 0000000000000000 R12: ffff8800456115a0
R13: 0000000000000000 R14: 000000c12ce6a080 R15: ffff8800456116a0
ORIG_RAX: ffffffffffffff9d CS: 0010 SS: 0018
#20 [ffffffff81a01ed0] cpuidle_idle_call at ffffffff81407997
#21 [ffffffff81a01ef0] cpu_idle at ffffffff81009e06
bin> bt -H
Backtrace:
# 0: [RSP: 0xffff880045603980, RIP: 0xffffffff8103284b] machine_kexec (struct kimage *
arg = 0xffff88082659b400)
# 1: [RSP: 0xffff880045603a50, RIP: 0xffffffff810ba972] crash_kexec (struct pt_regs *
arg = 0xffff880045603c88)
# 2: [RSP: 0xffff880045603a80, RIP: 0xffffffff81501860] oops_end (long unsigned int arg
= unknown, struct pt_regs * arg = 0xffff880045603c88, int arg = 0x9)
# 3: [RSP: 0xffff880045603ad0, RIP: 0xffffffff81043bfb] no_context (struct pt_regs * arg
= 0xffff880045603c88, long unsigned int arg = 0x10, long unsigned int arg = 0x0)
# 4: [RSP: 0xffff880045603b20, RIP: 0xffffffff81043e85] __bad_area_nosemaphore (struct
pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0x10, long unsigned int arg =
0x0, int arg = 0x30001)
# 5: [RSP: 0xffff880045603b30, RIP: 0xffffffff81043f53] bad_area_nosemaphore (struct
pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0x10, long unsigned int arg =
0x0)
# 6: [RSP: 0xffff880045603c50, RIP: 0xffffffff810446b1] __do_page_fault (struct pt_regs
* arg = 0xffff880045603c88, long unsigned int arg = 0x4b2, long unsigned int arg = 0x10)
# 7: [RSP: 0xffff880045603c80, RIP: 0xffffffff8150383e] do_page_fault (struct pt_regs *
arg = 0xffff880045603c88, long unsigned int arg = 0xffffffffffffffff)
# 8: [RSP: 0xffff880045603d30, RIP: 0xffffffff81500bf5] page_fault ()
# 9: [RSP: 0xffff880045603d60, RIP: 0xffffffff81053916] enqueue_task (struct rq * arg =
0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1)
# 10: [RSP: 0xffff880045603d70, RIP: 0xffffffff81053953] activate_task (struct rq * arg =
0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1)
# 11: [RSP: 0xffff880045603de0, RIP: 0xffffffff810600c0] try_to_wake_up (struct
task_struct * arg = 0xffff88082b83eae0, unsigned int arg = 0x3, int arg = 0x0)
# 12: [RSP: 0xffff880045603df0, RIP: 0xffffffff810602c2] default_wake_function
(wait_queue_t * arg = 0xffff8800456c3b40, unsigned int arg = 0x3, int arg = 0x0, void *
arg = 0xc1416b9c06)
# 13: [RSP: 0xffff880045603e40, RIP: 0xffffffff8104e369] __wake_up_common
(wait_queue_head_t * arg = 0xffff880825e6a2a8, unsigned int arg = 0x3, int arg = 0x1, int
arg = 0x0, void * arg = 0x0)
# 14: [RSP: 0xffff880045603e70, RIP: 0xffffffff81053347] complete (struct completion *
arg = 0xffff880825e6a2a0)
# 15: [RSP: 0xffff880045603ed0, RIP: 0xffffffffa00a8f74] qla24xx_msix_default (int arg =
0x3e, void * arg = 0xffff880826bd55c0)
# 16: [RSP: 0xffff880045603f20, RIP: 0xffffffff810dbb10] handle_IRQ_event (unsigned int
arg = 0x3e, struct irqaction * arg = 0xffff880826bd5840)
# 17: [RSP: 0xffff880045603f60, RIP: 0xffffffff810de29e] handle_edge_irq (unsigned int
arg = 0x3e, struct irq_desc * arg = 0xffff880824563ec0)
# 18: [RSP: 0xffff880045603f80, RIP: 0xffffffff8100df09] handle_irq (unsigned int arg =
0x3e, struct pt_regs * arg = 0xffffffff81a01da8)
# 19: [RSP: 0xffff880045603fb0, RIP: 0xffffffff815060bc] do_IRQ (struct pt_regs * arg =
0xffffffff81a01da8)
# 20: [RSP: 0xffffffff81a01e50, RIP: 0xffffffff8100ba53] common_interrupt ()
# 21: [RSP: 0xffffffff81a01ed0, RIP: 0xffffffff812cdc0e] intel_idle (struct
cpuidle_device * arg = 0xffff88004561dcc0, struct cpuidle_state * arg =
0xffff88004561de50)
# 22: [RSP: 0xffffffff81a01ef0, RIP: 0xffffffff81407997] cpuidle_idle_call ()
Some of the arguments are not accurate, but many of them are correct:
bin> p/x *(( struct pt_regs * ) 0xffffffff81a01da8 )
$54 = {
r15 = 0xffff8800456116a0,
r14 = 0xc12ce6a080,
r13 = 0x0,
r12 = 0xffff8800456115a0,
bp = 0xffffffff8100ba4e,
bx = 0xffffffff81a01ec8,
r11 = 0x0,
r10 = 0xc26ede6d76,
r9 = 0x6d,
r8 = 0x5,
ax = 0x0,
cx = 0x0,
dx = 0x116c,
si = 0x0,
di = 0x441165,
orig_ax = 0xffffffffffffff9d,
ip = 0xffffffff812cdc0e,
cs = 0x10,
flags = 0x206,
sp = 0xffffffff81a01e58,
ss = 0x18
}
bin> p/x *((struct irq_desc * ) 0xffff880824563ec0)
$55 = {
irq = 0x3e,
timer_rand_state = 0x0,
kstat_irqs = 0xffff880825d7f6c0,
irq_2_iommu = 0xffff880825e70880,
handle_irq = 0xffffffff810de1c0,
chip = 0xffffffff81a934e0,
msi_desc = 0xffff880826bd56c0,
handler_data = 0x0,
chip_data = 0xffff880825e708a0,
action = 0xffff880826bd5840,
status = 0x1000100,
depth = 0x0,
wake_depth = 0x0,
irq_count = 0x2cd,
last_unhandled = 0x0,
irqs_unhandled = 0x0,
lock = {
raw_lock = {
slock = 0x5bc05bc
}
},
affinity = 0xffff880826a0f800,
affinity_hint = 0x0,
node = 0x0,
pending_mask = 0xffff88082b765a00,
threads_active = {
counter = 0x0
},
wait_for_threads = {
lock = {
raw_lock = {
slock = 0x0
}
},
task_list = {
next = 0xffff880824563f60,
prev = 0xffff880824563f60
}
},
dir = 0xffff880824563d40,
name = 0xffffffff8177f1ce
}
=======> CPU #6
bin> bt
PID: 0 TASK: ffff88082b83eae0 CPU: 6 COMMAND: "swapper"
#0 [ffff8800456c7e90] crash_nmi_callback at ffffffff81029df6
#1 [ffff8800456c7ea0] notifier_call_chain at ffffffff815038f5
#2 [ffff8800456c7ee0] atomic_notifier_call_chain at ffffffff8150395a
#3 [ffff8800456c7ef0] notify_die at ffffffff810981ee
#4 [ffff8800456c7f20] do_nmi at ffffffff81501573
#5 [ffff8800456c7f50] nmi at ffffffff81500e80
[exception RIP: _spin_lock_irq+37]
RIP: ffffffff81500615 RSP: ffff8800456c3940 RFLAGS: 00000097
RAX: 00000000000008a4 RBX: ffff8800456d6680 RCX: 000000000000c14e
RDX: 00000000000008a3 RSI: 0000000000000001 RDI: ffff8800456d6680
RBP: ffff8800456c3940 R8: 0000000000013679 R9: 00000000fffffffa
R10: 0000000000000002 R11: 0000000000000002 R12: ffff8800456d6680
R13: 0000000000000001 R14: 00000000ffffffff R15: 00000000ffffffff
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8800456c3940] _spin_lock_irq at ffffffff81500615
#7 [ffff8800456c3948] dequeue_task_idle at ffffffff81053f19
#8 [ffff8800456c3968] dequeue_task at ffffffff8105385e
#9 [ffff8800456c3998] deactivate_task at ffffffff810538a3
#10 [ffff8800456c39a8] thread_return at ffffffff814fded9
#11 [ffff8800456c3a68] schedule_timeout at ffffffff814febf2
#12 [ffff8800456c3b18] wait_for_common at ffffffff814fe8f3
#13 [ffff8800456c3ba8] wait_for_completion_timeout at ffffffff814fe9e3
#14 [ffff8800456c3bb8] qla2x00_mailbox_command at ffffffffa0098d1a [qla2xxx]
#15 [ffff8800456c3c68] qla2x00_issue_iocb_timeout at ffffffffa009cc3b [qla2xxx]
#16 [ffff8800456c3cf8] qla2x00_issue_iocb at ffffffffa009ccd4 [qla2xxx]
#17 [ffff8800456c3d08] qla24xx_abort_command at ffffffffa009f5b8 [qla2xxx]
#18 [ffff8800456c3d68] qla24xx_bsg_timeout at ffffffffa00c0ba6 [qla2xxx]
#19 [ffff8800456c3da8] fc_bsg_job_timeout at ffffffffa00718f7 [scsi_transport_fc]
#20 [ffff8800456c3dd8] blk_rq_timed_out at ffffffff8125dccb
#21 [ffff8800456c3df8] blk_rq_timed_out_timer at ffffffff8125df88
#22 [ffff8800456c3e48] run_timer_softirq at ffffffff8107e927
#23 [ffff8800456c3ed8] __do_softirq at ffffffff81073f51
#24 [ffff8800456c3f48] call_softirq at ffffffff8100c24c
#25 [ffff8800456c3f60] do_softirq at ffffffff8100de85
#26 [ffff8800456c3f80] irq_exit at ffffffff81073d35
#27 [ffff8800456c3f90] smp_apic_timer_interrupt at ffffffff815061b0
#28 [ffff8800456c3fb0] apic_timer_interrupt at ffffffff8100bc13
--- <IRQ stack> ---
#29 [ffff88082b847db8] apic_timer_interrupt at ffffffff8100bc13
[exception RIP: intel_idle+222]
RIP: ffffffff812cdc0e RSP: ffff88082b847e68 RFLAGS: 00000206
RAX: 0000000000000000 RBX: ffff88082b847ed8 RCX: 0000000000000000
RDX: 0000000000000f69 RSI: 0000000000000000 RDI: 00000000003c35fa
RBP: ffffffff8100bc0e R8: 0000000000000005 R9: 000000000000006d
R10: 000000c130c95aec R11: 0000000000000000 R12: ffff8800456d15a0
R13: 0000000000000000 R14: 000000c12d23a980 R15: ffff8800456d16a0
ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018
#30 [ffff88082b847ee0] cpuidle_idle_call at ffffffff81407997
#31 [ffff88082b847f00] cpu_idle at ffffffff81009e06
bin> bt -H
# 0: [RSP: 0xffff8800456c7ea0, RIP: 0xffffffff81029df6] crash_nmi_callback (struct
notifier_block * arg = 0xffffffff81aa5260, long unsigned int arg = 0xc, void * arg =
0xffff8800456c7ef8)
# 1: [RSP: 0xffff8800456c7ee0, RIP: 0xffffffff815038f5] notifier_call_chain (struct
notifier_block ** arg = 0xffffffff81ea87a8, long unsigned int arg = 0xc, void * arg =
0xffff8800456c7ef8, int arg = 0xffffffff, int * arg = 0x0)
# 2: [RSP: 0xffff8800456c7ef0, RIP: 0xffffffff8150395a] atomic_notifier_call_chain
(struct atomic_notifier_head * arg = 0xffffffff81ea87a0, long unsigned int arg = 0xc, void
* arg = 0xffff8800456c7ef8)
# 3: [RSP: 0xffff8800456c7f20, RIP: 0xffffffff810981ee] notify_die (enum die_val arg =
0xc, const char * arg = 0xffffffff8177d9a2, struct pt_regs * arg = 0xffff8800456c7f58,
long int arg = 0x0, int arg = 0x2, int arg = 0x2)
# 4: [RSP: 0xffff8800456c7f50, RIP: 0xffffffff81501573] do_nmi (struct pt_regs * arg =
0xffff8800456c7f58, long int arg = 0xffffffffffffffff)
# 5: [RSP: 0xffff8800456c8000, RIP: 0xffffffff81500e80] nmi ()
# 6: [RSP: 0xffff8800456c3948, RIP: 0xffffffff81500615] _spin_lock_irq (spinlock_t * arg
= 0xffff8800456d6680)
# 7: [RSP: 0xffff8800456c3968, RIP: 0xffffffff81053f19] dequeue_task_idle (struct rq *
arg = 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1)
# 8: [RSP: 0xffff8800456c3998, RIP: 0xffffffff8105385e] dequeue_task (struct rq * arg =
0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1)
# 9: [RSP: 0xffff8800456c39a8, RIP: 0xffffffff810538a3] deactivate_task (struct rq * arg
= 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1)
# 10: [RSP: 0xffff8800456c3a68, RIP: 0xffffffff814fded9] thread_return ()
# 11: [RSP: 0xffff8800456c3b18, RIP: 0xffffffff814febf2] schedule_timeout (long int arg =
0x7530)
# 12: [RSP: 0xffff8800456c3ba8, RIP: 0xffffffff814fe8f3] wait_for_common (struct
completion * arg = 0xffff880825e6a2a0, long int arg = 0x7530, int arg = 0x2)
# 13: [RSP: 0xffff8800456c3bb8, RIP: 0xffffffff814fe9e3] wait_for_completion_timeout
(struct completion * arg = 0xffff880825e6a2a0, long unsigned int arg = 0x7530)
# 14: [RSP: 0xffff8800456c3c68, RIP: 0xffffffffa0098d1a] qla2x00_mailbox_command
(scsi_qla_host_t * arg = 0xffff880824562de0, mbx_cmd_t * arg = 0xffff8800456c3c70)
# 15: [RSP: 0xffff8800456c3cf8, RIP: 0xffffffffa009cc3b] qla2x00_issue_iocb_timeout
(scsi_qla_host_t * arg = 0xffff880824562de0, void * arg = 0xffff880824561100, dma_addr_t
arg = 0x824561100, size_t arg = 0x0, uint32_t arg = 0x1e)
# 16: [RSP: 0xffff8800456c3d08, RIP: 0xffffffffa009ccd4] qla2x00_issue_iocb
(scsi_qla_host_t * arg = 0xffff880824562de0, void * arg = 0xffff880824561100, dma_addr_t
arg = 0x824561100, size_t arg = 0x0)
# 17: [RSP: 0xffff8800456c3d68, RIP: 0xffffffffa009f5b8] qla24xx_abort_command (srb_t *
arg = 0xffff8808145db480)
# 18: [RSP: 0xffff8800456c3da8, RIP: 0xffffffffa00c0ba6] qla24xx_bsg_timeout (struct
fc_bsg_job * arg = 0xffff8808268a2740)
# 19: [RSP: 0xffff8800456c3dd8, RIP: 0xffffffffa00718f7] fc_bsg_job_timeout (struct
request * arg = 0xffff88082446b360)
# 20: [RSP: 0xffff8800456c3df8, RIP: 0xffffffff8125dccb] blk_rq_timed_out (struct request
* arg = 0xffff88082446b360)
# 21: [RSP: 0xffff8800456c3e48, RIP: 0xffffffff8125df88] blk_rq_timed_out_timer (long
unsigned int arg = 0xffff88082477cea0)
# 22: [RSP: 0xffff8800456c3ed8, RIP: 0xffffffff8107e927] run_timer_softirq (struct
softirq_action * arg = 0xffffffff81a83088)
# 23: [RSP: 0xffff8800456c3f48, RIP: 0xffffffff81073f51] __do_softirq ()
# 24: [RSP: 0xffff8800456c3f60, RIP: 0xffffffff8100c24c] call_softirq ()
# 25: [RSP: 0xffff8800456c3f80, RIP: 0xffffffff8100de85] do_softirq ()
# 26: [RSP: 0xffff8800456c3f90, RIP: 0xffffffff81073d35] irq_exit ()
# 27: [RSP: 0xffff8800456c3fb0, RIP: 0xffffffff815061b0] smp_apic_timer_interrupt (struct
pt_regs * arg = 0xffff88082b847db8)
# 28: [RSP: 0xffff88082b847e60, RIP: 0xffffffff8100bc13] apic_timer_interrupt ()
# 29: [RSP: 0xffff88082b847ee0, RIP: 0xffffffff812cdc0e] intel_idle (struct
cpuidle_device * arg = 0xffff8800456ddcc0, struct cpuidle_state * arg =
0xffff8800456dde50)
# 30: [RSP: 0xffff88082b847f00, RIP: 0xffffffff81407997] cpuidle_idle_call ()
Please feel free to submit any found issues via email or github.