On 2023/02/06 23:43, lijiang wrote:
>>>> On Mon, Feb 6, 2023 at 2:19 PM HAGIO KAZUHITO(萩尾　一仁) <k-hagio-ab@nec.com <mailto:k-hagio-ab@nec.com>> wrote:
>>>>
>>>>
>>>> On 2023/02/06 11:04, lijiang wrote:
>>>>
>>>> >> I tried to hardcode and get the expected eframe information, but the backtrace is incorrect:
>>>>
>>>> ok, it looks the expected one. The irq_eframe_link value has been
>>>> hardcoded for a long time, it might be hard to calculate it in the
>>>> current implementation..
>>>>
>>>> btw, what is the kernel version of your vmcore this case?
>>>>
>>>>
>>>> My kernel version is upstream kernel 5.7.

The kernel 5.7 doesn't have fa5e5c409213, so the patch is added?

I built the upstream kernel based on the commit fa5e5c409213, so it displays the kernel

version is 5.7+.

$ git describe --contains fa5e5c409213
v5.8-rc1~21^2~43

Yes, you are right.

>>>> For my side, the irq_eframe_link is set to -56 as below:

I see, thanks for trying.

>>>>
>>>> diff --git a/x86_64.c b/x86_64.c
>>>> index 7a5d6f050c89..61ecf8dbdb65 100644
>>>> --- a/x86_64.c
>>>> +++ b/x86_64.c
>>>> @@ -3938,6 +3938,11 @@ in_exception_stack:
>>>> if (irq_eframe) {
>>>> bt->flags |= BT_EXCEPTION_FRAME;
>>>> i = (irq_eframe - bt->stackbase)/sizeof(ulong);
>>>> + if (symbol_exists("asm_common_interrupt")) {
>>>> + i -= 1;
>>>> + up = (ulong *)(&bt->stackbuf[i*sizeof(ulong)]);
>>>> + bt->instptr = *up;
>>>> + }
>>>> x86_64_print_stack_entry(bt, ofp, level, i, bt->instptr);
>>>> bt->flags &= ~(ulonglong)BT_EXCEPTION_FRAME;
>>>> cs = x86_64_exception_frame(EFRAME_PRINT|EFRAME_CS, 0,
>>>> @@ -6521,6 +6526,11 @@ x86_64_irq_eframe_link_init(void)
>>>> else
>>>> return;
>>>>
>>>> + if (symbol_exists("asm_common_interrupt")) {
>>>> + machdep->machspec->irq_eframe_link = -56;
>>>> + return;
>>>> + }
>>>> +
>>>> if (THIS_KERNEL_VERSION < LINUX(2,6,9))
>>>> return;
>>>>
>>>> And get the following results:
>>>> crash> bt
>>>> PID: 0 TASK: ffff9e7a47e32f00 CPU: 3 COMMAND: "swapper/3"
>>>> #0 [ffffba7900118bb8] machine_kexec at ffffffff87e5c2c7
>>>> #1 [ffffba7900118c08] __crash_kexec at ffffffff87f9500d
>>>> #2 [ffffba7900118cd0] panic at ffffffff87edfff9
>>>> #3 [ffffba7900118d50] sysrq_handle_crash at ffffffff883ce2c1
>>>> #4 [ffffba7900118d58] __handle_sysrq.cold.15 at ffffffff883ceb56
>>>> #5 [ffffba7900118d88] sysrq_filter at ffffffff883ce9a2
>>>> #6 [ffffba7900118dc0] input_to_handler at ffffffff884fb0bf
>>>> #7 [ffffba7900118df8] input_pass_values at ffffffff884fc1b7
>>>> #8 [ffffba7900118e20] input_handle_event at ffffffff884fe278
>>>> #9 [ffffba7900118e50] input_event at ffffffff884fe74b
>>>> #10 [ffffba7900118e88] atkbd_interrupt at ffffffff88504e2f
>>>> #11 [ffffba7900118ee0] serio_interrupt at ffffffff884f7516
>>>> #12 [ffffba7900118f10] i8042_interrupt at ffffffff884f8b04
>>>> #13 [ffffba7900118f50] __handle_irq_event_percpu at ffffffff87f51430
>>>> #14 [ffffba7900118f90] handle_irq_event_percpu at ffffffff87f51590
>>>> #15 [ffffba7900118fb8] handle_irq_event at ffffffff87f51616
>>>> #16 [ffffba7900118fd8] handle_edge_irq at ffffffff87f559f2
>>>> #17 [ffffba7900118ff0] asm_call_on_stack at ffffffff88800fa2
>>>> --- <IRQ stack> ---
>>>> #18 [ffffba790008be08] asm_common_interrupt at ffffffff88800c1e
>>>> [exception RIP: default_idle+37]
>>>> RIP: ffffffff8876ffc5 RSP: ffffba790008beb8 RFLAGS: 00000246
>>>> RAX: ffffffff8876ffa0 RBX: 0000000000000003 RCX: 0000000000000001
>>>> RDX: 000000000001a6de RSI: 0000000000000087 RDI: 0000000000000003
>>>> RBP: 0000000000000003 R8: 000000146ccbc1f0 R9: 0000000000000000
>>>> R10: 0000000000000000 R11: 0000000000000124 R12: ffffffffffffffff
>>>> R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
>>>> ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
>>>> #19 [ffffba790008bed8] do_idle at ffffffff87f1982d
>>>> #20 [ffffba790008bf20] cpu_startup_entry at ffffffff87f19a29
>>>> #21 [ffffba790008bf30] start_secondary at ffffffff87e5118f
>>>> #22 [ffffba790008bf50] secondary_startup_64 at ffffffff87e000e6
>>>>
>>>> The above eframe information looks correct, but the backtrace seems problematic, it doesn't display the
>>>> common_interrupt() according to the kernel log:

yes, it's expected. The current crash unwinder prints only one return
address with the exception frame. Probably we can show them more, but
more important thing is how we determine the irq_eframe_link.

The following patch can work on upstream kernel vmcore and RHEL9 vmcore.

Maybe we can check the symbols asm_common_interrupt and asm_call_on_stack as below:

diff --git a/x86_64.c b/x86_64.c
index 7a5d6f050c89..62036f71f632 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -3938,6 +3938,11 @@ in_exception_stack:
if (irq_eframe) {
bt->flags |= BT_EXCEPTION_FRAME;
i = (irq_eframe - bt->stackbase)/sizeof(ulong);
+ if (symbol_exists("asm_common_interrupt")) {
+ i -= 1;
+ up = (ulong *)(&bt->stackbuf[i*sizeof(ulong)]);
+ bt->instptr = *up;
+ }
x86_64_print_stack_entry(bt, ofp, level, i, bt->instptr);
bt->flags &= ~(ulonglong)BT_EXCEPTION_FRAME;
cs = x86_64_exception_frame(EFRAME_PRINT|EFRAME_CS, 0,
@@ -6521,6 +6526,16 @@ x86_64_irq_eframe_link_init(void)
else
return;

+ if (symbol_exists("asm_common_interrupt") && !symbol_exists("asm_call_on_stack")) {
+ machdep->machspec->irq_eframe_link =-32;
+ return;
+ }
+
+ if (symbol_exists("asm_common_interrupt") && symbol_exists("asm_call_on_stack")) {
+ machdep->machspec->irq_eframe_link =-56;
+ return;
+ }
+
if (THIS_KERNEL_VERSION < LINUX(2,6,9))
return;

I will also think about it, could I have some outputs for reference?

Sure.

crash> rd ffffba7900118fd0 4

ffffba7900118fd0: 0000000000000024 ffffffff87f559f2 $........Y......
ffffba7900118fe0: ffffba790008be08 ffffba790008bdc8 ....y.......y...

crash> dis asm_common_interrupt

0xffffffff88800c00 <asm_common_interrupt>: data16 xchg %ax,%ax
0xffffffff88800c03 <asm_common_interrupt+3>: call 0xffffffff88801130 <error_entry>
0xffffffff88800c08 <asm_common_interrupt+8>: mov %rsp,%rdi
0xffffffff88800c0b <asm_common_interrupt+11>: mov 0x78(%rsp),%rsi
0xffffffff88800c10 <asm_common_interrupt+16>: movq $0xffffffffffffffff,0x78(%rsp)
0xffffffff88800c19 <asm_common_interrupt+25>: call 0xffffffff887604a0 <common_interrupt>
0xffffffff88800c1e <asm_common_interrupt+30>: jmp 0xffffffff88801210 <error_return>

crash> dis common_interrupt

0xffffffff887604a0 <common_interrupt>: push %r14
0xffffffff887604a2 <common_interrupt+2>: push %r13
0xffffffff887604a4 <common_interrupt+4>: push %r12
0xffffffff887604a6 <common_interrupt+6>: mov %rsi,%r12
0xffffffff887604a9 <common_interrupt+9>: push %rbp
0xffffffff887604aa <common_interrupt+10>: push %rbx
0xffffffff887604ab <common_interrupt+11>: mov %rdi,%rbx
0xffffffff887604ae <common_interrupt+14>: call 0xffffffff8875f730 <idtentry_enter_cond_rcu>
0xffffffff887604b3 <common_interrupt+19>: mov %eax,%ebp
0xffffffff887604b5 <common_interrupt+21>: call 0xffffffff87ee6820 <irq_enter_rcu>
0xffffffff887604ba <common_interrupt+26>: movzbl %r12b,%eax
0xffffffff887604be <common_interrupt+30>: mov %gs:0x778c9f32(%rip),%r13 # 0x2a3f8
0xffffffff887604c6 <common_interrupt+38>: movb $0x1,%gs:0x778cb4f4(%rip) # 0x2b9c2
0xffffffff887604ce <common_interrupt+46>: mov %rbx,%gs:0x778c9f22(%rip) # 0x2a3f8
0xffffffff887604d6 <common_interrupt+54>: mov %gs:0x17340(,%rax,8),%rdx
0xffffffff887604df <common_interrupt+63>: test %rdx,%rdx
0xffffffff887604e2 <common_interrupt+66>: je 0xffffffff88760587 <common_interrupt+231>
0xffffffff887604e8 <common_interrupt+72>: cmp $0xfffffffffffff000,%rdx
0xffffffff887604ef <common_interrupt+79>: ja 0xffffffff8876055e <common_interrupt+190>
0xffffffff887604f1 <common_interrupt+81>: mov 0x68(%rdx),%rsi
0xffffffff887604f5 <common_interrupt+85>: test %rbx,%rbx
0xffffffff887604f8 <common_interrupt+88>: je 0xffffffff8876052c <common_interrupt+140>
0xffffffff887604fa <common_interrupt+90>: testb $0x3,0x88(%rbx)
0xffffffff88760501 <common_interrupt+97>: je 0xffffffff8876052c <common_interrupt+140>
0xffffffff88760503 <common_interrupt+99>: mov %rdx,%rdi
0xffffffff88760506 <common_interrupt+102>: call 0xffffffff88a01040 <__x86_indirect_thunk_rsi>
0xffffffff8876050b <common_interrupt+107>: mov %r13,%gs:0x778c9ee5(%rip) # 0x2a3f8
0xffffffff88760513 <common_interrupt+115>: call 0xffffffff87ee6890 <irq_exit_rcu>
0xffffffff88760518 <common_interrupt+120>: movzbl %bpl,%esi
0xffffffff8876051c <common_interrupt+124>: mov %rbx,%rdi
0xffffffff8876051f <common_interrupt+127>: pop %rbx
0xffffffff88760520 <common_interrupt+128>: pop %rbp
0xffffffff88760521 <common_interrupt+129>: pop %r12
0xffffffff88760523 <common_interrupt+131>: pop %r13
0xffffffff88760525 <common_interrupt+133>: pop %r14
0xffffffff88760527 <common_interrupt+135>: jmp 0xffffffff8875f770 <idtentry_exit_cond_rcu>
0xffffffff8876052c <common_interrupt+140>: mov %gs:0x778b79d9(%rip),%eax # 0x17f0c
0xffffffff88760533 <common_interrupt+147>: cmp $0xffffffff,%eax
0xffffffff88760536 <common_interrupt+150>: sete %al
0xffffffff88760539 <common_interrupt+153>: test %al,%al
0xffffffff8876053b <common_interrupt+155>: je 0xffffffff88760503 <common_interrupt+99>
0xffffffff8876053d <common_interrupt+157>: mov %gs:0x778b79cb(%rip),%rdi # 0x17f10
0xffffffff88760545 <common_interrupt+165>: sub $0x8,%rdi
0xffffffff88760549 <common_interrupt+169>: incl %gs:0x778b79bc(%rip) # 0x17f0c
0xffffffff88760550 <common_interrupt+176>: call 0xffffffff88800f90 <asm_call_on_stack>
0xffffffff88760555 <common_interrupt+181>: decl %gs:0x778b79b0(%rip) # 0x17f0c
0xffffffff8876055c <common_interrupt+188>: jmp 0xffffffff8876050b <common_interrupt+107>
0xffffffff8876055e <common_interrupt+190>: mov 0x89699b(%rip),%rax # 0xffffffff88ff6f00 <apic>
0xffffffff88760565 <common_interrupt+197>: xor %esi,%esi
0xffffffff88760567 <common_interrupt+199>: mov $0xb0,%edi
0xffffffff8876056c <common_interrupt+204>: movzbl %r12b,%r12d
0xffffffff88760570 <common_interrupt+208>: mov (%rax),%rax
0xffffffff88760573 <common_interrupt+211>: call 0xffffffff88a00fc0 <__x86_indirect_thunk_rax>
0xffffffff88760578 <common_interrupt+216>: movq $0x0,%gs:0x17340(,%r12,8)
0xffffffff88760585 <common_interrupt+229>: jmp 0xffffffff8876050b <common_interrupt+107>
0xffffffff88760587 <common_interrupt+231>: mov %rax,%r14
0xffffffff8876058a <common_interrupt+234>: mov 0x89696f(%rip),%rax # 0xffffffff88ff6f00 <apic>
0xffffffff88760591 <common_interrupt+241>: xor %esi,%esi
0xffffffff88760593 <common_interrupt+243>: mov $0xb0,%edi
0xffffffff88760598 <common_interrupt+248>: mov (%rax),%rax
0xffffffff8876059b <common_interrupt+251>: call 0xffffffff88a00fc0 <__x86_indirect_thunk_rax>
0xffffffff887605a0 <common_interrupt+256>: mov $0xffffffff88c02b80,%rsi
0xffffffff887605a7 <common_interrupt+263>: mov $0xffffffff8921f560,%rdi
0xffffffff887605ae <common_interrupt+270>: call 0xffffffff882d6eb0 <___ratelimit>
0xffffffff887605b3 <common_interrupt+275>: test %eax,%eax
0xffffffff887605b5 <common_interrupt+277>: je 0xffffffff8876050b <common_interrupt+107>
0xffffffff887605bb <common_interrupt+283>: mov %r14d,%ecx
0xffffffff887605be <common_interrupt+286>: mov $0xffffffff88c02b80,%rsi
0xffffffff887605c5 <common_interrupt+293>: mov $0xffffffff88f0ce20,%rdi
0xffffffff887605cc <common_interrupt+300>: mov %gs:0x778b1d8d(%rip),%edx # 0x12360
0xffffffff887605d3 <common_interrupt+307>: call 0xffffffff87f500be <printk>
0xffffffff887605d8 <common_interrupt+312>: jmp 0xffffffff8876050b <common_interrupt+107>

crash> dis asm_call_on_stack

0xffffffff88800f90 <asm_call_on_stack>: push %rbp
0xffffffff88800f91 <asm_call_on_stack+1>: mov %rsp,%rbp
0xffffffff88800f94 <asm_call_on_stack+4>: mov %rsp,(%rdi)
0xffffffff88800f97 <asm_call_on_stack+7>: mov %rdi,%rsp
0xffffffff88800f9a <asm_call_on_stack+10>: mov %rdx,%rdi
0xffffffff88800f9d <asm_call_on_stack+13>: call 0xffffffff88a01045 <__x86_retpoline_rsi>
0xffffffff88800fa2 <asm_call_on_stack+18>: leave
0xffffffff88800fa3 <asm_call_on_stack+19>: ret

crash> bt -D -5 -I asm_common_interrupt+30

orc_dump: ffffffff88800c1e / asm_common_interrupt+30
--------
ip: ffffffff895886d4 -> ffffffff88800c08 / asm_common_interrupt+8 -> orc: ffffffff89790094 spo: 0 bpo: 0 spr: 5 bpr: 0 type: 1 end: 0
ip: ffffffff895886d8 -> ffffffff88800c23 / asm_common_interrupt+35 -> orc: ffffffff8979009a spo: 0 bpo: 0 spr: 0 bpr: 0 type: 0 end: 0

crash> bt -D -5 -I common_interrupt+181

orc_dump: ffffffff88760555 / common_interrupt+181

--------
ip: ffffffff895859d8 -> ffffffff8876052c / common_interrupt+140 -> orc: ffffffff8978bd1a spo: 48 bpo: -40 spr: 5 bpr: 1 type: 0 end: 0
ip: ffffffff895859dc -> ffffffff887605dd / common_interrupt+317 -> orc: ffffffff8978bd20 spo: 0 bpo: 0 spr: 0 bpr: 0 type: 0 end: 0

crash> bt -D -5 -I ffffffff88800fa2

orc_dump: ffffffff88800fa2 / asm_call_on_stack+18
--------
ip: ffffffff89588868 -> ffffffff88800f94 / asm_call_on_stack+4 -> orc: ffffffff897902f2 spo: 16 bpo: -16 spr: 4 bpr: 1 type: 0 end: 0
ip: ffffffff8958886c -> ffffffff88800fa3 / asm_call_on_stack+19 -> orc: ffffffff897902f8 spo: 8 bpo: 0 spr: 5 bpr: 0 type: 0 end: 0
ip: ffffffff89588870 -> ffffffff88800fa4 / asm_call_on_stack+20 -> orc: ffffffff897902fe spo: 0 bpo: 0 spr: 0 bpr: 0 type: 0 end: 0

Thanks

Lianbo

Thanks,
Kazu

>>>>
>>>> [ 57.552655] Call Trace:
>>>> [ 57.553994] <IRQ>
>>>> [ 57.555143] dump_stack+0x57/0x70
>>>> [ 57.556930] panic+0xfb/0x2d7
>>>> [ 57.558516] ? printk+0x58/0x6f
>>>> [ 57.560194] sysrq_handle_crash+0x11/0x20
>>>> [ 57.562371] __handle_sysrq.cold.15+0x48/0x102
>>>> [ 57.564663] sysrq_filter+0x312/0x3c0
>>>> [ 57.566579] input_to_handler+0x4f/0xf0
>>>> [ 57.568583] input_pass_values.part.7+0x117/0x130
>>>> [ 57.571188] input_handle_event+0x138/0x5c0
>>>> [ 57.573380] input_event+0x4b/0x70
>>>> [ 57.575173] atkbd_interrupt+0x61f/0x6a0
>>>> [ 57.577228] serio_interrupt+0x46/0x90
>>>> [ 57.579192] i8042_interrupt+0x154/0x240
>>>> [ 57.581328] __handle_irq_event_percpu+0x40/0x170
>>>> [ 57.583788] handle_irq_event_percpu+0x30/0x80
>>>> [ 57.586081] handle_irq_event+0x36/0x53
>>>> [ 57.588071] handle_edge_irq+0x82/0x190
>>>> [ 57.590096] asm_call_on_stack+0x12/0x20
>>>> [ 57.592142] </IRQ>
>>>> [ 57.593264] common_interrupt+0xb5/0x140
>>>> [ 57.595311] asm_common_interrupt+0x1e/0x40
>>>> [ 57.597474] RIP: 0010:default_idle+0x25/0x150
>>>> [ 57.599735] Code: cc cc cc cc cc 66 66 66 66 90 41 55 41 54 55 65 8b 2d af 23 8a 77 53 66 66 66 66 90 e9 07 00 00 00 0f 00 2d 9f 37 49 00 fb f4 <65> 8b 2d 94 23 8a 77 66 66 66 66 90 5b 5d 41 5c 41 5d c3 65 8b 05
>>>> [ 57.609196] RSP: 0018:ffffba790008beb8 EFLAGS: 00000246
>>>> [ 57.611875] RAX: ffffffff8876ffa0 RBX: 0000000000000003 RCX: 0000000000000001
>>>> [ 57.615506] RDX: 000000000001a6de RSI: 0000000000000087 RDI: 0000000000000003
>>>> [ 57.619142] RBP: 0000000000000003 R08: 000000146ccbc1f0 R09: 0000000000000000
>>>> [ 57.622770] R10: 0000000000000000 R11: 0000000000000124 R12: ffffffffffffffff
>>>> [ 57.626386] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
>>>> [ 57.630029] ? __cpuidle_text_start+0x8/0x8
>>>> [ 57.632209] do_idle+0x1bd/0x240
>>>> [ 57.633914] cpu_startup_entry+0x19/0x20
>>>> [ 57.635966] start_secondary+0x15f/0x1b0
>>>> [ 57.638021] secondary_startup_64+0xb6/0xc0
>>>> crash>
>>>>
>>>> But I can see it in the raw stack data:
>>>>
>>>> ffffba790008bdc0: 0000000000000001 0000000000000001
>>>> ffffba790008bdd0: common_interrupt+181 0000000000000000 <---- Why is it not in the backtrace?
>>>> ffffba790008bde0: 0000000000000000 0000000000000000
>>>> ffffba790008bdf0: 0000000000000000 0000000000000000
>>>> ffffba790008be00: asm_common_interrupt+30 0000000000000000
>>>> ffffba790008be10: 0000000000000000 0000000000000000
>>>> ffffba790008be20: ffffffffffffffff 0000000000000003
>>>> ffffba790008be30: 0000000000000003 0000000000000124
>>>> ffffba790008be40: 0000000000000000 0000000000000000
>>>> ffffba790008be50: 000000146ccbc1f0 default_idle
>>>> ffffba790008be60: 0000000000000001 000000000001a6de
>>>> ffffba790008be70: 0000000000000087 0000000000000003
>>>> ffffba790008be80: ffffffffffffffff default_idle+37
>>>> ffffba790008be90: 0000000000000010 0000000000000246
>>>> ffffba790008bea0: ffffba790008beb8 0000000000000018
>>>> ffffba790008beb0: 0000000000000003 0000000000000003
>>>> ffffba790008bec0: __cpu_online_mask ffffffffffffffff
>>>> ffffba790008bed0: 0000000000000000 do_idle+445
>>>> ffffba790008bee0: ffffffffffffff04 c4a71aecbd899c00
>>>> ffffba790008bef0: 000000000000008f 0000000000000000
>>>> ffffba790008bf00: 0000000000000000 0000000000000000
>>>> ffffba790008bf10: 0000000000000000 0000000000000000
>>>> ffffba790008bf20: cpu_startup_entry+25 ffffba790008bf38
>>>> ffffba790008bf30: start_secondary+351 c4a71aecbd899c00
>>>> ffffba790008bf40: 0000000000000000 0000000000000000
>>>> ffffba790008bf50: secondary_startup_64+182 0000000000000000
>>>>
>>>>
>>>> And for RHEL9, this trial patch looks good, how does this work for
>>>> your vmcore?
>>>> https://github.com/k-hagio/crash/commit/0719360a2b76b3c03b1f7ad06caf2ce14e2bfcbf.patch <https://github.com/k-hagio/crash/commit/0719360a2b76b3c03b1f7ad06caf2ce14e2bfcbf.patch>
>>>>
>>>> Unfortunately, it still doesn't work(with the above patch):
>>>>
>>>> crash> bt
>>>> PID: 0 TASK: ffff9e7a47e32f00 CPU: 3 COMMAND: "swapper/3"
>>>> #0 [ffffba7900118bb8] machine_kexec at ffffffff87e5c2c7
>>>> #1 [ffffba7900118c08] __crash_kexec at ffffffff87f9500d
>>>> #2 [ffffba7900118cd0] panic at ffffffff87edfff9
>>>> #3 [ffffba7900118d50] sysrq_handle_crash at ffffffff883ce2c1
>>>> #4 [ffffba7900118d58] __handle_sysrq.cold.15 at ffffffff883ceb56
>>>> #5 [ffffba7900118d88] sysrq_filter at ffffffff883ce9a2
>>>> #6 [ffffba7900118dc0] input_to_handler at ffffffff884fb0bf
>>>> #7 [ffffba7900118df8] input_pass_values at ffffffff884fc1b7
>>>> #8 [ffffba7900118e20] input_handle_event at ffffffff884fe278
>>>> #9 [ffffba7900118e50] input_event at ffffffff884fe74b
>>>> #10 [ffffba7900118e88] atkbd_interrupt at ffffffff88504e2f
>>>> #11 [ffffba7900118ee0] serio_interrupt at ffffffff884f7516
>>>> #12 [ffffba7900118f10] i8042_interrupt at ffffffff884f8b04
>>>> #13 [ffffba7900118f50] __handle_irq_event_percpu at ffffffff87f51430
>>>> #14 [ffffba7900118f90] handle_irq_event_percpu at ffffffff87f51590
>>>> #15 [ffffba7900118fb8] handle_irq_event at ffffffff87f51616
>>>> #16 [ffffba7900118fd8] handle_edge_irq at ffffffff87f559f2
>>>> #17 [ffffba7900118ff0] asm_call_on_stack at ffffffff88800fa2
>>>> --- <IRQ stack> ---
>>>> RIP: 000000000001a6de RSP: ffffffffffffffff RFLAGS: 00000003
>>>> RAX: 0000000000000124 RBX: 0000000000000000 RCX: 0000000000000000
>>>> RDX: 0000000000000000 RSI: 000000146ccbc1f0 RDI: ffffffff8876ffa0
>>>> RBP: 0000000000000000 R8: 0000000000000003 R9: 0000000000000003
>>>> R10: ffffffffffffffff R11: 0000000000000000 R12: ffffffff88800c1e
>>>> R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
>>>> ORIG_RAX: 0000000000000001 CS: 0087 SS: ffffffff8876ffc5
>>>> bt: WARNING: possibly bogus exception frame
>>>> crash>
>>>>
>>>> Thanks.
>>>> Lianbo
>>>>
>>>> Thanks,
>>>> Kazu
>>>>
>>>> >>
>>>> >> crash> bt
>>>> >> PID: 0 TASK: ffff9e7a47e32f00 CPU: 3 COMMAND: "swapper/3"
>>>> >> #0 [ffffba7900118bb8] machine_kexec at ffffffff87e5c2c7
>>>> >> #1 [ffffba7900118c08] __crash_kexec at ffffffff87f9500d
>>>> >> #2 [ffffba7900118cd0] panic at ffffffff87edfff9
>>>> >> #3 [ffffba7900118d50] sysrq_handle_crash at ffffffff883ce2c1
>>>> >> #4 [ffffba7900118d58] __handle_sysrq.cold.15 at ffffffff883ceb56
>>>> >> #5 [ffffba7900118d88] sysrq_filter at ffffffff883ce9a2
>>>> >> #6 [ffffba7900118dc0] input_to_handler at ffffffff884fb0bf
>>>> >> #7 [ffffba7900118df8] input_pass_values at ffffffff884fc1b7
>>>> >> #8 [ffffba7900118e20] input_handle_event at ffffffff884fe278
>>>> >> #9 [ffffba7900118e50] input_event at ffffffff884fe74b
>>>> >> #10 [ffffba7900118e88] atkbd_interrupt at ffffffff88504e2f
>>>> >> #11 [ffffba7900118ee0] serio_interrupt at ffffffff884f7516
>>>> >> #12 [ffffba7900118f10] i8042_interrupt at ffffffff884f8b04
>>>> >> #13 [ffffba7900118f50] __handle_irq_event_percpu at ffffffff87f51430
>>>> >> #14 [ffffba7900118f90] handle_irq_event_percpu at ffffffff87f51590
>>>> >> #15 [ffffba7900118fb8] handle_irq_event at ffffffff87f51616
>>>> >> #16 [ffffba7900118fd8] handle_edge_irq at ffffffff87f559f2
>>>> >> #17 [ffffba7900118ff0] asm_call_on_stack at ffffffff88800fa2
>>>> >> --- <IRQ stack> ---
>>>> >> #18 [ffffba790008be08] asm_call_on_stack at ffffffff88800fa2
>>>> >> [exception RIP: default_idle+37]
>>>> >> RIP: ffffffff8876ffc5 RSP: ffffba790008beb8 RFLAGS: 00000246
>>>> >> RAX: ffffffff8876ffa0 RBX: 0000000000000003 RCX: 0000000000000001
>>>> >> RDX: 000000000001a6de RSI: 0000000000000087 RDI: 0000000000000003
>>>> >> RBP: 0000000000000003 R8: 000000146ccbc1f0 R9: 0000000000000000
>>>> >> R10: 0000000000000000 R11: 0000000000000124 R12: ffffffffffffffff
>>>> >> R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
>>>> >> ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
>>>> >> #19 [ffffba790008bed8] do_idle at ffffffff87f1982d
>>>> >> #20 [ffffba790008bf20] cpu_startup_entry at ffffffff87f19a29
>>>> >> #21 [ffffba790008bf30] start_secondary at ffffffff87e5118f
>>>> >> #22 [ffffba790008bf50] secondary_startup_64 at ffffffff87e000e6
>>>> >> crash>>> Let me debug it more and to see what happened.
>>>> >>
>>>> >> What is printed as the panic message in "log" command?
>>>> >>
>>>> >>
>>>> >> The kernel panic message:
>>>> >> [ 57.542538] Kernel panic - not syncing: sysrq triggered crash
>>>> >> [ 57.545512] CPU: 3 PID: 0 Comm: swapper/3 Kdump: loaded Not tainted 5.7.0+ #7
>>>> >> [ 57.549160] Hardware name: Red Hat KVM/RHEL-AV, BIOS 0.0.0 02/06/2015
>>>> >> [ 57.552655] Call Trace:
>>>> >> [ 57.553994] <IRQ>
>>>> >> [ 57.555143] dump_stack+0x57/0x70
>>>> >> [ 57.556930] panic+0xfb/0x2d7
>>>> >> [ 57.558516] ? printk+0x58/0x6f
>>>> >> [ 57.560194] sysrq_handle_crash+0x11/0x20
>>>> >> [ 57.562371] __handle_sysrq.cold.15+0x48/0x102
>>>> >> [ 57.564663] sysrq_filter+0x312/0x3c0
>>>> >> [ 57.566579] input_to_handler+0x4f/0xf0
>>>> >> [ 57.568583] input_pass_values.part.7+0x117/0x130
>>>> >> [ 57.571188] input_handle_event+0x138/0x5c0
>>>> >> [ 57.573380] input_event+0x4b/0x70
>>>> >> [ 57.575173] atkbd_interrupt+0x61f/0x6a0
>>>> >> [ 57.577228] serio_interrupt+0x46/0x90
>>>> >> [ 57.579192] i8042_interrupt+0x154/0x240
>>>> >> [ 57.581328] __handle_irq_event_percpu+0x40/0x170
>>>> >> [ 57.583788] handle_irq_event_percpu+0x30/0x80
>>>> >> [ 57.586081] handle_irq_event+0x36/0x53
>>>> >> [ 57.588071] handle_edge_irq+0x82/0x190
>>>> >> [ 57.590096] asm_call_on_stack+0x12/0x20
>>>> >> [ 57.592142] </IRQ>
>>>> >> [ 57.593264] common_interrupt+0xb5/0x140
>>>> >> [ 57.595311] asm_common_interrupt+0x1e/0x40
>>>> >> [ 57.597474] RIP: 0010:default_idle+0x25/0x150
>>>> >> [ 57.599735] Code: cc cc cc cc cc 66 66 66 66 90 41 55 41 54 55 65 8b 2d af 23 8a 77 53 66 66 66 66 90 e9 07 00 00 00 0f 00 2d 9f 37 49 00 fb f4 <65> 8b 2d 94 23 8a 77 66 66 66 66 90 5b 5d 41 5c 41 5d c3 65 8b 05
>>>> >> [ 57.609196] RSP: 0018:ffffba790008beb8 EFLAGS: 00000246
>>>> >> [ 57.611875] RAX: ffffffff8876ffa0 RBX: 0000000000000003 RCX: 0000000000000001
>>>> >> [ 57.615506] RDX: 000000000001a6de RSI: 0000000000000087 RDI: 0000000000000003
>>>> >> [ 57.619142] RBP: 0000000000000003 R08: 000000146ccbc1f0 R09: 0000000000000000
>>>> >> [ 57.622770] R10: 0000000000000000 R11: 0000000000000124 R12: ffffffffffffffff
>>>> >> [ 57.626386] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
>>>> >> [ 57.630029] ? __cpuidle_text_start+0x8/0x8
>>>> >> [ 57.632209] do_idle+0x1bd/0x240
>>>> >> [ 57.633914] cpu_startup_entry+0x19/0x20
>>>> >> [ 57.635966] start_secondary+0x15f/0x1b0
>>>> >> [ 57.638021] secondary_startup_64+0xb6/0xc0
>>>> >> Thanks.
>>>> >> Lianbo
>>>> >>
>>>>