On 2023/02/09 21:35, lijiang wrote:> On Tue, Feb 7, 2023 at 6:08 PM lijiang <lijiang@redhat.com <mailto:lijiang@redhat.com>> wrote:
>
>         more important thing is how we determine the irq_eframe_link.
>
>     The following patch can work on upstream kernel vmcore and RHEL9 vmcore.
>     Maybe we can check the symbols asm_common_interrupt and asm_call_on_stack as below:
>
>     diff --git a/x86_64.c b/x86_64.c
>     index 7a5d6f050c89..62036f71f632 100644
>     --- a/x86_64.c
>     +++ b/x86_64.c
>     @@ -3938,6 +3938,11 @@ in_exception_stack:
>               if (irq_eframe) {
>                       bt->flags |= BT_EXCEPTION_FRAME;
>                       i = (irq_eframe - bt->stackbase)/sizeof(ulong);
>     + if (symbol_exists("asm_common_interrupt")) {
>     + i -= 1;
>     + up = (ulong *)(&bt->stackbuf[i*sizeof(ulong)]);
>     + bt->instptr = *up;
>     + }
>                       x86_64_print_stack_entry(bt, ofp, level, i, bt->instptr);
>                       bt->flags &= ~(ulonglong)BT_EXCEPTION_FRAME;
>                       cs = x86_64_exception_frame(EFRAME_PRINT|EFRAME_CS, 0,
>     @@ -6521,6 +6526,16 @@ x86_64_irq_eframe_link_init(void)
>        else
>        return;
>
>     + if (symbol_exists("asm_common_interrupt") && !symbol_exists("asm_call_on_stack")) {
>     + machdep->machspec->irq_eframe_link =-32;
>     + return;
>     + }
>     +
>     + if (symbol_exists("asm_common_interrupt") && symbol_exists("asm_call_on_stack")) {
>     + machdep->machspec->irq_eframe_link =-56;
>     + return;
>     + }
>     +
>        if (THIS_KERNEL_VERSION < LINUX(2,6,9))
>        return;
>
>
> Do you have any other comments about the above changes? Or still looking for a better solution, any thoughts? Kazu.

Thank you for the information.  I've looked for a way to follow
kernel changes, but it would be hard with the current unwinder.

So for now, let's go with this?

+       if (symbol_exists("asm_common_interrupt")) {
+               if (symbol_exists("asm_call_on_stack"))
+                       machdep->machspec->irq_eframe_link = -64;
+               else
+                       machdep->machspec->irq_eframe_link = -32;
+               return;
+       }

Probably the actual irq_eframe_link for your kernel is -64, I
think it's adjusted in x86_64_irq_eframe_link().  Please check

"help -m".  e.g. a 5.8 vmcore on hand:

crash-dev> bt
...
crash-dev> help -m | grep irq_eframe
           irq_eframe_link: -64


The following is the reason why the irq_eframe_link is -32 on
RHEL9.1 (5.14.0-162.6.1.el9_1.x86_64).

crash-dev> dis asm_common_interrupt
0xffffffff87a00cc0 <asm_common_interrupt>:      clac
0xffffffff87a00cc3 <asm_common_interrupt+3>:    cld
0xffffffff87a00cc4 <asm_common_interrupt+4>:    call   0xffffffff87a012b0 <error_entry> [1]
0xffffffff87a00cc9 <asm_common_interrupt+9>:    mov    %rax,%rsp                        [2]
0xffffffff87a00ccc <asm_common_interrupt+12>:   mov    %rsp,%rdi
0xffffffff87a00ccf <asm_common_interrupt+15>:   mov    0x78(%rsp),%rsi
0xffffffff87a00cd4 <asm_common_interrupt+20>:   movq   $0xffffffffffffffff,0x78(%rsp)
0xffffffff87a00cdd <asm_common_interrupt+29>:   call   0xffffffff878d8370 <common_interrupt> [3]
0xffffffff87a00ce2 <asm_common_interrupt+34>:   jmp    0xffffffff87a013d0 <error_return>

The asm_common_interrupt calls the error_entry first [1], which pushes
registers (pt_regs) to the stack and returns the address:

         leaq    8(%rsp), %rax                   /* return pt_regs pointer */

Then it writes the address in %rax to %rsp [2].  After that, it calls
the common_interrupt [3], so now *(%rsp) = asm_common_interrupt+34 and
*(%rsp+8) = pt_regs.

crash-dev> dis common_interrupt
0xffffffff878d8370 <common_interrupt>:          push   %r13       [4]
0xffffffff878d8372 <common_interrupt+2>:        push   %r12       [4]
0xffffffff878d8374 <common_interrupt+4>:        mov    %rsi,%r12
0xffffffff878d8377 <common_interrupt+7>:        push   %rbp       [4]
0xffffffff878d8378 <common_interrupt+8>:        mov    %rdi,%rbp
0xffffffff878d837b <common_interrupt+11>:       movzbl %r12b,%r12d
0xffffffff878d837f <common_interrupt+15>:       call   0xffffffff878dab00 <irqentry_enter>
0xffffffff878d8384 <common_interrupt+20>:       mov    %eax,%r13d
0xffffffff878d8387 <common_interrupt+23>:       movb   $0x1,%gs:0x787522f3(%rip)        # 0x2a682
0xffffffff878d838f <common_interrupt+31>:       testb  $0x3,0x88(%rbp)
0xffffffff878d8396 <common_interrupt+38>:       jne    0xffffffff878d83a3 <common_interrupt+51>  [5]
0xffffffff878d8398 <common_interrupt+40>:       mov    %gs:0x78740b65(%rip),%al        # 0x18f04
0xffffffff878d839f <common_interrupt+47>:       test   %al,%al
0xffffffff878d83a1 <common_interrupt+49>:       je     0xffffffff878d83c8 <common_interrupt+88>  [5]
0xffffffff878d83a3 <common_interrupt+51>:       call   0xffffffff86efb8a0 <irq_enter_rcu>
0xffffffff878d83a8 <common_interrupt+56>:       mov    %r12d,%esi
0xffffffff878d83ab <common_interrupt+59>:       mov    %rbp,%rdi
0xffffffff878d83ae <common_interrupt+62>:       call   0xffffffff86e26630 <__common_interrupt>
0xffffffff878d83b3 <common_interrupt+67>:       call   0xffffffff86efb920 <irq_exit_rcu>
0xffffffff878d83b8 <common_interrupt+72>:       mov    %r13d,%esi
0xffffffff878d83bb <common_interrupt+75>:       mov    %rbp,%rdi
0xffffffff878d83be <common_interrupt+78>:       pop    %rbp
0xffffffff878d83bf <common_interrupt+79>:       pop    %r12
0xffffffff878d83c1 <common_interrupt+81>:       pop    %r13
0xffffffff878d83c3 <common_interrupt+83>:       jmp    0xffffffff878dab80 <irqentry_exit>
0xffffffff878d83c8 <common_interrupt+88>:       movb   $0x1,%gs:0x78740b34(%rip)        # 0x18f04 [6]
0xffffffff878d83d0 <common_interrupt+96>:       mov    %gs:0x78740b30(%rip),%r11        # 0x18f08 [7]
0xffffffff878d83d8 <common_interrupt+104>:      mov    %rsp,(%r11)                                [7]
0xffffffff878d83db <common_interrupt+107>:      mov    %r11,%rsp                                  [7]
0xffffffff878d83de <common_interrupt+110>:      call   0xffffffff86efb8a0 <irq_enter_rcu>
0xffffffff878d83e3 <common_interrupt+115>:      mov    %rbp,%rdi
0xffffffff878d83e6 <common_interrupt+118>:      mov    %r12d,%esi
0xffffffff878d83e9 <common_interrupt+121>:      call   0xffffffff86e26630 <__common_interrupt>
0xffffffff878d83ee <common_interrupt+126>:      call   0xffffffff86efb920 <irq_exit_rcu>
0xffffffff878d83f3 <common_interrupt+131>:      pop    %rsp
0xffffffff878d83f4 <common_interrupt+132>:      movb   $0x0,%gs:0x78740b08(%rip)        # 0x18f04 [8]
0xffffffff878d83fc <common_interrupt+140>:      mov    %r13d,%esi
0xffffffff878d83ff <common_interrupt+143>:      mov    %rbp,%rdi
0xffffffff878d8402 <common_interrupt+146>:      pop    %rbp
0xffffffff878d8403 <common_interrupt+147>:      pop    %r12
0xffffffff878d8405 <common_interrupt+149>:      pop    %r13
0xffffffff878d8407 <common_interrupt+151>:      jmp    0xffffffff878dab80 <irqentry_exit>

The common_interrupt pushes 3 registers [4], so now *(%rsp+32) = pt_regs.
[5]s look the following condition, the else block goes to [6].  Then,
writes the current %rsp to the address in hardirq_stack_ptr, which
points to the end of the irq stack, and switches %rsp to it [7].

#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...)  \
{                                                                       \
         /*                                                              \
          * User mode entry and interrupt on the irq stack do not        \
          * switch stacks. If from user mode the task stack is empty.    \
          */                                                             \
         if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {  \ [5]
                 irq_enter_rcu();                                        \
                 func(c_args);                                           \
                 irq_exit_rcu();                                         \
         } else {                                                        \
                 /*                                                      \
                  * Mark the irq stack inuse _before_ and unmark _after_ \
                  * switching stacks. Interrupts are disabled in both    \
                  * places. Invoke the stack switch macro with the call  \
                  * sequence which matches the above direct invocation.  \
                  */                                                     \
                 __this_cpu_write(hardirq_stack_inuse, true);            \ [6]
                 call_on_irqstack(func, asm_call, constr);               \ [7]
                 __this_cpu_write(hardirq_stack_inuse, false);           \ [8]
         }                                                               \
}

crash-dev> sym -l
...
18f04 (D) hardirq_stack_inuse
18f08 (D) hardirq_stack_ptr
...
crash-dev> p hardirq_stack_ptr:a
per_cpu(hardirq_stack_ptr, 0) = $2 = (void *) 0xffffa5ab40003ff8
per_cpu(hardirq_stack_ptr, 1) = $3 = (void *) 0xffffa5ab400ecff8
per_cpu(hardirq_stack_ptr, 2) = $4 = (void *) 0xffffa5ab40118ff8
per_cpu(hardirq_stack_ptr, 3) = $5 = (void *) 0xffffa5ab40144ff8


On the other hand, the current crash reads the saved %rsp here:

     up = (ulong *)(&bt->stackbuf[stacktop - bt->stackbase]);
     up -= 1;     <<<-- the same as hardirq_stack_ptr, the saved %rsp
     irq_eframe = rsp = bt->stkptr = x86_64_irq_eframe_link(*up, bt, ofp);
     up -= 1;
     bt->instptr = *up;

So the irq_eframe_link should be -32, because *(%rsp+32) is pt_regs.
And the return address from the common_interrupt is *(%rsp+24),
the following code gets it.

     if (symbol_exists("asm_common_interrupt")) {
             i -= 1;
             up = (ulong *)(&bt->stackbuf[i*sizeof(ulong)]);
             bt->instptr = *up;
     }


With the same way, I think the irq_eframe_link for kernel 5.8 to
5.11 will be -64.

Thanks,
Kazu