11:57 p.m.
Changes in v5:
* add a workaround against an exceptional case of backtracing
(See "FIXME" in arm64_back_trace_cmd())
* update comments in arm64_unwind_frame()
* add patch[4/4]
Changes in v4:
* use arm64_on_irq_stack() to check whether or not we are on IRQ stack.
mistakendly used 'flags & IRQ_STACKS'.
* fix a bug at critical timing around switching stacks in arm64_unwind_frame().
We need to take are of a possibility that we get back to process stack,
but fp still points to IRQ stack.
* add patch[2/3]
This should be applied even withtout patch[1/3]
* add patch[3/3]
applying this patch would be a discussion.
AKASHI Takahiro (4):
arm64: more improvement of bt -f
arm64: find a correct starting stackframe at bt
arm64: correct a PC shown in bt
arm64: add VHE support
arm64.c | 603 ++++++++++++++++++++++++++++++++++++++++++++--------------------
defs.h | 6 +
2 files changed, 423 insertions(+), 186 deletions(-)
--
2.9.0
11:57 p.m.
New subject: [PATCH v5 1/4] arm64: more improvement of bt -f
Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
---
arm64.c | 546 ++++++++++++++++++++++++++++++++++++++++++++--------------------
defs.h | 6 +
2 files changed, 386 insertions(+), 166 deletions(-)
diff --git a/arm64.c b/arm64.c
index 06676d1..0ac5894 100644
--- a/arm64.c
+++ b/arm64.c
@@ -42,18 +42,18 @@ static void arm64_irq_stack_init(void);
static void arm64_stackframe_init(void);
static int arm64_eframe_search(struct bt_info *);
static int arm64_is_kernel_exception_frame(struct bt_info *, ulong);
-static int arm64_in_exception_text(ulong);
+static int arm64_in_exp_entry(ulong);
static void arm64_back_trace_cmd(struct bt_info *);
static void arm64_print_text_symbols(struct bt_info *, struct arm64_stackframe *, FILE
*);
static int arm64_print_stackframe_entry(struct bt_info *, int, struct arm64_stackframe *,
FILE *);
-static void arm64_display_full_frame(struct bt_info *, ulong);
-static int arm64_unwind_frame(struct bt_info *, struct arm64_stackframe *);
+static void arm64_display_full_frame(struct bt_info *, struct arm64_stackframe *, struct
arm64_stackframe *);
+static int arm64_unwind_frame(struct bt_info *, struct arm64_stackframe *, FILE *);
static int arm64_get_dumpfile_stackframe(struct bt_info *, struct arm64_stackframe *);
static int arm64_in_kdump_text(struct bt_info *, struct arm64_stackframe *);
static int arm64_in_kdump_text_on_irq_stack(struct bt_info *);
-static int arm64_switch_stack(struct bt_info *, struct arm64_stackframe *, FILE *);
static int arm64_get_stackframe(struct bt_info *, struct arm64_stackframe *);
static void arm64_get_stack_frame(struct bt_info *, ulong *, ulong *);
+static void arm64_gen_hidden_frame(struct bt_info *bt, ulong, struct arm64_stackframe
*);
static void arm64_print_exception_frame(struct bt_info *, ulong, int, FILE *);
static void arm64_do_bt_reference_check(struct bt_info *, ulong, char *);
static int arm64_translate_pte(ulong, void *, ulonglong);
@@ -580,6 +580,10 @@ arm64_dump_machdep_table(ulong arg)
fprintf(fp, " __exception_text_end: %lx\n", ms->__exception_text_end);
fprintf(fp, " __irqentry_text_start: %lx\n", ms->__irqentry_text_start);
fprintf(fp, " __irqentry_text_end: %lx\n", ms->__irqentry_text_end);
+ fprintf(fp, " exp_entry1_start: %lx\n", ms->exp_entry1_start);
+ fprintf(fp, " exp_entry1_end: %lx\n", ms->exp_entry1_end);
+ fprintf(fp, " exp_entry2_start: %lx\n", ms->exp_entry2_start);
+ fprintf(fp, " exp_entry2_end: %lx\n", ms->exp_entry2_end);
fprintf(fp, " panic_task_regs: %lx\n", (ulong)ms->panic_task_regs);
fprintf(fp, " PTE_PROT_NONE: %lx\n", ms->PTE_PROT_NONE);
fprintf(fp, " PTE_FILE: ");
@@ -1286,6 +1290,15 @@ arm64_stackframe_init(void)
machdep->machspec->__irqentry_text_start = sp1->value;
machdep->machspec->__irqentry_text_end = sp2->value;
}
+ if ((sp1 = kernel_symbol_search("vectors")) &&
+ (sp1n = kernel_symbol_search("cpu_switch_to")) &&
+ (sp2 = kernel_symbol_search("ret_fast_syscall")) &&
+ (sp2n = kernel_symbol_search("sys_rt_sigreturn_wrapper"))) {
+ machdep->machspec->exp_entry1_start = sp1->value;
+ machdep->machspec->exp_entry1_end = sp1n->value;
+ machdep->machspec->exp_entry2_start = sp2->value;
+ machdep->machspec->exp_entry2_end = sp2n->value;
+ }
if ((sp1 = kernel_symbol_search("crash_kexec")) &&
(sp1n = next_symbol(NULL, sp1)) &&
@@ -1472,25 +1485,20 @@ arm64_eframe_search(struct bt_info *bt)
}
static int
-arm64_in_exception_text(ulong ptr)
+arm64_in_exp_entry(ulong addr)
{
- struct machine_specific *ms = machdep->machspec;
+ struct machine_specific *ms;
- if ((ptr >= ms->__exception_text_start) &&
- (ptr < ms->__exception_text_end))
+ ms = machdep->machspec;
+ if ((ms->exp_entry1_start <= addr) && (addr < ms->exp_entry1_end))
return TRUE;
-
- if (ms->__irqentry_text_start && ms->__irqentry_text_end &&
- ((ptr >= ms->__irqentry_text_start) &&
- (ptr < ms->__irqentry_text_end)))
+ if ((ms->exp_entry2_start <= addr) && (addr < ms->exp_entry2_end))
return TRUE;
-
return FALSE;
}
#define BACKTRACE_CONTINUE (1)
#define BACKTRACE_COMPLETE_KERNEL (2)
-#define BACKTRACE_COMPLETE_USER (3)
static int
arm64_print_stackframe_entry(struct bt_info *bt, int level, struct arm64_stackframe
*frame, FILE *ofp)
@@ -1511,11 +1519,6 @@ arm64_print_stackframe_entry(struct bt_info *bt, int level, struct
arm64_stackfr
value_to_symstr(frame->pc, buf, bt->radix);
}
- if ((bt->flags & BT_FULL) && level) {
- arm64_display_full_frame(bt, frame->fp);
- bt->frameptr = frame->fp;
- }
-
fprintf(ofp, "%s#%d [%8lx] %s at %lx", level < 10 ? " " :
"", level,
frame->fp ? frame->fp : bt->stacktop - USER_EFRAME_OFFSET,
name_plus_offset ? name_plus_offset : name, frame->pc);
@@ -1534,7 +1537,8 @@ arm64_print_stackframe_entry(struct bt_info *bt, int level, struct
arm64_stackfr
fprintf(ofp, " %s\n", buf);
}
- if (STREQ(name, "start_kernel") || STREQ(name,
"secondary_start_kernel") ||
+ if (STREQ(name, "start_kernel") ||
+ STREQ(name, "secondary_start_kernel") ||
STREQ(name, "kthread") || STREQ(name, "kthreadd"))
return BACKTRACE_COMPLETE_KERNEL;
@@ -1542,46 +1546,185 @@ arm64_print_stackframe_entry(struct bt_info *bt, int level,
struct arm64_stackfr
}
static void
-arm64_display_full_frame(struct bt_info *bt, ulong sp)
+arm64_display_full_frame(struct bt_info *bt, struct arm64_stackframe *cur,
+ struct arm64_stackframe *next)
{
+ struct machine_specific *ms;
+ ulong next_fp, stackbase;
+ char *stackbuf;
int i, u_idx;
ulong *up;
ulong words, addr;
char buf[BUFSIZE];
- if (bt->frameptr == sp)
- return;
+ stackbase = bt->stackbase;
+ stackbuf = bt->stackbuf;
+ ms = machdep->machspec;
- if (!INSTACK(sp, bt) || !INSTACK(bt->frameptr, bt)) {
- if (sp == 0)
- sp = bt->stacktop - USER_EFRAME_OFFSET;
- else
- return;
- }
+ /* Calc next fp for dump */
+ if (next->fp == 0)
+ /* last stackframe on kernel tack */
+ next_fp = bt->stacktop - 0x10;
+ else if (!INSTACK(cur->sp, bt)) {
+ /* We have just switched over stacks */
+ next_fp = ms->irq_stacks[bt->tc->processor]
+ + ms->irq_stack_size - 0x10;
+
+ /*
+ * We are already buffering a process stack.
+ * So use an old buffer for IRQ stack.
+ */
+ stackbase = ms->irq_stacks[bt->tc->processor];
+ stackbuf = ms->irq_stackbuf;
+ } else
+ next_fp = next->fp;
+
+ if (CRASHDEBUG(1))
+ fprintf(fp, " frame <%016lx:%016lx>\n", cur->fp, next_fp);
- words = (sp - bt->frameptr) / sizeof(ulong);
+ /* Check here because we want to see a debug message above. */
+ if (!(bt->flags & BT_FULL))
+ return;
+ if (next_fp <= cur->fp)
+ return;
- addr = bt->frameptr;
- u_idx = (bt->frameptr - bt->stackbase)/sizeof(ulong);
+ /* Dump */
+ words = (next_fp - cur->fp) / sizeof(ulong);
+ addr = cur->fp;
+ u_idx = (cur->fp - stackbase)/sizeof(ulong);
for (i = 0; i < words; i++, u_idx++) {
if (!(i & 1))
fprintf(fp, "%s %lx: ", i ? "\n" : "", addr);
- up = (ulong *)(&bt->stackbuf[u_idx*sizeof(ulong)]);
+ up = (ulong *)(&stackbuf[u_idx*sizeof(ulong)]);
fprintf(fp, "%s ", format_stack_entry(bt, buf, *up, 0));
addr += sizeof(ulong);
}
fprintf(fp, "\n");
+
+ if (stackbuf == ms->irq_stackbuf)
+ FREEBUF(stackbuf);
}
-static int
-arm64_unwind_frame(struct bt_info *bt, struct arm64_stackframe *frame)
+/*
+ * The following figure shows how unwinding can be done.
+ * Here we assume that the callstack order is:
+ * #(X-1) ppc (previous PC)
+ * #X cpc (current PC)
+ * < #(X+ 1) epc (Exception entry) >
+ * #(X+1/2) npc (Next PC)
+ * #(X+2/3) Npc (One before Next)
+ * #(X+3/4) NNpc (One before 'Npc')
+ * and unwind frames from #X to #(X+1).
+ * When we add a faked frame for exception entry (exception frame)
+ * as #(X+1), the next frame for npc will be recognized as #(x+2).
+ *
+ * (1)Normal stackframe:
+ * +------+
+ * | pfp |
+ * | cpc |
+ * psp + +
+ * | |
+ * | |
+ * pfp +------+ <--- :prev stackframe = <pfp, psp, ppc>
+ * | cfp |
+ * | npc |
+ * csp + +
+ * | |
+ * | |
+ * cfp +------+ <--- :curr stackframe = <cfp, csp, cpc>
+ * | nfp | cfp = *pfp
+ * | Npc | csp = pfp + 0x10
+ * nsp + +
+ * | |
+ * | |
+ * nfp +------+ <--- :next stackframe = <nfp, nsp, npc>
+ * | |
+ *
+ * (2)Exception on the same (IRQ or process) stack:
+ * +------+
+ * | pfp |
+ * | cpc |
+ * psp + +
+ * | |
+ * | |
+ * pfp +------+ <--- :prev stackframe = <pfp, psp, ppc>
+ * | cfp |
+ * | npc |
+ * csp + +
+ * | |
+ * | |
+ * cfp +------+ <--- :curr stackframe = <cfp, csp, cpc>
+ * | nfp |
+ * | epc |
+ * + +
+ * | |
+ * | | faked(*)
+ * esp +------+ <--- :excp stackframe = <---, esp, epc
+ * | | esp = nsp - sizeof(pt_regs)
+ * | |
+ * | Npc | (*) If we didn't add this frame, the next frame
+ * | nfp | would be
+ * | nsp | <nfp, nfp + 0x10, epc>
+ * | npc | and the frame below for npc would be lost.
+ * nsp + +
+ * | |
+ * nfp +------+ <--- :task stackframe = <nfp, nsp, npc>
+ * | Nfp |
+ * | NNpc |
+ * Nsp + +
+ * | |
+ * Nfp +------+ <--- :task stackframe = <Nfp, Nsp, Npc>
+ * | NNfp |
+ *
+ * (3)Interrupt:
+ * +------+
+ * | cfp |
+ * | ipc |
+ * csp + +
+ * | |
+ * | |
+ * cfp +------+ <--- :curr stackframe = <cfp, csp, cpc>
+ * | ifp |
+ * | epc |
+ * isp + +
+ * | |
+ * | | (*)
+ * ifp +------+ <--- :irq stackframe = <ifp, isp, epc>
+ * | nfp | ifp == IRQ_STACK_PTR
+ * | esp | (*) Before the kernel enters an irq handler, frame
+ * top +------+ pointer moves to the top of IRQ stack.
+ * IRQ stack So we have to skip this frame in unwinding.
+ *
+ * faked
+ * esp +------+ <--- :excp stackframe = <---, esp, epc>
+ * | | esp = nsp - sizeof(pt_regs)
+ * | |
+ * | Npc |
+ * | nfp |
+ * | nsp |
+ * | npc |
+ * nsp + +
+ * | |
+ * nfp +------+ <--- :task stackframe = <nfp, nsp, npc>
+ * | Nfp |
+ * | NNpc |
+ * Nsp + +
+ * | |
+ * Nfp +------+ <--- :task stackframe = <Nfp, Nsp, Npc>
+ * | NNfp |
+ */
+
+static struct arm64_stackframe ext_frame;
+
+static int
+arm64_unwind_frame(struct bt_info *bt, struct arm64_stackframe *frame,
+ FILE *ofp)
{
unsigned long high, low, fp;
unsigned long stack_mask;
- unsigned long irq_stack_ptr, orig_sp;
- struct arm64_pt_regs *ptregs;
+ unsigned long irq_stack_ptr;
struct machine_specific *ms;
stack_mask = (unsigned long)(ARM64_STACK_SIZE) - 1;
@@ -1593,54 +1736,137 @@ arm64_unwind_frame(struct bt_info *bt, struct arm64_stackframe
*frame)
if (fp < low || fp > high || fp & 0xf)
return FALSE;
- frame->sp = fp + 0x10;
- frame->fp = GET_STACK_ULONG(fp);
+ if (CRASHDEBUG(1))
+ fprintf(ofp, " cur fp:%016lx sp:%016lx pc:%016lx\n",
+ frame->fp, frame->sp, frame->pc);
+
+ if (ext_frame.pc) {
+ /*
+ * The previous frame was a dummy for exception entry.
+ * So complement a missing (task) stackframe now.
+ */
+ frame->fp = ext_frame.fp;
+ frame->sp = ext_frame.sp;
+ frame->pc = ext_frame.pc;
+
+ ext_frame.pc = 0; /* back to normal unwinding */
+
+ goto unwind_done;
+ }
+
frame->pc = GET_STACK_ULONG(fp + 8);
+ if (!arm64_in_exp_entry(frame->pc)) {
+ /* (1) Normal stack frame */
- /*
- * The kernel's manner of determining the end of the IRQ stack:
- *
- * #define THREAD_SIZE 16384
- * #define THREAD_START_SP (THREAD_SIZE - 16)
- * #define IRQ_STACK_START_SP THREAD_START_SP
- * #define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) +
IRQ_STACK_START_SP)
- * #define IRQ_STACK_TO_TASK_STACK(ptr) (*((unsigned long *)((ptr) - 0x08)))
- *
- * irq_stack_ptr = IRQ_STACK_PTR(raw_smp_processor_id());
- * orig_sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); (pt_regs pointer on process
stack)
- */
- if (machdep->flags & IRQ_STACKS) {
- ms = machdep->machspec;
- irq_stack_ptr = ms->irq_stacks[bt->tc->processor] + ms->irq_stack_size -
16;
-
- if (frame->sp == irq_stack_ptr) {
- orig_sp = GET_STACK_ULONG(irq_stack_ptr - 8);
- arm64_set_process_stack(bt);
- if (INSTACK(orig_sp, bt) && (INSTACK(frame->fp, bt) || (frame->fp ==
0))) {
- ptregs = (struct arm64_pt_regs
*)&bt->stackbuf[(ulong)(STACK_OFFSET_TYPE(orig_sp))];
- frame->sp = orig_sp;
- frame->pc = ptregs->pc;
- bt->bptr = fp;
- if (CRASHDEBUG(1))
- error(INFO,
- "arm64_unwind_frame: switch stacks: fp: %lx sp: %lx pc: %lx\n",
- frame->fp, frame->sp, frame->pc);
+ frame->sp = fp + 0x10;
+ frame->fp = GET_STACK_ULONG(fp);
+ } else {
+ /*
+ * We are in exception entry code, and so
+ * - add a faked frame for exception entry, and
+ * - prepare for a stackframe hidden by exception
+ */
+
+ ext_frame.fp = GET_STACK_ULONG(fp);
+ /*
+ * Note:
+ * In the following code, we determine a stack pointer for
+ * exception entry based on ext_frame.fp because we have
+ * no way to know a ext_frame.sp.
+ * Fortunately, this will work fine for most functions
+ * in the kernel.
+ */
+ if (ext_frame.fp == 0) {
+ /*
+ * (2)
+ * Either on process stack or on IRQ stack,
+ * the next frame is the last one on process stack.
+ */
+
+ frame->sp = bt->stacktop
+ - sizeof(struct arm64_pt_regs) - 0x10;
+ frame->fp = frame->sp;
+ } else if (!arm64_on_irq_stack(bt->tc->processor, frame->sp)) {
+ /*
+ * (2)
+ * We are on process stack. Just add a faked frame
+ */
+
+ if (!arm64_on_irq_stack(bt->tc->processor, ext_frame.fp))
+ frame->sp = ext_frame.fp
+ - sizeof(struct arm64_pt_regs);
+ else {
+ /*
+ * FIXME: very exceptional case
+ * We are already back on process stack, but
+ * a saved frame pointer indicates that we are
+ * on IRQ stack. Unfortunately this can happen
+ * when some functions are called after
+ * an irq handler is done because irq_exit()
+ * doesn't restore a frame pointer (x29).
+ * Those functions include
+ * - do_notify_resume()
+ * - trace_hardirqs_off()
+ * - schedule()
+ *
+ * We have no perfect way to determine a true
+ * stack pointer value here.
+ * 0x20 is a stackframe size of schedule().
+ * Really ugly
+ */
+ frame->sp = frame->fp + 0x20;
+ fprintf(ofp, " (Next exception frame might be wrong)\n");
+ }
+
+ frame->fp = frame->sp;
+ } else {
+ /* We are on IRQ stack */
+
+ ms = machdep->machspec;
+ irq_stack_ptr = ms->irq_stacks[bt->tc->processor]
+ + ms->irq_stack_size - 0x20;
+ if (ext_frame.fp != irq_stack_ptr) {
+ /* (2) Just add a faked frame */
+
+ frame->sp = ext_frame.fp
+ - sizeof(struct arm64_pt_regs);
+ frame->fp = frame->sp;
} else {
- error(WARNING,
- "arm64_unwind_frame: on IRQ stack: oriq_sp: %lx%s fp: %lx%s\n",
- orig_sp, INSTACK(orig_sp, bt) ? "" : " (?)",
- frame->fp, INSTACK(frame->fp, bt) ? "" : " (?)");
- return FALSE;
+ /*
+ * (3)
+ * Switch from IRQ stack to process stack
+ */
+
+ frame->sp = GET_STACK_ULONG(irq_stack_ptr + 8);
+ frame->fp = frame->sp;
+
+ /*
+ * Keep a buffer for a while until
+ * displaying the last frame on IRQ stack
+ * at next arm64_print_stackframe_entry()
+ */
+ if (bt->flags | BT_FULL)
+ ms->irq_stackbuf = bt->stackbuf;
+
+ arm64_set_process_stack(bt);
}
}
+
+ /* prepare for a stackframe hidden by exception */
+ arm64_gen_hidden_frame(bt, frame->sp, &ext_frame);
}
+unwind_done:
+ if (CRASHDEBUG(1))
+ fprintf(ofp, " nxt fp:%016lx sp:%016lx pc:%016lx\n",
+ frame->fp, frame->sp, frame->pc);
+
return TRUE;
}
-/*
+/*
* A layout of a stack frame in a function looks like:
- *
+ *
* stack grows to lower addresses.
* /|\
* |
@@ -1658,7 +1884,7 @@ arm64_unwind_frame(struct bt_info *bt, struct arm64_stackframe
*frame)
* | vars |
* old fp +------+
* | |
- *
+ *
* - On function entry, sp is decremented down to new fp.
*
* - and old fp and sp are saved into this stack frame.
@@ -1680,13 +1906,13 @@ arm64_unwind_frame(struct bt_info *bt, struct arm64_stackframe
*frame)
* sp shows "callee's static local variables", old fp and sp.
*
* Diagram and explanation courtesy of Takahiro Akashi
- */
+ */
static void
arm64_back_trace_cmd(struct bt_info *bt)
{
- struct arm64_stackframe stackframe;
- int level;
+ struct arm64_stackframe stackframe, cur_frame;
+ int level, mode;
ulong exception_frame;
FILE *ofp;
@@ -1708,17 +1934,7 @@ arm64_back_trace_cmd(struct bt_info *bt)
stackframe.fp = GET_STACK_ULONG(bt->bptr - 8);
stackframe.pc = GET_STACK_ULONG(bt->bptr);
stackframe.sp = bt->bptr + 8;
- bt->frameptr = stackframe.sp;
- } else if (bt->hp && bt->hp->esp) {
- if (arm64_on_irq_stack(bt->tc->processor, bt->hp->esp)) {
- arm64_set_irq_stack(bt);
- bt->flags |= BT_IRQSTACK;
- }
- stackframe.fp = GET_STACK_ULONG(bt->hp->esp - 8);
- stackframe.pc = bt->hp->eip ?
- bt->hp->eip : GET_STACK_ULONG(bt->hp->esp);
- stackframe.sp = bt->hp->esp + 8;
- bt->flags &= ~BT_REGS_NOT_FOUND;
+ bt->frameptr = stackframe.fp;
} else {
stackframe.sp = bt->stkptr;
stackframe.pc = bt->instptr;
@@ -1739,8 +1955,14 @@ arm64_back_trace_cmd(struct bt_info *bt)
return;
if (!(bt->flags & BT_KDUMP_ADJUST)) {
- if (bt->flags & BT_USER_SPACE)
- goto complete_user;
+ if (bt->flags & BT_USER_SPACE) {
+ exception_frame = bt->stacktop - USER_EFRAME_OFFSET;
+ arm64_print_exception_frame(bt, exception_frame,
+ USER_MODE, ofp);
+ fprintf(ofp, "\n #0 [user space]\n");
+
+ return;
+ }
if (DUMPFILE() && is_task_active(bt->task)) {
exception_frame = stackframe.fp - SIZE(pt_regs);
@@ -1750,53 +1972,64 @@ arm64_back_trace_cmd(struct bt_info *bt)
}
}
- level = exception_frame = 0;
- while (1) {
+ for (level = 0;; level++) {
bt->instptr = stackframe.pc;
- switch (arm64_print_stackframe_entry(bt, level, &stackframe, ofp))
- {
- case BACKTRACE_COMPLETE_KERNEL:
- return;
- case BACKTRACE_COMPLETE_USER:
- goto complete_user;
- case BACKTRACE_CONTINUE:
+ /*
+ * Show one-line stackframe info
+ */
+ if (arm64_print_stackframe_entry(bt, level, &stackframe, ofp)
+ == BACKTRACE_COMPLETE_KERNEL)
break;
- }
-
- if (exception_frame) {
- arm64_print_exception_frame(bt, exception_frame, KERNEL_MODE, ofp);
- exception_frame = 0;
- }
- if (!arm64_unwind_frame(bt, &stackframe))
+ cur_frame = stackframe;
+ if (!arm64_unwind_frame(bt, &stackframe, ofp))
break;
- if (arm64_in_exception_text(bt->instptr) && INSTACK(stackframe.fp, bt)) {
- if (!(bt->flags & BT_IRQSTACK) ||
- (((stackframe.sp + SIZE(pt_regs)) < bt->stacktop)))
- exception_frame = stackframe.fp - SIZE(pt_regs);
- }
+ /*
+ * Dump the contents of the current stackframe.
+ * We need to know the next stackframe to determine
+ * the dump range:
+ * <cur_frame.fp:stackframe.fp>
+ */
+ arm64_display_full_frame(bt, &cur_frame, &stackframe);
- if ((bt->flags & BT_IRQSTACK) &&
- !arm64_on_irq_stack(bt->tc->processor, stackframe.sp)) {
- bt->flags &= ~BT_IRQSTACK;
- if (arm64_switch_stack(bt, &stackframe, ofp) == USER_MODE)
- break;
- }
+ /*
+ * If we are in a normal stackframe, just continue,
+ * otherwise show an exception frame.
+ * Since exception entry code doesn't have a real
+ * stackframe, we fake a dummy frame here.
+ */
+ if (!arm64_in_exp_entry(stackframe.pc))
+ continue;
+ if (!INSTACK(cur_frame.sp, bt))
+ fprintf(ofp, "--- <IRQ stack> ---\n");
- level++;
- }
+ arm64_print_stackframe_entry(bt, ++level, &stackframe, ofp);
+ cur_frame = stackframe;
+ arm64_unwind_frame(bt, &stackframe, ofp);
- if (is_kernel_thread(bt->tc->task))
- return;
+ /*
+ * and don't show the contenxts. Instead,
+ * show an exception frame below
+ */
-complete_user:
- exception_frame = bt->stacktop - USER_EFRAME_OFFSET;
- arm64_print_exception_frame(bt, exception_frame, USER_MODE, ofp);
- if ((bt->flags & (BT_USER_SPACE|BT_KDUMP_ADJUST)) == BT_USER_SPACE)
- fprintf(ofp, " #0 [user space]\n");
+ if (!INSTACK(cur_frame.sp, bt)) {
+ /* This check is a safeguard. See unwind_frame(). */
+ error(WARNING,
+ "stack pointer for exception frame is wrong\n");
+ return;
+ }
+ mode = (stackframe.pc < machdep->machspec->userspace_top) ?
+ USER_MODE : KERNEL_MODE;
+ fprintf(ofp, "--- <Exception in %s> ---\n",
+ mode == KERNEL_MODE ? "kernel" : "user");
+ arm64_print_exception_frame(bt, cur_frame.sp, mode, ofp);
+
+ if (mode == USER_MODE)
+ break;
+ }
}
static void
@@ -1932,41 +2165,6 @@ arm64_in_kdump_text_on_irq_stack(struct bt_info *bt)
return FALSE;
}
-static int
-arm64_switch_stack(struct bt_info *bt, struct arm64_stackframe *frame, FILE *ofp)
-{
- int i;
- ulong stacktop, words, addr;
- ulong *stackbuf;
- char buf[BUFSIZE];
- struct machine_specific *ms = machdep->machspec;
-
- if (bt->flags & BT_FULL) {
- stacktop = ms->irq_stacks[bt->tc->processor] + ms->irq_stack_size;
- words = (stacktop - bt->bptr) / sizeof(ulong);
- stackbuf = (ulong *)GETBUF(words * sizeof(ulong));
- readmem(bt->bptr, KVADDR, stackbuf, words * sizeof(long),
- "top of IRQ stack", FAULT_ON_ERROR);
-
- addr = bt->bptr;
- for (i = 0; i < words; i++) {
- if (!(i & 1))
- fprintf(ofp, "%s %lx: ", i ? "\n" : "", addr);
- fprintf(ofp, "%s ", format_stack_entry(bt, buf, stackbuf[i], 0));
- addr += sizeof(ulong);
- }
- fprintf(ofp, "\n");
- FREEBUF(stackbuf);
- }
- fprintf(ofp, "--- <IRQ stack> ---\n");
-
- if (frame->fp == 0)
- return USER_MODE;
-
- arm64_print_exception_frame(bt, frame->sp, KERNEL_MODE, ofp);
- return KERNEL_MODE;
-}
-
static int
arm64_get_dumpfile_stackframe(struct bt_info *bt, struct arm64_stackframe *frame)
{
@@ -2047,6 +2245,20 @@ arm64_get_stack_frame(struct bt_info *bt, ulong *pcp, ulong *spp)
}
static void
+arm64_gen_hidden_frame(struct bt_info *bt, ulong sp,
+ struct arm64_stackframe *frame)
+{
+ struct arm64_pt_regs *ptregs;
+
+ ptregs = (struct arm64_pt_regs *)
+ &bt->stackbuf[(ulong)(STACK_OFFSET_TYPE(sp))];
+
+ frame->pc = ptregs->pc;
+ frame->fp = ptregs->regs[29];
+ frame->sp = ptregs->sp;
+}
+
+static void
arm64_print_exception_frame(struct bt_info *bt, ulong pt_regs, int mode, FILE *ofp)
{
int i, r, rows, top_reg, is_64_bit;
@@ -2058,7 +2270,8 @@ arm64_print_exception_frame(struct bt_info *bt, ulong pt_regs, int
mode, FILE *o
if (CRASHDEBUG(1))
fprintf(ofp, "pt_regs: %lx\n", pt_regs);
- regs = (struct arm64_pt_regs
*)&bt->stackbuf[(ulong)(STACK_OFFSET_TYPE(pt_regs))];
+ regs = (struct arm64_pt_regs *)
+ &bt->stackbuf[(ulong)(STACK_OFFSET_TYPE(pt_regs))];
if ((mode == USER_MODE) && (regs->pstate & PSR_MODE32_BIT)) {
LR = regs->regs[14];
@@ -2130,10 +2343,11 @@ arm64_print_exception_frame(struct bt_info *bt, ulong pt_regs, int
mode, FILE *o
}
if (is_64_bit) {
- fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
- (ulong)regs->orig_x0, (ulong)regs->syscallno);
- if (mode == USER_MODE)
+ if (mode == USER_MODE) {
+ fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
+ (ulong)regs->orig_x0, (ulong)regs->syscallno);
fprintf(ofp, " PSTATE: %08lx", (ulong)regs->pstate);
+ }
fprintf(ofp, "\n");
}
diff --git a/defs.h b/defs.h
index d6f719c..f7ea5a0 100644
--- a/defs.h
+++ b/defs.h
@@ -3058,8 +3058,14 @@ struct machine_specific {
ulong kernel_flags;
ulong irq_stack_size;
ulong *irq_stacks;
+ char *irq_stackbuf;
ulong __irqentry_text_start;
ulong __irqentry_text_end;
+ /* for exception vector code */
+ ulong exp_entry1_start;
+ ulong exp_entry1_end;
+ ulong exp_entry2_start;
+ ulong exp_entry2_end;
/* only needed for v4.6 or later kernel */
ulong kimage_voffset;
ulong kimage_text;
--
2.9.0
11:57 p.m.
New subject: [PATCH v5 2/4] arm64: find a correct starting stackframe at bt
For example, crash_kexec() calls several sub functions.
So even if we find some address which belongs to one of those functions
as a saved lr in a task's stack, it may not be the *last* stackframe.
For our purpose of backtracing, it is not necessary to find the last
frame, but the saved frame pointer should be sane for later use.
This patch adds some check.
Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
---
arm64.c | 32 +++++++++++++++++++++-----------
1 file changed, 21 insertions(+), 11 deletions(-)
diff --git a/arm64.c b/arm64.c
index 0ac5894..b74d2c4 100644
--- a/arm64.c
+++ b/arm64.c
@@ -1931,9 +1931,9 @@ arm64_back_trace_cmd(struct bt_info *bt)
arm64_set_irq_stack(bt);
bt->flags |= BT_IRQSTACK;
}
- stackframe.fp = GET_STACK_ULONG(bt->bptr - 8);
- stackframe.pc = GET_STACK_ULONG(bt->bptr);
- stackframe.sp = bt->bptr + 8;
+ stackframe.fp = GET_STACK_ULONG(bt->bptr);
+ stackframe.pc = GET_STACK_ULONG(bt->bptr + 8);
+ stackframe.sp = bt->bptr + 16;
bt->frameptr = stackframe.fp;
} else {
stackframe.sp = bt->stkptr;
@@ -2098,14 +2098,20 @@ arm64_in_kdump_text(struct bt_info *bt, struct arm64_stackframe
*frame)
ms = machdep->machspec;
for (ptr = start - 8; ptr >= base; ptr--) {
- if ((*ptr >= ms->crash_kexec_start) && (*ptr <
ms->crash_kexec_end)) {
- bt->bptr = ((ulong)ptr - (ulong)base) + bt->tc->thread_info;
+ if ((*ptr >= ms->crash_kexec_start) &&
+ (*ptr < ms->crash_kexec_end) &&
+ INSTACK(*(ptr - 1), bt)) {
+ bt->bptr = ((ulong)(ptr - 1) - (ulong)base)
+ + bt->tc->thread_info;
if (CRASHDEBUG(1))
fprintf(fp, "%lx: %lx (crash_kexec)\n", bt->bptr, *ptr);
return TRUE;
}
- if ((*ptr >= ms->crash_save_cpu_start) && (*ptr <
ms->crash_save_cpu_end)) {
- bt->bptr = ((ulong)ptr - (ulong)base) + bt->tc->thread_info;
+ if ((*ptr >= ms->crash_save_cpu_start) &&
+ (*ptr < ms->crash_save_cpu_end) &&
+ INSTACK(*(ptr - 1), bt)) {
+ bt->bptr = ((ulong)(ptr - 1) - (ulong)base)
+ + bt->tc->thread_info;
if (CRASHDEBUG(1))
fprintf(fp, "%lx: %lx (crash_save_cpu)\n", bt->bptr, *ptr);
return TRUE;
@@ -2143,16 +2149,20 @@ arm64_in_kdump_text_on_irq_stack(struct bt_info *bt)
start = (ulong *)(stackbuf + ms->irq_stack_size);
for (ptr = start - 8; ptr >= base; ptr--) {
- if ((*ptr >= ms->crash_kexec_start) && (*ptr <
ms->crash_kexec_end)) {
- bt->bptr = ((ulong)ptr - (ulong)base) + stackbase;
+ if ((*ptr >= ms->crash_kexec_start) &&
+ (*ptr < ms->crash_kexec_end) &&
+ INSTACK(*(ptr - 1), bt)) {
+ bt->bptr = ((ulong)(ptr - 1) - (ulong)base) + stackbase;
if (CRASHDEBUG(1))
fprintf(fp, "%lx: %lx (crash_kexec on IRQ stack)\n",
bt->bptr, *ptr);
FREEBUF(stackbuf);
return TRUE;
}
- if ((*ptr >= ms->crash_save_cpu_start) && (*ptr <
ms->crash_save_cpu_end)) {
- bt->bptr = ((ulong)ptr - (ulong)base) + stackbase;
+ if ((*ptr >= ms->crash_save_cpu_start) &&
+ (*ptr < ms->crash_save_cpu_end) &&
+ INSTACK(*(ptr - 1), bt)) {
+ bt->bptr = ((ulong)(ptr - 1) - (ulong)base) + stackbase;
if (CRASHDEBUG(1))
fprintf(fp, "%lx: %lx (crash_save_cpu on IRQ stack)\n",
bt->bptr, *ptr);
--
2.9.0
11:57 p.m.
New subject: [PATCH v5 3/4] arm64: correct a PC shown in bt
On arm64, the link register (LR) holds a return address, which is the one
just after a branch instruction. So using a saved lr as PC for backtracing
might cause some confusion.
For example, in kernel/entry.S,
work_resched:
...
bl schedule
ret_to_user:
...
The current code shows "ret_o_user", instead of "work_resched",
as a caller of schedule().
This patch corrects a PC by decrementing it by 4.
But please note that this change may also make people a bit confused
because a value of LR in the stack dump of "bt -f" doesn't match with
an address in one-line summary.
#2 [ffffcc7511407eb0] schedule at ffff0000d628aee0
ffffcc7511407eb0: ffffcc6d22f23080 ffff0000d5b44d6c <= LR
ffffcc7511407ec0: ffffcc7511407ed0 0000000000000000
#3 [ffffcc7511407ed0] work_resched at ffff0000d5b44d68 <= correcrted PC
Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
---
arm64.c | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/arm64.c b/arm64.c
index b74d2c4..bad36aa 100644
--- a/arm64.c
+++ b/arm64.c
@@ -1504,35 +1504,40 @@ static int
arm64_print_stackframe_entry(struct bt_info *bt, int level, struct arm64_stackframe
*frame, FILE *ofp)
{
char *name, *name_plus_offset;
- ulong symbol_offset;
+ ulong pc, symbol_offset;
struct syment *sp;
struct load_module *lm;
char buf[BUFSIZE];
- name = closest_symbol(frame->pc);
+ /*
+ * if pc comes from a saved lr, it actually points to an instruction
+ * after branch. To avoid any confusion, decrement pc by 4.
+ * See, for example, "bl schedule" before ret_to_user().
+ */
+ pc = frame->pc - 0x4;
+ name = closest_symbol(pc);
name_plus_offset = NULL;
if (bt->flags & BT_SYMBOL_OFFSET) {
- sp = value_search(frame->pc, &symbol_offset);
+ sp = value_search(pc, &symbol_offset);
if (sp && symbol_offset)
- name_plus_offset =
- value_to_symstr(frame->pc, buf, bt->radix);
+ name_plus_offset = value_to_symstr(pc, buf, bt->radix);
}
fprintf(ofp, "%s#%d [%8lx] %s at %lx", level < 10 ? " " :
"", level,
frame->fp ? frame->fp : bt->stacktop - USER_EFRAME_OFFSET,
- name_plus_offset ? name_plus_offset : name, frame->pc);
+ name_plus_offset ? name_plus_offset : name, pc);
if (BT_REFERENCE_CHECK(bt))
- arm64_do_bt_reference_check(bt, frame->pc, name);
+ arm64_do_bt_reference_check(bt, pc, name);
- if (module_symbol(frame->pc, NULL, &lm, NULL, 0))
+ if (module_symbol(pc, NULL, &lm, NULL, 0))
fprintf(ofp, " [%s]", lm->mod_name);
fprintf(ofp, "\n");
if (bt->flags & BT_LINE_NUMBERS) {
- get_line_number(frame->pc, buf, FALSE);
+ get_line_number(pc, buf, FALSE);
if (strlen(buf))
fprintf(ofp, " %s\n", buf);
}
--
2.9.0
11:57 p.m.
New subject: [PATCH v5 4/4] arm64: add VHE support
On VHE-enabled system, the kernel runs in EL2.
Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
---
arm64.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arm64.c b/arm64.c
index bad36aa..b9fa14d 100644
--- a/arm64.c
+++ b/arm64.c
@@ -1414,6 +1414,8 @@ arm64_is_kernel_exception_frame(struct bt_info *bt, ulong stkptr)
{
case PSR_MODE_EL1t:
case PSR_MODE_EL1h:
+ case PSR_MODE_EL2t:
+ case PSR_MODE_EL2h:
return TRUE;
}
}
--
2.9.0
2:35 p.m.
New subject: [PATCH v5 4/4] arm64: add VHE support
Takahiro,
Without this patch -- aside from in-kernel exception frames not
getting found and displayed -- does everything else work OK?
Dave
----- Original Message -----
On VHE-enabled system, the kernel runs in EL2.
Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
---
arm64.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arm64.c b/arm64.c
index bad36aa..b9fa14d 100644
--- a/arm64.c
+++ b/arm64.c
@@ -1414,6 +1414,8 @@ arm64_is_kernel_exception_frame(struct bt_info *bt,
ulong stkptr)
{
case PSR_MODE_EL1t:
case PSR_MODE_EL1h:
+ case PSR_MODE_EL2t:
+ case PSR_MODE_EL2h:
return TRUE;
}
}
--
2.9.0
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
2:48 p.m.
New subject: [PATCH v5 3/4] arm64: correct a PC shown in bt
----- Original Message -----
On arm64, the link register (LR) holds a return address, which is the
one
just after a branch instruction. So using a saved lr as PC for backtracing
might cause some confusion.
For example, in kernel/entry.S,
work_resched:
...
bl schedule
ret_to_user:
...
The current code shows "ret_o_user", instead of "work_resched",
as a caller of schedule().
I see...
This patch corrects a PC by decrementing it by 4.
But please note that this change may also make people a bit confused
because a value of LR in the stack dump of "bt -f" doesn't match with
an address in one-line summary.
#2 [ffffcc7511407eb0] schedule at ffff0000d628aee0
ffffcc7511407eb0: ffffcc6d22f23080 ffff0000d5b44d6c <= LR
ffffcc7511407ec0: ffffcc7511407ed0 0000000000000000
#3 [ffffcc7511407ed0] work_resched at ffff0000d5b44d68 <= correcrted PC
That's probably OK -- and at least "bt -F" would clarify it somewhat.
Thanks,
Dave
Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
---
arm64.c | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/arm64.c b/arm64.c
index b74d2c4..bad36aa 100644
--- a/arm64.c
+++ b/arm64.c
@@ -1504,35 +1504,40 @@ static int
arm64_print_stackframe_entry(struct bt_info *bt, int level, struct
arm64_stackframe *frame, FILE *ofp)
{
char *name, *name_plus_offset;
- ulong symbol_offset;
+ ulong pc, symbol_offset;
struct syment *sp;
struct load_module *lm;
char buf[BUFSIZE];
- name = closest_symbol(frame->pc);
+ /*
+ * if pc comes from a saved lr, it actually points to an instruction
+ * after branch. To avoid any confusion, decrement pc by 4.
+ * See, for example, "bl schedule" before ret_to_user().
+ */
+ pc = frame->pc - 0x4;
+ name = closest_symbol(pc);
name_plus_offset = NULL;
if (bt->flags & BT_SYMBOL_OFFSET) {
- sp = value_search(frame->pc, &symbol_offset);
+ sp = value_search(pc, &symbol_offset);
if (sp && symbol_offset)
- name_plus_offset =
- value_to_symstr(frame->pc, buf, bt->radix);
+ name_plus_offset = value_to_symstr(pc, buf, bt->radix);
}
fprintf(ofp, "%s#%d [%8lx] %s at %lx", level < 10 ? " " :
"", level,
frame->fp ? frame->fp : bt->stacktop - USER_EFRAME_OFFSET,
- name_plus_offset ? name_plus_offset : name, frame->pc);
+ name_plus_offset ? name_plus_offset : name, pc);
if (BT_REFERENCE_CHECK(bt))
- arm64_do_bt_reference_check(bt, frame->pc, name);
+ arm64_do_bt_reference_check(bt, pc, name);
- if (module_symbol(frame->pc, NULL, &lm, NULL, 0))
+ if (module_symbol(pc, NULL, &lm, NULL, 0))
fprintf(ofp, " [%s]", lm->mod_name);
fprintf(ofp, "\n");
if (bt->flags & BT_LINE_NUMBERS) {
- get_line_number(frame->pc, buf, FALSE);
+ get_line_number(pc, buf, FALSE);
if (strlen(buf))
fprintf(ofp, " %s\n", buf);
}
--
2.9.0
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
7:11 p.m.
New subject: [PATCH v5 3/4] arm64: correct a PC shown in bt
On 29 June 2016 at 04:48, Dave Anderson <anderson(a)redhat.com> wrote:
> On arm64, the link register (LR) holds a return address, which is the
one
> just after a branch instruction. So using a saved lr as PC for
backtracing
> might cause some confusion.
> For example, in kernel/entry.S,
> work_resched:
> ...
> bl schedule
>
> ret_to_user:
> ...
>
> The current code shows "ret_o_user", instead of "work_resched",
> as a caller of schedule().
I see...
​FYI, this can potentially be seen in the kernel's ftrace-based stack
tracer or
panic dump.
In the past, ​PC calculation was modified to be "LR-4", but this patch
was reverted. See
commit 9702970c7bd3(Revert "ARM64: unwind: Fix PC calculation")
​So I said that adding this patch was a discussion topic.​
>
> This patch corrects a PC by decrementing it by 4.
> But please note that this change may also make people a bit confused
> because a value of LR in the stack dump of "bt -f" doesn't match with
> an address in one-line summary.
>
> #2 [ffffcc7511407eb0] schedule at ffff0000d628aee0
> ffffcc7511407eb0: ffffcc6d22f23080 ffff0000d5b44d6c <= LR
> ffffcc7511407ec0: ffffcc7511407ed0 0000000000000000
> #3 [ffffcc7511407ed0] work_resched at ffff0000d5b44d68 <= correcrted
PC
That's probably OK -- and at least "bt -F" would clarify it somewhat.
​and "bt -Fs"​
​-Takahiro AKASHI​
Thanks,
Dave
>
> Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
> ---
> arm64.c | 23 ++++++++++++++---------
> 1 file changed, 14 insertions(+), 9 deletions(-)
>
> diff --git a/arm64.c b/arm64.c
> index b74d2c4..bad36aa 100644
> --- a/arm64.c
> +++ b/arm64.c
> @@ -1504,35 +1504,40 @@ static int
> arm64_print_stackframe_entry(struct bt_info *bt, int level, struct
> arm64_stackframe *frame, FILE *ofp)
> {
> char *name, *name_plus_offset;
> - ulong symbol_offset;
> + ulong pc, symbol_offset;
> struct syment *sp;
> struct load_module *lm;
> char buf[BUFSIZE];
>
> - name = closest_symbol(frame->pc);
> + /*
> + * if pc comes from a saved lr, it actually points to an
instruction
> + * after branch. To avoid any confusion, decrement pc by
4.
> + * See, for example, "bl schedule" before ret_to_user().
> + */
> + pc = frame->pc - 0x4;
> + name = closest_symbol(pc);
> name_plus_offset = NULL;
>
> if (bt->flags & BT_SYMBOL_OFFSET) {
> - sp = value_search(frame->pc, &symbol_offset);
> + sp = value_search(pc, &symbol_offset);
> if (sp && symbol_offset)
> - name_plus_offset =
> - value_to_symstr(frame->pc, buf,
bt->radix);
> + name_plus_offset = value_to_symstr(pc,
buf,
bt->radix);
> }
>
> fprintf(ofp, "%s#%d [%8lx] %s at %lx", level < 10 ? "
" : "",
level,
> frame->fp ? frame->fp : bt->stacktop -
USER_EFRAME_OFFSET,
> - name_plus_offset ? name_plus_offset : name,
frame->pc);
> + name_plus_offset ? name_plus_offset : name, pc);
>
> if (BT_REFERENCE_CHECK(bt))
> - arm64_do_bt_reference_check(bt, frame->pc, name);
> + arm64_do_bt_reference_check(bt, pc, name);
>
> - if (module_symbol(frame->pc, NULL, &lm, NULL, 0))
> + if (module_symbol(pc, NULL, &lm, NULL, 0))
> fprintf(ofp, " [%s]", lm->mod_name);
>
> fprintf(ofp, "\n");
>
> if (bt->flags & BT_LINE_NUMBERS) {
> - get_line_number(frame->pc, buf, FALSE);
> + get_line_number(pc, buf, FALSE);
> if (strlen(buf))
> fprintf(ofp, " %s\n", buf);
> }
> --
> 2.9.0
>
> --
> Crash-utility mailing list
> Crash-utility(a)redhat.com
> https://www.redhat.com/mailman/listinfo/crash-utility
>
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
7:22 p.m.
New subject: [PATCH v5 4/4] arm64: add VHE support
On 29 June 2016 at 04:35, Dave Anderson <anderson(a)redhat.com> wrote:
Takahiro,
Without this patch -- aside from in-kernel exception frames not
getting found and displayed -- does everything else work OK?
​All what I can say is that this is the only issue that I have noticed so
far
on VHE-enabled system.​
So the "subject" may be overstated.
Thanks,
-Takahiro AKASHI
Dave
----- Original Message -----
> On VHE-enabled system, the kernel runs in EL2.
>
> Signed-off-by: AKASHI Takahiro <takahiro.akashi(a)linaro.org>
> ---
> arm64.c | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/arm64.c b/arm64.c
> index bad36aa..b9fa14d 100644
> --- a/arm64.c
> +++ b/arm64.c
> @@ -1414,6 +1414,8 @@ arm64_is_kernel_exception_frame(struct bt_info
*bt,
> ulong stkptr)
> {
> case PSR_MODE_EL1t:
> case PSR_MODE_EL1h:
> + case PSR_MODE_EL2t:
> + case PSR_MODE_EL2h:
> return TRUE;
> }
> }
> --
> 2.9.0
>
> --
> Crash-utility mailing list
> Crash-utility(a)redhat.com
> https://www.redhat.com/mailman/listinfo/crash-utility
>
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
3:09 p.m.
Hi Takahiro,
I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
believe the results are what you intended?
For example, taking the 4.6 vmcore that you gave to me, here is the
current crash utility's output of "bt -a", where the crashing task
entered crash_kexec() via the sysrq-c page fault exception, and the
tasks on the other cpus have all entered crash_save_cpu() on their
IRQ stack as a result of the shutdown IPI, one from user-space and
the others from the kernel:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
#0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f43010] handle_IPI at ffff00000808e718
#2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f43080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff000008dc3f10 PSTATE: 60400149
X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
#5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
#6 [ffff000008dc3f80] rest_init at ffff0000087c7930
#7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
#0 [ffff800022f56e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f57010] handle_IPI at ffff00000808e718
#2 [ffff800022f57040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f57080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218cff60 PSTATE: 60000349
X29: ffff8000218cff60 X28: ffff8000218cc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218cc000 X22: ffff000008bd0270 X21: ffff8000218cc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000005
X14: 0000000c80096000 X13: ffff8000212b6600 X12: 0000000000047ae2
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218cc000
X8: 000000010000aa07 X7: 0000000000000243 X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
#0 [ffff800022f6ae50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f6b010] handle_IPI at ffff00000808e718
#2 [ffff800022f6b040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f6b080] el0_irq_naked at ffff000008084c4c
--- <IRQ stack> ---
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE: 20000000
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
#1 [ffff800020b0ba90] die at ffff000008088ce8
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
#3 [ffff800020b0bb00] do_page_fault at ffff000008096814
#4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
#7 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
#8 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
#9 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
#10 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
#11 [ffff800020b0be50] vfs_write at ffff0000081c5588
#12 [ffff800020b0be90] sys_write at ffff0000081c68fc
#13 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
#0 [ffff800022f92e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f93010] handle_IPI at ffff00000808e718
#2 [ffff800022f93040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f93080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dbf60 PSTATE: 60000349
X29: ffff8000218dbf60 X28: ffff8000218d8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218d8000 X22: ffff000008bd0270 X21: ffff8000218d8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000000
X14: 0000000000000000 X13: 00000000000003e7 X12: 0000000000000016
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218d8000
X8: ffff8000218c3ab0 X7: 7fffffffffffffff X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
#0 [ffff800022fa6e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fa7010] handle_IPI at ffff00000808e718
#2 [ffff800022fa7040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fa7080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dff60 PSTATE: 60000349
X29: ffff8000218dff60 X28: ffff8000218dc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218dc000 X22: ffff000008bd0270 X21: ffff8000218dc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000007
X14: 0000000000000000 X13: 000000000000039d X12: 0000000000000000
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218dc000
X8: 000000010000a479 X7: 7fffffffffffffff X6: 0016694e00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
#0 [ffff800022fbae50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fbb010] handle_IPI at ffff00000808e718
#2 [ffff800022fbb040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fbb080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218ebf60 PSTATE: 60000349
X29: ffff8000218ebf60 X28: ffff8000218e8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218e8000 X22: ffff000008bd0270 X21: ffff8000218e8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 000000000000000a
X14: 0000000000000000 X13: 00000000000003a1 X12: 0000000000000000
X11: ffff8000213a0d00 X10: 0000000000000850 X9: ffff8000218e8000
X8: ffff8000218c53b0 X7: 00000000000001c4 X6: 002887fa00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
#0 [ffff800022fcee50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fcf010] handle_IPI at ffff00000808e718
#2 [ffff800022fcf040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fcf080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218eff60 PSTATE: 60400149
X29: ffff8000218eff60 X28: ffff8000218ec000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218ec000 X22: ffff000008bd0270 X21: ffff8000218ec000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: fffffffffffffffb
X17: 0000ffff9dc8ba48 X16: ffff0000081da048 X15: 003b9aca00000000
X14: 0000000000000000 X13: 0000000000000001 X12: 0000000000000000
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218ec000
X8: 000000010000a280 X7: ffff8000209b39c8 X6: ffff8000209b39c9
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
crash>
And that's precisely what we want to see. (other than perhaps the
in-kernel exception frame's dump of the ORIG_X0 and SYSCALL registers)
Now, with v5's patches 1/2 and 2/2 patches applied, this is what gets displayed:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
#0 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
#1 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
#2 [ffff000008dc3f80] rest_init at ffff0000087c7930
#3 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
#0 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE: 20000000
#0 [user space]
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
#1 [ffff800020b0ba90] die at ffff000008088ce8
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
#3 [ffff800020b0bb00] do_page_fault at ffff000008096814
#4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
#6 [ffff800020b0bc20] el1_da at ffff000008084568
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
#7 [ffff800020b0bd40] sysrq_handle_crash at ffff000008457fc8
#8 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
#9 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
#10 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
#11 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
#12 [ffff800020b0be50] vfs_write at ffff0000081c5588
#13 [ffff800020b0be90] sys_write at ffff0000081c68fc
#14 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
--- <Exception in user> ---
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
#0 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
#0 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
#0 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
#0 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
crash>
What happened? This can't possibly be what you intended, is it?
Dave
3:44 p.m.
----- Original Message -----
Hi Takahiro,
I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
believe the results are what you intended?
Obviously I meant 1/4 and 2/4 above.
However, I was under the impression that the 3/4 patch was a standalone
patch that only served to change the text address displayed, and that
"adding this patch was a discussion topic": ​
On arm64, the link register (LR) holds a return address, which is
the one
just after a branch instruction. So using a saved lr as PC for backtracing
might cause some confusion.
For example, in kernel/entry.S,
work_resched:
...
bl schedule
ret_to_user:
...
The current code shows "ret_o_user", instead of "work_resched",
as a caller of schedule().
This patch corrects a PC by decrementing it by 4.
But please note that this change may also make people a bit confused
because a value of LR in the stack dump of "bt -f" doesn't match with
an address in one-line summary.
#2 [ffffcc7511407eb0] schedule at ffff0000d628aee0
ffffcc7511407eb0: ffffcc6d22f23080 ffff0000d5b44d6c <= LR
ffffcc7511407ec0: ffffcc7511407ed0 0000000000000000
#3 [ffffcc7511407ed0] work_resched at ffff0000d5b44d68 <= correcrted PC
Signed-off-by: AKASHI Takahiro <takahiro akashi linaro org>
...and as you subsequently mentioned, "adding this patch was a discussion
topic".
But anyway, for the hell of it, I subsequently applied 3/4, and now I at least see
the IPI exception frames:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff000008dc3f10 PSTATE: 60400149
X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857bc
#1 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26c8
#2 [ffff000008dc3f80] rest_init at ffff0000087c792c
#3 [ffff000008dc3fa0] start_kernel at ffff000008b10b6c
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218cff60 PSTATE: 60000349
X29: ffff8000218cff60 X28: ffff8000218cc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218cc000 X22: ffff000008bd0270 X21: ffff8000218cc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000005
X14: 0000000c80096000 X13: ffff8000212b6600 X12: 0000000000047ae2
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218cc000
X8: 000000010000aa07 X7: 0000000000000243 X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218cff60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1e8
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE: 20000000
#0 [user space]
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0a8
#1 [ffff800020b0ba90] die at ffff000008088ce4
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fa8
#3 [ffff800020b0bb00] do_page_fault at ffff000008096810
#4 [ffff800020b0bb60] do_translation_fault at ffff000008096908
#5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
#6 [ffff800020b0bc20] el1_da at ffff000008084564
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
#7 [ffff800020b0bd40] sysrq_handle_crash at ffff000008457fc4
#8 [ffff800020b0bd50] __handle_sysrq at ffff000008458640
#9 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458abc
#10 [ffff800020b0bdb0] proc_reg_write at ffff000008225418
#11 [ffff800020b0bdd0] __vfs_write at ffff0000081c4770
#12 [ffff800020b0be50] vfs_write at ffff0000081c5584
#13 [ffff800020b0be90] sys_write at ffff0000081c68f8
#14 [ffff800020b0bed0] el0_svc_naked at ffff000008084e6c
--- <Exception in user> ---
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dbf60 PSTATE: 60000349
X29: ffff8000218dbf60 X28: ffff8000218d8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218d8000 X22: ffff000008bd0270 X21: ffff8000218d8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000000
X14: 0000000000000000 X13: 00000000000003e7 X12: 0000000000000016
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218d8000
X8: ffff8000218c3ab0 X7: 7fffffffffffffff X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1e8
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dff60 PSTATE: 60000349
X29: ffff8000218dff60 X28: ffff8000218dc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218dc000 X22: ffff000008bd0270 X21: ffff8000218dc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000007
X14: 0000000000000000 X13: 000000000000039d X12: 0000000000000000
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218dc000
X8: 000000010000a479 X7: 7fffffffffffffff X6: 0016694e00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218dff60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1e8
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218ebf60 PSTATE: 60000349
X29: ffff8000218ebf60 X28: ffff8000218e8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218e8000 X22: ffff000008bd0270 X21: ffff8000218e8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 000000000000000a
X14: 0000000000000000 X13: 00000000000003a1 X12: 0000000000000000
X11: ffff8000213a0d00 X10: 0000000000000850 X9: ffff8000218e8000
X8: ffff8000218c53b0 X7: 00000000000001c4 X6: 002887fa00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1e8
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218eff60 PSTATE: 60400149
X29: ffff8000218eff60 X28: ffff8000218ec000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218ec000 X22: ffff000008bd0270 X21: ffff8000218ec000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: fffffffffffffffb
X17: 0000ffff9dc8ba48 X16: ffff0000081da048 X15: 003b9aca00000000
X14: 0000000000000000 X13: 0000000000000001 X12: 0000000000000000
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218ec000
X8: 000000010000a280 X7: ffff8000209b39c8 X6: ffff8000209b39c9
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218eff60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1e8
crash>
But why do you think that it is an improvement to leave out the transition
to the IRQ stack?
In any case, I haven't even started looking at the "bt -f" part of the
patch
because quite frankly, this patchset is so complex that I haven't even
begun to try to understand it. When you said it would be "easier to
maintain", well, perhaps for *you* maybe, but certainly not for me! Without
the kernel's backtrace code from which the current code is based, I don't have
anything to work from anymore. I'm really not sure whether the coverage of the
"corner cases" you referred to make this effort worth it. I would like to see
examples
of how the current code fails. But anyway, I will continue to test it to see
if there actually is any significant upgrade from what we already have in place.
From a kernel debugging perspective, all we really need is a basic
unwinder,
and optionally the full dump of the stack data in between those frames.
And the
current code does do that at a minimum, and there's much to be said for simplicity.
Dave
For example, taking the 4.6 vmcore that you gave to me, here is the
current crash utility's output of "bt -a", where the crashing task
entered crash_kexec() via the sysrq-c page fault exception, and the
tasks on the other cpus have all entered crash_save_cpu() on their
IRQ stack as a result of the shutdown IPI, one from user-space and
the others from the kernel:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
#0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f43010] handle_IPI at ffff00000808e718
#2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f43080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff000008dc3f10 PSTATE: 60400149
X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
#5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
#6 [ffff000008dc3f80] rest_init at ffff0000087c7930
#7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
#0 [ffff800022f56e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f57010] handle_IPI at ffff00000808e718
#2 [ffff800022f57040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f57080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218cff60 PSTATE: 60000349
X29: ffff8000218cff60 X28: ffff8000218cc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218cc000 X22: ffff000008bd0270 X21: ffff8000218cc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000005
X14: 0000000c80096000 X13: ffff8000212b6600 X12: 0000000000047ae2
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218cc000
X8: 000000010000aa07 X7: 0000000000000243 X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
#0 [ffff800022f6ae50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f6b010] handle_IPI at ffff00000808e718
#2 [ffff800022f6b040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f6b080] el0_irq_naked at ffff000008084c4c
--- <IRQ stack> ---
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE:
20000000
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
#1 [ffff800020b0ba90] die at ffff000008088ce8
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
#3 [ffff800020b0bb00] do_page_fault at ffff000008096814
#4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
#7 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
#8 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
#9 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
#10 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
#11 [ffff800020b0be50] vfs_write at ffff0000081c5588
#12 [ffff800020b0be90] sys_write at ffff0000081c68fc
#13 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
#0 [ffff800022f92e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f93010] handle_IPI at ffff00000808e718
#2 [ffff800022f93040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f93080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dbf60 PSTATE: 60000349
X29: ffff8000218dbf60 X28: ffff8000218d8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218d8000 X22: ffff000008bd0270 X21: ffff8000218d8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000000
X14: 0000000000000000 X13: 00000000000003e7 X12: 0000000000000016
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218d8000
X8: ffff8000218c3ab0 X7: 7fffffffffffffff X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
#0 [ffff800022fa6e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fa7010] handle_IPI at ffff00000808e718
#2 [ffff800022fa7040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fa7080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dff60 PSTATE: 60000349
X29: ffff8000218dff60 X28: ffff8000218dc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218dc000 X22: ffff000008bd0270 X21: ffff8000218dc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000007
X14: 0000000000000000 X13: 000000000000039d X12: 0000000000000000
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218dc000
X8: 000000010000a479 X7: 7fffffffffffffff X6: 0016694e00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
#0 [ffff800022fbae50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fbb010] handle_IPI at ffff00000808e718
#2 [ffff800022fbb040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fbb080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218ebf60 PSTATE: 60000349
X29: ffff8000218ebf60 X28: ffff8000218e8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218e8000 X22: ffff000008bd0270 X21: ffff8000218e8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 000000000000000a
X14: 0000000000000000 X13: 00000000000003a1 X12: 0000000000000000
X11: ffff8000213a0d00 X10: 0000000000000850 X9: ffff8000218e8000
X8: ffff8000218c53b0 X7: 00000000000001c4 X6: 002887fa00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
#0 [ffff800022fcee50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fcf010] handle_IPI at ffff00000808e718
#2 [ffff800022fcf040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fcf080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218eff60 PSTATE: 60400149
X29: ffff8000218eff60 X28: ffff8000218ec000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218ec000 X22: ffff000008bd0270 X21: ffff8000218ec000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: fffffffffffffffb
X17: 0000ffff9dc8ba48 X16: ffff0000081da048 X15: 003b9aca00000000
X14: 0000000000000000 X13: 0000000000000001 X12: 0000000000000000
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218ec000
X8: 000000010000a280 X7: ffff8000209b39c8 X6: ffff8000209b39c9
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
crash>
And that's precisely what we want to see. (other than perhaps the
in-kernel exception frame's dump of the ORIG_X0 and SYSCALL registers)
Now, with v5's patches 1/2 and 2/2 patches applied, this is what gets
displayed:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
#0 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
#1 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
#2 [ffff000008dc3f80] rest_init at ffff0000087c7930
#3 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
#0 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE:
20000000
#0 [user space]
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
#1 [ffff800020b0ba90] die at ffff000008088ce8
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
#3 [ffff800020b0bb00] do_page_fault at ffff000008096814
#4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
#6 [ffff800020b0bc20] el1_da at ffff000008084568
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
#7 [ffff800020b0bd40] sysrq_handle_crash at ffff000008457fc8
#8 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
#9 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
#10 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
#11 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
#12 [ffff800020b0be50] vfs_write at ffff0000081c5588
#13 [ffff800020b0be90] sys_write at ffff0000081c68fc
#14 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
--- <Exception in user> ---
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
#0 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
#0 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
#0 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
#0 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
crash>
What happened? This can't possibly be what you intended, is it?
Dave
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
4:25 p.m.
Hi Takahiro,
Here is another thing that I would prefer not to change/omit.
In the current code, the raw exception frame data is dumped as
part of the "bt -[fF]" output, just prior to it being translated
as an exception frame:
crash> bt -F
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
... [ cut ] ...
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
ffff800020b0bc20: 0000000000000001 0000000000000000
ffff800020b0bc30: log_wait+8 0000000000000000
ffff800020b0bc40: 0000000000000000 00000000000047d4
ffff800020b0bc50: ffff800022f337a4 0000000000000000
ffff800020b0bc60: 0000000000000106 0000000000000001
ffff800020b0bc70: 0000000000000002 0000000000000106
ffff800020b0bc80: log_buf_len cont
ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
ffff800020b0bca0: sys_write 0000ffff83d266c0
ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
ffff800020b0bcd0: 0000000000000009 0000000000000000
ffff800020b0bce0: 0000000000000015 0000000000000120
ffff800020b0bcf0: 0000000000000040 sys_call_table
ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
...
whereas with the v5 patchset, the exception frame only gets translated,
but the actual raw memory never gets dumped:
crash> bt -F
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
... [ cut ] ...
#5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
#6 [ffff800020b0bc20] el1_da at ffff000008084564
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
...
It's actually helpful being able to dump the exception frame data with
"bt -F" to show symbolic references that are found within the exception
frame itself.
Dave
6:32 p.m.
Dave,
If you don't like my patches, that is OK.
Applying them or not is totally up to you.
I will *never* submit my patches again.
Having said so, I think I would better explain my intentions on the code.
On Wed, Jun 29, 2016 at 04:09:01PM -0400, Dave Anderson wrote:
Hi Takahiro,
I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
believe the results are what you intended?
For example, taking the 4.6 vmcore that you gave to me, here is the
current crash utility's output of "bt -a", where the crashing task
entered crash_kexec() via the sysrq-c page fault exception, and the
tasks on the other cpus have all entered crash_save_cpu() on their
IRQ stack as a result of the shutdown IPI, one from user-space and
the others from the kernel:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
#0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f43010] handle_IPI at ffff00000808e718
#2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f43080] el1_irq at ffff000008084720
--- <IRQ stack> ---
This part of dump shown above is the result of "kdump" mechanisms,
and is *not* useful for you to understand what was happening on this
CPU when a crash occurred on the other CPU, #3 in this case.
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff000008dc3f10 PSTATE: 60400149
X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
#5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
#6 [ffff000008dc3f80] rest_init at ffff0000087c7930
#7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
All you need to know is that CPU#0 was idle at the crash point.
Whatever else do you want to know?
-Takahiro AKASHI
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND:
"swapper/1"
#0 [ffff800022f56e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f57010] handle_IPI at ffff00000808e718
#2 [ffff800022f57040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f57080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218cff60 PSTATE: 60000349
X29: ffff8000218cff60 X28: ffff8000218cc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218cc000 X22: ffff000008bd0270 X21: ffff8000218cc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000005
X14: 0000000c80096000 X13: ffff8000212b6600 X12: 0000000000047ae2
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218cc000
X8: 000000010000aa07 X7: 0000000000000243 X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
#0 [ffff800022f6ae50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f6b010] handle_IPI at ffff00000808e718
#2 [ffff800022f6b040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f6b080] el0_irq_naked at ffff000008084c4c
--- <IRQ stack> ---
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE: 20000000
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
#1 [ffff800020b0ba90] die at ffff000008088ce8
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
#3 [ffff800020b0bb00] do_page_fault at ffff000008096814
#4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
#7 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
#8 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
#9 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
#10 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
#11 [ffff800020b0be50] vfs_write at ffff0000081c5588
#12 [ffff800020b0be90] sys_write at ffff0000081c68fc
#13 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
#0 [ffff800022f92e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022f93010] handle_IPI at ffff00000808e718
#2 [ffff800022f93040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022f93080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dbf60 PSTATE: 60000349
X29: ffff8000218dbf60 X28: ffff8000218d8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218d8000 X22: ffff000008bd0270 X21: ffff8000218d8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000000
X14: 0000000000000000 X13: 00000000000003e7 X12: 0000000000000016
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218d8000
X8: ffff8000218c3ab0 X7: 7fffffffffffffff X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
#0 [ffff800022fa6e50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fa7010] handle_IPI at ffff00000808e718
#2 [ffff800022fa7040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fa7080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dff60 PSTATE: 60000349
X29: ffff8000218dff60 X28: ffff8000218dc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218dc000 X22: ffff000008bd0270 X21: ffff8000218dc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000007
X14: 0000000000000000 X13: 000000000000039d X12: 0000000000000000
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218dc000
X8: 000000010000a479 X7: 7fffffffffffffff X6: 0016694e00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
#0 [ffff800022fbae50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fbb010] handle_IPI at ffff00000808e718
#2 [ffff800022fbb040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fbb080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218ebf60 PSTATE: 60000349
X29: ffff8000218ebf60 X28: ffff8000218e8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218e8000 X22: ffff000008bd0270 X21: ffff8000218e8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 000000000000000a
X14: 0000000000000000 X13: 00000000000003a1 X12: 0000000000000000
X11: ffff8000213a0d00 X10: 0000000000000850 X9: ffff8000218e8000
X8: ffff8000218c53b0 X7: 00000000000001c4 X6: 002887fa00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
#0 [ffff800022fcee50] crash_save_cpu at ffff00000812ae44
#1 [ffff800022fcf010] handle_IPI at ffff00000808e718
#2 [ffff800022fcf040] gic_handle_irq at ffff0000080815f8
#3 [ffff800022fcf080] el1_irq at ffff000008084720
--- <IRQ stack> ---
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218eff60 PSTATE: 60400149
X29: ffff8000218eff60 X28: ffff8000218ec000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218ec000 X22: ffff000008bd0270 X21: ffff8000218ec000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: fffffffffffffffb
X17: 0000ffff9dc8ba48 X16: ffff0000081da048 X15: 003b9aca00000000
X14: 0000000000000000 X13: 0000000000000001 X12: 0000000000000000
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218ec000
X8: 000000010000a280 X7: ffff8000209b39c8 X6: ffff8000209b39c9
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
#4 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
#5 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
#6 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
crash>
And that's precisely what we want to see. (other than perhaps the
in-kernel exception frame's dump of the ORIG_X0 and SYSCALL registers)
Now, with v5's patches 1/2 and 2/2 patches applied, this is what gets displayed:
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
#0 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
#1 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
#2 [ffff000008dc3f80] rest_init at ffff0000087c7930
#3 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
#0 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE: 20000000
#0 [user space]
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
#1 [ffff800020b0ba90] die at ffff000008088ce8
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
#3 [ffff800020b0bb00] do_page_fault at ffff000008096814
#4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
#6 [ffff800020b0bc20] el1_da at ffff000008084568
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
#7 [ffff800020b0bd40] sysrq_handle_crash at ffff000008457fc8
#8 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
#9 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
#10 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
#11 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
#12 [ffff800020b0be50] vfs_write at ffff0000081c5588
#13 [ffff800020b0be90] sys_write at ffff0000081c68fc
#14 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
--- <Exception in user> ---
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
#0 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
#0 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
#0 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
#0 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
#1 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
#2 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
crash>
What happened? This can't possibly be what you intended, is it?
Dave
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
7:16 p.m.
On Wed, Jun 29, 2016 at 04:44:41PM -0400, Dave Anderson wrote:
> Hi Takahiro,
>
> I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
> believe the results are what you intended?
Obviously I meant 1/4 and 2/4 above.
However, I was under the impression that the 3/4 patch was a standalone
patch that only served to change the text address displayed,
Right. I might have made some mistake when I squashed up the changes.
and that
"adding this patch was a discussion topic": ???
> On arm64, the link register (LR) holds a return address, which is the one
> just after a branch instruction. So using a saved lr as PC for backtracing
> might cause some confusion.
> For example, in kernel/entry.S,
> work_resched:
> ...
> bl schedule
>
> ret_to_user:
> ...
>
> The current code shows "ret_o_user", instead of
"work_resched",
> as a caller of schedule().
>
> This patch corrects a PC by decrementing it by 4.
> But please note that this change may also make people a bit confused
> because a value of LR in the stack dump of "bt -f" doesn't match
with
> an address in one-line summary.
>
> #2 [ffffcc7511407eb0] schedule at ffff0000d628aee0
> ffffcc7511407eb0: ffffcc6d22f23080 ffff0000d5b44d6c <= LR
> ffffcc7511407ec0: ffffcc7511407ed0 0000000000000000
> #3 [ffffcc7511407ed0] work_resched at ffff0000d5b44d68 <= correcrted PC
>
> Signed-off-by: AKASHI Takahiro <takahiro akashi linaro org>
...and as you subsequently mentioned, "adding this patch was a discussion
topic".
But anyway, for the hell of it, I subsequently applied 3/4, and now I at least see
the IPI exception frames:
Sure that I did make this change recently.
I mean, it is intentional but it should not have gone with patch#3 but patch#1.
crash> bt -a
PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff000008dc3f10 PSTATE: 60400149
X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857bc
#1 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26c8
#2 [ffff000008dc3f80] rest_init at ffff0000087c792c
#3 [ffff000008dc3fa0] start_kernel at ffff000008b10b6c
PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218cff60 PSTATE: 60000349
X29: ffff8000218cff60 X28: ffff8000218cc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218cc000 X22: ffff000008bd0270 X21: ffff8000218cc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000005
X14: 0000000c80096000 X13: ffff8000212b6600 X12: 0000000000047ae2
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218cc000
X8: 000000010000aa07 X7: 0000000000000243 X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218cff60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1e8
PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE: 20000000
#0 [user space]
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
#0 [ffff800020b0ba70] crash_kexec at ffff00000812b0a8
#1 [ffff800020b0ba90] die at ffff000008088ce4
#2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fa8
#3 [ffff800020b0bb00] do_page_fault at ffff000008096810
#4 [ffff800020b0bb60] do_translation_fault at ffff000008096908
#5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
#6 [ffff800020b0bc20] el1_da at ffff000008084564
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
#7 [ffff800020b0bd40] sysrq_handle_crash at ffff000008457fc4
#8 [ffff800020b0bd50] __handle_sysrq at ffff000008458640
#9 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458abc
#10 [ffff800020b0bdb0] proc_reg_write at ffff000008225418
#11 [ffff800020b0bdd0] __vfs_write at ffff0000081c4770
#12 [ffff800020b0be50] vfs_write at ffff0000081c5584
#13 [ffff800020b0be90] sys_write at ffff0000081c68f8
#14 [ffff800020b0bed0] el0_svc_naked at ffff000008084e6c
--- <Exception in user> ---
PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dbf60 PSTATE: 60000349
X29: ffff8000218dbf60 X28: ffff8000218d8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218d8000 X22: ffff000008bd0270 X21: ffff8000218d8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000000
X14: 0000000000000000 X13: 00000000000003e7 X12: 0000000000000016
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218d8000
X8: ffff8000218c3ab0 X7: 7fffffffffffffff X6: 0015752a00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1e8
PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218dff60 PSTATE: 60000349
X29: ffff8000218dff60 X28: ffff8000218dc000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218dc000 X22: ffff000008bd0270 X21: ffff8000218dc000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000007
X14: 0000000000000000 X13: 000000000000039d X12: 0000000000000000
X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218dc000
X8: 000000010000a479 X7: 7fffffffffffffff X6: 0016694e00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218dff60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1e8
PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218ebf60 PSTATE: 60000349
X29: ffff8000218ebf60 X28: ffff8000218e8000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218e8000 X22: ffff000008bd0270 X21: ffff8000218e8000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 000000000000000a
X14: 0000000000000000 X13: 00000000000003a1 X12: 0000000000000000
X11: ffff8000213a0d00 X10: 0000000000000850 X9: ffff8000218e8000
X8: ffff8000218c53b0 X7: 00000000000001c4 X6: 002887fa00000000
X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1e8
PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
PC: ffff0000080857c0 [arch_cpu_idle+16]
LR: ffff0000080857bc [arch_cpu_idle+12]
SP: ffff8000218eff60 PSTATE: 60400149
X29: ffff8000218eff60 X28: ffff8000218ec000 X27: 0000000000000000
X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
X23: ffff8000218ec000 X22: ffff000008bd0270 X21: ffff8000218ec000
X20: ffff000008dc5b88 X19: 0000000000000000 X18: fffffffffffffffb
X17: 0000ffff9dc8ba48 X16: ffff0000081da048 X15: 003b9aca00000000
X14: 0000000000000000 X13: 0000000000000001 X12: 0000000000000000
X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218ec000
X8: 000000010000a280 X7: ffff8000209b39c8 X6: ffff8000209b39c9
X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
#0 [ffff8000218eff60] arch_cpu_idle at ffff0000080857bc
#1 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26c8
#2 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1e8
crash>
But why do you think that it is an improvement to leave out the transition
to the IRQ stack?
I explained the reason in my previous e-mail.
I still believe that it is an improvement.
In any case, I haven't even started looking at the "bt
-f" part of the patch
because quite frankly, this patchset is so complex that I haven't even
begun to try to understand it. When you said it would be "easier to
maintain", well, perhaps for *you* maybe, but certainly not for me!
Do you think so?
It is because you are the author.
The changes I made were big and look complex to you,
but the resulting code is simple enough for others IMO.
I tried to make the main loop of arm64_back_trace_cmd() simple and
quite resemble the counterpart of dump_backtrace() in the kernel.
The differences come mainly from the facts:
1.you (crash util) need to display not only a one-line frame summary
(with a value of stack/frame pointer), but also a full dump of stack
2.your current code will miss a *very important* stack frame that is
the one exactly when an interrupt takes place.
I believe that the latter is a big improvement.
Yeah, due to the changes, arm64_unwind_frame() may get a bit complicated,
but it is very naive implementation based on the nature of the kernel's
stack usage (or PCS for ARM), the complexity is not the result of my poor
skill of coding.
In other words, the complexity is now *encapsulated* in that function.
In addition, I wrote down that function step-by-step, from generic cases
to exceptional cases, with a bunch of comments for better understandings.
And you will be able to remove such exceptional cases *if* you don't like
them.
Given those stuffs, I think that my code is easier to maintain.
Again, surely up to you.
-Takahiro AKASHI
Without
the kernel's backtrace code from which the current code is based, I don't have
anything to work from anymore. I'm really not sure whether the coverage of the
"corner cases" you referred to make this effort worth it. I would like to see
examples
of how the current code fails. But anyway, I will continue to test it to see
if there actually is any significant upgrade from what we already have in place.
>From a kernel debugging perspective, all we really need is a basic unwinder,
and optionally the full dump of the stack data in between those frames. And the
current code does do that at a minimum, and there's much to be said for simplicity.
Dave
>
> For example, taking the 4.6 vmcore that you gave to me, here is the
> current crash utility's output of "bt -a", where the crashing task
> entered crash_kexec() via the sysrq-c page fault exception, and the
> tasks on the other cpus have all entered crash_save_cpu() on their
> IRQ stack as a result of the shutdown IPI, one from user-space and
> the others from the kernel:
>
> crash> bt -a
> PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
> #0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022f43010] handle_IPI at ffff00000808e718
> #2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022f43080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff000008dc3f10 PSTATE: 60400149
> X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
> X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
> X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
> X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
> X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
> X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
> #5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff000008dc3f80] rest_init at ffff0000087c7930
> #7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
>
> PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
> #0 [ffff800022f56e50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022f57010] handle_IPI at ffff00000808e718
> #2 [ffff800022f57040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022f57080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff8000218cff60 PSTATE: 60000349
> X29: ffff8000218cff60 X28: ffff8000218cc000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff8000218cc000 X22: ffff000008bd0270 X21: ffff8000218cc000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
> X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000005
> X14: 0000000c80096000 X13: ffff8000212b6600 X12: 0000000000047ae2
> X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218cc000
> X8: 000000010000aa07 X7: 0000000000000243 X6: 0015752a00000000
> X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
> #5 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
> #0 [ffff800022f6ae50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022f6b010] handle_IPI at ffff00000808e718
> #2 [ffff800022f6b040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022f6b080] el0_irq_naked at ffff000008084c4c
> --- <IRQ stack> ---
> PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
> X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
> X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
> X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
> X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
> X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
> X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
> X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
> X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
> X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
> X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
> ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE:
> 20000000
>
> PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> #0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
> #1 [ffff800020b0ba90] die at ffff000008088ce8
> #2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
> #3 [ffff800020b0bb00] do_page_fault at ffff000008096814
> #4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
> #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> #6 [ffff800020b0bd40] el1_da at ffff000008084568
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
> ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
> #7 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
> #8 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
> #9 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
> #10 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
> #11 [ffff800020b0be50] vfs_write at ffff0000081c5588
> #12 [ffff800020b0be90] sys_write at ffff0000081c68fc
> #13 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
> PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
> X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
> X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
> X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
> X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
> X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
> X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
> X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
> X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
> X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
> X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
> ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
>
> PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
> #0 [ffff800022f92e50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022f93010] handle_IPI at ffff00000808e718
> #2 [ffff800022f93040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022f93080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff8000218dbf60 PSTATE: 60000349
> X29: ffff8000218dbf60 X28: ffff8000218d8000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff8000218d8000 X22: ffff000008bd0270 X21: ffff8000218d8000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
> X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000000
> X14: 0000000000000000 X13: 00000000000003e7 X12: 0000000000000016
> X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218d8000
> X8: ffff8000218c3ab0 X7: 7fffffffffffffff X6: 0015752a00000000
> X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
> #5 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
> #0 [ffff800022fa6e50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022fa7010] handle_IPI at ffff00000808e718
> #2 [ffff800022fa7040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022fa7080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff8000218dff60 PSTATE: 60000349
> X29: ffff8000218dff60 X28: ffff8000218dc000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff8000218dc000 X22: ffff000008bd0270 X21: ffff8000218dc000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
> X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 0000000000000007
> X14: 0000000000000000 X13: 000000000000039d X12: 0000000000000000
> X11: ffff0000087ef470 X10: 0000000000000850 X9: ffff8000218dc000
> X8: 000000010000a479 X7: 7fffffffffffffff X6: 0016694e00000000
> X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
> #5 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
> #0 [ffff800022fbae50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022fbb010] handle_IPI at ffff00000808e718
> #2 [ffff800022fbb040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022fbb080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff8000218ebf60 PSTATE: 60000349
> X29: ffff8000218ebf60 X28: ffff8000218e8000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff8000218e8000 X22: ffff000008bd0270 X21: ffff8000218e8000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: 000000000000016d
> X17: 0000007f8b122780 X16: ffffffc0001adf68 X15: 000000000000000a
> X14: 0000000000000000 X13: 00000000000003a1 X12: 0000000000000000
> X11: ffff8000213a0d00 X10: 0000000000000850 X9: ffff8000218e8000
> X8: ffff8000218c53b0 X7: 00000000000001c4 X6: 002887fa00000000
> X5: 0100000000000000 X4: 00000000000003c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
> #5 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
> #0 [ffff800022fcee50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022fcf010] handle_IPI at ffff00000808e718
> #2 [ffff800022fcf040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022fcf080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff8000218eff60 PSTATE: 60400149
> X29: ffff8000218eff60 X28: ffff8000218ec000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff8000218ec000 X22: ffff000008bd0270 X21: ffff8000218ec000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: fffffffffffffffb
> X17: 0000ffff9dc8ba48 X16: ffff0000081da048 X15: 003b9aca00000000
> X14: 0000000000000000 X13: 0000000000000001 X12: 0000000000000000
> X11: ffff800021089980 X10: 0000000000000850 X9: ffff8000218ec000
> X8: 000000010000a280 X7: ffff8000209b39c8 X6: ffff8000209b39c9
> X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
> #5 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
> crash>
>
> And that's precisely what we want to see. (other than perhaps the
> in-kernel exception frame's dump of the ORIG_X0 and SYSCALL registers)
>
> Now, with v5's patches 1/2 and 2/2 patches applied, this is what gets
> displayed:
>
> crash> bt -a
> PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
> #0 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
> #1 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
> #2 [ffff000008dc3f80] rest_init at ffff0000087c7930
> #3 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
>
> PID: 0 TASK: ffff8000218c0c80 CPU: 1 COMMAND: "swapper/1"
> #0 [ffff8000218cff60] arch_cpu_idle at ffff0000080857c0
> #1 [ffff8000218cff70] cpu_startup_entry at ffff0000080f26cc
> #2 [ffff8000218cffd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 1324 TASK: ffff80002018be80 CPU: 2 COMMAND: "dhry"
> PC: 00000000004016a4 LR: 00000000004016a4 SP: 0000ffffc10c40a0
> X29: 0000ffffc10c40a0 X28: 0000000000000000 X27: 0000000000000000
> X26: 0000000000000000 X25: 0000000000402138 X24: 00000000004021f0
> X23: 0000000000000000 X22: 0000000000000000 X21: 00000000004001a0
> X20: 0000000000000000 X19: 0000000000000000 X18: 0000000000000000
> X17: 0000000000000001 X16: 0000000000000000 X15: 0000000000493000
> X14: 0000000000498000 X13: ffffffffffffffff X12: 0000000000000005
> X11: 000000000000001e X10: 0101010101010101 X9: fffffffff59a9190
> X8: 7f7f7f7f7f7f7f7f X7: 1f535226301f2b4c X6: 00000003001d1000
> X5: 00101d0003000000 X4: 0000000000000000 X3: 4952545320454d4f
> X2: 0000000010c35b40 X1: 0000000000000011 X0: 0000000010c35b40
> ORIG_X0: 0000000000498700 SYSCALLNO: ffffffffffffffff PSTATE:
> 20000000
>
> #0 [user space]
>
> PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> #0 [ffff800020b0ba70] crash_kexec at ffff00000812b0ac
> #1 [ffff800020b0ba90] die at ffff000008088ce8
> #2 [ffff800020b0bad0] __do_kernel_fault at ffff000008098fac
> #3 [ffff800020b0bb00] do_page_fault at ffff000008096814
> #4 [ffff800020b0bb60] do_translation_fault at ffff00000809690c
> #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> #6 [ffff800020b0bc20] el1_da at ffff000008084568
> --- <Exception in kernel> ---
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
>
> #7 [ffff800020b0bd40] sysrq_handle_crash at ffff000008457fc8
> #8 [ffff800020b0bd50] __handle_sysrq at ffff000008458644
> #9 [ffff800020b0bd90] write_sysrq_trigger at ffff000008458ac0
> #10 [ffff800020b0bdb0] proc_reg_write at ffff00000822541c
> #11 [ffff800020b0bdd0] __vfs_write at ffff0000081c4774
> #12 [ffff800020b0be50] vfs_write at ffff0000081c5588
> #13 [ffff800020b0be90] sys_write at ffff0000081c68fc
> #14 [ffff800020b0bed0] el0_svc_naked at ffff000008084e70
> --- <Exception in user> ---
> PC: 0000ffff83d7b1e8 LR: 0000ffff83d2d700 SP: 0000fffffe197460
> X29: 0000fffffe197460 X28: 0000000000000000 X27: 0000000000000000
> X26: 000000001e9e40c8 X25: 00000000004c9be8 X24: 00000000004ec000
> X23: 0000000000000001 X22: 0000000000000002 X21: 0000ffff83e02500
> X20: 000000001e9da808 X19: 0000000000000002 X18: 0000000000000015
> X17: 0000ffff83d266c0 X16: 0000000000000000 X15: 0015a60d0a000000
> X14: 0000ffff83cc28f0 X13: 0000000000000018 X12: 0000000000000008
> X11: 0101010101010101 X10: 7f7f7f7f7f7f7f7f X9: fefefefeff00ff62
> X8: 0000000000000040 X7: fefefefefefefefe X6: 000000001e9da808
> X5: 0000ffff83d055c8 X4: 0000000000000000 X3: 0000000000000000
> X2: 0000000000000002 X1: 000000001e9da808 X0: 0000000000000001
> ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
>
> PID: 0 TASK: ffff8000218c3200 CPU: 4 COMMAND: "swapper/4"
> #0 [ffff8000218dbf60] arch_cpu_idle at ffff0000080857c0
> #1 [ffff8000218dbf70] cpu_startup_entry at ffff0000080f26cc
> #2 [ffff8000218dbfd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 0 TASK: ffff8000218c3e80 CPU: 5 COMMAND: "swapper/5"
> #0 [ffff8000218dff60] arch_cpu_idle at ffff0000080857c0
> #1 [ffff8000218dff70] cpu_startup_entry at ffff0000080f26cc
> #2 [ffff8000218dffd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 0 TASK: ffff8000218c4b00 CPU: 6 COMMAND: "swapper/6"
> #0 [ffff8000218ebf60] arch_cpu_idle at ffff0000080857c0
> #1 [ffff8000218ebf70] cpu_startup_entry at ffff0000080f26cc
> #2 [ffff8000218ebfd0] secondary_start_kernel at ffff00000808e1ec
>
> PID: 0 TASK: ffff8000218c5780 CPU: 7 COMMAND: "swapper/7"
> #0 [ffff8000218eff60] arch_cpu_idle at ffff0000080857c0
> #1 [ffff8000218eff70] cpu_startup_entry at ffff0000080f26cc
> #2 [ffff8000218effd0] secondary_start_kernel at ffff00000808e1ec
> crash>
>
> What happened? This can't possibly be what you intended, is it?
>
> Dave
>
> --
> Crash-utility mailing list
> Crash-utility(a)redhat.com
> https://www.redhat.com/mailman/listinfo/crash-utility
>
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
7:32 p.m.
On Wed, Jun 29, 2016 at 05:25:27PM -0400, Dave Anderson wrote:
Hi Takahiro,
Here is another thing that I would prefer not to change/omit.
In the current code, the raw exception frame data is dumped as
part of the "bt -[fF]" output, just prior to it being translated
as an exception frame:
crash> bt -F
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
... [ cut ] ...
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
ffff800020b0bc20: 0000000000000001 0000000000000000
ffff800020b0bc30: log_wait+8 0000000000000000
ffff800020b0bc40: 0000000000000000 00000000000047d4
ffff800020b0bc50: ffff800022f337a4 0000000000000000
ffff800020b0bc60: 0000000000000106 0000000000000001
ffff800020b0bc70: 0000000000000002 0000000000000106
ffff800020b0bc80: log_buf_len cont
ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
ffff800020b0bca0: sys_write 0000ffff83d266c0
ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
ffff800020b0bcd0: 0000000000000009 0000000000000000
ffff800020b0bce0: 0000000000000015 0000000000000120
ffff800020b0bcf0: 0000000000000040 sys_call_table
ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
...
whereas with the v5 patchset, the exception frame only gets translated,
but the actual raw memory never gets dumped:
I surely remember that you said that would not be an issue
when I submitted older version, maybe v1 or v2.
crash> bt -F
PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
... [ cut ] ...
#5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
#6 [ffff800020b0bc20] el1_da at ffff000008084564
Do you think that those symbolic display are still useful
though it is not quite easy to recognize which register has what value?
Even more, <ffff800020b0bb80-ffff800020b0bc10> is *not* a stack for
do_mem_abort(). It is just wrong and will confuse people.
So this is another example of improvement on my patches.
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
...
It's actually helpful being able to dump the exception frame data with
"bt -F" to show symbolic references that are found within the exception
frame itself.
IMO I think that you'd better improve the output of
arm64_print_exception_frame() for bt -F.
And this is an totally independent issue from other parts of my patches.
-Takahiro AKASHI
Dave
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
1:20 p.m.
New subject: [PATCH v5 4/4] arm64: add VHE support
----- Original Message -----
On 29 June 2016 at 04:35, Dave Anderson < anderson(a)redhat.com >
wrote:
>
>
> Takahiro,
>
> Without this patch -- aside from in-kernel exception frames not
> getting found and displayed -- does everything else work OK?
​All what I can say is that this is the only issue that I have noticed so far on
VHE-enabled system.​
So the "subject" may be overstated.
Thanks,
-Takahiro AKASHI
This one's low-hanging fruit -- queued for crash-7.1.6:
https://github.com/crash-utility/crash/commit/7bdaf442dcadefb924805b5db30...
Dave
> Dave
>
>
> ----- Original Message -----
> > On VHE-enabled system, the kernel runs in EL2.
> >
> > Signed-off-by: AKASHI Takahiro < takahiro.akashi(a)linaro.org >
> > ---
> > arm64.c | 2 ++
> > 1 file changed, 2 insertions(+)
> >
> > diff --git a/arm64.c b/arm64.c
> > index bad36aa..b9fa14d 100644
> > --- a/arm64.c
> > +++ b/arm64.c
> > @@ -1414,6 +1414,8 @@ arm64_is_kernel_exception_frame(struct bt_info *bt,
> > ulong stkptr)
> > {
> > case PSR_MODE_EL1t:
> > case PSR_MODE_EL1h:
> > + case PSR_MODE_EL2t:
> > + case PSR_MODE_EL2h:
> > return TRUE;
> > }
> > }
> > --
> > 2.9.0
> >
> > --
> > Crash-utility mailing list
> > Crash-utility(a)redhat.com
> > https://www.redhat.com/mailman/listinfo/crash-utility
> >
>
> --
> Crash-utility mailing list
> Crash-utility(a)redhat.com
> https://www.redhat.com/mailman/listinfo/crash-utility
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
1:59 p.m.
----- Original Message -----
On Wed, Jun 29, 2016 at 05:25:27PM -0400, Dave Anderson wrote:
>
>
> Hi Takahiro,
>
> Here is another thing that I would prefer not to change/omit.
>
> In the current code, the raw exception frame data is dumped as
> part of the "bt -[fF]" output, just prior to it being translated
> as an exception frame:
>
> crash> bt -F
> PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> ... [ cut ] ...
> #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> ffff800020b0bba0: 0000000000000002 textbuf.34610
> ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> ffff800020b0bbc0: 000000000000000f 0000000100000000
> ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> ffff800020b0bc00: 0000000000000000 0000000000000015
> ffff800020b0bc10: 0000000000000120 0000000000000040
> ffff800020b0bc20: 0000000000000001 0000000000000000
> ffff800020b0bc30: log_wait+8 0000000000000000
> ffff800020b0bc40: 0000000000000000 00000000000047d4
> ffff800020b0bc50: ffff800022f337a4 0000000000000000
> ffff800020b0bc60: 0000000000000106 0000000000000001
> ffff800020b0bc70: 0000000000000002 0000000000000106
> ffff800020b0bc80: log_buf_len cont
> ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
> ffff800020b0bca0: sys_write 0000ffff83d266c0
> ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
> ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
> ffff800020b0bcd0: 0000000000000009 0000000000000000
> ffff800020b0bce0: 0000000000000015 0000000000000120
> ffff800020b0bcf0: 0000000000000040 sys_call_table
> ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
> ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
> ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
> ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
> #6 [ffff800020b0bd40] el1_da at ffff000008084568
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
> ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
> ...
>
> whereas with the v5 patchset, the exception frame only gets translated,
> but the actual raw memory never gets dumped:
I surely remember that you said that would not be an issue
when I submitted older version, maybe v1 or v2.
> crash> bt -F
> PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> ... [ cut ] ...
> #5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
> ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> ffff800020b0bba0: 0000000000000002 textbuf.34610
> ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> ffff800020b0bbc0: 000000000000000f 0000000100000000
> ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> ffff800020b0bc00: 0000000000000000 0000000000000015
> ffff800020b0bc10: 0000000000000120 0000000000000040
> #6 [ffff800020b0bc20] el1_da at ffff000008084564
Do you think that those symbolic display are still useful
though it is not quite easy to recognize which register has what value?
Even more, <ffff800020b0bb80-ffff800020b0bc10> is *not* a stack for
do_mem_abort(). It is just wrong and will confuse people.
So this is another example of improvement on my patches.
> --- <Exception in kernel> ---
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
> ...
>
> It's actually helpful being able to dump the exception frame data with
> "bt -F" to show symbolic references that are found within the exception
> frame itself.
IMO I think that you'd better improve the output of
arm64_print_exception_frame() for bt -F.
And this is an totally independent issue from other parts of my patches.
-Takahiro AKASHI
> Dave
>
> --
> Crash-utility mailing list
> Crash-utility(a)redhat.com
> https://www.redhat.com/mailman/listinfo/crash-utility
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
2:52 p.m.
----- Original Message -----
Takahiro,
Sorry for the previous empty response -- I sent it by mistake while
putting this 3-message collaborative response together.
Dave,
If you don't like my patches, that is OK.
Applying them or not is totally up to you.
I will *never* submit my patches again.
Well, that's certainly unfortunate.
Having said so, I think I would better explain my intentions on the
code.
On Wed, Jun 29, 2016 at 04:09:01PM -0400, Dave Anderson wrote:
>
>
> Hi Takahiro,
>
> I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
> believe the results are what you intended?
>
> For example, taking the 4.6 vmcore that you gave to me, here is the
> current crash utility's output of "bt -a", where the crashing task
> entered crash_kexec() via the sysrq-c page fault exception, and the
> tasks on the other cpus have all entered crash_save_cpu() on their
> IRQ stack as a result of the shutdown IPI, one from user-space and
> the others from the kernel:
>
> crash> bt -a
> PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
> #0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
> #1 [ffff800022f43010] handle_IPI at ffff00000808e718
> #2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
> #3 [ffff800022f43080] el1_irq at ffff000008084720
> --- <IRQ stack> ---
This part of dump shown above is the result of "kdump" mechanisms,
and is *not* useful for you to understand what was happening on this
CPU when a crash occurred on the other CPU, #3 in this case.
> PC: ffff0000080857c0 [arch_cpu_idle+16]
> LR: ffff0000080857bc [arch_cpu_idle+12]
> SP: ffff000008dc3f10 PSTATE: 60400149
> X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
> X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
> X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
> X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
> X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
> X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
> X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
> X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
> X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> #4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
> #5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
> #6 [ffff000008dc3f80] rest_init at ffff0000087c7930
> #7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
All you need to know is that CPU#0 was idle at the crash point.
Whatever else do you want to know?
Things like:
(1) I want to know whether the active cpu was properly shut down by
the kdump mechanism.
(2) I would like to actually "test" that the crash utility's transition from
the IRQ stack back to the process stack works as expected. (especially given
the relative rarity of kernel crashes that actually occur while operating
on IRQ stacks)
(3) There are several other dumping mechanisms that can be utilized on ARM64
systems, so again, I'd like it to be obvious that this is kdump.
(4) When doing a filtered "foreach bt" it would be helpful to simply make it
obvious that a task was an active, non-crashing, task -- with more information
than a "hanging" exception frame dump.
(5) All of the other architectures do it.
... [ cut ] ...
> But why do you think that it is an improvement to leave out the
transition
> to the IRQ stack?
I explained the reason in my previous e-mail.
I still believe that it is an improvement.
OK, we're just going to have to agree to disagree.
> In any case, I haven't even started looking at the "bt
-f" part of the patch
> because quite frankly, this patchset is so complex that I haven't even
> begun to try to understand it. When you said it would be "easier to
> maintain", well, perhaps for *you* maybe, but certainly not for me!
Do you think so?
It is because you are the author.
Maybe so, but I absolutely do think so.
The changes I made were big and look complex to you,
but the resulting code is simple enough for others IMO.
I tried to make the main loop of arm64_back_trace_cmd() simple and
quite resemble the counterpart of dump_backtrace() in the kernel.
From my perspective, the current arm64_back_trace_cmd() is far
simpler,
and is essentially a backport of the kernel's dump_backtrace() code.
The differences come mainly from the facts:
1.you (crash util) need to display not only a one-line frame summary
(with a value of stack/frame pointer), but also a full dump of stack
Right, but the current loop remains basically the same as the kernel's.
2.your current code will miss a *very important* stack frame that is
the one exactly when an interrupt takes place.
I believe that the latter is a big improvement.
I'm not following when you say it "will miss" a frame. It's just
following
the links established by the architecture. If you line up the backtrace output
of the current vs. your patch, there are no missing frames, but rather you
essentially move the "el1_da()" stack frame entry to a lower stack address,
and your "bt -f" implementation doesn't dump the raw exception frame data.
Yeah, due to the changes, arm64_unwind_frame() may get a bit
complicated,
but it is very naive implementation based on the nature of the kernel's
stack usage (or PCS for ARM), the complexity is not the result of my poor
skill of coding.
In other words, the complexity is now *encapsulated* in that function.
I understand what you're saying, and certainly appreciate the depth of
knowledge that you have, but quite frankly I don't find the changes
significant enough to reinvent the wheel.
In addition, I wrote down that function step-by-step, from generic
cases
to exceptional cases, with a bunch of comments for better understandings.
And you will be able to remove such exceptional cases *if* you don't like
them.
Given those stuffs, I think that my code is easier to maintain.
Again, surely up to you.
> Without the kernel's backtrace code from which the current code is based, I
don't
> have anything to work from anymore. I'm really not sure whether the coverage
of
> the "corner cases" you referred to make this effort worth it. I would
like to
> see examples of how the current code fails. But anyway, I will continue to test it
to
> see if there actually is any significant upgrade from what we already have in
> place. From a kernel debugging perspective, all we really need is a basic
unwinder,
> and optionally the full dump of the stack data in between those frames. And the
> current code does do that at a minimum, and there's much to be said for
> simplicity.
>
... [ cut ] ...
> Here is another thing that I would prefer not to change/omit.
>
> In the current code, the raw exception frame data is dumped as
> part of the "bt -[fF]" output, just prior to it being translated
> as an exception frame:
>
> crash> bt -F
> PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> ... [ cut ] ...
> #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> ffff800020b0bba0: 0000000000000002 textbuf.34610
> ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> ffff800020b0bbc0: 000000000000000f 0000000100000000
> ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> ffff800020b0bc00: 0000000000000000 0000000000000015
> ffff800020b0bc10: 0000000000000120 0000000000000040
> ffff800020b0bc20: 0000000000000001 0000000000000000
> ffff800020b0bc30: log_wait+8 0000000000000000
> ffff800020b0bc40: 0000000000000000 00000000000047d4
> ffff800020b0bc50: ffff800022f337a4 0000000000000000
> ffff800020b0bc60: 0000000000000106 0000000000000001
> ffff800020b0bc70: 0000000000000002 0000000000000106
> ffff800020b0bc80: log_buf_len cont
> ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
> ffff800020b0bca0: sys_write 0000ffff83d266c0
> ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
> ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
> ffff800020b0bcd0: 0000000000000009 0000000000000000
> ffff800020b0bce0: 0000000000000015 0000000000000120
> ffff800020b0bcf0: 0000000000000040 sys_call_table
> ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
> ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
> ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
> ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
> #6 [ffff800020b0bd40] el1_da at ffff000008084568
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
> ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
> ...
>
> whereas with the v5 patchset, the exception frame only gets translated,
> but the actual raw memory never gets dumped:
I surely remember that you said that would not be an issue
when I submitted older version, maybe v1 or v2.
Well, yes, that's true in the case of an exception frame generated
by the IPI interrupt (or any interrupt), where the eframe contents are
incidental to the interrupt's arrival, and not nearly as interesting
as an in-line kernel exception frame generated by a NULL pointer
reference, BUG(), etc., where the contents of the exception frame
are completely related to the problem/bug at hand. And in that case,
the current code does dump the raw contents just before the register
translation.
Do you think that those symbolic display are still useful
though it is not quite easy to recognize which register has what value?
It's not a major item, but it certainly just dumps the raw memory
in between frame like anywhere else in the "bt -f" output. Seems
strange to just omit it there.
Even more, <ffff800020b0bb80-ffff800020b0bc10> is *not* a stack for
do_mem_abort(). It is just wrong and will confuse people.
So this is another example of improvement on my patches.
I'm not following -- your patch (as well as the current version) both
show ffff800020b0bb80-ffff800020b0bc10 below do_mem_abort():
...
#5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
#6 [ffff800020b0bc20] el1_da at ffff000008084564
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
...
The current code does the above, but dumps the raw exception frame
before printing a stack entry for el1_da():
...
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
ffff800020b0bc20: 0000000000000001 0000000000000000
ffff800020b0bc30: log_wait+8 0000000000000000
ffff800020b0bc40: 0000000000000000 00000000000047d4
ffff800020b0bc50: ffff800022f337a4 0000000000000000
ffff800020b0bc60: 0000000000000106 0000000000000001
ffff800020b0bc70: 0000000000000002 0000000000000106
ffff800020b0bc80: log_buf_len cont
ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
ffff800020b0bca0: sys_write 0000ffff83d266c0
ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
ffff800020b0bcd0: 0000000000000009 0000000000000000
ffff800020b0bce0: 0000000000000015 0000000000000120
ffff800020b0bcf0: 0000000000000040 sys_call_table
ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
...
Yes, again I understand that you feel that the el1_da() frame
should be at ffff800020b0bc20, but on the other hand, the
link register/pc under do_mem_abort() points to ffff800020b0bd40,
and I kind of prefer to show that. I *know* you disagree.
... [ cut ] ...
IMO I think that you'd better improve the output of
arm64_print_exception_frame() for bt -F.
And this is an totally independent issue from other parts of my patches.
I don't see where "bt -F" gets involved in arm64_print_exception_frame()?
Are you referring to this, which I agree is OK:
@@ -2130,10 +2343,11 @@ arm64_print_exception_frame(struct bt_info *bt, ulong pt_regs, int
mode, FILE *o
}
if (is_64_bit) {
- fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
- (ulong)regs->orig_x0, (ulong)regs->syscallno);
- if (mode == USER_MODE)
+ if (mode == USER_MODE) {
+ fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
+ (ulong)regs->orig_x0, (ulong)regs->syscallno);
fprintf(ofp, " PSTATE: %08lx",
(ulong)regs->pstate);
+ }
fprintf(ofp, "\n");
}
Anyway, I'm going to be away for the next few days, and will revisit
this all again next week.
But here's a thought:
There already are "bt -o" and "bt -O" for x86 and x86_64, which allow
for the existence of coexisting but separate back trace methods.
I'm thinking that we can keep the current code, while also introducing
your patchset as an "optional" back trace method. There would be
arm64_back_trace_cmd_v1() and arm64_back_trace_cmd_v2(), arm64_unwind_frame_v1()
and arm64_unwind_frame_v2(), and any other functions that can't comfortably
be utilized by both methods. And then "bt -o" would use the optional
version, or the default method selection could be toggled with "bt -O".
And again, I hope we don't lose you. During the KASLR negotiations, you've
already seen what an obstinate ass I can be. But it's really part of my
job as a gatekeeper.
Thanks,
Dave
"Simplicity, simplicity, simplicity."
- Henry David Thoreau
3:49 p.m.
Nakahiro,
As I had last proposed, I have kept the original backtrace method in place,
while making your redesigned method available as a bt option accessible by
using "bt -o", or by using "bt -O" to toggle the default method from
the
original to the optional.
There were a couple bugs in your backtrace method.
(1) Although I can't find a sample vmcore where this piece gets executed,
this part was certainly incorrect:
+ /*
+ * Keep a buffer for a while until
+ * displaying the last frame on IRQ stack
+ * at next arm64_print_stackframe_entry()
+ */
+ if (bt->flags | BT_FULL)
+ ms->irq_stackbuf = bt->stackbuf;
(2) And on "older" kernels, your backtrace generates a segmentation violation
on any user-space task. I'm not sure when things changed, but at least on
3.19 kernels, the fp of the fp/pc stackframe pair that references the
kernel-entry exception is not 0, but rather contains the user-space stack
address.
For example, in a 4.5-based kernel, here's a typical syscall-generated
fp/pc stackframe, where the fp is 0:
crash> bt -F 2513
... [ cut ] ...
#11 [ffff8003dbf23e80] sys_write at ffff800000237898
ffff8003dbf23e80: 0000000000000000 __sys_trace_return <==== fp 0
ffff8003dbf23e90: 0000000000000200 0000ffff76e30000
ffff8003dbf23ea0: ffffffffffffffff __sys_trace+28
ffff8003dbf23eb0: 0000000000000002 0000ffff76e30000
ffff8003dbf23ec0: ffffffffffffffff 0000000000000000
#12 [ffff8003dbf23ed0] __sys_trace at ffff800000091a88
PC: 0000ffff7d7dbda8 LR: 0000ffff7d7835d4 SP: 0000fffff90fe1b0
X29: 0000fffff90fe1b0 X28: 0000000000000000 X27: 00000000004fb000
X26: 00000000004bb420 X25: 0000000000000001 X24: 00000000004f8000
X23: 0000000000000000 X22: 0000000000000002 X21: 0000ffff7d881168
X20: 0000ffff76e30000 X19: 0000000000000002 X18: 0000000000000000
X17: 0000ffff7d780e20 X16: 0000000000000000 X15: 00192ea0bab15d05
X14: 0000000000000000 X13: 0000000000000000 X12: 0000000000000001
X11: 000000001c1fc6a0 X10: 00000000004fd000 X9: 0000fffff90fe130
X8: 0000000000000040 X7: 0000000000000001 X6: 0000ffff7d759a98
X5: 0000000000000001 X4: 00000000fbad2a84 X3: 0000000000000000
X2: 0000000000000002 X1: 0000ffff76e30000 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
crash>
But on a 3.19 kernel, it looks like this, where the fp is the user-space
stack address seen in the exception frame:
crash> bt -F 27755
... [ cut ] ...
#10 [fffffe035de9be80] sys_write at fffffe00001e8720
fffffe035de9be80: 000003ffed468290 __sys_trace_return <== fp 3ffed468290
fffffe035de9be90: 0000000000000200 000003ff955f0000
fffffe035de9bea0: ffffffffffffffff __sys_trace+28
fffffe035de9beb0: 0000000000000002 000003ff955f0000
fffffe035de9bec0: ffffffffffffffff 0000000000000000
#11 [fffffe035de9bed0] __sys_trace at fffffe00000939c8
PC: 000003ff9bf9c668 LR: 000003ff9bf439cc SP: 000003ffed468290
X29: 000003ffed468290 X28: 0000000000000000 X27: 00000000004fb000
X26: 00000000004bb330 X25: 0000000000000001 X24: 00000000004f8000
X23: 0000000000000000 X22: 0000000000000002 X21: 000003ff9c041178
X20: 000003ff955f0000 X19: 0000000000000002 X18: 0000000000000000
X17: 000003ff9bf41218 X16: 0000000000000000 X15: 0016431976760606
X14: 0000000000000000 X13: 0000000000000000 X12: 0000000000000001
X11: 0000000000c48820 X10: 00000000004fd000 X9: 000003ffed468220
X8: 0000000000000040 X7: 0000000000000001 X6: 000003ff9bf19a98
X5: 0000000000000001 X4: 00000000fbad2a84 X3: 0000000000000000
X2: 0000000000000002 X1: 000003ff955f0000 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
Same thing if the kernel was entered via interrupt -- here the 4.5-based kernel:
crash> bt -F 1
... [ cut ] ...
#6 [ffff8003dc103e70] sys_epoll_pwait at ffff8000002806c8
ffff8003dc103e70: 0000000000000000 el0_svc_naked+36 <=== fp 0
ffff8003dc103e80: 0000000000000000 0000000000000000
ffff8003dc103e90: ffffffffffffffff 0000ffff9455a48c
ffff8003dc103ea0: 0000000060000000 0000000039000a80
ffff8003dc103eb0: 0000000000000008 0000000000000000
ffff8003dc103ec0: 0000000000000014 000000000ecb0180
#7 [ffff8003dc103ed0] el0_svc_naked at ffff800000091a2c
PC: 0000ffff9455a48c LR: 0000aaaad308fb48 SP: 0000fffff45629b0
X29: 0000fffff45629b0 X28: 0000fffff4562ec8 X27: 0000aaaad313f000
X26: 0000aaaad30bfd70 X25: 0000aaaad313f000 X24: 0000aaaad30c0248
X23: 0000000000000001 X22: 0000aaaaebf2ea50 X21: 0000fffff45629f0
X20: 0000000000000000 X19: 0000000000000004 X18: 0000aaaad30e0050
X17: 0000ffff9455a228 X16: 0000aaaad313f640 X15: 00000000000000d8
X14: 00e53b496f58b0a8 X13: 0000000000000000 X12: 0000000000000010
X11: 0000000000000067 X10: 0000000000000065 X9: 0000000000000074
X8: 0000000000000016 X7: 000000000000cbf6 X6: 000000000003d090
X5: 0000000000000008 X4: 0000000000000000 X3: ffffffffffffffff
X2: 0000000000000022 X1: 0000fffff45629f0 X0: 0000000000000004
ORIG_X0: 0000000000000004 SYSCALLNO: 16 PSTATE: 60000000
crash>
And then here on the 3.19 kernel:
crash> bt -F 1
... [ cut ] ...
#6 [fffffe035c083e80] sys_epoll_pwait at fffffe0000229bc8
fffffe035c083e80: 000003ffe63d3610 el0_svc_naked+36 <=== fp 3ffe63d3610
fffffe035c083e90: 0000000000000000 0000000000000000
fffffe035c083ea0: ffffffffffffffff 000003ff7bc9ad4c
fffffe035c083eb0: 000003ffe63d3470 work_pending+28
fffffe035c083ec0: 0000000000000000 0000000000000000
#7 [fffffe035c083ed0] el0_svc_naked at fffffe000009396c
PC: 000003ff7bc9ad4c LR: 000003ff7c11711c SP: 000003ffe63d3610
X29: 000003ffe63d3610 X28: 0000000000000001 X27: 000003ff7c21f000
X26: 000003ff7c195618 X25: 000003ff7c195f88 X24: 000003ff7c1943e8
X23: 000003ff8e21dc20 X22: 0000000000000000 X21: 000003ffe63d3780
X20: 0000000000000000 X19: 0000000000000004 X18: 0000000000000800
X17: 000003ff7bc9aae8 X16: 000003ff7c21f598 X15: 003b9aca00000000
X14: 002ca73218000000 X13: ffffffffa90e8b35 X12: 0000000000000018
X11: 00000000127f8e50 X10: 0000000000001102 X9: 000000000001f418
X8: 0000000000000016 X7: 0000000000000000 X6: 000003ff7c090000
X5: 0000000000000008 X4: 0000000000000000 X3: ffffffffffffffff
X2: 0000000000000001 X1: 000003ffe63d3780 X0: 0000000000000004
ORIG_X0: 0000000000000004 SYSCALLNO: 16 PSTATE: 60000000
Anyway, that kernel anomoly caused a segmentation violation in your
arm64_gen_hidden_frame() function when it tries to use the user
space stack address in STACK_OFFSET_TYPE(). That turned out to be
my first attempt at maintenance of your patch, and while I've got
it to work without a SIGSEGV, my hack/kludge of a fix doesn't fit
very nicely into your arm64_back_trace_cmd_v2 for() loop. And
on the old kernel, "bt -f" does not display stack memory at and
above the fp/pc stackframe that contain the user-stack fp. Please
feel free to send a follow-up patch.
Other changes I made to your backtrace method:
(1) I removed the "--- <Exception in kernel> ---", "--- <Exception
in user> ---",
lines. I felt that they are unnecessary given that that it's obvious which
is which, and no other arches print such lines.
(2) I removed the "#0 [user space]" line, which was meant for non-kdump kernels
that
did not have a user-space exception frame available, so instead of printing
nothing for a task that was in user-space, it just showed that single line.
(3) I removed the empty linefeed after in-kernel exception frames so that each task's
backtrace is a single blurb without any intervening linefeeds.
I did make a couple changes to the original backtrace method that
adopts code from your new optional method:
(1) ORIG_X0 and SYSCALLNO registers are not displayed in kernel
exception frames.
(2) stackframe entry text locations are modified to be the PC address
of the branch instruction instead of the subsequent "return" PC
address.
That being the case, these are the essential differences between
the original and optional methods:
(1) original: the starting point of backtraces for the active, non-crashing,
tasks, will continue to have crash_save_cpu() on the IRQ stack as the
starting point;
optional: backtrace will start with the IPI exception frame located
on the process stack
(2) optional: exception entry stackframe adjusted to be located farther
down in the IRQ stack
(3) optional: bt -f does not display IRQ stack memory above the adjusted
exception entry stackframe
(4) optional: may display "(Next exception frame might be wrong)"
The combined patchset is queued for crash-7.1.6:
https://github.com/crash-utility/crash/commit/09fdac65d1ff47267d1ebbe7405...
I appreciate all of your help.
Thanks,
Dave
----- Original Message -----
----- Original Message -----
Takahiro,
Sorry for the previous empty response -- I sent it by mistake while
putting this 3-message collaborative response together.
> Dave,
>
> If you don't like my patches, that is OK.
> Applying them or not is totally up to you.
> I will *never* submit my patches again.
Well, that's certainly unfortunate.
> Having said so, I think I would better explain my intentions on the code.
>
> On Wed, Jun 29, 2016 at 04:09:01PM -0400, Dave Anderson wrote:
> >
> >
> > Hi Takahiro,
> >
> > I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
> > believe the results are what you intended?
> >
> > For example, taking the 4.6 vmcore that you gave to me, here is the
> > current crash utility's output of "bt -a", where the crashing
task
> > entered crash_kexec() via the sysrq-c page fault exception, and the
> > tasks on the other cpus have all entered crash_save_cpu() on their
> > IRQ stack as a result of the shutdown IPI, one from user-space and
> > the others from the kernel:
> >
> > crash> bt -a
> > PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND: "swapper/0"
> > #0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
> > #1 [ffff800022f43010] handle_IPI at ffff00000808e718
> > #2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
> > #3 [ffff800022f43080] el1_irq at ffff000008084720
> > --- <IRQ stack> ---
>
> This part of dump shown above is the result of "kdump" mechanisms,
> and is *not* useful for you to understand what was happening on this
> CPU when a crash occurred on the other CPU, #3 in this case.
>
> > PC: ffff0000080857c0 [arch_cpu_idle+16]
> > LR: ffff0000080857bc [arch_cpu_idle+12]
> > SP: ffff000008dc3f10 PSTATE: 60400149
> > X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
> > X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> > X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
> > X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
> > X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
> > X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
> > X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
> > X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
> > X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
> > X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> > ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> > #4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
> > #5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
> > #6 [ffff000008dc3f80] rest_init at ffff0000087c7930
> > #7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
>
> All you need to know is that CPU#0 was idle at the crash point.
> Whatever else do you want to know?
Things like:
(1) I want to know whether the active cpu was properly shut down by
the kdump mechanism.
(2) I would like to actually "test" that the crash utility's transition
from
the IRQ stack back to the process stack works as expected. (especially
given
the relative rarity of kernel crashes that actually occur while operating
on IRQ stacks)
(3) There are several other dumping mechanisms that can be utilized on ARM64
systems, so again, I'd like it to be obvious that this is kdump.
(4) When doing a filtered "foreach bt" it would be helpful to simply make it
obvious that a task was an active, non-crashing, task -- with more
information
than a "hanging" exception frame dump.
(5) All of the other architectures do it.
... [ cut ] ...
> > But why do you think that it is an improvement to leave out the
> > transition
> > to the IRQ stack?
>
> I explained the reason in my previous e-mail.
> I still believe that it is an improvement.
OK, we're just going to have to agree to disagree.
> > In any case, I haven't even started looking at the "bt -f" part
of the
> > patch
> > because quite frankly, this patchset is so complex that I haven't even
> > begun to try to understand it. When you said it would be "easier to
> > maintain", well, perhaps for *you* maybe, but certainly not for me!
>
> Do you think so?
> It is because you are the author.
Maybe so, but I absolutely do think so.
> The changes I made were big and look complex to you,
> but the resulting code is simple enough for others IMO.
>
> I tried to make the main loop of arm64_back_trace_cmd() simple and
> quite resemble the counterpart of dump_backtrace() in the kernel.
>From my perspective, the current arm64_back_trace_cmd() is far simpler,
and is essentially a backport of the kernel's dump_backtrace() code.
> The differences come mainly from the facts:
> 1.you (crash util) need to display not only a one-line frame summary
> (with a value of stack/frame pointer), but also a full dump of stack
Right, but the current loop remains basically the same as the kernel's.
> 2.your current code will miss a *very important* stack frame that is
> the one exactly when an interrupt takes place.
>
> I believe that the latter is a big improvement.
I'm not following when you say it "will miss" a frame. It's just
following
the links established by the architecture. If you line up the backtrace
output
of the current vs. your patch, there are no missing frames, but rather you
essentially move the "el1_da()" stack frame entry to a lower stack address,
and your "bt -f" implementation doesn't dump the raw exception frame data.
> Yeah, due to the changes, arm64_unwind_frame() may get a bit complicated,
> but it is very naive implementation based on the nature of the kernel's
> stack usage (or PCS for ARM), the complexity is not the result of my poor
> skill of coding.
> In other words, the complexity is now *encapsulated* in that function.
I understand what you're saying, and certainly appreciate the depth of
knowledge that you have, but quite frankly I don't find the changes
significant enough to reinvent the wheel.
> In addition, I wrote down that function step-by-step, from generic cases
> to exceptional cases, with a bunch of comments for better understandings.
> And you will be able to remove such exceptional cases *if* you don't like
> them.
>
> Given those stuffs, I think that my code is easier to maintain.
>
> Again, surely up to you.
>
> > Without the kernel's backtrace code from which the current code is based,
> > I don't
> > have anything to work from anymore. I'm really not sure whether the
> > coverage of
> > the "corner cases" you referred to make this effort worth it. I
would
> > like to
> > see examples of how the current code fails. But anyway, I will continue
> > to test it to
> > see if there actually is any significant upgrade from what we already
> > have in
> > place. From a kernel debugging perspective, all we really need is a
> > basic unwinder,
> > and optionally the full dump of the stack data in between those frames.
> > And the
> > current code does do that at a minimum, and there's much to be said for
> > simplicity.
> >
... [ cut ] ...
> > Here is another thing that I would prefer not to change/omit.
> >
> > In the current code, the raw exception frame data is dumped as
> > part of the "bt -[fF]" output, just prior to it being translated
> > as an exception frame:
> >
> > crash> bt -F
> > PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> > ... [ cut ] ...
> > #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> > ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> > ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> > ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> > ffff800020b0bba0: 0000000000000002 textbuf.34610
> > ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> > ffff800020b0bbc0: 000000000000000f 0000000100000000
> > ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> > ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> > ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> > ffff800020b0bc00: 0000000000000000 0000000000000015
> > ffff800020b0bc10: 0000000000000120 0000000000000040
> > ffff800020b0bc20: 0000000000000001 0000000000000000
> > ffff800020b0bc30: log_wait+8 0000000000000000
> > ffff800020b0bc40: 0000000000000000 00000000000047d4
> > ffff800020b0bc50: ffff800022f337a4 0000000000000000
> > ffff800020b0bc60: 0000000000000106 0000000000000001
> > ffff800020b0bc70: 0000000000000002 0000000000000106
> > ffff800020b0bc80: log_buf_len cont
> > ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
> > ffff800020b0bca0: sys_write 0000ffff83d266c0
> > ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
> > ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
> > ffff800020b0bcd0: 0000000000000009 0000000000000000
> > ffff800020b0bce0: 0000000000000015 0000000000000120
> > ffff800020b0bcf0: 0000000000000040 sys_call_table
> > ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
> > ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
> > ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
> > ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
> > #6 [ffff800020b0bd40] el1_da at ffff000008084568
> > PC: ffff000008457fc8 [sysrq_handle_crash+32]
> > LR: ffff000008457fb4 [sysrq_handle_crash+12]
> > SP: ffff800020b0bd40 PSTATE: 60400149
> > X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> > X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> > X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> > X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> > X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> > X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> > X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> > X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> > X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> > X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
> > ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
> > ...
> >
> > whereas with the v5 patchset, the exception frame only gets translated,
> > but the actual raw memory never gets dumped:
>
> I surely remember that you said that would not be an issue
> when I submitted older version, maybe v1 or v2.
Well, yes, that's true in the case of an exception frame generated
by the IPI interrupt (or any interrupt), where the eframe contents are
incidental to the interrupt's arrival, and not nearly as interesting
as an in-line kernel exception frame generated by a NULL pointer
reference, BUG(), etc., where the contents of the exception frame
are completely related to the problem/bug at hand. And in that case,
the current code does dump the raw contents just before the register
translation.
> Do you think that those symbolic display are still useful
> though it is not quite easy to recognize which register has what value?
It's not a major item, but it certainly just dumps the raw memory
in between frame like anywhere else in the "bt -f" output. Seems
strange to just omit it there.
>
> Even more, <ffff800020b0bb80-ffff800020b0bc10> is *not* a stack for
> do_mem_abort(). It is just wrong and will confuse people.
> So this is another example of improvement on my patches.
I'm not following -- your patch (as well as the current version) both
show ffff800020b0bb80-ffff800020b0bc10 below do_mem_abort():
...
#5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
#6 [ffff800020b0bc20] el1_da at ffff000008084564
--- <Exception in kernel> ---
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
...
The current code does the above, but dumps the raw exception frame
before printing a stack entry for el1_da():
...
#5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
ffff800020b0bb70: ffff800020b0bd40 el1_da+24
ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
ffff800020b0bba0: 0000000000000002 textbuf.34610
ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
ffff800020b0bbc0: 000000000000000f 0000000100000000
ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
ffff800020b0bbf0: sysrq_crash_op 0000000000000009
ffff800020b0bc00: 0000000000000000 0000000000000015
ffff800020b0bc10: 0000000000000120 0000000000000040
ffff800020b0bc20: 0000000000000001 0000000000000000
ffff800020b0bc30: log_wait+8 0000000000000000
ffff800020b0bc40: 0000000000000000 00000000000047d4
ffff800020b0bc50: ffff800022f337a4 0000000000000000
ffff800020b0bc60: 0000000000000106 0000000000000001
ffff800020b0bc70: 0000000000000002 0000000000000106
ffff800020b0bc80: log_buf_len cont
ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
ffff800020b0bca0: sys_write 0000ffff83d266c0
ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
ffff800020b0bcd0: 0000000000000009 0000000000000000
ffff800020b0bce0: 0000000000000015 0000000000000120
ffff800020b0bcf0: 0000000000000040 sys_call_table
ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
#6 [ffff800020b0bd40] el1_da at ffff000008084568
PC: ffff000008457fc8 [sysrq_handle_crash+32]
LR: ffff000008457fb4 [sysrq_handle_crash+12]
SP: ffff800020b0bd40 PSTATE: 60400149
...
Yes, again I understand that you feel that the el1_da() frame
should be at ffff800020b0bc20, but on the other hand, the
link register/pc under do_mem_abort() points to ffff800020b0bd40,
and I kind of prefer to show that. I *know* you disagree.
... [ cut ] ...
> IMO I think that you'd better improve the output of
> arm64_print_exception_frame() for bt -F.
> And this is an totally independent issue from other parts of my patches.
I don't see where "bt -F" gets involved in arm64_print_exception_frame()?
Are you referring to this, which I agree is OK:
@@ -2130,10 +2343,11 @@ arm64_print_exception_frame(struct bt_info *bt, ulong
pt_regs, int mode, FILE *o
}
if (is_64_bit) {
- fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
- (ulong)regs->orig_x0, (ulong)regs->syscallno);
- if (mode == USER_MODE)
+ if (mode == USER_MODE) {
+ fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
+ (ulong)regs->orig_x0,
(ulong)regs->syscallno);
fprintf(ofp, " PSTATE: %08lx",
(ulong)regs->pstate);
+ }
fprintf(ofp, "\n");
}
Anyway, I'm going to be away for the next few days, and will revisit
this all again next week.
But here's a thought:
There already are "bt -o" and "bt -O" for x86 and x86_64, which
allow
for the existence of coexisting but separate back trace methods.
I'm thinking that we can keep the current code, while also introducing
your patchset as an "optional" back trace method. There would be
arm64_back_trace_cmd_v1() and arm64_back_trace_cmd_v2(),
arm64_unwind_frame_v1()
and arm64_unwind_frame_v2(), and any other functions that can't comfortably
be utilized by both methods. And then "bt -o" would use the optional
version, or the default method selection could be toggled with "bt -O".
And again, I hope we don't lose you. During the KASLR negotiations, you've
already seen what an obstinate ass I can be. But it's really part of my
job as a gatekeeper.
Thanks,
Dave
"Simplicity, simplicity, simplicity."
- Henry David Thoreau
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
8:54 p.m.
Dave,
My apologies for slow response since I was attending LinuxCon last week.
On Wed, Jul 13, 2016 at 04:49:31PM -0400, Dave Anderson wrote:
Nakahiro,
As I had last proposed, I have kept the original backtrace method in place,
while making your redesigned method available as a bt option accessible by
using "bt -o", or by using "bt -O" to toggle the default method from
the
original to the optional.
IMHO, add an "optional" method is not a good idea if you don't see good
advantages over the original. The option should be reserved for
dwarf-based back-tracer in case that somebody wants to implement it
in the future :)
There were a couple bugs in your backtrace method.
Anyway,
I will try to review your changes, but cannot give it a priority
in the next few weeks.
Thanks,
-Takahiro AKASHI
(1) Although I can't find a sample vmcore where this piece gets executed,
this part was certainly incorrect:
+ /*
+ * Keep a buffer for a while until
+ * displaying the last frame on IRQ stack
+ * at next arm64_print_stackframe_entry()
+ */
+ if (bt->flags | BT_FULL)
+ ms->irq_stackbuf = bt->stackbuf;
(2) And on "older" kernels, your backtrace generates a segmentation violation
on any user-space task. I'm not sure when things changed, but at least on
3.19 kernels, the fp of the fp/pc stackframe pair that references the
kernel-entry exception is not 0, but rather contains the user-space stack
address.
For example, in a 4.5-based kernel, here's a typical syscall-generated
fp/pc stackframe, where the fp is 0:
crash> bt -F 2513
... [ cut ] ...
#11 [ffff8003dbf23e80] sys_write at ffff800000237898
ffff8003dbf23e80: 0000000000000000 __sys_trace_return <==== fp 0
ffff8003dbf23e90: 0000000000000200 0000ffff76e30000
ffff8003dbf23ea0: ffffffffffffffff __sys_trace+28
ffff8003dbf23eb0: 0000000000000002 0000ffff76e30000
ffff8003dbf23ec0: ffffffffffffffff 0000000000000000
#12 [ffff8003dbf23ed0] __sys_trace at ffff800000091a88
PC: 0000ffff7d7dbda8 LR: 0000ffff7d7835d4 SP: 0000fffff90fe1b0
X29: 0000fffff90fe1b0 X28: 0000000000000000 X27: 00000000004fb000
X26: 00000000004bb420 X25: 0000000000000001 X24: 00000000004f8000
X23: 0000000000000000 X22: 0000000000000002 X21: 0000ffff7d881168
X20: 0000ffff76e30000 X19: 0000000000000002 X18: 0000000000000000
X17: 0000ffff7d780e20 X16: 0000000000000000 X15: 00192ea0bab15d05
X14: 0000000000000000 X13: 0000000000000000 X12: 0000000000000001
X11: 000000001c1fc6a0 X10: 00000000004fd000 X9: 0000fffff90fe130
X8: 0000000000000040 X7: 0000000000000001 X6: 0000ffff7d759a98
X5: 0000000000000001 X4: 00000000fbad2a84 X3: 0000000000000000
X2: 0000000000000002 X1: 0000ffff76e30000 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
crash>
But on a 3.19 kernel, it looks like this, where the fp is the user-space
stack address seen in the exception frame:
crash> bt -F 27755
... [ cut ] ...
#10 [fffffe035de9be80] sys_write at fffffe00001e8720
fffffe035de9be80: 000003ffed468290 __sys_trace_return <== fp 3ffed468290
fffffe035de9be90: 0000000000000200 000003ff955f0000
fffffe035de9bea0: ffffffffffffffff __sys_trace+28
fffffe035de9beb0: 0000000000000002 000003ff955f0000
fffffe035de9bec0: ffffffffffffffff 0000000000000000
#11 [fffffe035de9bed0] __sys_trace at fffffe00000939c8
PC: 000003ff9bf9c668 LR: 000003ff9bf439cc SP: 000003ffed468290
X29: 000003ffed468290 X28: 0000000000000000 X27: 00000000004fb000
X26: 00000000004bb330 X25: 0000000000000001 X24: 00000000004f8000
X23: 0000000000000000 X22: 0000000000000002 X21: 000003ff9c041178
X20: 000003ff955f0000 X19: 0000000000000002 X18: 0000000000000000
X17: 000003ff9bf41218 X16: 0000000000000000 X15: 0016431976760606
X14: 0000000000000000 X13: 0000000000000000 X12: 0000000000000001
X11: 0000000000c48820 X10: 00000000004fd000 X9: 000003ffed468220
X8: 0000000000000040 X7: 0000000000000001 X6: 000003ff9bf19a98
X5: 0000000000000001 X4: 00000000fbad2a84 X3: 0000000000000000
X2: 0000000000000002 X1: 000003ff955f0000 X0: 0000000000000001
ORIG_X0: 0000000000000001 SYSCALLNO: 40 PSTATE: 20000000
Same thing if the kernel was entered via interrupt -- here the 4.5-based kernel:
crash> bt -F 1
... [ cut ] ...
#6 [ffff8003dc103e70] sys_epoll_pwait at ffff8000002806c8
ffff8003dc103e70: 0000000000000000 el0_svc_naked+36 <=== fp 0
ffff8003dc103e80: 0000000000000000 0000000000000000
ffff8003dc103e90: ffffffffffffffff 0000ffff9455a48c
ffff8003dc103ea0: 0000000060000000 0000000039000a80
ffff8003dc103eb0: 0000000000000008 0000000000000000
ffff8003dc103ec0: 0000000000000014 000000000ecb0180
#7 [ffff8003dc103ed0] el0_svc_naked at ffff800000091a2c
PC: 0000ffff9455a48c LR: 0000aaaad308fb48 SP: 0000fffff45629b0
X29: 0000fffff45629b0 X28: 0000fffff4562ec8 X27: 0000aaaad313f000
X26: 0000aaaad30bfd70 X25: 0000aaaad313f000 X24: 0000aaaad30c0248
X23: 0000000000000001 X22: 0000aaaaebf2ea50 X21: 0000fffff45629f0
X20: 0000000000000000 X19: 0000000000000004 X18: 0000aaaad30e0050
X17: 0000ffff9455a228 X16: 0000aaaad313f640 X15: 00000000000000d8
X14: 00e53b496f58b0a8 X13: 0000000000000000 X12: 0000000000000010
X11: 0000000000000067 X10: 0000000000000065 X9: 0000000000000074
X8: 0000000000000016 X7: 000000000000cbf6 X6: 000000000003d090
X5: 0000000000000008 X4: 0000000000000000 X3: ffffffffffffffff
X2: 0000000000000022 X1: 0000fffff45629f0 X0: 0000000000000004
ORIG_X0: 0000000000000004 SYSCALLNO: 16 PSTATE: 60000000
crash>
And then here on the 3.19 kernel:
crash> bt -F 1
... [ cut ] ...
#6 [fffffe035c083e80] sys_epoll_pwait at fffffe0000229bc8
fffffe035c083e80: 000003ffe63d3610 el0_svc_naked+36 <=== fp 3ffe63d3610
fffffe035c083e90: 0000000000000000 0000000000000000
fffffe035c083ea0: ffffffffffffffff 000003ff7bc9ad4c
fffffe035c083eb0: 000003ffe63d3470 work_pending+28
fffffe035c083ec0: 0000000000000000 0000000000000000
#7 [fffffe035c083ed0] el0_svc_naked at fffffe000009396c
PC: 000003ff7bc9ad4c LR: 000003ff7c11711c SP: 000003ffe63d3610
X29: 000003ffe63d3610 X28: 0000000000000001 X27: 000003ff7c21f000
X26: 000003ff7c195618 X25: 000003ff7c195f88 X24: 000003ff7c1943e8
X23: 000003ff8e21dc20 X22: 0000000000000000 X21: 000003ffe63d3780
X20: 0000000000000000 X19: 0000000000000004 X18: 0000000000000800
X17: 000003ff7bc9aae8 X16: 000003ff7c21f598 X15: 003b9aca00000000
X14: 002ca73218000000 X13: ffffffffa90e8b35 X12: 0000000000000018
X11: 00000000127f8e50 X10: 0000000000001102 X9: 000000000001f418
X8: 0000000000000016 X7: 0000000000000000 X6: 000003ff7c090000
X5: 0000000000000008 X4: 0000000000000000 X3: ffffffffffffffff
X2: 0000000000000001 X1: 000003ffe63d3780 X0: 0000000000000004
ORIG_X0: 0000000000000004 SYSCALLNO: 16 PSTATE: 60000000
Anyway, that kernel anomoly caused a segmentation violation in your
arm64_gen_hidden_frame() function when it tries to use the user
space stack address in STACK_OFFSET_TYPE(). That turned out to be
my first attempt at maintenance of your patch, and while I've got
it to work without a SIGSEGV, my hack/kludge of a fix doesn't fit
very nicely into your arm64_back_trace_cmd_v2 for() loop. And
on the old kernel, "bt -f" does not display stack memory at and
above the fp/pc stackframe that contain the user-stack fp. Please
feel free to send a follow-up patch.
Other changes I made to your backtrace method:
(1) I removed the "--- <Exception in kernel> ---", "---
<Exception in user> ---",
lines. I felt that they are unnecessary given that that it's obvious which
is which, and no other arches print such lines.
(2) I removed the "#0 [user space]" line, which was meant for non-kdump kernels
that
did not have a user-space exception frame available, so instead of printing
nothing for a task that was in user-space, it just showed that single line.
(3) I removed the empty linefeed after in-kernel exception frames so that each
task's
backtrace is a single blurb without any intervening linefeeds.
I did make a couple changes to the original backtrace method that
adopts code from your new optional method:
(1) ORIG_X0 and SYSCALLNO registers are not displayed in kernel
exception frames.
(2) stackframe entry text locations are modified to be the PC address
of the branch instruction instead of the subsequent "return" PC
address.
That being the case, these are the essential differences between
the original and optional methods:
(1) original: the starting point of backtraces for the active, non-crashing,
tasks, will continue to have crash_save_cpu() on the IRQ stack as the
starting point;
optional: backtrace will start with the IPI exception frame located
on the process stack
(2) optional: exception entry stackframe adjusted to be located farther
down in the IRQ stack
(3) optional: bt -f does not display IRQ stack memory above the adjusted
exception entry stackframe
(4) optional: may display "(Next exception frame might be wrong)"
The combined patchset is queued for crash-7.1.6:
https://github.com/crash-utility/crash/commit/09fdac65d1ff47267d1ebbe7405...
I appreciate all of your help.
Thanks,
Dave
----- Original Message -----
> ----- Original Message -----
>
> Takahiro,
>
> Sorry for the previous empty response -- I sent it by mistake while
> putting this 3-message collaborative response together.
>
> > Dave,
> >
> > If you don't like my patches, that is OK.
> > Applying them or not is totally up to you.
> > I will *never* submit my patches again.
>
> Well, that's certainly unfortunate.
>
> > Having said so, I think I would better explain my intentions on the code.
> >
> > On Wed, Jun 29, 2016 at 04:09:01PM -0400, Dave Anderson wrote:
> > >
> > >
> > > Hi Takahiro,
> > >
> > > I applied patches 1/2 and 2/2 from the v5 patchset. But I can't
> > > believe the results are what you intended?
> > >
> > > For example, taking the 4.6 vmcore that you gave to me, here is the
> > > current crash utility's output of "bt -a", where the
crashing task
> > > entered crash_kexec() via the sysrq-c page fault exception, and the
> > > tasks on the other cpus have all entered crash_save_cpu() on their
> > > IRQ stack as a result of the shutdown IPI, one from user-space and
> > > the others from the kernel:
> > >
> > > crash> bt -a
> > > PID: 0 TASK: ffff000008dcd900 CPU: 0 COMMAND:
"swapper/0"
> > > #0 [ffff800022f42e50] crash_save_cpu at ffff00000812ae44
> > > #1 [ffff800022f43010] handle_IPI at ffff00000808e718
> > > #2 [ffff800022f43040] gic_handle_irq at ffff0000080815f8
> > > #3 [ffff800022f43080] el1_irq at ffff000008084720
> > > --- <IRQ stack> ---
> >
> > This part of dump shown above is the result of "kdump" mechanisms,
> > and is *not* useful for you to understand what was happening on this
> > CPU when a crash occurred on the other CPU, #3 in this case.
> >
> > > PC: ffff0000080857c0 [arch_cpu_idle+16]
> > > LR: ffff0000080857bc [arch_cpu_idle+12]
> > > SP: ffff000008dc3f10 PSTATE: 60400149
> > > X29: ffff000008dc3f10 X28: ffff000008dc0000 X27: 0000000000000000
> > > X26: ffff000008dc5000 X25: ffff000008dc5c1c X24: ffff000008dc5000
> > > X23: ffff000008dc0000 X22: ffff000008bd0270 X21: ffff000008dc0000
> > > X20: ffff000008dc5b88 X19: 0000000000000000 X18: 00000000a632f641
> > > X17: 0000ffff7da57880 X16: ffff0000081d9838 X15: 00000000383a0a79
> > > X14: 00000000b2b0b162 X13: 000000005f2cbeec X12: 0000000000045a9e
> > > X11: ffff8000213bd800 X10: 0000000000000850 X9: ffff000008dc0000
> > > X8: 000000010000aa07 X7: 000000000000003d X6: 0015752a00000000
> > > X5: 0100000000000000 X4: 00000000000001c0 X3: 0000000000000000
> > > X2: 0000000000000001 X1: 0000000000000000 X0: 0000000000000000
> > > ORIG_X0: 0000000000000000 SYSCALLNO: 7fffffffffffffff
> > > #4 [ffff000008dc3f10] arch_cpu_idle at ffff0000080857c0
> > > #5 [ffff000008dc3f20] cpu_startup_entry at ffff0000080f26cc
> > > #6 [ffff000008dc3f80] rest_init at ffff0000087c7930
> > > #7 [ffff000008dc3fa0] start_kernel at ffff000008b10b70
> >
> > All you need to know is that CPU#0 was idle at the crash point.
> > Whatever else do you want to know?
>
> Things like:
>
> (1) I want to know whether the active cpu was properly shut down by
> the kdump mechanism.
> (2) I would like to actually "test" that the crash utility's
transition from
> the IRQ stack back to the process stack works as expected. (especially
> given
> the relative rarity of kernel crashes that actually occur while operating
> on IRQ stacks)
> (3) There are several other dumping mechanisms that can be utilized on ARM64
> systems, so again, I'd like it to be obvious that this is kdump.
> (4) When doing a filtered "foreach bt" it would be helpful to simply make
it
> obvious that a task was an active, non-crashing, task -- with more
> information
> than a "hanging" exception frame dump.
> (5) All of the other architectures do it.
>
> ... [ cut ] ...
>
> > > But why do you think that it is an improvement to leave out the
> > > transition
> > > to the IRQ stack?
> >
> > I explained the reason in my previous e-mail.
> > I still believe that it is an improvement.
>
> OK, we're just going to have to agree to disagree.
>
> > > In any case, I haven't even started looking at the "bt -f"
part of the
> > > patch
> > > because quite frankly, this patchset is so complex that I haven't
even
> > > begun to try to understand it. When you said it would be "easier to
> > > maintain", well, perhaps for *you* maybe, but certainly not for me!
> >
> > Do you think so?
> > It is because you are the author.
>
> Maybe so, but I absolutely do think so.
>
> > The changes I made were big and look complex to you,
> > but the resulting code is simple enough for others IMO.
> >
> > I tried to make the main loop of arm64_back_trace_cmd() simple and
> > quite resemble the counterpart of dump_backtrace() in the kernel.
>
> >From my perspective, the current arm64_back_trace_cmd() is far simpler,
> and is essentially a backport of the kernel's dump_backtrace() code.
>
> > The differences come mainly from the facts:
> > 1.you (crash util) need to display not only a one-line frame summary
> > (with a value of stack/frame pointer), but also a full dump of stack
>
> Right, but the current loop remains basically the same as the kernel's.
>
> > 2.your current code will miss a *very important* stack frame that is
> > the one exactly when an interrupt takes place.
> >
> > I believe that the latter is a big improvement.
>
> I'm not following when you say it "will miss" a frame. It's just
following
> the links established by the architecture. If you line up the backtrace
> output
> of the current vs. your patch, there are no missing frames, but rather you
> essentially move the "el1_da()" stack frame entry to a lower stack
address,
> and your "bt -f" implementation doesn't dump the raw exception frame
data.
>
> > Yeah, due to the changes, arm64_unwind_frame() may get a bit complicated,
> > but it is very naive implementation based on the nature of the kernel's
> > stack usage (or PCS for ARM), the complexity is not the result of my poor
> > skill of coding.
> > In other words, the complexity is now *encapsulated* in that function.
>
> I understand what you're saying, and certainly appreciate the depth of
> knowledge that you have, but quite frankly I don't find the changes
> significant enough to reinvent the wheel.
>
> > In addition, I wrote down that function step-by-step, from generic cases
> > to exceptional cases, with a bunch of comments for better understandings.
> > And you will be able to remove such exceptional cases *if* you don't like
> > them.
> >
> > Given those stuffs, I think that my code is easier to maintain.
> >
> > Again, surely up to you.
> >
> > > Without the kernel's backtrace code from which the current code is
based,
> > > I don't
> > > have anything to work from anymore. I'm really not sure whether the
> > > coverage of
> > > the "corner cases" you referred to make this effort worth it. I
would
> > > like to
> > > see examples of how the current code fails. But anyway, I will continue
> > > to test it to
> > > see if there actually is any significant upgrade from what we already
> > > have in
> > > place. From a kernel debugging perspective, all we really need is a
> > > basic unwinder,
> > > and optionally the full dump of the stack data in between those frames.
> > > And the
> > > current code does do that at a minimum, and there's much to be said
for
> > > simplicity.
> > >
>
> ... [ cut ] ...
>
> > > Here is another thing that I would prefer not to change/omit.
> > >
> > > In the current code, the raw exception frame data is dumped as
> > > part of the "bt -[fF]" output, just prior to it being
translated
> > > as an exception frame:
> > >
> > > crash> bt -F
> > > PID: 1223 TASK: ffff800020ef5780 CPU: 3 COMMAND: "sh"
> > > ... [ cut ] ...
> > > #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> > > ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> > > ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> > > ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> > > ffff800020b0bba0: 0000000000000002 textbuf.34610
> > > ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> > > ffff800020b0bbc0: 000000000000000f 0000000100000000
> > > ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> > > ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> > > ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> > > ffff800020b0bc00: 0000000000000000 0000000000000015
> > > ffff800020b0bc10: 0000000000000120 0000000000000040
> > > ffff800020b0bc20: 0000000000000001 0000000000000000
> > > ffff800020b0bc30: log_wait+8 0000000000000000
> > > ffff800020b0bc40: 0000000000000000 00000000000047d4
> > > ffff800020b0bc50: ffff800022f337a4 0000000000000000
> > > ffff800020b0bc60: 0000000000000106 0000000000000001
> > > ffff800020b0bc70: 0000000000000002 0000000000000106
> > > ffff800020b0bc80: log_buf_len cont
> > > ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
> > > ffff800020b0bca0: sys_write 0000ffff83d266c0
> > > ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
> > > ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
> > > ffff800020b0bcd0: 0000000000000009 0000000000000000
> > > ffff800020b0bce0: 0000000000000015 0000000000000120
> > > ffff800020b0bcf0: 0000000000000040 sys_call_table
> > > ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
> > > ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
> > > ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
> > > ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
> > > #6 [ffff800020b0bd40] el1_da at ffff000008084568
> > > PC: ffff000008457fc8 [sysrq_handle_crash+32]
> > > LR: ffff000008457fb4 [sysrq_handle_crash+12]
> > > SP: ffff800020b0bd40 PSTATE: 60400149
> > > X29: ffff800020b0bd40 X28: ffff800020b08000 X27: ffff0000087e2000
> > > X26: 0000000000000040 X25: 0000000000000120 X24: 0000000000000015
> > > X23: 0000000000000000 X22: 0000000000000009 X21: ffff000008e071b0
> > > X20: 0000000000000063 X19: ffff000008dda000 X18: 0000000000000006
> > > X17: 0000ffff83d266c0 X16: ffff0000081c68b8 X15: ffff000008e6cc95
> > > X14: 0000ffff83cc28f0 X13: ffff000008e6c758 X12: ffff000008dda7a0
> > > X11: 0000000000000106 X10: 0000000000000002 X9: 0000000000000001
> > > X8: 0000000000000106 X7: 0000000000000000 X6: ffff800022f337a4
> > > X5: 00000000000047d4 X4: 0000000000000000 X3: 0000000000000000
> > > X2: ffff000008dda7b8 X1: 0000000000000000 X0: 0000000000000001
> > > ORIG_X0: ffff000008dda000 SYSCALLNO: ffff80002104d418
> > > ...
> > >
> > > whereas with the v5 patchset, the exception frame only gets translated,
> > > but the actual raw memory never gets dumped:
> >
> > I surely remember that you said that would not be an issue
> > when I submitted older version, maybe v1 or v2.
>
> Well, yes, that's true in the case of an exception frame generated
> by the IPI interrupt (or any interrupt), where the eframe contents are
> incidental to the interrupt's arrival, and not nearly as interesting
> as an in-line kernel exception frame generated by a NULL pointer
> reference, BUG(), etc., where the contents of the exception frame
> are completely related to the problem/bug at hand. And in that case,
> the current code does dump the raw contents just before the register
> translation.
>
> > Do you think that those symbolic display are still useful
> > though it is not quite easy to recognize which register has what value?
>
> It's not a major item, but it certainly just dumps the raw memory
> in between frame like anywhere else in the "bt -f" output. Seems
> strange to just omit it there.
>
> >
> > Even more, <ffff800020b0bb80-ffff800020b0bc10> is *not* a stack for
> > do_mem_abort(). It is just wrong and will confuse people.
> > So this is another example of improvement on my patches.
>
> I'm not following -- your patch (as well as the current version) both
> show ffff800020b0bb80-ffff800020b0bc10 below do_mem_abort():
>
> ...
> #5 [ffff800020b0bb70] do_mem_abort at ffff000008081288
> ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> ffff800020b0bba0: 0000000000000002 textbuf.34610
> ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> ffff800020b0bbc0: 000000000000000f 0000000100000000
> ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> ffff800020b0bc00: 0000000000000000 0000000000000015
> ffff800020b0bc10: 0000000000000120 0000000000000040
> #6 [ffff800020b0bc20] el1_da at ffff000008084564
> --- <Exception in kernel> ---
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> ...
>
> The current code does the above, but dumps the raw exception frame
> before printing a stack entry for el1_da():
>
> ...
> #5 [ffff800020b0bb70] do_mem_abort at ffff00000808128c
> ffff800020b0bb70: ffff800020b0bd40 el1_da+24
> ffff800020b0bb80: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bb90: ffff800020b0bd40 sysrq_handle_crash+32
> ffff800020b0bba0: 0000000000000002 textbuf.34610
> ffff800020b0bbb0: ffff800020b0bbd0 kallsyms_token_index+43128
> ffff800020b0bbc0: 000000000000000f 0000000100000000
> ffff800020b0bbd0: ffff800020b0bc70 vprintk_default+56
> ffff800020b0bbe0: cpu_cgrp_subsys+152 0000000000000063
> ffff800020b0bbf0: sysrq_crash_op 0000000000000009
> ffff800020b0bc00: 0000000000000000 0000000000000015
> ffff800020b0bc10: 0000000000000120 0000000000000040
> ffff800020b0bc20: 0000000000000001 0000000000000000
> ffff800020b0bc30: log_wait+8 0000000000000000
> ffff800020b0bc40: 0000000000000000 00000000000047d4
> ffff800020b0bc50: ffff800022f337a4 0000000000000000
> ffff800020b0bc60: 0000000000000106 0000000000000001
> ffff800020b0bc70: 0000000000000002 0000000000000106
> ffff800020b0bc80: log_buf_len cont
> ffff800020b0bc90: 0000ffff83cc28f0 text.34829+13
> ffff800020b0bca0: sys_write 0000ffff83d266c0
> ffff800020b0bcb0: 0000000000000006 cpu_cgrp_subsys+152
> ffff800020b0bcc0: 0000000000000063 sysrq_crash_op
> ffff800020b0bcd0: 0000000000000009 0000000000000000
> ffff800020b0bce0: 0000000000000015 0000000000000120
> ffff800020b0bcf0: 0000000000000040 sys_call_table
> ffff800020b0bd00: ffff800020b08000 ffff800020b0bd40
> ffff800020b0bd10: sysrq_handle_crash+12 ffff800020b0bd40
> ffff800020b0bd20: sysrq_handle_crash+32 0000000060400149
> ffff800020b0bd30: cpu_cgrp_subsys+152 [kmalloc-1024]
> #6 [ffff800020b0bd40] el1_da at ffff000008084568
> PC: ffff000008457fc8 [sysrq_handle_crash+32]
> LR: ffff000008457fb4 [sysrq_handle_crash+12]
> SP: ffff800020b0bd40 PSTATE: 60400149
> ...
>
> Yes, again I understand that you feel that the el1_da() frame
> should be at ffff800020b0bc20, but on the other hand, the
> link register/pc under do_mem_abort() points to ffff800020b0bd40,
> and I kind of prefer to show that. I *know* you disagree.
>
> ... [ cut ] ...
>
> > IMO I think that you'd better improve the output of
> > arm64_print_exception_frame() for bt -F.
> > And this is an totally independent issue from other parts of my patches.
>
> I don't see where "bt -F" gets involved in
arm64_print_exception_frame()?
> Are you referring to this, which I agree is OK:
>
> @@ -2130,10 +2343,11 @@ arm64_print_exception_frame(struct bt_info *bt, ulong
> pt_regs, int mode, FILE *o
> }
>
> if (is_64_bit) {
> - fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
> - (ulong)regs->orig_x0, (ulong)regs->syscallno);
> - if (mode == USER_MODE)
> + if (mode == USER_MODE) {
> + fprintf(ofp, "ORIG_X0: %016lx SYSCALLNO: %lx",
> + (ulong)regs->orig_x0,
> (ulong)regs->syscallno);
> fprintf(ofp, " PSTATE: %08lx",
(ulong)regs->pstate);
> + }
> fprintf(ofp, "\n");
> }
>
> Anyway, I'm going to be away for the next few days, and will revisit
> this all again next week.
>
> But here's a thought:
>
> There already are "bt -o" and "bt -O" for x86 and x86_64, which
allow
> for the existence of coexisting but separate back trace methods.
> I'm thinking that we can keep the current code, while also introducing
> your patchset as an "optional" back trace method. There would be
> arm64_back_trace_cmd_v1() and arm64_back_trace_cmd_v2(),
> arm64_unwind_frame_v1()
> and arm64_unwind_frame_v2(), and any other functions that can't comfortably
> be utilized by both methods. And then "bt -o" would use the optional
> version, or the default method selection could be toggled with "bt -O".
>
> And again, I hope we don't lose you. During the KASLR negotiations, you've
> already seen what an obstinate ass I can be. But it's really part of my
> job as a gatekeeper.
>
> Thanks,
> Dave
>
>
> "Simplicity, simplicity, simplicity."
> - Henry David Thoreau
>
> --
> Crash-utility mailing list
> Crash-utility(a)redhat.com
> https://www.redhat.com/mailman/listinfo/crash-utility
>
--
Crash-utility mailing list
Crash-utility(a)redhat.com
https://www.redhat.com/mailman/listinfo/crash-utility
3273
days inactive
3294
days old
devel@lists.crash-utility.osci.io
19 comments
3 participants
participants (3)
-
AKASHI Takahiro -
AKASHI, Takahiro
-
Dave Anderson