[PATCH] Fix a "Bus error" issue caused by 'crash --osrelease' or crash loading
by Lianbo Jiang
Sometimes, in a production environment, there are still some vmcores
that are incomplete, such as partial header or the data is corrupted.
When crash tool attempts to parse such vmcores, it may fail as below:
$ ./crash --osrelease vmcore
Bus error (core dumped)
or
$ crash vmlinux vmcore
...
Bus error (core dumped)
$
The gdb sees that crash tool reads out a null bitmap from the header in
this vmcore, when executing memcpy(), emits a SIGBUS error as below:
$ gdb /home/lijiang/src/crash/crash /tmp/core.126301
Core was generated by `./crash --osrelease /home/lijiang/src/39317/vmcore'.
Program terminated with signal SIGBUS, Bus error.
#0 __memcpy_evex_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:831
831 LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
(gdb) bt
#0 __memcpy_evex_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:831
#1 0x0000000000651096 in read_dump_header (file=0x7ffc59ddff5f "/home/lijiang/src/39317/vmcore") at diskdump.c:820
#2 0x0000000000651cf3 in is_diskdump (file=0x7ffc59ddff5f "/home/lijiang/src/39317/vmcore") at diskdump.c:1042
#3 0x0000000000502ac9 in get_osrelease (dumpfile=0x7ffc59ddff5f "/home/lijiang/src/39317/vmcore") at main.c:1938
#4 0x00000000004fb2e8 in main (argc=3, argv=0x7ffc59dde3a8) at main.c:271
(gdb) frame 1
#1 0x0000000000651096 in read_dump_header (file=0x7ffc59ddff5f "/home/lijiang/src/39317/vmcore") at diskdump.c:820
820 memcpy(dd->dumpable_bitmap, dd->bitmap + bitmap_len/2,
(gdb) p dd->dumpable_bitmap
$1 = 0x7f8e89800010 ""
(gdb) p dd->bitmap
$2 = 0x7f8e87e09000 ""
(gdb) p dd->bitmap + bitmap_len/2
$3 = 0x7f8e88a17000 ""
(gdb) p *(char*)(dd->bitmap+bitmap_len/2)
$4 = 0 '\000'
(gdb) p bitmap_len/2
$5 = 12640256
Let's add a sanity check for such cases to avoid causing a SIGBUS error.
With the patch:
$ crash -s vmlinux vmcore
crash: vmcore: not a supported file format
...
Enter "crash -h" for details.
$ crash --osrelease vmcore
unknown
Reported-by: Buland Kumar Singh <bsingh(a)redhat.com>
Signed-off-by: Lianbo Jiang <lijiang(a)redhat.com>
---
diskdump.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/diskdump.c b/diskdump.c
index 1f7118cacfc6..c31eab01aa05 100644
--- a/diskdump.c
+++ b/diskdump.c
@@ -814,10 +814,12 @@ restart:
madvise(dd->bitmap, bitmap_len, MADV_WILLNEED);
}
- if (dump_is_partial(header))
+ if (dump_is_partial(header)) {
+ if (*(char*)(dd->bitmap + bitmap_len/2) == '\0')
+ goto err;
memcpy(dd->dumpable_bitmap, dd->bitmap + bitmap_len/2,
bitmap_len/2);
- else
+ } else
memcpy(dd->dumpable_bitmap, dd->bitmap, bitmap_len);
dd->data_offset
--
2.45.1
1 day, 5 hours
[PATCH] Fix "irq -a" exceeding the memory range issue
by Tao Liu
Previously without the patch, there was an error observed as follows:
crash> irq -a
IRQ NAME AFFINITY
0 timer 0-191
4 ttyS0 0-23,96-119
...
84 smartpqi 72-73,168
irq: page excluded: kernel virtual address: ffff97d03ffff000 type: "irq_desc affinity"
The reason is the reading of irq affinity exceeded the memory range, see
the following debug info:
Thread 1 "crash" hit Breakpoint 1, generic_get_irq_affinity (irq=85) at kernel.c:7373
7375 irq_desc_addr = get_irq_desc_addr(irq);
(gdb) p/x irq_desc_addr
$1 = 0xffff97d03f21e800
crash> struct irq_desc 0xffff97d03f21e800
struct irq_desc {
irq_common_data = {
state_use_accessors = 425755136,
node = 3,
handler_data = 0x0,
msi_desc = 0xffff97ca51b83480,
affinity = 0xffff97d03fffee60,
effective_affinity = 0xffff97d03fffe6c0
},
crash> whatis cpumask_t
typedef struct cpumask {
unsigned long bits[128];
} cpumask_t;
SIZE: 1024
In order to get the affinity, crash will read the memory range 0xffff97d03fffee60
~ 0xffff97d03fffee60 + 1024(0x400) by line:
readmem(affinity_ptr, KVADDR, affinity, len,
"irq_desc affinity", FAULT_ON_ERROR);
However the reading will exceed the effective memory range:
crash> kmem 0xffff97d03fffee60
CACHE OBJSIZE ALLOCATED TOTAL SLABS SSIZE NAME
ffff97c900044400 32 123297 162944 1273 4k kmalloc-32
SLAB MEMORY NODE TOTAL ALLOCATED FREE
fffffca460ffff80 ffff97d03fffe000 3 128 81 47
FREE / [ALLOCATED]
[ffff97d03fffee60]
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
fffffca460ffff80 83fffe000 dead000000000001 ffff97d03fffe340 1 d7ffffe0000800 slab
crash> kmem ffff97d03ffff000
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
fffffca460ffffc0 83ffff000 0 0 1 d7ffffe0004000 reserved
crash> dmesg
...
[ 0.000000] BIOS-e820: [mem 0x00000000fe000000-0x00000000fe00ffff] reserved
[ 0.000000] BIOS-e820: [mem 0x0000000100000000-0x000000083fffefff] usable
[ 0.000000] BIOS-e820: [mem 0x000000083ffff000-0x000000083fffffff] reserved
...
The beginning physical address, aka 0x83fffe000, is located in the usable
area and is readable, however the later physical address, starting from
0x83ffff000, is located in reserved region and not readable. In fact,
the affinity member is allocated by alloc_cpumask_var_node(), for the 192 CPUs
system, the allocated size is only 24, and we can see it is within
the kmalloc-32 slab. So it is incorrect to read 1024 length(given by
STRUCT_SIZE("cpumask_t")), only 24 is enough.
Since there are plenty of places in crash which takes the value of
STRUCT_SIZE("cpumask_t"), and works fine for the past, this patch will
not modify them all, but only this place which encountered the issue.
Signed-off-by: Tao Liu <ltao(a)redhat.com>
---
kernel.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/kernel.c b/kernel.c
index 8a9d498..464e877 100644
--- a/kernel.c
+++ b/kernel.c
@@ -7362,7 +7362,7 @@ void
generic_get_irq_affinity(int irq)
{
ulong irq_desc_addr;
- long len;
+ long len, len_cpumask;
ulong affinity_ptr;
ulong *affinity;
ulong tmp_addr;
@@ -7382,8 +7382,11 @@ generic_get_irq_affinity(int irq)
if (!action)
return;
- if ((len = STRUCT_SIZE("cpumask_t")) < 0)
- len = DIV_ROUND_UP(kt->cpus, BITS_PER_LONG) * sizeof(ulong);
+ len = DIV_ROUND_UP(kt->cpus, BITS_PER_LONG) * sizeof(ulong);
+ len_cpumask = STRUCT_SIZE("cpumask_t");
+ if (len_cpumask > 0) {
+ len = len_cpumask > len ? len : len_cpumask;
+ }
affinity = (ulong *)GETBUF(len);
if (VALID_MEMBER(irq_common_data_affinity))
--
2.40.1
2 days, 9 hours
arm64: Fix bt command show wrong stacktrace on ramdump source
by cb126yx
Hi all,
I may found a potential bug when using qcom arm64 ramdump to parse backtrace.
Unfortunately, I actually found no processes can use the bt command correctly.
Ex: when start crash tool to do analyse: # crash vmlinux --kaslr=xxx DDRCS0_0.BIN@0x0000000080000000,... --machdep vabits_actual=39
Then seen below misleading backtrace information :
crash> bt 16930
PID: 16930 TASK: ffffff89b3eada00 CPU: 2 COMMAND: "Firebase Backgr"
#0 [ffffffc034c437f0] __switch_to at ffffffe0036832d4
#1 [ffffffc034c43850] __kvm_nvhe_$d.2314 at 6be732e004cf05a0
#2 [ffffffc034c438b0] __kvm_nvhe_$d.2314 at 86c54c6004ceff80
#3 [ffffffc034c43950] __kvm_nvhe_$d.2314 at 55d6f96003a7b120
#4 [ffffffc034c439f0] __kvm_nvhe_$d.2314 at 9ccec46003a80a64
#5 [ffffffc034c43ac0] __kvm_nvhe_$d.2314 at 8cf41e6003a945c4
#6 [ffffffc034c43b10] __kvm_nvhe_$d.2314 at a8f181e00372c818
#7 [ffffffc034c43b40] __kvm_nvhe_$d.2314 at 6dedde600372c0d0
#8 [ffffffc034c43b90] __kvm_nvhe_$d.2314 at 62cc07e00373d0ac
#9 [ffffffc034c43c00] __kvm_nvhe_$d.2314 at 72fb1de00373bedc
...
PC: 00000073f5294840 LR: 00000070d8f39ba4 SP: 00000070d4afd5d0
X29: 00000070d4afd600 X28: b4000071efcda7f0 X27: 00000070d4afe000
X26: 0000000000000000 X25: 00000070d9616000 X24: 0000000000000000
X23: 0000000000000000 X22: 0000000000000000 X21: 0000000000000000
X20: b40000728fd27520 X19: b40000728fd27550 X18: 000000702daba000
X17: 00000073f5294820 X16: 00000070d940f9d8 X15: 00000000000000bf
X14: 0000000000000000 X13: 00000070d8ad2fac X12: b40000718fce5040
X11: 0000000000000000 X10: 0000000000000070 X9: 0000000000000001
X8: 0000000000000062 X7: 0000000000000020 X6: 0000000000000000
X5: 0000000000000000 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 0000000000000080 X0: b40000728fd27550
ORIG_X0: b40000728fd27550 SYSCALLNO: ffffffff PSTATE: 40001000
By checking the raw data below, will see the lr (fp+8) data show the pointer which already been replaced by PAC prefix.
crash> bt -f
PID: 16930 TASK: ffffff89b3eada00 CPU: 2 COMMAND: "Firebase Backgr"
#0 [ffffffc034c437f0] __switch_to at ffffffe0036832d4
ffffffc034c437f0: ffffffc034c43850 6be732e004cf05a4
ffffffc034c43800: ffffffe006186108 a0ed07e004cf09c4
ffffffc034c43810: ffffff8a1a340000 ffffff8a8d343c00
ffffffc034c43820: ffffff89b3eada00 ffffff8b780db540
ffffffc034c43830: ffffff89b3eada00 0000000000000000
ffffffc034c43840: 0000000000000004 712b828118484a00
#1 [ffffffc034c43850] __kvm_nvhe_$d.2314 at 6be732e004cf05a0
ffffffc034c43850: ffffffc034c438b0 86c54c6004ceff84
ffffffc034c43860: 000000708070f000 ffffffc034c43938
ffffffc034c43870: ffffff88bd822878 ffffff89b3eada00
...
So we check the CONFIG_ARM64_PTR_AUTH and CONFIG_ARM64_PTR_AUTH_KERNEL to double check if pac mechanism been enabled on this ramdump.
Then we use vabits to figure it out.
Fix then show the right backtrace below:
crash> bt 16930
PID: 16930 TASK: ffffff89b3eada00 CPU: 2 COMMAND: "Firebase Backgr"
#0 [ffffffc034c437f0] __switch_to at ffffffe0036832d4
#1 [ffffffc034c43850] __schedule at ffffffe004cf05a0
#2 [ffffffc034c438b0] preempt_schedule_common at ffffffe004ceff80
#3 [ffffffc034c43950] unmap_page_range at ffffffe003a7b120
#4 [ffffffc034c439f0] unmap_vmas at ffffffe003a80a64
#5 [ffffffc034c43ac0] exit_mmap at ffffffe003a945c4
#6 [ffffffc034c43b10] __mmput at ffffffe00372c818
#7 [ffffffc034c43b40] mmput at ffffffe00372c0d0
#8 [ffffffc034c43b90] exit_mm at ffffffe00373d0ac
#9 [ffffffc034c43c00] do_exit at ffffffe00373bedc
PC: 00000073f5294840 LR: 00000070d8f39ba4 SP: 00000070d4afd5d0
X29: 00000070d4afd600 X28: b4000071efcda7f0 X27: 00000070d4afe000
X26: 0000000000000000 X25: 00000070d9616000 X24: 0000000000000000
X23: 0000000000000000 X22: 0000000000000000 X21: 0000000000000000
X20: b40000728fd27520 X19: b40000728fd27550 X18: 000000702daba000
X17: 00000073f5294820 X16: 00000070d940f9d8 X15: 00000000000000bf
X14: 0000000000000000 X13: 00000070d8ad2fac X12: b40000718fce5040
X11: 0000000000000000 X10: 0000000000000070 X9: 0000000000000001
X8: 0000000000000062 X7: 0000000000000020 X6: 0000000000000000
X5: 0000000000000000 X4: 0000000000000000 X3: 0000000000000000
X2: 0000000000000002 X1: 0000000000000080 X0: b40000728fd27550
ORIG_X0: b40000728fd27550 SYSCALLNO: ffffffff PSTATE: 40001000
Let's use GENMASK to replace the pac pointer to fix it.
gki related commit url here:
https://lore.kernel.org/all/20230412160134.306148-4-mark.rutland@arm.com/
2 days, 15 hours
[PATCH v2] list: Enable LIST_HEAD_FORMAT for -r option
by Li Zhijian
Previously, if LIST_HEAD_FORMAT was not set, `list -r` will traverse the
list in order, that doesn't obey the -r(reverse) semantics.
Per the code, -r(LIST_HEAD_REVERSE) only work with LIST_HEAD_FORMAT and
'-r' usage also says that "For a list linked with list_head structures",
so it's reasonable to enable LIST_HEAD_FORMAT for it.
Cc: Lianbo Jiang <lijiang(a)redhat.com>
Cc: Tao Liu <ltao(a)redhat.com>
Signed-off-by: Li Zhijian <lizhijian(a)fujitsu.com>
---
V2: enable LIST_HEAD_FORMAT for -r option instead of fatal when -h/-H is
not specified for -r. # Lianbo
---
tools.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/tools.c b/tools.c
index 0f2db108838a..1022d579071a 100644
--- a/tools.c
+++ b/tools.c
@@ -3370,6 +3370,7 @@ cmd_list(void)
break;
case 'r':
+ ld->flags |= LIST_HEAD_FORMAT;
ld->flags |= LIST_HEAD_REVERSE;
break;
--
2.29.2
3 days, 12 hours
Re: [PATCH] list: fatal if -r isn't used in conjunction with -H or -h
by lijiang
On Thu, Jun 27, 2024 at 4:31 PM <devel-request(a)lists.crash-utility.osci.io>
wrote:
> Date: Thu, 27 Jun 2024 11:18:29 +0800
> From: Li Zhijian <lizhijian(a)fujitsu.com>
> Subject: [Crash-utility] [PATCH] list: fatal if -r isn't used in
> conjunction with -H or -h
> To: devel(a)lists.crash-utility.osci.io
> Cc: Li Zhijian <lizhijian(a)fujitsu.com>
> Message-ID: <20240627031829.235256-1-lizhijian(a)fujitsu.com>
>
> Per the code, -r(LIST_HEAD_REVERSE) only work with LIST_HEAD_FORMAT
> which is set by -H or -h.
>
> Previously, if LIST_HEAD_FORMAT was not set, `list -r` will traverse the
> list in order, that doesn't obey the -r(reverse) semantics.
>
>
According to the above descriptions, I guess that it should be:
diff --git a/tools.c b/tools.c
index 0f2db108838a..1022d579071a 100644
--- a/tools.c
+++ b/tools.c
@@ -3370,6 +3370,7 @@ cmd_list(void)
break;
case 'r':
+ ld->flags |= LIST_HEAD_FORMAT;
ld->flags |= LIST_HEAD_REVERSE;
break;
Also let's see the following code:
if (ld->flags & LIST_HEAD_FORMAT) {
ld->list_head_offset = ld->member_offset;
if (ld->flags & LIST_HEAD_REVERSE)
ld->member_offset = sizeof(void *);
else
ld->member_offset = 0;
Furthermore, the help page is:
SYNOPSIS
list [[-o] offset][-e end][-[s|S] struct[.member[,member] [-l offset]]
-[x|d]]
[-r|-B] [-h [-O head_offset]|-H] start
The '-r' looks like an optional option, just like '-h' and '-H'.
Can you help double check?
Thanks
Lianbo
> Add a further check to ensure -r is used in conjunction with -H or -h.
>
> Signed-off-by: Li Zhijian <lizhijian(a)fujitsu.com>
> ---
> help.c | 3 ++-
> tools.c | 3 +++
> 2 files changed, 5 insertions(+), 1 deletion(-)
>
> diff --git a/help.c b/help.c
> index d80e843703c1..6f7e093cbef1 100644
> --- a/help.c
> +++ b/help.c
> @@ -5977,7 +5977,8 @@ char *help__list[] = {
> " ",
> " -x Override the default output format with hexadecimal
> format.",
> " -d Override the default output format with decimal format.",
> -" -r For a list linked with list_head structures, traverse the
> list",
> +" -r Must be used in conjunction with either -H or -h.",
> +" For a list linked with list_head structures, traverse the
> list",
> " in the reverse order by using the \"prev\" pointer
> instead",
> " of \"next\".",
> " -B Use the algorithm from R. P. Brent to detect loops
> instead of",
> diff --git a/tools.c b/tools.c
> index 0f2db108838a..67977605c276 100644
> --- a/tools.c
> +++ b/tools.c
> @@ -3451,6 +3451,9 @@ cmd_list(void)
> }
> }
>
> + if (ld->flags & LIST_HEAD_REVERSE && !(ld->flags &
> LIST_HEAD_FORMAT))
> + error(FATAL, "-r must be used in conjunction with -H or
> -h\n");
> +
> if (argerrs)
> cmd_usage(pc->curcmd, SYNOPSIS);
>
> --
> 2.29.2
>
4 days, 7 hours
