vtop returns stale page table entries due to FILL_PUD caching logic
by Anderson Nascimento
Hello,
I have been using the crash tool to teach paging. It is an excellent
tool for simplifying page table walks for students. However, I have
encountered a persistent issue regarding stale data when inspecting
mappings that are populated mid-session.
When using the vtop command on a mapping that is not yet populated,
and then running it again after a memory operation has occurred, vtop
continues to return NULL or stale entries. This happens because the
FILL_PUD (and similar) macros check if the current PUD address matches
the last_pud_read address. If they match, the tool skips the readmem()
call, even if the underlying physical memory has changed.
In the debugging session below, I demonstrate that rd -p showed the
populated PUD entry, but vtop still reported 0. I was able to resolve
this by manually forcing a re-read in GDB by resetting the cache
variable:
(gdb) set machdep->last_pud_read=1
992 #define IS_LAST_PUD_READ(pud) ((ulong)(pud) == machdep->last_pud_read)
...
1001 #define FILL_PUD(PUD, TYPE, SIZE)
\
1002 if (!IS_LAST_PUD_READ(PUD)) {
\
1003 readmem((ulonglong)((ulong)(PUD)), TYPE,
machdep->pud, \
1004 SIZE, "pud page", FAULT_ON_ERROR);
\
1005 machdep->last_pud_read = (ulong)(PUD);
\
1006 }
Steps to Reproduce:
1) Run vtop on an unpopulated user address.
2) Trigger a page fault/memory access in the target process to
populate the entry.
3) Run vtop again; it will still show "(not mapped)" despite the
physical memory being updated.
Is this caching behavior intended for performance, or should a way to
invalidate this cache for live sessions be implemented?
crash> vtop -c 5084 0x41414000
[Detaching after fork from child process 5085]
VIRTUAL PHYSICAL
41414000 (not mapped)
PGD: 6e80000 => 7fc8067
PUD: 7fc8008 => 0
VMA START END FLAGS FILE
ffff88801ec382b8 41414000 41415000 8100073
crash> rd -p 7fc8008
[Detaching after fork from child process 5086]
7fc8008: 0000000000000000 ........
crash> vtop -c 5084 0x41414000
[Detaching after fork from child process 5087]
VIRTUAL PHYSICAL
41414000 (not mapped)
PGD: 6e80000 => 7fc8067
PUD: 7fc8008 => 0
VMA START END FLAGS FILE
ffff88801ec382b8 41414000 41415000 8100073
crash> rd -p 7fc8008
[Detaching after fork from child process 5088]
7fc8008: 000000000e576067 g`W.....
crash>
Thread 1 "crash" received signal SIGINT, Interrupt.
0x00007ffff629d141 in pselect () from /lib64/libc.so.6
=> 0x00007ffff629d141 <pselect+193>: 48 3d 00 f0 ff ff cmp
$0xfffffffffffff000,%rax
(gdb) en 2
(gdb) c
Continuing.
vtop -c 5084 0x41414000
[Detaching after fork from child process 5089]
VIRTUAL PHYSICAL
Thread 1 "crash" hit Breakpoint 2, x86_64_pud_offset
(pgd_pte=<optimized out>, vaddr=1094795264, verbose=0, IS_XEN=0) at
x86_64.c:1970
1970 FILL_PUD(pud_paddr, PHYSADDR, PAGESIZE());
=> 0x00005555557f5da5 <x86_64_pud_offset+85>: 48 8b 96 40 01 00 00 mov
0x140(%rsi),%rdx
0x00005555557f5dac <x86_64_pud_offset+92>: 48 39 9e 20 01 00 00 cmp
%rbx,0x120(%rsi)
0x00005555557f5db3 <x86_64_pud_offset+99>: 74 32 je
0x5555557f5de7 <x86_64_pud_offset+151>
0x00005555557f5db5 <x86_64_pud_offset+101>: 8b 4e 18 mov 0x18(%rsi),%ecx
0x00005555557f5db8 <x86_64_pud_offset+104>: 41 b9 01 00 00 00 mov
$0x1,%r9d
0x00005555557f5dbe <x86_64_pud_offset+110>: be 04 00 00 00 mov $0x4,%esi
0x00005555557f5dc3 <x86_64_pud_offset+115>: 48 89 df mov %rbx,%rdi
0x00005555557f5dc6 <x86_64_pud_offset+118>: 4c 8d 05 6c a8 58 00
lea 0x58a86c(%rip),%r8 # 0x555555d80639
0x00005555557f5dcd <x86_64_pud_offset+125>: e8 8e e2 f5 ff callq
0x555555754060 <readmem>
0x00005555557f5dd2 <x86_64_pud_offset+130>: 48 8b 35 87 8e a9 00
mov 0xa98e87(%rip),%rsi # 0x55555628ec60 <machdep>
0x00005555557f5dd9 <x86_64_pud_offset+137>: 48 89 9e 20 01 00 00
mov %rbx,0x120(%rsi)
0x00005555557f5de0 <x86_64_pud_offset+144>: 48 8b 96 40 01 00 00
mov 0x140(%rsi),%rdx
(gdb) set machdep->last_pud_read=1 <- This forces the PUD to be re-read
(gdb) dis
(gdb) c
Continuing.
41414000 f67e000
PGD: 6e80000 => 7fc8067
PUD: 7fc8008 => e576067
PMD: e576050 => aa1c067
PTE: aa1c0a0 => 800000000f67e867
PAGE: f67e000
PTE PHYSICAL FLAGS
800000000f67e867 f67e000 (PRESENT|RW|USER|ACCESSED|DIRTY|NX)
VMA START END FLAGS FILE
ffff88801ec382b8 41414000 41415000 8100073
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
ffffea00003d9f80 f67e000 ffff8880142d40c1 41414 1 fffffc0040028
uptodate,lru,swapbacked
crash>
Best regards,
--
Anderson Nascimento
Allele Security Intelligence
https://www.allelesecurity.com
2 months, 2 weeks
- 2
- 4
[PATCH 1/2] Fix "kmem -i" option to display swap usage on Linux 6.18 and later
by HAGIO KAZUHITO(萩尾 一仁)
From: Kazuhito Hagio <k-hagio-ab(a)nec.com>
Kernel commit 8578e0c00dcf ("mm, swap: use the swap table to track the
swap count"), which is contained in Linux 6.18 and later kernels,
removed swapper_spaces symbol.
As a result, "kmem -i" skips swap usage output because the existing
check only looks for swapper_space/swapper_spaces.
Also check for the swap_info symbol so dump_swap_info() is called on
newer kernels as well.
Signed-off-by: Kazuhito Hagko <k-hagio-ab(a)nec.com>
---
memory.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/memory.c b/memory.c
index 38c6a139e984..15946c58eaf2 100644
--- a/memory.c
+++ b/memory.c
@@ -8871,7 +8871,8 @@ dump_kmeminfo(struct meminfo *mi)
* get swap data from dump_swap_info().
*/
fprintf(fp, "\n");
- if (symbol_exists("swapper_space") || symbol_exists("swapper_spaces")) {
+ if (symbol_exists("swap_info") ||
+ symbol_exists("swapper_space") || symbol_exists("swapper_spaces")) {
if (dump_swap_info(RETURN_ON_ERROR, &totalswap_pages,
&totalused_pages)) {
fprintf(fp, "%13s %7ld %11s ----\n",
--
2.31.1
2 months, 2 weeks
- 2
- 3
[PATCH] x86_64: Fix "bt" command to use correct ORC register values on Linux 7.1 and later
by HAGIO KAZUHITO(萩尾 一仁)
From: Kazuhito Hagio <k-hagio-ab(a)nec.com>
Kernel commit 1735858caa4b ("objtool/x86: Reorder ORC register numbering")
changed the ORC register numbering. Without the patch, crash can interpret ORC
entry incorrectly and the "bt" command may generate broken backtraces on Linux
7.1 and later kernels like this:
crash> bt 1
PID: 1 TASK: ffff8ab0009cd100 CPU: 2 COMMAND: "systemd"
#0 [ffffd2218003b9c8] __schedule at ffffffffaa9d862b
#1 [ffffd2218003ba20] schedule at ffffffffaa9d8993
#2 [ffffd2218003ba30] schedule_hrtimeout_range_clock at ffffffffaa9df77b
#3 [ffffd2218003bab0] ep_poll at ffffffffaa1231e4
#4 [ffffd2218003bb50] do_epoll_wait at ffffffffaa123272
#5 [ffffd2218003bb88] __x64_sys_epoll_wait at ffffffffaa123b1f
#6 [ffffd2218003bbd8] do_syscall_64 at ffffffffaa9cca6c
#7 [ffffd2218003bc58] __memcg_slab_free_hook at ffffffffaa079da3
#8 [ffffd2218003bcf0] __memcg_slab_free_hook at ffffffffaa079da3
#9 [ffffd2218003bd50] __x64_sys_gettid at ffffffffa9ce1656
#10 [ffffd2218003bd58] do_syscall_64 at ffffffffaa9ccaa4
#11 [ffffd2218003bdc0] update_cfs_rq_load_avg at ffffffffa9d1bf59
#12 [ffffd2218003be00] __update_blocked_fair at ffffffffa9d214b8
#13 [ffffd2218003be70] sched_clock at ffffffffa9c460dc
#14 [ffffd2218003be78] sched_clock_cpu at ffffffffa9d4aeab
#15 [ffffd2218003be98] irqtime_account_irq at ffffffffa9d3af0d
#16 [ffffd2218003bec0] handle_softirqs at ffffffffa9cce5ac
#17 [ffffd2218003bf40] entry_SYSCALL_64_after_hwframe at ffffffffa9a0012b
Fix this by making ORC_REG_SP and ORC_REG_PREV_SP depend on kernel version, as
no other way was found.
Signed-off-by: Kazuhito Hagio <k-hagio-ab(a)nec.com>
---
Hi,
I could not find another way except for using kernel version, is there any idea?
defs.h | 6 ++++--
x86_64.c | 11 +++++++++++
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/defs.h b/defs.h
index 6373ee1831af..e4bd77a518aa 100644
--- a/defs.h
+++ b/defs.h
@@ -6649,6 +6649,8 @@ struct ORC_data {
orc_entry orc_entry_data;
int has_signal;
int has_end;
+ int reg_sp;
+ int reg_prev_sp;
};
#define ORC_TYPE_CALL ((machdep->flags & ORC_6_4) ? 2 : 0)
@@ -6659,11 +6661,11 @@ struct ORC_data {
#define UNWIND_HINT_TYPE_RESTORE 4
#define ORC_REG_UNDEFINED 0
-#define ORC_REG_PREV_SP 1
+#define ORC_REG_PREV_SP (machdep->machspec->orc.reg_prev_sp)
#define ORC_REG_DX 2
#define ORC_REG_DI 3
#define ORC_REG_BP 4
-#define ORC_REG_SP 5
+#define ORC_REG_SP (machdep->machspec->orc.reg_sp)
#define ORC_REG_R10 6
#define ORC_REG_R13 7
#define ORC_REG_BP_INDIRECT 8
diff --git a/x86_64.c b/x86_64.c
index b2cddbf8ba3d..ec3c0e87fa06 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -999,6 +999,8 @@ x86_64_dump_machdep_table(ulong arg)
fprintf(fp, " module_ORC: %s\n", ms->orc.module_ORC ? "TRUE" : "FALSE");
fprintf(fp, " has_signal: %s\n", ms->orc.has_signal ? "TRUE" : "FALSE");
fprintf(fp, " has_end: %s\n", ms->orc.has_end ? "TRUE" : "FALSE");
+ fprintf(fp, " reg_sp: %d\n", ms->orc.reg_sp);
+ fprintf(fp, " reg_prev_sp: %d\n", ms->orc.reg_prev_sp);
fprintf(fp, " lookup_num_blocks: %d\n", ms->orc.lookup_num_blocks);
fprintf(fp, " __start_orc_unwind_ip: %lx\n", ms->orc.__start_orc_unwind_ip);
fprintf(fp, " __stop_orc_unwind_ip: %lx\n", ms->orc.__stop_orc_unwind_ip);
@@ -6720,6 +6722,15 @@ x86_64_ORC_init(void)
if (orc->has_signal && !orc->has_end)
machdep->flags |= ORC_6_4;
+ /* See kernel commit 1735858caa4b */
+ if (THIS_KERNEL_VERSION >= LINUX(7,1,0)) {
+ ORC_REG_SP = 3;
+ ORC_REG_PREV_SP = 8;
+ } else {
+ ORC_REG_SP = 5;
+ ORC_REG_PREV_SP = 1;
+ }
+
machdep->flags |= ORC;
}
--
2.31.1
2 months, 2 weeks
- 2
- 2
[PATCH] x86_64: Fix "bt" command for noreturn functions
by HAGIO KAZUHITO(萩尾 一仁)
From: Kazuhito Hagio <k-hagio-ab(a)nec.com>
On x86_64, the "bt" command resolves saved return addresses with
value_search(textaddr). However, a return address is the instruction
pointer after the call, not the call site itself.
This becomes a problem when the caller ends with a call to a noreturn
function. In that case, the saved return address can match the start
address of the following symbol, and "bt" loses track of the call chain
and this can lead to very long session initialization.
The same issue also affects symbol+offset formatting, line number
lookup, and ORC-based frame size resolution.
Fix it by resolving normal backtrace return addresses with textaddr-1,
while keeping exact textaddr handling for real RIP values saved in
exception frames. Add value_to_symstr_trace() so the displayed
symbol+offset still reflects the original return address value.
Suggested-by: Kosuke Tatsukawa <tatsu-ab1(a)nec.com>
Signed-off-by: Kazuhito Hagio <k-hagio-ab(a)nec.com>
---
defs.h | 1 +
symbols.c | 24 +++++++++++++++++++++---
x86_64.c | 31 ++++++++++++++++++++++++++-----
3 files changed, 48 insertions(+), 8 deletions(-)
diff --git a/defs.h b/defs.h
index 89044b18cdbe..79969df2a8c2 100644
--- a/defs.h
+++ b/defs.h
@@ -5798,6 +5798,7 @@ struct syment *prev_symbol(char *, struct syment *);
void get_symbol_data(char *, long, void *);
int try_get_symbol_data(char *, long, void *);
char *value_to_symstr(ulong, char *, ulong);
+char *value_to_symstr_trace(ulong, char *, ulong);
char *value_symbol(ulong);
ulong symbol_value(char *);
ulong symbol_value_module(char *, char *);
diff --git a/symbols.c b/symbols.c
index 3c62f54d4a93..372d0b230b18 100644
--- a/symbols.c
+++ b/symbols.c
@@ -104,6 +104,7 @@ static void free_structure(struct struct_elem *);
static unsigned char is_right_brace(const char *);
static struct struct_elem *find_node(struct struct_elem *, char *);
static void dump_node(struct struct_elem *, char *, unsigned char, unsigned char);
+static char *_value_to_symstr(ulong value, char *buf, ulong radix, int trace);
static int module_mem_type(ulong, struct load_module *);
static ulong module_mem_end(ulong, struct load_module *);
@@ -5973,14 +5974,25 @@ generic_machdep_value_to_symbol(ulong value, ulong *offset)
return NULL;
}
+char *
+value_to_symstr(ulong value, char *buf, ulong radix)
+{
+ return _value_to_symstr(value, buf, radix, 0);
+}
+
+char *
+value_to_symstr_trace(ulong value, char *buf, ulong radix)
+{
+ return _value_to_symstr(value, buf, radix, 1);
+}
/*
* For a given value, format a string containing the nearest symbol name
* plus the offset if appropriate. Display the offset in the specified
* radix (10 or 16) -- if it's 0, set it to the current pc->output_radix.
*/
-char *
-value_to_symstr(ulong value, char *buf, ulong radix)
+static char *
+_value_to_symstr(ulong value, char *buf, ulong radix, int trace)
{
struct syment *sp;
ulong offset;
@@ -5996,7 +6008,13 @@ value_to_symstr(ulong value, char *buf, ulong radix)
if ((radix != 10) && (radix != 16))
radix = 16;
- if ((sp = value_search(value, &offset))) {
+ if (trace) {
+ sp = value_search(value-1, &offset);
+ offset++;
+ } else
+ sp = value_search(value, &offset);
+
+ if (sp) {
if (offset)
sprintf(buf, radix == 16 ? "%s+0x%lx" : "%s+%ld",
sp->name, offset);
diff --git a/x86_64.c b/x86_64.c
index b2cddbf8ba3d..ff283ed68191 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -3229,14 +3229,23 @@ x86_64_print_stack_entry(struct bt_info *bt, FILE *ofp, int level,
if (!(bt->flags & BT_SAVE_EFRAME_IP))
bt->eframe_ip = 0;
offset = 0;
- sp = value_search(text, &offset);
+ if (bt->flags & BT_SAVE_EFRAME_IP)
+ sp = value_search(text, &offset);
+ else {
+ sp = value_search(text-1, &offset);
+ offset++;
+ }
if (!sp)
return BACKTRACE_ENTRY_IGNORED;
name = sp->name;
if (offset && (bt->flags & BT_SYMBOL_OFFSET))
- name_plus_offset = value_to_symstr(text, buf2, bt->radix);
+ if (bt->flags & BT_SAVE_EFRAME_IP)
+ name_plus_offset = value_to_symstr(text, buf2, bt->radix);
+ else
+ /* text-1 is used in the function */
+ name_plus_offset = value_to_symstr_trace(text, buf2, bt->radix);
else
name_plus_offset = NULL;
@@ -3337,7 +3346,10 @@ x86_64_print_stack_entry(struct bt_info *bt, FILE *ofp, int level,
fprintf(ofp, "\n");
if (bt->flags & BT_LINE_NUMBERS) {
- get_line_number(text, buf1, FALSE);
+ if (bt->flags & BT_SAVE_EFRAME_IP)
+ get_line_number(text, buf1, FALSE);
+ else
+ get_line_number(text-1, buf1, FALSE);
if (strlen(buf1))
fprintf(ofp, " %s\n", buf1);
}
@@ -3864,8 +3876,10 @@ in_exception_stack:
}
level++;
+ bt->flags |= BT_SAVE_EFRAME_IP;
if ((framesize = x86_64_get_framesize(bt, bt->instptr, rsp, NULL)) >= 0)
rsp += framesize;
+ bt->flags &= ~BT_SAVE_EFRAME_IP;
}
}
@@ -8811,7 +8825,13 @@ x86_64_get_framesize(struct bt_info *bt, ulong textaddr, ulong rsp, char *stack_
return 0;
}
- if (!(sp = value_search(textaddr, &offset))) {
+ if (bt->flags & BT_SAVE_EFRAME_IP)
+ sp = value_search(textaddr, &offset);
+ else {
+ sp = value_search(textaddr-1, &offset);
+ offset++;
+ }
+ if (!sp) {
if (!(bt->flags & BT_FRAMESIZE_DEBUG))
bt->flags |= BT_FRAMESIZE_DISABLE;
return 0;
@@ -8887,7 +8907,8 @@ x86_64_get_framesize(struct bt_info *bt, ulong textaddr, ulong rsp, char *stack_
if ((sp->value >= kt->init_begin) && (sp->value < kt->init_end))
return 0;
- if ((machdep->flags & ORC) && (korc = orc_find(textaddr))) {
+ if ((machdep->flags & ORC) &&
+ (korc = orc_find(bt->flags & BT_SAVE_EFRAME_IP ? textaddr : textaddr-1))) {
if (CRASHDEBUG(1)) {
struct ORC_data *orc = &machdep->machspec->orc;
fprintf(fp,
--
2.31.1
2 months, 2 weeks
- 2
- 2