Re: [Crash-utility] [PATCH] vmware_vmss: read vCPUs regs and show them in 'bt'

VMSS dump files contain the state of each vCPU at the time of suspending the VM. This change enables 'crash' to read some relevant registers from each vCPU state and display them in 'bt'.

This is also the first step towards implementing kaslr offset calculation for VMSS dump files.

--- defs.h | 3 ++ kernel.c | 2 + vmware_vmss.c | 148 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- vmware_vmss.h | 28 +++++++++++ x86_64.c | 13 ++++-- 5 files changed, 184 insertions(+), 10 deletions(-) diff --git a/defs.h b/defs.h index 7998ebf..0ebd38b 100644 --- a/defs.h +++ b/defs.h @@ -283,6 +283,7 @@ struct number_option { #define LKCD_KERNTYPES() (pc->flags & KERNTYPES) #define KVMDUMP_DUMPFILE() (pc->flags & KVMDUMP) #define SADUMP_DUMPFILE() (pc->flags & SADUMP) +#define VMSS_DUMPFILE() (pc->flags & VMWARE_VMSS) #define NETDUMP_LOCAL (0x1) /* netdump_data flags */ #define NETDUMP_REMOTE (0x2) @@ -6388,6 +6389,8 @@ int vmware_vmss_init(char *filename, FILE *ofp); uint vmware_vmss_page_size(void); int read_vmware_vmss(int, void *, int, ulong, physaddr_t); int write_vmware_vmss(int, void *, int, ulong, physaddr_t); +void vmware_vmss_display_regs(int, FILE *); +void get_vmware_vmss_regs(struct bt_info *, ulong *, ulong *); /* * gnu_binutils.c diff --git a/kernel.c b/kernel.c index 1bf6251..7642217 100644 --- a/kernel.c +++ b/kernel.c @@ -2969,6 +2969,8 @@ back_trace(struct bt_info *bt) get_xendump_regs(bt, &eip, &esp); else if (SADUMP_DUMPFILE()) get_sadump_regs(bt, &eip, &esp); + else if (VMSS_DUMPFILE()) + get_vmware_vmss_regs(bt, &eip, &esp); else if (REMOTE_PAUSED()) { if (!is_task_active(bt->task) || !get_remote_regs(bt, &eip, &esp)) machdep->get_stack_frame(bt, &eip, &esp); diff --git a/vmware_vmss.c b/vmware_vmss.c index 667676a..10fbe9e 100644 --- a/vmware_vmss.c +++ b/vmware_vmss.c @@ -24,6 +24,9 @@ /* VMware only supports X86/X86_64 virtual machines. */ #define VMW_PAGE_SIZE (4096) #define VMW_PAGE_SHIFT (12) +#define VMW_GPREGS_SIZE (128) +#define VMW_CR64_SIZE (72) +#define VMW_IDTR_SIZE (10) static vmssdata vmss = { 0 }; @@ -128,7 +131,8 @@ vmware_vmss_init(char *filename, FILE *ofp) DEBUG_PARSE_PRINT((ofp, LOGPRX"Group: %-20s offset=%#llx size=0x%#llx.\n", grps[i].name, (ulonglong)grps[i].position, (ulonglong)grps[i].size)); - if (strcmp(grps[i].name, "memory") != 0) { + if (strcmp(grps[i].name, "memory") != 0 && + (strcmp(grps[i].name, "cpu") != 0 || !machine_type("X86_64"))) { continue; } @@ -198,12 +202,6 @@ vmware_vmss_init(char *filename, FILE *ofp) } blockpos += padsize; - if (fseek(fp, blockpos + nbytes, SEEK_SET) == -1) { - error(INFO, LOGPRX"Cannot seek past block at %#llx.\n", - (ulonglong)(blockpos + nbytes)); - break; - } - if (strcmp(name, "Memory") == 0) { /* The things that we really care about...*/ vmss.memoffset = blockpos; @@ -217,11 +215,46 @@ vmware_vmss_init(char *filename, FILE *ofp) result = FALSE; goto exit; } + + if (fseek(fp, blockpos + nbytes, SEEK_SET) == -1) { + error(INFO, LOGPRX"Cannot seek past block at %#llx.\n", + (ulonglong)(blockpos + nbytes)); + break; + } + } else if (strcmp(name, "gpregs") == 0 && + nbytes == VMW_GPREGS_SIZE && + idx[0] < vmss.num_vcpus) { + int cpu = idx[0]; + + fread(vmss.regs64[cpu], nbytes, 1, fp); + } else if (strcmp(name, "CR64") == 0 && + nbytes == VMW_CR64_SIZE && + idx[0] < vmss.num_vcpus) { + int cpu = idx[0]; + uint64_t regs[9]; + + fread(&regs[0], nbytes, 1, fp); + vmss.regs64[cpu]->cr3 = regs[3]; + } else if (strcmp(name, "IDTR") == 0 && + nbytes == VMW_IDTR_SIZE && + idx[0] < vmss.num_vcpus) { + int cpu = idx[0]; + char buf[10]; + + fread(&buf[0], nbytes, 1, fp); + vmss.regs64[cpu]->cr3 = *((uint64_t *)(&buf[0] + 2)); + } else { + if (fseek(fp, blockpos + nbytes, SEEK_SET) == -1) { + error(INFO, LOGPRX"Cannot seek past block at %#llx.\n", + (ulonglong)(blockpos + nbytes)); + break; + } } } else { union { uint8_t val[TAG_VALSIZE_MASK]; uint32_t val32; + uint64_t val64; } u; unsigned k; unsigned valsize = TAG_VALSIZE(tag); @@ -253,6 +286,30 @@ vmware_vmss_init(char *filename, FILE *ofp) if (strcmp(name, "align_mask") == 0) { vmss.alignmask = u.val32; } + } else if (strcmp(grps[i].name, "cpu") == 0) { + if (strcmp(name, "cpu:numVCPUs") == 0) { + if (vmss.regs64 != NULL) { + error(INFO, LOGPRX"Duplicated cpu:numVCPUs entry.\n"); + break; + } + + vmss.num_vcpus = u.val32; + vmss.regs64 = malloc(vmss.num_vcpus * sizeof(void *)); + + for (k = 0; k < vmss.num_vcpus; k++) { + vmss.regs64[k] = malloc(sizeof(vmssregs64)); + memset(vmss.regs64[k], 0, sizeof(vmssregs64)); + } + } else if (strcmp(name, "rip") == 0) { + int cpu = idx[0]; + vmss.regs64[cpu]->rip = u.val64; + } else if (strcmp(name, "eflags") == 0) { + int cpu = idx[0]; + vmss.regs64[cpu]->eflags |= u.val32; + } else if (strcmp(name, "EFLAGS") == 0) { + int cpu = idx[0]; + vmss.regs64[cpu]->eflags |= u.val32; + } } DEBUG_PARSE_PRINT((ofp, "\n")); @@ -350,3 +407,80 @@ write_vmware_vmss(int fd, void *bufptr, int cnt, ulong addr, physaddr_t paddr) return SEEK_ERROR; } +void +vmware_vmss_display_regs(int cpu, FILE *ofp) +{ + if (cpu >= vmss.num_vcpus) + return; + + if (machine_type("X86_64")) { + fprintf(ofp, + " RIP: %016llx RSP: %016llx RFLAGS: %08llx\n" + " RAX: %016llx RBX: %016llx RCX: %016llx\n" + " RDX: %016llx RSI: %016llx RDI: %016llx\n" + " RBP: %016llx R8: %016llx R9: %016llx\n" + " R10: %016llx R11: %016llx R12: %016llx\n" + " R13: %016llx R14: %016llx R15: %016llx\n", + vmss.regs64[cpu]->rip, + vmss.regs64[cpu]->rsp, + vmss.regs64[cpu]->eflags, + vmss.regs64[cpu]->rax, + vmss.regs64[cpu]->rbx, + vmss.regs64[cpu]->rcx, + vmss.regs64[cpu]->rdx, + vmss.regs64[cpu]->rsi, + vmss.regs64[cpu]->rdi, + vmss.regs64[cpu]->rbp, + vmss.regs64[cpu]->r8, + vmss.regs64[cpu]->r9, + vmss.regs64[cpu]->r10, + vmss.regs64[cpu]->r11, + vmss.regs64[cpu]->r12, + vmss.regs64[cpu]->r13, + vmss.regs64[cpu]->r14, + vmss.regs64[cpu]->r15 + ); + } +} + +void +get_vmware_vmss_regs(struct bt_info *bt, ulong *ipp, ulong *spp) +{ + ulong ip, sp; + struct register_set *rp; + + ip = sp = 0; + + if (!is_task_active(bt->task)) { + machdep->get_stack_frame(bt, ipp, spp); + return; + } + + bt->flags |= BT_DUMPFILE_SEARCH; + if (machine_type("X86_64")) + machdep->get_stack_frame(bt, ipp, spp); + else if (machine_type("X86")) + get_netdump_regs_x86(bt, ipp, spp); + if (bt->flags & BT_DUMPFILE_SEARCH) + return; + + if ((vmss.regs64 == NULL) || + (bt->tc->processor >= vmss.num_vcpus)) + return; + + ip = (ulong)vmss.regs64[bt->tc->processor]->rip; + sp = (ulong)vmss.regs64[bt->tc->processor]->rsp; + if (is_kernel_text(ip) && + (((sp >= GET_STACKBASE(bt->task)) && + (sp < GET_STACKTOP(bt->task))) || + in_alternate_stack(bt->tc->processor, sp))) { + *ipp = ip; + *spp = sp; + bt->flags |= BT_KERNEL_SPACE; + return; + } + + if (!is_kernel_text(ip) && + in_user_stack(bt->tc->task, sp)) + bt->flags |= BT_USER_SPACE; +} diff --git a/vmware_vmss.h b/vmware_vmss.h index a4b8937..3c69a82 100644 --- a/vmware_vmss.h +++ b/vmware_vmss.h @@ -90,6 +90,32 @@ struct memregion { typedef struct memregion memregion; #define MAX_REGIONS 3 +struct vmssregs64 { + /* read from vmss */ + uint64_t rax; + uint64_t rcx; + uint64_t rdx; + uint64_t rbx; + uint64_t rbp; + uint64_t rsp; + uint64_t rsi; + uint64_t rdi; + uint64_t r8; + uint64_t r9; + uint64_t r10; + uint64_t r11; + uint64_t r12; + uint64_t r13; + uint64_t r14; + uint64_t r15; + /* manually managed */ + uint64_t idtr; + uint64_t cr3; + uint64_t rip; + uint64_t eflags; +}; +typedef struct vmssregs64 vmssregs64; + struct vmssdata { int32_t cpt64bit; FILE *dfp; @@ -99,6 +125,8 @@ struct vmssdata { memregion regions[MAX_REGIONS]; uint64_t memoffset; uint64_t memsize; + uint64_t num_vcpus; + vmssregs64 **regs64; }; typedef struct vmssdata vmssdata; diff --git a/x86_64.c b/x86_64.c index 0d5e150..7b02761 100644 --- a/x86_64.c +++ b/x86_64.c @@ -3273,6 +3273,8 @@ x86_64_low_budget_back_trace_cmd(struct bt_info *bt_in) diskdump_display_regs(bt->tc->processor, ofp); else if (SADUMP_DUMPFILE()) sadump_display_regs(bt->tc->processor, ofp); + else if (VMSS_DUMPFILE()) + vmware_vmss_display_regs(bt->tc->processor, ofp); return; } @@ -3295,13 +3297,16 @@ x86_64_low_budget_back_trace_cmd(struct bt_info *bt_in) diskdump_display_regs(bt->tc->processor, ofp); else if (SADUMP_DUMPFILE()) sadump_display_regs(bt->tc->processor, ofp); + else if (VMSS_DUMPFILE()) + vmware_vmss_display_regs(bt->tc->processor, ofp); else if (pc->flags2 & QEMU_MEM_DUMP_ELF) display_regs_from_elf_notes(bt->tc->processor, ofp); return; } else if ((bt->flags & BT_KERNEL_SPACE) && (KVMDUMP_DUMPFILE() || (ELF_NOTES_VALID() && DISKDUMP_DUMPFILE()) || - SADUMP_DUMPFILE() || (pc->flags2 & QEMU_MEM_DUMP_ELF))) { + SADUMP_DUMPFILE() || (pc->flags2 & QEMU_MEM_DUMP_ELF) || + VMSS_DUMPFILE())) { fprintf(ofp, " [exception RIP: "); if ((sp = value_search(bt->instptr, &offset))) { fprintf(ofp, "%s", sp->name); @@ -3317,6 +3322,8 @@ x86_64_low_budget_back_trace_cmd(struct bt_info *bt_in) diskdump_display_regs(bt->tc->processor, ofp); else if (SADUMP_DUMPFILE()) sadump_display_regs(bt->tc->processor, ofp); + else if (VMSS_DUMPFILE()) + vmware_vmss_display_regs(bt->tc->processor, ofp); else if (pc->flags2 & QEMU_MEM_DUMP_ELF) display_regs_from_elf_notes(bt->tc->processor, ofp); @@ -4941,7 +4948,7 @@ skip_stage: if (halt_rip && halt_rsp) { *rip = halt_rip; *rsp = halt_rsp; - if (KVMDUMP_DUMPFILE() || SADUMP_DUMPFILE()) + if (KVMDUMP_DUMPFILE() || SADUMP_DUMPFILE() || VMSS_DUMPFILE()) bt_in->flags &= ~(ulonglong)BT_DUMPFILE_SEARCH; return; } @@ -4986,7 +4993,7 @@ skip_stage: machdep->get_stack_frame(bt, rip, rsp); - if (KVMDUMP_DUMPFILE() || SADUMP_DUMPFILE()) + if (KVMDUMP_DUMPFILE() || SADUMP_DUMPFILE() || VMSS_DUMPFILE()) bt_in->flags &= ~(ulonglong)BT_DUMPFILE_SEARCH; } -- 2.14.3