2:43 a.m.
vmware_guestdump is extension to vmware_vmss with ability to debug
debug.guest and debug.vmem files.
debug.guest.gz and debug.vmem.gz can be obtained using following
.vmx options from VM running in debug mode:
monitor.mini-suspend_on_panic = TRUE
monitor.suspend_on_triplefault = TRUE
guestdump (debug.guest) is simplified version of *.vmss which does
not contain full VM state, but minimal guest state, such as memory
layout and CPUs state, needed for debugger. is_vmware_guestdump()
and vmware_guestdump_init() functions parse guestdump header and
populate vmss data structure (from vmware_vmss.c). As result, all
handlers (except mempry_dump) from vmware_vmss.c can be reused.
How to use: $ crash /path/to/debug_file.guest vmlinux
Companion debug_file.vmem must be present in the same folder as
debug_file.guest. Otherwise crash will shot a message:
vmw: Open the companion vmem file: /path/to/debug_file.vmem
crash: vmw: /path/to/debug_file.vmem: No such file or directory
Signed-off-by: Alexey Makhalov <amakhalov(a)vmware.com>
---
Makefile | 7 +-
crash.8 | 6 +
defs.h | 8 ++
filesys.c | 12 +-
main.c | 14 +++
memory.c | 8 +-
vmware_guestdump.c | 315 +++++++++++++++++++++++++++++++++++++++++++++++++++++
vmware_vmss.c | 8 +-
vmware_vmss.h | 8 ++
9 files changed, 372 insertions(+), 14 deletions(-)
create mode 100644 vmware_guestdump.c
diff --git a/Makefile b/Makefile
index 7455410..d185719 100644
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,7 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
unwind_x86_32_64.c unwind_arm.c \
xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \
xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c sadump.c ipcs.c \
- ramdump.c vmware_vmss.c \
+ ramdump.c vmware_vmss.c vmware_guestdump.c \
xen_dom0.c kaslr_helper.c
SOURCE_FILES=${CFILES} ${GENERIC_HFILES} ${MCORE_HFILES} \
@@ -89,7 +89,7 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o
task.o \
unwind_x86_32_64.o unwind_arm.o \
xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \
xen_hyper_dump_tables.o kvmdump.o qemu.o qemu-load.o sadump.o ipcs.o \
- ramdump.o vmware_vmss.o \
+ ramdump.o vmware_vmss.o vmware_guestdump.o \
xen_dom0.o kaslr_helper.o
MEMORY_DRIVER_FILES=memory_driver/Makefile memory_driver/crash.c memory_driver/README
@@ -518,6 +518,9 @@ ramdump.o: ${GENERIC_HFILES} ${REDHAT_HFILES} ramdump.c
vmware_vmss.o: ${GENERIC_HFILES} ${VMWARE_HFILES} vmware_vmss.c
${CC} -c ${CRASH_CFLAGS} vmware_vmss.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+vmware_guestdump.o: ${GENERIC_HFILES} ${VMWARE_HFILES} vmware_guestdump.c
+ ${CC} -c ${CRASH_CFLAGS} vmware_guestdump.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+
kaslr_helper.o: ${GENERIC_HFILES} kaslr_helper.c
${CC} -c ${CRASH_CFLAGS} kaslr_helper.c ${WARNING_OPTIONS} ${WARNING_ERROR}
diff --git a/crash.8 b/crash.8
index 136ae78..783fdf7 100644
--- a/crash.8
+++ b/crash.8
@@ -116,6 +116,12 @@ or
.I kvmdump
facilities.
+The
+.B crash
+utility is able to process VMware VM memory dump generated by VM suspend
+or guest core dump. In that case, .vmss or .guest file should be used as
+a MEMORY-IMAGE and .vmem file must be located in the same folder.
+
If a MEMORY-IMAGE argument is not entered, the session will be invoked on
the live system, which typically requires root privileges because of
the device file used to access system RAM. By default,
diff --git a/defs.h b/defs.h
index c899fe2..3386709 100644
--- a/defs.h
+++ b/defs.h
@@ -544,6 +544,7 @@ struct program_context {
#define is_excluded_vmemmap() (pc->flags2 & EXCLUDED_VMEMMAP)
#define MEMSRC_LOCAL (0x80000ULL)
#define REDZONE (0x100000ULL)
+#define VMWARE_VMSS_GUESTDUMP (0x200000ULL)
char *cleanup;
char *namelist_orig;
char *namelist_debug_orig;
@@ -6679,6 +6680,13 @@ int vmware_vmss_phys_base(ulong *phys_base);
int vmware_vmss_set_phys_base(ulong);
/*
+ * vmware_guestdump.c
+ */
+int is_vmware_guestdump(char *filename);
+int vmware_guestdump_init(char *filename, FILE *ofp);
+int vmware_guestdump_memory_dump(FILE *);
+
+/*
* kaslr_helper.c
*/
int calc_kaslr_offset(ulong *, ulong *);
diff --git a/filesys.c b/filesys.c
index 2ec2b31..3361b6c 100644
--- a/filesys.c
+++ b/filesys.c
@@ -253,9 +253,15 @@ memory_source_init(void)
error(FATAL, "%s: initialization failed\n",
pc->dumpfile);
} else if (pc->flags & VMWARE_VMSS) {
- if (!vmware_vmss_init(pc->dumpfile, fp))
- error(FATAL, "%s: initialization failed\n",
- pc->dumpfile);
+ if (pc->flags2 & VMWARE_VMSS_GUESTDUMP) {
+ if (!vmware_guestdump_init(pc->dumpfile, fp))
+ error(FATAL, "%s: initialization failed\n",
+ pc->dumpfile);
+ } else {
+ if (!vmware_vmss_init(pc->dumpfile, fp))
+ error(FATAL, "%s: initialization failed\n",
+ pc->dumpfile);
+ }
}
}
}
diff --git a/main.c b/main.c
index 7f562e6..388ac46 100644
--- a/main.c
+++ b/main.c
@@ -671,6 +671,18 @@ main(int argc, char **argv)
pc->readmem = read_vmware_vmss;
pc->writemem = write_vmware_vmss;
+ } else if (is_vmware_guestdump(argv[optind])) {
+ if (pc->flags & MEMORY_SOURCES) {
+ error(INFO,
+ "too many dumpfile arguments\n");
+ program_usage(SHORT_FORM);
+ }
+ pc->flags |= VMWARE_VMSS;
+ pc->flags2 |= VMWARE_VMSS_GUESTDUMP;
+ pc->dumpfile = argv[optind];
+ pc->readmem = read_vmware_vmss;
+ pc->writemem = write_vmware_vmss;
+
} else {
error(INFO,
"%s: not a supported file format\n",
@@ -1486,6 +1498,8 @@ dump_program_context(void)
fprintf(fp, "%sMEMSRC_LOCAL", others++ ? "|" : "");
if (pc->flags2 & REDZONE)
fprintf(fp, "%sREDZONE", others++ ? "|" : "");
+ if (pc->flags2 & VMWARE_VMSS_GUESTDUMP)
+ fprintf(fp, "%sVMWARE_VMSS_GUESTDUMP", others++ ? "|" :
"");
fprintf(fp, ")\n");
fprintf(fp, " namelist: %s\n", pc->namelist);
diff --git a/memory.c b/memory.c
index c951827..0848097 100644
--- a/memory.c
+++ b/memory.c
@@ -17115,8 +17115,12 @@ dumpfile_memory(int cmd)
retval = kcore_memory_dump(fp);
else if (pc->flags & SADUMP)
retval = sadump_memory_dump(fp);
- else if (pc->flags & VMWARE_VMSS)
- retval = vmware_vmss_memory_dump(fp);
+ else if (pc->flags & VMWARE_VMSS) {
+ if (pc->flags2 & VMWARE_VMSS_GUESTDUMP)
+ retval = vmware_guestdump_memory_dump(fp);
+ else
+ retval = vmware_vmss_memory_dump(fp);
+ }
break;
case DUMPFILE_ENVIRONMENT:
diff --git a/vmware_guestdump.c b/vmware_guestdump.c
new file mode 100644
index 0000000..d3fac59
--- /dev/null
+++ b/vmware_guestdump.c
@@ -0,0 +1,315 @@
+/*
+ * vmware_guestdump.c
+ *
+ * Copyright (c) 2020 VMware, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Author: Alexey Makhalov <amakhalov(a)vmware.com>
+ */
+
+#include "defs.h"
+#include "vmware_vmss.h"
+
+#define LOGPRX "vmw: "
+
+#define GUESTDUMP_VERSION 4
+#define GUESTDUMP_MAGIC1 1
+#define GUESTDUMP_MAGIC2 0
+
+struct guestdumpheader {
+ uint32_t version;
+ uint32_t num_vcpus;
+ uint8_t magic1;
+ uint8_t reserved1;
+ uint32_t cpu_vendor;
+ uint64_t magic2;
+ uint64_t last_addr;
+ uint64_t memsize_in_pages;
+ uint32_t reserved2;
+ uint32_t mem_holes;
+ struct memhole {
+ uint64_t ppn;
+ uint64_t pages;
+ } holes[2];
+} __attribute__((packed));
+
+struct vcpu_state {
+ uint32_t cr0;
+ uint64_t cr2;
+ uint64_t cr3;
+ uint64_t cr4;
+ uint64_t reserved1[10];
+ uint64_t idt_base;
+ uint16_t reserved2[21];
+ struct x86_64_pt_regs {
+ uint64_t r15;
+ uint64_t r14;
+ uint64_t r13;
+ uint64_t r12;
+ uint64_t rbp;
+ uint64_t rbx;
+ uint64_t r11;
+ uint64_t r10;
+ uint64_t r9;
+ uint64_t r8;
+ uint64_t rax;
+ uint64_t rcx;
+ uint64_t rdx;
+ uint64_t rsi;
+ uint64_t rdi;
+ uint64_t orig_rax;
+ uint64_t rip;
+ uint64_t cs;
+ uint64_t eflags;
+ uint64_t rsp;
+ uint64_t ss;
+ } regs64;
+ uint8_t reserved3[65];
+} __attribute__((packed));
+
+
+/*
+ * vmware_guestdump is extension to vmware_vmss with ability to debug
+ * debug.guest and debug.vmem files.
+ *
+ * debug.guest.gz and debug.vmem.gz can be obtained using following
+ * .vmx options from VM running in debug mode:
+ * monitor.mini-suspend_on_panic = TRUE
+ * monitor.suspend_on_triplefault = TRUE
+ *
+ * guestdump (debug.guest) is simplified version of *.vmss which does
+ * not contain full VM state, but minimal guest state, such as memory
+ * layout and CPUs state, needed for debugger. is_vmware_guestdump()
+ * and vmware_guestdump_init() functions parse guestdump header and
+ * populate vmss data structure (from vmware_vmss.c). As result, all
+ * handlers (except mempry_dump) from vmware_vmss.c can be reused.
+ *
+ * debug.guest does not have dedicated header magic or signature for
+ * its format. To probe debug.guest we need to perform header fields
+ * and file size validity. In addition, check for the filename
+ * extension, which must be ".guest".
+ */
+
+int
+is_vmware_guestdump(char *filename)
+{
+ struct guestdumpheader hdr;
+ FILE *fp;
+ uint64_t filesize, holes_sum = 0;
+ int i;
+
+ if (strcmp(filename + strlen(filename) - 6, ".guest"))
+ return FALSE;
+
+ if ((fp = fopen(filename, "r")) == NULL) {
+ error(INFO, LOGPRX"Failed to open '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ return FALSE;
+ }
+
+ if (fread(&hdr, sizeof(struct guestdumpheader), 1, fp) != 1) {
+ error(INFO, LOGPRX"Failed to read '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ fclose(fp);
+ return FALSE;
+ }
+
+ if (fseek(fp, 0L, SEEK_END) == -1) {
+ error(INFO, LOGPRX"Failed to fseek '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ fclose(fp);
+ return FALSE;
+ }
+ filesize = ftell(fp);
+ fclose(fp);
+
+ if (hdr.mem_holes > 2)
+ goto unrecognized;
+
+ for (i = 0; i < hdr.mem_holes; i++) {
+ /* hole start page */
+ vmss.regions[i].startpagenum = hdr.holes[i].ppn;
+ /* hole end page */
+ vmss.regions[i].startppn = hdr.holes[i].ppn + hdr.holes[i].pages;
+ holes_sum += hdr.holes[i].pages;
+ }
+
+ if (hdr.version != GUESTDUMP_VERSION ||
+ hdr.magic1 != GUESTDUMP_MAGIC1 ||
+ hdr.magic2 != GUESTDUMP_MAGIC2 ||
+ (hdr.last_addr + 1) != ((hdr.memsize_in_pages + holes_sum) << VMW_PAGE_SHIFT)
||
+ filesize != sizeof(struct guestdumpheader) +
+ hdr.num_vcpus * (sizeof (struct vcpu_state) + VMW_PAGE_SIZE))
+ goto unrecognized;
+
+ vmss.memsize = hdr.memsize_in_pages << VMW_PAGE_SHIFT;
+ vmss.regionscount = hdr.mem_holes + 1;
+ vmss.memoffset = 0;
+ vmss.num_vcpus = hdr.num_vcpus;
+ return TRUE;
+
+unrecognized:
+ if (CRASHDEBUG(1))
+ error(INFO, LOGPRX"Unrecognized debug.guest file.\n");
+ return FALSE;
+}
+
+int
+vmware_guestdump_init(char *filename, FILE *ofp)
+{
+ FILE *fp = NULL;
+ int i, result = TRUE;
+ char *vmem_filename = NULL;
+ struct vcpu_state vs;
+ char *p;
+
+ if (!machine_type("X86") && !machine_type("X86_64")) {
+ error(INFO,
+ LOGPRX"Invalid or unsupported host architecture for .vmss file: %s\n",
+ MACHINE_TYPE);
+ result = FALSE;
+ goto exit;
+ }
+
+ if ((fp = fopen(filename, "r")) == NULL) {
+ error(INFO, LOGPRX"Failed to open '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+
+ if (fseek(fp, sizeof(struct guestdumpheader), SEEK_SET) == -1) {
+ error(INFO, LOGPRX"Failed to fseek '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+
+ vmss.vcpu_regs = malloc(vmss.num_vcpus * sizeof(uint32_t));
+ vmss.regs64 = calloc(vmss.num_vcpus, sizeof(void *));
+ if (!vmss.vcpu_regs || !vmss.regs64) {
+ error(INFO, LOGPRX"Failed to allocate memory\n");
+ result = FALSE;
+ goto exit;
+ }
+
+ for (i = 0; i < vmss.num_vcpus; i++) {
+ if (fread(&vs, sizeof(struct vcpu_state), 1, fp) != 1) {
+ error(INFO, LOGPRX"Failed to read '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+ vmss.regs64[i] = calloc(1, sizeof(vmssregs64));
+ if (!vmss.regs64[i]) {
+ error(INFO, LOGPRX"Failed to allocate memory\n");
+ result = FALSE;
+ goto exit;
+ }
+ vmss.vcpu_regs[i] = 0;
+
+ vmss.regs64[i]->rax = vs.regs64.rax;
+ vmss.regs64[i]->rcx = vs.regs64.rcx;
+ vmss.regs64[i]->rdx = vs.regs64.rdx;
+ vmss.regs64[i]->rbx = vs.regs64.rbx;
+ vmss.regs64[i]->rbp = vs.regs64.rbp;
+ vmss.regs64[i]->rsp = vs.regs64.rsp;
+ vmss.regs64[i]->rsi = vs.regs64.rsi;
+ vmss.regs64[i]->rdi = vs.regs64.rdi;
+ vmss.regs64[i]->r8 = vs.regs64.r8;
+ vmss.regs64[i]->r9 = vs.regs64.r9;
+ vmss.regs64[i]->r10 = vs.regs64.r10;
+ vmss.regs64[i]->r11 = vs.regs64.r11;
+ vmss.regs64[i]->r12 = vs.regs64.r12;
+ vmss.regs64[i]->r13 = vs.regs64.r13;
+ vmss.regs64[i]->r14 = vs.regs64.r14;
+ vmss.regs64[i]->r15 = vs.regs64.r15;
+ vmss.regs64[i]->idtr = vs.idt_base;
+ vmss.regs64[i]->cr[0] = vs.cr0;
+ vmss.regs64[i]->cr[2] = vs.cr2;
+ vmss.regs64[i]->cr[3] = vs.cr3;
+ vmss.regs64[i]->cr[4] = vs.cr4;
+ vmss.regs64[i]->rip = vs.regs64.rip;
+ vmss.regs64[i]->rflags = vs.regs64.eflags;
+
+ vmss.vcpu_regs[i] = REGS_PRESENT_ALL;
+ }
+
+ vmem_filename = strdup(filename);
+ p = vmem_filename + strlen(vmem_filename) - 5;
+ if (strcmp(p, "guest") != 0) {
+ result = FALSE;
+ goto exit;
+ }
+ strcpy(p, "vmem");
+
+ fprintf(ofp, LOGPRX"Open the companion vmem file: %s\n", vmem_filename);
+ if ((vmss.dfp = fopen(vmem_filename, "r")) == NULL) {
+ error(INFO, LOGPRX"%s: %s\n", vmem_filename, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+ fseek(vmss.dfp, 0L, SEEK_END);
+ if (vmss.memsize != ftell(vmss.dfp)) {
+ error(INFO, LOGPRX"%s: unexpected size\n", vmem_filename);
+ result = FALSE;
+ goto exit;
+ }
+ fseek(vmss.dfp, 0L, SEEK_SET);
+ fprintf(ofp, LOGPRX"vmem file: %s\n\n", vmem_filename);
+
+exit:
+ if (fp)
+ fclose(fp);
+ if (vmem_filename)
+ free(vmem_filename);
+ if (result == FALSE) {
+ if (vmss.dfp)
+ fclose(vmss.dfp);
+ if (vmss.regs64) {
+ for (i = 0; i < vmss.num_vcpus; i++) {
+ if (vmss.regs64[i])
+ free(vmss.regs64[i]);
+ }
+ free(vmss.regs64);
+ }
+ if (vmss.vcpu_regs)
+ free(vmss.vcpu_regs);
+ }
+ return result;
+}
+
+int
+vmware_guestdump_memory_dump(FILE *ofp)
+{
+ fprintf(ofp, "vmware_guestdump:\n");
+ fprintf(ofp, " Header: version=%d num_vcpus=%ld\n",
+ GUESTDUMP_VERSION, vmss.num_vcpus);
+ fprintf(ofp, "Total memory: %ld\n", vmss.memsize);
+
+ if (vmss.regionscount > 1) {
+ uint64_t holes_sum = 0;
+ unsigned i;
+
+ fprintf(ofp, "Memory regions[%d]:\n", vmss.regionscount);
+ fprintf(ofp, " [0x%016x-", 0);
+ for (i = 0; i < vmss.regionscount - 1; i++) {
+ fprintf(ofp, "0x%016lx]\n", (uint64_t)vmss.regions[i].startpagenum <<
VMW_PAGE_SHIFT);
+ fprintf(ofp, " [0x%016lx-", (uint64_t)vmss.regions[i].startppn <<
VMW_PAGE_SHIFT);
+ holes_sum += vmss.regions[i].startppn - vmss.regions[i].startpagenum;
+ }
+ fprintf(ofp, "0x%016lx]\n", vmss.memsize + (holes_sum <<
VMW_PAGE_SHIFT));
+ }
+
+ return TRUE;
+}
+
diff --git a/vmware_vmss.c b/vmware_vmss.c
index 252bfa2..b168f29 100644
--- a/vmware_vmss.c
+++ b/vmware_vmss.c
@@ -23,13 +23,7 @@
#define LOGPRX "vmw: "
-/* VMware only supports X86/X86_64 virtual machines. */
-#define VMW_PAGE_SIZE (4096)
-#define VMW_PAGE_SHIFT (12)
-
-#define MAX_BLOCK_DUMP (128)
-
-static vmssdata vmss = { 0 };
+vmssdata vmss = { 0 };
int
is_vmware_vmss(char *filename)
diff --git a/vmware_vmss.h b/vmware_vmss.h
index a5828a0..01d9446 100644
--- a/vmware_vmss.h
+++ b/vmware_vmss.h
@@ -165,6 +165,14 @@ struct vmssdata {
};
typedef struct vmssdata vmssdata;
+/* VMware only supports X86/X86_64 virtual machines. */
+#define VMW_PAGE_SIZE (4096)
+#define VMW_PAGE_SHIFT (12)
+
+#define MAX_BLOCK_DUMP (128)
+
+extern vmssdata vmss;
+
#define DEBUG_PARSE_PRINT(x) \
do { \
if (CRASHDEBUG(1)) { \
--
2.11.0