1:35 p.m.
vmware_guestdump is extension to vmware_vmss with ability to debug
debug.guest and debug.vmem files.
debug.guest.gz and debug.vmem.gz can be obtained using following
.vmx options from VM running in debug mode:
monitor.mini-suspend_on_panic = TRUE
monitor.suspend_on_triplefault = TRUE
guestdump (debug.guest) is simplified version of *.vmss which does
not contain full VM state, but minimal guest state, such as memory
layout and CPUs state, needed for debugger. is_vmware_guestdump()
and vmware_guestdump_init() functions parse guestdump header and
populate vmss data structure (from vmware_vmss.c). As result, all
handlers (except mempry_dump) from vmware_vmss.c can be reused.
How to use: $ crash /path/to/debug_file.guest vmlinux
Companion debug_file.vmem must be present in the same folder as
debug_file.guest. Otherwise crash will shot a message:
vmw: Open the companion vmem file: /path/to/debug_file.vmem
crash: vmw: /path/to/debug_file.vmem: No such file or directory
Signed-off-by: Alexey Makhalov <amakhalov(a)vmware.com>
---
Makefile | 7 +-
crash.8 | 12 +-
defs.h | 8 ++
filesys.c | 12 +-
main.c | 14 +++
memory.c | 8 +-
vmware_guestdump.c | 315 +++++++++++++++++++++++++++++++++++++++++++++++++++++
vmware_vmss.c | 8 +-
vmware_vmss.h | 8 ++
9 files changed, 376 insertions(+), 16 deletions(-)
create mode 100644 vmware_guestdump.c
diff --git a/Makefile b/Makefile
index 7455410..d185719 100644
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,7 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
unwind_x86_32_64.c unwind_arm.c \
xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \
xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c sadump.c ipcs.c \
- ramdump.c vmware_vmss.c \
+ ramdump.c vmware_vmss.c vmware_guestdump.c \
xen_dom0.c kaslr_helper.c
SOURCE_FILES=${CFILES} ${GENERIC_HFILES} ${MCORE_HFILES} \
@@ -89,7 +89,7 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o
task.o \
unwind_x86_32_64.o unwind_arm.o \
xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \
xen_hyper_dump_tables.o kvmdump.o qemu.o qemu-load.o sadump.o ipcs.o \
- ramdump.o vmware_vmss.o \
+ ramdump.o vmware_vmss.o vmware_guestdump.o \
xen_dom0.o kaslr_helper.o
MEMORY_DRIVER_FILES=memory_driver/Makefile memory_driver/crash.c memory_driver/README
@@ -518,6 +518,9 @@ ramdump.o: ${GENERIC_HFILES} ${REDHAT_HFILES} ramdump.c
vmware_vmss.o: ${GENERIC_HFILES} ${VMWARE_HFILES} vmware_vmss.c
${CC} -c ${CRASH_CFLAGS} vmware_vmss.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+vmware_guestdump.o: ${GENERIC_HFILES} ${VMWARE_HFILES} vmware_guestdump.c
+ ${CC} -c ${CRASH_CFLAGS} vmware_guestdump.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+
kaslr_helper.o: ${GENERIC_HFILES} kaslr_helper.c
${CC} -c ${CRASH_CFLAGS} kaslr_helper.c ${WARNING_OPTIONS} ${WARNING_ERROR}
diff --git a/crash.8 b/crash.8
index 136ae78..5020ce1 100644
--- a/crash.8
+++ b/crash.8
@@ -21,8 +21,9 @@ core dump has been created by the
.I LKCD,
.I kdump,
.I xendump
-or
.I kvmdump
+or
+.I VMware
facilities. It is loosely based on the SVR4 UNIX crash
command, but has been significantly enhanced
by completely merging it with the
@@ -112,8 +113,9 @@ A kernel core dump file created by the
.I LKCD
.I kdump,
.I xendump
-or
.I kvmdump
+or
+.I VMware
facilities.
If a MEMORY-IMAGE argument is not entered, the session will be invoked on
@@ -144,6 +146,12 @@ in /var/tmp, which will only exist during the crash session. If a
raw RAM
dumpile represents a live memory source, such as that specified by the QEMU
mem-path argument of a memory-backend-file object, then "live:" must be
prepended to the MEMORY-IMAGE name.
+
+As VMware facility, the
+.B crash
+utility is able to process VMware VM memory dump generated by VM suspend
+or guest core dump. In that case, .vmss or .guest file should be used as
+a MEMORY-IMAGE and .vmem file must be located in the same folder.
.TP
.BI mapfile
If the NAMELIST file is not the same kernel that is
diff --git a/defs.h b/defs.h
index c899fe2..3386709 100644
--- a/defs.h
+++ b/defs.h
@@ -544,6 +544,7 @@ struct program_context {
#define is_excluded_vmemmap() (pc->flags2 & EXCLUDED_VMEMMAP)
#define MEMSRC_LOCAL (0x80000ULL)
#define REDZONE (0x100000ULL)
+#define VMWARE_VMSS_GUESTDUMP (0x200000ULL)
char *cleanup;
char *namelist_orig;
char *namelist_debug_orig;
@@ -6679,6 +6680,13 @@ int vmware_vmss_phys_base(ulong *phys_base);
int vmware_vmss_set_phys_base(ulong);
/*
+ * vmware_guestdump.c
+ */
+int is_vmware_guestdump(char *filename);
+int vmware_guestdump_init(char *filename, FILE *ofp);
+int vmware_guestdump_memory_dump(FILE *);
+
+/*
* kaslr_helper.c
*/
int calc_kaslr_offset(ulong *, ulong *);
diff --git a/filesys.c b/filesys.c
index 2ec2b31..3361b6c 100644
--- a/filesys.c
+++ b/filesys.c
@@ -253,9 +253,15 @@ memory_source_init(void)
error(FATAL, "%s: initialization failed\n",
pc->dumpfile);
} else if (pc->flags & VMWARE_VMSS) {
- if (!vmware_vmss_init(pc->dumpfile, fp))
- error(FATAL, "%s: initialization failed\n",
- pc->dumpfile);
+ if (pc->flags2 & VMWARE_VMSS_GUESTDUMP) {
+ if (!vmware_guestdump_init(pc->dumpfile, fp))
+ error(FATAL, "%s: initialization failed\n",
+ pc->dumpfile);
+ } else {
+ if (!vmware_vmss_init(pc->dumpfile, fp))
+ error(FATAL, "%s: initialization failed\n",
+ pc->dumpfile);
+ }
}
}
}
diff --git a/main.c b/main.c
index 7f562e6..388ac46 100644
--- a/main.c
+++ b/main.c
@@ -671,6 +671,18 @@ main(int argc, char **argv)
pc->readmem = read_vmware_vmss;
pc->writemem = write_vmware_vmss;
+ } else if (is_vmware_guestdump(argv[optind])) {
+ if (pc->flags & MEMORY_SOURCES) {
+ error(INFO,
+ "too many dumpfile arguments\n");
+ program_usage(SHORT_FORM);
+ }
+ pc->flags |= VMWARE_VMSS;
+ pc->flags2 |= VMWARE_VMSS_GUESTDUMP;
+ pc->dumpfile = argv[optind];
+ pc->readmem = read_vmware_vmss;
+ pc->writemem = write_vmware_vmss;
+
} else {
error(INFO,
"%s: not a supported file format\n",
@@ -1486,6 +1498,8 @@ dump_program_context(void)
fprintf(fp, "%sMEMSRC_LOCAL", others++ ? "|" : "");
if (pc->flags2 & REDZONE)
fprintf(fp, "%sREDZONE", others++ ? "|" : "");
+ if (pc->flags2 & VMWARE_VMSS_GUESTDUMP)
+ fprintf(fp, "%sVMWARE_VMSS_GUESTDUMP", others++ ? "|" :
"");
fprintf(fp, ")\n");
fprintf(fp, " namelist: %s\n", pc->namelist);
diff --git a/memory.c b/memory.c
index c951827..0848097 100644
--- a/memory.c
+++ b/memory.c
@@ -17115,8 +17115,12 @@ dumpfile_memory(int cmd)
retval = kcore_memory_dump(fp);
else if (pc->flags & SADUMP)
retval = sadump_memory_dump(fp);
- else if (pc->flags & VMWARE_VMSS)
- retval = vmware_vmss_memory_dump(fp);
+ else if (pc->flags & VMWARE_VMSS) {
+ if (pc->flags2 & VMWARE_VMSS_GUESTDUMP)
+ retval = vmware_guestdump_memory_dump(fp);
+ else
+ retval = vmware_vmss_memory_dump(fp);
+ }
break;
case DUMPFILE_ENVIRONMENT:
diff --git a/vmware_guestdump.c b/vmware_guestdump.c
new file mode 100644
index 0000000..d3fac59
--- /dev/null
+++ b/vmware_guestdump.c
@@ -0,0 +1,315 @@
+/*
+ * vmware_guestdump.c
+ *
+ * Copyright (c) 2020 VMware, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Author: Alexey Makhalov <amakhalov(a)vmware.com>
+ */
+
+#include "defs.h"
+#include "vmware_vmss.h"
+
+#define LOGPRX "vmw: "
+
+#define GUESTDUMP_VERSION 4
+#define GUESTDUMP_MAGIC1 1
+#define GUESTDUMP_MAGIC2 0
+
+struct guestdumpheader {
+ uint32_t version;
+ uint32_t num_vcpus;
+ uint8_t magic1;
+ uint8_t reserved1;
+ uint32_t cpu_vendor;
+ uint64_t magic2;
+ uint64_t last_addr;
+ uint64_t memsize_in_pages;
+ uint32_t reserved2;
+ uint32_t mem_holes;
+ struct memhole {
+ uint64_t ppn;
+ uint64_t pages;
+ } holes[2];
+} __attribute__((packed));
+
+struct vcpu_state {
+ uint32_t cr0;
+ uint64_t cr2;
+ uint64_t cr3;
+ uint64_t cr4;
+ uint64_t reserved1[10];
+ uint64_t idt_base;
+ uint16_t reserved2[21];
+ struct x86_64_pt_regs {
+ uint64_t r15;
+ uint64_t r14;
+ uint64_t r13;
+ uint64_t r12;
+ uint64_t rbp;
+ uint64_t rbx;
+ uint64_t r11;
+ uint64_t r10;
+ uint64_t r9;
+ uint64_t r8;
+ uint64_t rax;
+ uint64_t rcx;
+ uint64_t rdx;
+ uint64_t rsi;
+ uint64_t rdi;
+ uint64_t orig_rax;
+ uint64_t rip;
+ uint64_t cs;
+ uint64_t eflags;
+ uint64_t rsp;
+ uint64_t ss;
+ } regs64;
+ uint8_t reserved3[65];
+} __attribute__((packed));
+
+
+/*
+ * vmware_guestdump is extension to vmware_vmss with ability to debug
+ * debug.guest and debug.vmem files.
+ *
+ * debug.guest.gz and debug.vmem.gz can be obtained using following
+ * .vmx options from VM running in debug mode:
+ * monitor.mini-suspend_on_panic = TRUE
+ * monitor.suspend_on_triplefault = TRUE
+ *
+ * guestdump (debug.guest) is simplified version of *.vmss which does
+ * not contain full VM state, but minimal guest state, such as memory
+ * layout and CPUs state, needed for debugger. is_vmware_guestdump()
+ * and vmware_guestdump_init() functions parse guestdump header and
+ * populate vmss data structure (from vmware_vmss.c). As result, all
+ * handlers (except mempry_dump) from vmware_vmss.c can be reused.
+ *
+ * debug.guest does not have dedicated header magic or signature for
+ * its format. To probe debug.guest we need to perform header fields
+ * and file size validity. In addition, check for the filename
+ * extension, which must be ".guest".
+ */
+
+int
+is_vmware_guestdump(char *filename)
+{
+ struct guestdumpheader hdr;
+ FILE *fp;
+ uint64_t filesize, holes_sum = 0;
+ int i;
+
+ if (strcmp(filename + strlen(filename) - 6, ".guest"))
+ return FALSE;
+
+ if ((fp = fopen(filename, "r")) == NULL) {
+ error(INFO, LOGPRX"Failed to open '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ return FALSE;
+ }
+
+ if (fread(&hdr, sizeof(struct guestdumpheader), 1, fp) != 1) {
+ error(INFO, LOGPRX"Failed to read '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ fclose(fp);
+ return FALSE;
+ }
+
+ if (fseek(fp, 0L, SEEK_END) == -1) {
+ error(INFO, LOGPRX"Failed to fseek '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ fclose(fp);
+ return FALSE;
+ }
+ filesize = ftell(fp);
+ fclose(fp);
+
+ if (hdr.mem_holes > 2)
+ goto unrecognized;
+
+ for (i = 0; i < hdr.mem_holes; i++) {
+ /* hole start page */
+ vmss.regions[i].startpagenum = hdr.holes[i].ppn;
+ /* hole end page */
+ vmss.regions[i].startppn = hdr.holes[i].ppn + hdr.holes[i].pages;
+ holes_sum += hdr.holes[i].pages;
+ }
+
+ if (hdr.version != GUESTDUMP_VERSION ||
+ hdr.magic1 != GUESTDUMP_MAGIC1 ||
+ hdr.magic2 != GUESTDUMP_MAGIC2 ||
+ (hdr.last_addr + 1) != ((hdr.memsize_in_pages + holes_sum) << VMW_PAGE_SHIFT)
||
+ filesize != sizeof(struct guestdumpheader) +
+ hdr.num_vcpus * (sizeof (struct vcpu_state) + VMW_PAGE_SIZE))
+ goto unrecognized;
+
+ vmss.memsize = hdr.memsize_in_pages << VMW_PAGE_SHIFT;
+ vmss.regionscount = hdr.mem_holes + 1;
+ vmss.memoffset = 0;
+ vmss.num_vcpus = hdr.num_vcpus;
+ return TRUE;
+
+unrecognized:
+ if (CRASHDEBUG(1))
+ error(INFO, LOGPRX"Unrecognized debug.guest file.\n");
+ return FALSE;
+}
+
+int
+vmware_guestdump_init(char *filename, FILE *ofp)
+{
+ FILE *fp = NULL;
+ int i, result = TRUE;
+ char *vmem_filename = NULL;
+ struct vcpu_state vs;
+ char *p;
+
+ if (!machine_type("X86") && !machine_type("X86_64")) {
+ error(INFO,
+ LOGPRX"Invalid or unsupported host architecture for .vmss file: %s\n",
+ MACHINE_TYPE);
+ result = FALSE;
+ goto exit;
+ }
+
+ if ((fp = fopen(filename, "r")) == NULL) {
+ error(INFO, LOGPRX"Failed to open '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+
+ if (fseek(fp, sizeof(struct guestdumpheader), SEEK_SET) == -1) {
+ error(INFO, LOGPRX"Failed to fseek '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+
+ vmss.vcpu_regs = malloc(vmss.num_vcpus * sizeof(uint32_t));
+ vmss.regs64 = calloc(vmss.num_vcpus, sizeof(void *));
+ if (!vmss.vcpu_regs || !vmss.regs64) {
+ error(INFO, LOGPRX"Failed to allocate memory\n");
+ result = FALSE;
+ goto exit;
+ }
+
+ for (i = 0; i < vmss.num_vcpus; i++) {
+ if (fread(&vs, sizeof(struct vcpu_state), 1, fp) != 1) {
+ error(INFO, LOGPRX"Failed to read '%s': [Error %d] %s\n",
+ filename, errno, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+ vmss.regs64[i] = calloc(1, sizeof(vmssregs64));
+ if (!vmss.regs64[i]) {
+ error(INFO, LOGPRX"Failed to allocate memory\n");
+ result = FALSE;
+ goto exit;
+ }
+ vmss.vcpu_regs[i] = 0;
+
+ vmss.regs64[i]->rax = vs.regs64.rax;
+ vmss.regs64[i]->rcx = vs.regs64.rcx;
+ vmss.regs64[i]->rdx = vs.regs64.rdx;
+ vmss.regs64[i]->rbx = vs.regs64.rbx;
+ vmss.regs64[i]->rbp = vs.regs64.rbp;
+ vmss.regs64[i]->rsp = vs.regs64.rsp;
+ vmss.regs64[i]->rsi = vs.regs64.rsi;
+ vmss.regs64[i]->rdi = vs.regs64.rdi;
+ vmss.regs64[i]->r8 = vs.regs64.r8;
+ vmss.regs64[i]->r9 = vs.regs64.r9;
+ vmss.regs64[i]->r10 = vs.regs64.r10;
+ vmss.regs64[i]->r11 = vs.regs64.r11;
+ vmss.regs64[i]->r12 = vs.regs64.r12;
+ vmss.regs64[i]->r13 = vs.regs64.r13;
+ vmss.regs64[i]->r14 = vs.regs64.r14;
+ vmss.regs64[i]->r15 = vs.regs64.r15;
+ vmss.regs64[i]->idtr = vs.idt_base;
+ vmss.regs64[i]->cr[0] = vs.cr0;
+ vmss.regs64[i]->cr[2] = vs.cr2;
+ vmss.regs64[i]->cr[3] = vs.cr3;
+ vmss.regs64[i]->cr[4] = vs.cr4;
+ vmss.regs64[i]->rip = vs.regs64.rip;
+ vmss.regs64[i]->rflags = vs.regs64.eflags;
+
+ vmss.vcpu_regs[i] = REGS_PRESENT_ALL;
+ }
+
+ vmem_filename = strdup(filename);
+ p = vmem_filename + strlen(vmem_filename) - 5;
+ if (strcmp(p, "guest") != 0) {
+ result = FALSE;
+ goto exit;
+ }
+ strcpy(p, "vmem");
+
+ fprintf(ofp, LOGPRX"Open the companion vmem file: %s\n", vmem_filename);
+ if ((vmss.dfp = fopen(vmem_filename, "r")) == NULL) {
+ error(INFO, LOGPRX"%s: %s\n", vmem_filename, strerror(errno));
+ result = FALSE;
+ goto exit;
+ }
+ fseek(vmss.dfp, 0L, SEEK_END);
+ if (vmss.memsize != ftell(vmss.dfp)) {
+ error(INFO, LOGPRX"%s: unexpected size\n", vmem_filename);
+ result = FALSE;
+ goto exit;
+ }
+ fseek(vmss.dfp, 0L, SEEK_SET);
+ fprintf(ofp, LOGPRX"vmem file: %s\n\n", vmem_filename);
+
+exit:
+ if (fp)
+ fclose(fp);
+ if (vmem_filename)
+ free(vmem_filename);
+ if (result == FALSE) {
+ if (vmss.dfp)
+ fclose(vmss.dfp);
+ if (vmss.regs64) {
+ for (i = 0; i < vmss.num_vcpus; i++) {
+ if (vmss.regs64[i])
+ free(vmss.regs64[i]);
+ }
+ free(vmss.regs64);
+ }
+ if (vmss.vcpu_regs)
+ free(vmss.vcpu_regs);
+ }
+ return result;
+}
+
+int
+vmware_guestdump_memory_dump(FILE *ofp)
+{
+ fprintf(ofp, "vmware_guestdump:\n");
+ fprintf(ofp, " Header: version=%d num_vcpus=%ld\n",
+ GUESTDUMP_VERSION, vmss.num_vcpus);
+ fprintf(ofp, "Total memory: %ld\n", vmss.memsize);
+
+ if (vmss.regionscount > 1) {
+ uint64_t holes_sum = 0;
+ unsigned i;
+
+ fprintf(ofp, "Memory regions[%d]:\n", vmss.regionscount);
+ fprintf(ofp, " [0x%016x-", 0);
+ for (i = 0; i < vmss.regionscount - 1; i++) {
+ fprintf(ofp, "0x%016lx]\n", (uint64_t)vmss.regions[i].startpagenum <<
VMW_PAGE_SHIFT);
+ fprintf(ofp, " [0x%016lx-", (uint64_t)vmss.regions[i].startppn <<
VMW_PAGE_SHIFT);
+ holes_sum += vmss.regions[i].startppn - vmss.regions[i].startpagenum;
+ }
+ fprintf(ofp, "0x%016lx]\n", vmss.memsize + (holes_sum <<
VMW_PAGE_SHIFT));
+ }
+
+ return TRUE;
+}
+
diff --git a/vmware_vmss.c b/vmware_vmss.c
index 252bfa2..b168f29 100644
--- a/vmware_vmss.c
+++ b/vmware_vmss.c
@@ -23,13 +23,7 @@
#define LOGPRX "vmw: "
-/* VMware only supports X86/X86_64 virtual machines. */
-#define VMW_PAGE_SIZE (4096)
-#define VMW_PAGE_SHIFT (12)
-
-#define MAX_BLOCK_DUMP (128)
-
-static vmssdata vmss = { 0 };
+vmssdata vmss = { 0 };
int
is_vmware_vmss(char *filename)
diff --git a/vmware_vmss.h b/vmware_vmss.h
index a5828a0..01d9446 100644
--- a/vmware_vmss.h
+++ b/vmware_vmss.h
@@ -165,6 +165,14 @@ struct vmssdata {
};
typedef struct vmssdata vmssdata;
+/* VMware only supports X86/X86_64 virtual machines. */
+#define VMW_PAGE_SIZE (4096)
+#define VMW_PAGE_SHIFT (12)
+
+#define MAX_BLOCK_DUMP (128)
+
+extern vmssdata vmss;
+
#define DEBUG_PARSE_PRINT(x) \
do { \
if (CRASHDEBUG(1)) { \
--
2.11.0