5:46 a.m.
From: Don Slutz <dslutz(a)verizon.com>
Currently crash 4.0 and later using the code:
http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
There is some very limited documention on crashes remote ptotocol in:
http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
and some fixes in the attachment 0001-xen-crashd-Connect-crash-with-domain.patch
from the 1st link above.
How ever there are issues with the current code:
1) The remote protocol has a minor issue (which I have not been able to
happen) based on the fact TCP/IP is stream based protocol. This means
a RECV or a SEND may not do the fully requested size of data. In fact
the current code assumes that the amount of data that a SEND is called
with will all be read with a single RECV.
2) The most common mismatch between crash and older kernels is
phys_base. In the remote case, see if the remote server supports
vitrual memory access, and if so, see if phys_base can be retreived.
3) crash assumes that the remote system is active and can not return
currect IP and SP to do a better back trace.
4) enable a non-active mode of remote access.
This patch set attempts to fix these:
The fix for #1 I have called NIL mode (patch 1).
The fix for #2 uses get_remote_phys_base (patch 3).
The fix for #3 uses get_remote_regs (patch 5).
The fix for #4 uses special file /dev/xenmem (patch 7).
It also REMOTE_NIL() to indicate "remote paused system".
Don Slutz (7):
Add NIL mode to remote.
remote_proc_version: NULL terminate passed buffer on error.
Add get_remote_phys_base.
Add remote_vtop.
bt: get remote live registers if possible.
Add get_remote_cr3
Add support for non-live remote.
defs.h | 8 +-
kernel.c | 25 +++-
memory.c | 25 +++-
remote.c | 462 +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
x86_64.c | 24 ++++
5 files changed, 465 insertions(+), 79 deletions(-)
--
1.8.4
5:46 a.m.
New subject: [PATCH 1/7] Add NIL mode to remote.
From: Don Slutz <dslutz(a)verizon.com>
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 1 +
remote.c | 241 +++++++++++++++++++++++++++++++++++++++++++++------------------
2 files changed, 173 insertions(+), 69 deletions(-)
diff --git a/defs.h b/defs.h
index a5299cc..108ffc1 100755
--- a/defs.h
+++ b/defs.h
@@ -504,6 +504,7 @@ struct program_context {
#define GET_LOG (0x200ULL)
#define VMCOREINFO (0x400ULL)
#define ALLOW_FP (0x800ULL)
+#define REMOTE_NIL (0x1000ULL)
char *cleanup;
char *namelist_orig;
char *namelist_debug_orig;
diff --git a/remote.c b/remote.c
index fca08a1..e64f2bd 100755
--- a/remote.c
+++ b/remote.c
@@ -1808,6 +1808,10 @@ static int remote_file_type(char *);
static int remote_lkcd_dump_init(void);
static int remote_s390_dump_init(void);
static int remote_netdump_init(void);
+static int remote_tcp_read(int, const char *, size_t);
+static int remote_tcp_read_string(int, const char *, size_t, int);
+static int remote_tcp_write(int, const void *, size_t);
+static int remote_tcp_write_string(int, const char *);
/*
* Parse, verify and establish a connection with the network daemon
@@ -1921,6 +1925,16 @@ is_remote_daemon(char *dp)
remote_socket_options(pc->sockfd);
+ /*
+ * Try and use NIL mode.
+ */
+ BZERO(sendbuf, BUFSIZE);
+ BZERO(recvbuf, BUFSIZE);
+ sprintf(sendbuf, "NIL");
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ if (!strstr(recvbuf, "<FAIL>"))
+ pc->flags2 |= REMOTE_NIL;
/*
* Get the remote machine type and verify a match. The daemon pid
* is also used as a live system initial context.
@@ -1928,8 +1942,8 @@ is_remote_daemon(char *dp)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "MACHINE_PID");
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
p1 = strtok(recvbuf, " "); /* MACHINE */
p1 = strtok(NULL, " "); /* machine type */
if (CRASHDEBUG(1))
@@ -2032,8 +2046,8 @@ remote_file_type(char *file)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "TYPE %s", file);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>"))
error(FATAL, "invalid remote file name: %s\n", file);
@@ -2112,6 +2126,84 @@ remote_socket_options(int sockfd)
}
/*
+ * Wrapper around recv to read full length packet.
+ */
+static int
+remote_tcp_read(int sock, const char *pv_buffer, size_t cb_buffer)
+{
+ size_t cb_total = 0;
+
+ do
+ {
+ ssize_t cb_read = recv(sock, (void*)pv_buffer, cb_buffer, MSG_NOSIGNAL);
+
+ if (cb_read <= 0)
+ return cb_read;
+ cb_total += cb_read;
+ cb_buffer -= cb_read;
+ pv_buffer = (char *)pv_buffer + cb_read;
+ } while (cb_buffer);
+
+ return cb_total;
+}
+
+/*
+ * Wrapper around recv to read full string packet.
+ */
+static int
+remote_tcp_read_string(int sock, const char *pv_buffer, size_t cb_buffer, int nil_mode)
+{
+ size_t cb_total = 0;
+
+ do
+ {
+ ssize_t cb_read = recv(sock, (void*)pv_buffer, cb_buffer, MSG_NOSIGNAL);
+
+ if (cb_read <= 0)
+ return cb_read;
+ cb_total += cb_read;
+ if (!nil_mode && cb_total >= 4)
+ return cb_total;
+ if (!pv_buffer[cb_read - 1])
+ return cb_total;
+ cb_buffer -= cb_read;
+ pv_buffer = (char *)pv_buffer + cb_read;
+ } while (cb_buffer);
+
+ return cb_total;
+}
+
+/*
+ * Wrapper around send to send full packet.
+ */
+static int
+remote_tcp_write(int sock, const void *pv_buffer, size_t cb_buffer)
+{
+ do
+ {
+ size_t cb_now = cb_buffer;
+ ssize_t cb_written = send(sock, (const char *)pv_buffer, cb_now, MSG_NOSIGNAL);
+
+ if (cb_written < 0)
+ return 1;
+ cb_buffer -= cb_written;
+ pv_buffer = (char *)pv_buffer + cb_written;
+ } while (cb_buffer);
+
+ return 0;
+}
+
+/*
+ * Wrapper around tcp_write to send a string
+ */
+static int
+remote_tcp_write_string(int sock, const char *pv_buffer)
+{
+ return remote_tcp_write(sock, pv_buffer, strlen(pv_buffer) + 1);
+}
+
+
+/*
* Request that the daemon open a file.
*/
static int
@@ -2124,8 +2216,8 @@ remote_file_open(struct remote_file *rfp)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "OPEN %s", rfp->filename);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (CRASHDEBUG(1))
fprintf(fp, "remote_file_open: [%s]\n", recvbuf);
@@ -2159,8 +2251,8 @@ remote_file_close(struct remote_file *rfp)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "CLOSE %d", rfp->fd);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
return (strstr(recvbuf, "OK") ? TRUE : FALSE);
}
@@ -2177,8 +2269,8 @@ remote_proc_version(char *buf)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "PROC_VERSION");
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (STREQ(recvbuf, "<FAIL>"))
return FALSE;
strcpy(buf, recvbuf);
@@ -2359,8 +2451,8 @@ copy_to_local_namelist(struct remote_file *rfp)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "DEBUGGING_SYMBOLS %s", rfp->filename);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "NO_DEBUG")) {
sprintf(readbuf, "%s@%s", rfp->filename, pc->server);
pc->namelist = readbuf;
@@ -2472,8 +2564,8 @@ identical_namelist(char *file, struct remote_file *rfp)
BZERO(readbuf, BUFSIZE);
sprintf(sendbuf, "LINUX_VERSION %s", rfp->filename);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -2515,8 +2607,8 @@ remote_file_checksum(struct remote_file *rfp)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "SUM %s", rfp->filename);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>")) {
error(INFO, "%s: does not exist on server %s\n",
rfp->filename, pc->server);
@@ -2785,8 +2877,8 @@ remote_find_booted_kernel(struct remote_file *rfp)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "FIND_BOOTED_KERNEL");
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
strtok(recvbuf, " "); /* FIND_BOOTED_KERNEL */
p1 = strtok(NULL, " "); /* filename */
if (STREQ(p1, "<FAIL>"))
@@ -2806,8 +2898,8 @@ remote_lkcd_dump_init(void)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "LKCD_DUMP_INIT %d %s", pc->rmfd,
pc->server_memsrc);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -2844,8 +2936,8 @@ remote_s390_dump_init(void)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "S390_DUMP_INIT %d %s", pc->rmfd,
pc->server_memsrc);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -2880,8 +2972,8 @@ remote_netdump_init(void)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "NETDUMP_INIT %d %s", pc->rmfd,
pc->server_memsrc);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -2925,8 +3017,8 @@ remote_page_size(void)
error(FATAL,
"cannot determine remote page size (unknown memory source)\n");
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "FAIL"))
error(FATAL, "cannot determine remote page size\n");
strtok(recvbuf, " "); /* PAGESIZE */
@@ -2968,7 +3060,7 @@ copy_remote_file(struct remote_file *rfp, int fd, char *file, char
*ttystr)
BZERO(sendbuf, BUFSIZE);
sprintf(sendbuf, "READ %d %lx %ld", rfp->fd, offset, size);
- bytes = write(pc->sockfd, sendbuf, strlen(sendbuf));
+ bytes = write(pc->sockfd, sendbuf, strlen(sendbuf) + 1);
bzero(readbuf, READBUFSIZE);
@@ -3059,7 +3151,7 @@ copy_remote_gzip_file(struct remote_file *rfp, char *file, char
*ttystr)
BZERO(sendbuf, BUFSIZE);
sprintf(sendbuf, "READ_GZIP %ld %s", pc->rcvbufsize,
rfp->filename);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
bzero(readbuf, READBUFSIZE);
@@ -3170,8 +3262,8 @@ find_remote_module_objfile(struct load_module *lm, char *module,
char *retbuf)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "FIND_MODULE %s %s",
kt->utsname.release, module);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
if (strstr(recvbuf, "<FAIL>")) {
fprintf(fp, "find_remote_module_objfile: [%s]\n",
recvbuf);
@@ -3231,8 +3323,8 @@ remote_free_memory(void)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "MEMORY FREE %s", type);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
p1 = strtok(recvbuf, " "); /* MEMORY */
p1 = strtok(NULL, " "); /* FREE */
p1 = strtok(NULL, " "); /* MCLXCD, LKCD etc. */
@@ -3267,8 +3359,8 @@ remote_memory_used(void)
BZERO(sendbuf, BUFSIZE);
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "MEMORY USED %s", type);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
- recv(pc->sockfd, recvbuf, BUFSIZE-1, 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
p1 = strtok(recvbuf, " "); /* MEMORY */
p1 = strtok(NULL, " "); /* FREE */
p1 = strtok(NULL, " "); /* MCLXCD, LKCD, etc. */
@@ -3308,7 +3400,7 @@ remote_memory_dump(int verbose)
BZERO(sendbuf, BUFSIZE);
sprintf(sendbuf, "MEMORY_DUMP %ld %s%s", pc->rcvbufsize, type,
verbose ? "_VERBOSE" : "");
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
bzero(readbuf, READBUFSIZE);
done = total = 0;
@@ -3371,10 +3463,9 @@ int
remote_memory_read(int rfd, char *buffer, int cnt, physaddr_t address)
{
char sendbuf[BUFSIZE];
- char readbuf[READBUFSIZE];
char datahdr[DATA_HDRSIZE];
- char *bufptr, *p1;
- int ret, req, tot;
+ char *p1;
+ int ret, tot;
ulong addr;
addr = (ulong)address; /* may be virtual */
@@ -3391,48 +3482,49 @@ remote_memory_read(int rfd, char *buffer, int cnt, physaddr_t
address)
else
sprintf(sendbuf, "READ_LIVE %d %lx %d", rfd, addr, cnt);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
-
- bzero(readbuf, READBUFSIZE);
+ if (remote_tcp_write_string(pc->sockfd, sendbuf))
+ return -1;
/*
* Read request will come back with a singular header
* followed by the data.
*/
- req = cnt+DATA_HDRSIZE;
- tot = 0;
- errno = 0;
- bufptr = readbuf;
-
- while (req) {
- if ((ret = recv(pc->sockfd, bufptr, req, 0)) == -1)
- return -1;
- req -= ret;
- bufptr += ret;
- tot += ret;
- if ((tot >= DATA_HDRSIZE) && STRNEQ(readbuf, FAILMSG)) {
- p1 = strtok(readbuf, " "); /* FAIL */
- p1 = strtok(NULL, " "); /* errno */
- errno = atoi(p1);
- return -1;
- }
+ BZERO(datahdr, DATA_HDRSIZE);
+ ret = remote_tcp_read_string(pc->sockfd, datahdr, DATA_HDRSIZE, 1);
+ if (ret <= 0)
+ return -1;
+ if (CRASHDEBUG(3))
+ fprintf(fp, "remote_memory_read: [%s]\n", datahdr);
+ if (STRNEQ(datahdr, FAILMSG)) {
+ p1 = strtok(datahdr, " "); /* FAIL */
+ p1 = strtok(NULL, " "); /* errno */
+ errno = atoi(p1);
+ return -1;
}
-
- if (!STRNEQ(readbuf, DONEMSG) && !STRNEQ(readbuf, DATAMSG)) {
+ if (!STRNEQ(datahdr, DONEMSG) && !STRNEQ(datahdr, DATAMSG)) {
error(INFO, "out of sync with remote memory source\n");
return -1;
}
- strncpy(datahdr, readbuf, DATA_HDRSIZE);
- if (CRASHDEBUG(3))
- fprintf(fp, "remote_memory_read: [%s]\n", datahdr);
- p1 = strtok(datahdr, " "); /* DONE */
- p1 = strtok(NULL, " "); /* count */
+ p1 = strtok(datahdr, " "); /* DONE */
+ p1 = strtok(NULL, " "); /* count */
tot = atol(p1);
- BCOPY(readbuf+DATA_HDRSIZE, buffer, tot);
+ if (cnt != tot) {
+ error(FATAL,
+ "requested %d bytes remote memory return %d bytes\n",
+ cnt, tot);
+ return -1;
+ }
+ ret = remote_tcp_read(pc->sockfd, buffer, tot);
+ if (ret != tot) {
+ error(FATAL,
+ "requested %d bytes remote memory return %d bytes\n",
+ ret, tot);
+ return -1;
+ }
return tot;
}
@@ -3492,7 +3584,7 @@ remote_execute(void)
BZERO(sendbuf, BUFSIZE);
sprintf(sendbuf, "EXECUTE %ld %s", pc->rcvbufsize, command);
- send(pc->sockfd, sendbuf, strlen(sendbuf), 0);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
bzero(readbuf, READBUFSIZE);
done = total = 0;
@@ -3561,12 +3653,23 @@ remote_exit(void)
BZERO(buf, BUFSIZE);
sprintf(buf, "EXIT");
- send(pc->sockfd, buf, strlen(buf), 0);
+ remote_tcp_write_string(pc->sockfd, buf);
/*
* Read but ignore the return status -- we don't really care...
*/
- recv(pc->sockfd, buf, BUFSIZE-1, 0);
+ remote_tcp_read_string(pc->sockfd, buf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
}
#endif /* !DAEMON */
+/*
+ * Specify Emacs local variables so the formating
+ * of the code stays the same.
+ *
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 8
+ * indent-tabs-mode: t
+ * End:
+ */
--
1.8.4
5:46 a.m.
New subject: [PATCH 2/7] remote_proc_version: NULL terminate passed buffer on error.
From: Don Slutz <dslutz(a)verizon.com>
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 2 +-
memory.c | 4 ++--
remote.c | 8 ++++++--
3 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/defs.h b/defs.h
index 108ffc1..4cb2113 100755
--- a/defs.h
+++ b/defs.h
@@ -5536,7 +5536,7 @@ int remote_memory_used(void);
void remote_exit(void);
int remote_execute(void);
void remote_clear_pipeline(void);
-int remote_memory_read(int, char *, int, physaddr_t);
+int remote_memory_read(int, char *, int, physaddr_t, int);
/*
* gnu_binutils.c
diff --git a/memory.c b/memory.c
index d936b53..5ec454d 100755
--- a/memory.c
+++ b/memory.c
@@ -2492,7 +2492,7 @@ write_lkcd_dumpfile(int fd, void *bufptr, int cnt, ulong addr,
physaddr_t paddr)
int
read_daemon(int fd, void *bufptr, int cnt, ulong vaddr, physaddr_t paddr)
{
- if (remote_memory_read(pc->rmfd, bufptr, cnt, paddr) == cnt)
+ if (remote_memory_read(pc->rmfd, bufptr, cnt, paddr, -1) == cnt)
return cnt;
if (!IS_VMALLOC_ADDR(vaddr) || DUMPFILE())
@@ -2506,7 +2506,7 @@ read_daemon(int fd, void *bufptr, int cnt, ulong vaddr, physaddr_t
paddr)
* the only alternative is to read it from /dev/kmem.
*/
- if (BITS32() && remote_memory_read(pc->rkfd, bufptr, cnt, vaddr) == cnt)
+ if (BITS32() && remote_memory_read(pc->rkfd, bufptr, cnt, vaddr, -1) == cnt)
return cnt;
return READ_ERROR;
diff --git a/remote.c b/remote.c
index e64f2bd..823aba0 100755
--- a/remote.c
+++ b/remote.c
@@ -2271,8 +2271,10 @@ remote_proc_version(char *buf)
sprintf(sendbuf, "PROC_VERSION");
remote_tcp_write_string(pc->sockfd, sendbuf);
remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
- if (STREQ(recvbuf, "<FAIL>"))
+ if (STREQ(recvbuf, "<FAIL>")) {
+ buf[0] = 0;
return FALSE;
+ }
strcpy(buf, recvbuf);
return TRUE;
}
@@ -3460,7 +3462,7 @@ remote_memory_dump(int verbose)
* a page in length.
*/
int
-remote_memory_read(int rfd, char *buffer, int cnt, physaddr_t address)
+remote_memory_read(int rfd, char *buffer, int cnt, physaddr_t address, int vcpu)
{
char sendbuf[BUFSIZE];
char datahdr[DATA_HDRSIZE];
@@ -3479,6 +3481,8 @@ remote_memory_read(int rfd, char *buffer, int cnt, physaddr_t
address)
sprintf(sendbuf, "READ_LKCD %d %lx %d", rfd, addr, cnt);
else if (pc->flags & REM_S390D)
sprintf(sendbuf, "READ_S390D %d %lx %d", rfd, addr, cnt);
+ else if (vcpu >= 0)
+ sprintf(sendbuf, "READ_LIVE %d %lx %d %d", rfd, addr, cnt,
vcpu);
else
sprintf(sendbuf, "READ_LIVE %d %lx %d", rfd, addr, cnt);
--
1.8.4
5:46 a.m.
New subject: [PATCH 3/7] Add get_remote_phys_base.
From: Don Slutz <dslutz(a)verizon.com>
The most common mismatch between crash and older kernels is
phys_base. In the remote case, see if the remote server supports
vitrual memory access, and if so, see if phys_base can be retreived.
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 1 +
remote.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
x86_64.c | 24 +++++++++++++++++++++
3 files changed, 98 insertions(+), 2 deletions(-)
diff --git a/defs.h b/defs.h
index 4cb2113..e7073c6 100755
--- a/defs.h
+++ b/defs.h
@@ -5526,6 +5526,7 @@ void dump_sockets_workhorse(ulong, ulong, struct reference *);
* remote.c
*/
int is_remote_daemon(char *);
+physaddr_t get_remote_phys_base(physaddr_t, physaddr_t);
void remote_fd_init(void);
int get_remote_file(struct remote_file *);
uint remote_page_size(void);
diff --git a/remote.c b/remote.c
index 823aba0..58588ce 100755
--- a/remote.c
+++ b/remote.c
@@ -1797,6 +1797,7 @@ static void copy_to_local_namelist(struct remote_file *);
static char *create_local_namelist(struct remote_file *);
static int remote_find_booted_kernel(struct remote_file *);
static int remote_proc_version(char *);
+static int validate_phys_base(physaddr_t, physaddr_t, physaddr_t);
static int remote_file_open(struct remote_file *);
static int remote_file_close(struct remote_file *);
static int identical_namelist(char *, struct remote_file *);
@@ -1813,6 +1814,14 @@ static int remote_tcp_read_string(int, const char *, size_t, int);
static int remote_tcp_write(int, const void *, size_t);
static int remote_tcp_write_string(int, const char *);
+struct _remote_context {
+ int nCpus;
+ int vFd;
+ char remoteType[10];
+} remote_context;
+
+struct _remote_context *rc = &remote_context;
+
/*
* Parse, verify and establish a connection with the network daemon
* specified on the crash command line.
@@ -1848,7 +1857,7 @@ is_remote_daemon(char *dp)
pc->port = 0;
pc->server = pc->server_memsrc = NULL;
- pc->rmfd = pc->rkfd = -1;
+ rc->vFd = pc->rmfd = pc->rkfd = -1;
file1 = file2 = NULL;
if ((filep = strstr(dp, ","))) {
@@ -2280,6 +2289,59 @@ remote_proc_version(char *buf)
}
/*
+ * Check that virt_phys_base when accessed via
+ * phys_base - text_start is phys_base.
+ */
+static int
+validate_phys_base(physaddr_t phys_base, physaddr_t text_start, physaddr_t
virt_phys_base)
+{
+ ulong value;
+
+ if (CRASHDEBUG(3))
+ fprintf(fp, "validate_phys_base: virt_phys_base=0x%llx
phys_base=0x%llx text_start=0x%llx calc=0x%llx\n",
+ virt_phys_base, phys_base, text_start, virt_phys_base + phys_base
- text_start);
+
+ if (READMEM(pc->rmfd, (void*)&value, sizeof(value),
+ virt_phys_base, virt_phys_base + phys_base - text_start)
+ == sizeof(value)) {
+ if (value == phys_base)
+ return 1;
+ }
+ return 0;
+}
+
+/*
+ * Get remote phys_base based on virtual address of "phys_base".
+ */
+physaddr_t
+get_remote_phys_base(physaddr_t text_start, physaddr_t virt_phys_base)
+{
+ int vcpu;
+ ulong value;
+
+ if (rc->vFd < 0) {
+ struct remote_file remote_file, *rfp;
+
+ rfp = &remote_file;
+ BZERO(rfp, sizeof(struct remote_file));
+ rfp->filename = "/dev/vmem";
+ if (remote_file_open(rfp)) {
+ rc->vFd = rfp->fd;
+ } else
+ return 0;
+ }
+
+ for (vcpu = 0; vcpu < rc->nCpus; vcpu++)
+ if (remote_memory_read(rc->vFd, (void*)&value, sizeof(value),
+ virt_phys_base, vcpu) == sizeof(value)) {
+ if (validate_phys_base(value, text_start, virt_phys_base))
+ return value;
+ }
+
+ return 0;
+}
+
+/*
*
* Set up the file descriptors and file name strings if they haven't
* been set up before:
@@ -2999,7 +3061,7 @@ remote_page_size(void)
{
char sendbuf[BUFSIZE];
char recvbuf[BUFSIZE];
- char *p1;
+ char *p1, *p2, *p3;
uint psz;
BZERO(sendbuf, BUFSIZE);
@@ -3026,12 +3088,21 @@ remote_page_size(void)
strtok(recvbuf, " "); /* PAGESIZE */
p1 = strtok(NULL, " "); /* LIVE, MCLXCD or LKCD */
p1 = strtok(NULL, " "); /* page size */
+ p2 = strtok(NULL, " "); /* remote type */
+ p3 = strtok(NULL, " "); /* number of Cpus */
psz = atoi(p1);
if (psz > MAXRECVBUFSIZE)
error(FATAL,
"remote page size %d is larger than MAXRECVBUFSIZE!\n", psz);
+ if (p2) {
+ strncpy(rc->remoteType, p2, sizeof(rc->remoteType) - 1);
+ rc->remoteType[sizeof(rc->remoteType) - 1] = 0;
+ }
+ if (p3)
+ rc->nCpus = atoi(p3);
+
return psz;
}
diff --git a/x86_64.c b/x86_64.c
index e8196f8..e5f8f19 100755
--- a/x86_64.c
+++ b/x86_64.c
@@ -5838,6 +5838,18 @@ x86_64_calc_phys_base(void)
return;
else
text_start = symbol_value("_text");
+ if (REMOTE()) {
+ phys_base = get_remote_phys_base(text_start, symbol_value("phys_base"));
+ if (phys_base) {
+ machdep->machspec->phys_base = phys_base;
+ if (CRASHDEBUG(1)) {
+ fprintf(fp, "_text: %lx ", text_start);
+ fprintf(fp, "phys_base: %lx\n\n",
+ machdep->machspec->phys_base);
+ }
+ return;
+ }
+ }
}
if (ACTIVE()) {
@@ -7680,3 +7692,15 @@ x86_64_verify_paddr(uint64_t paddr)
}
#endif /* X86_64 */
+
+/*
+ * Specify Emacs local variables so the formating
+ * of the code stays the same.
+ *
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 8
+ * indent-tabs-mode: t
+ * End:
+ */
--
1.8.4
5:46 a.m.
New subject: [PATCH 4/7] Add remote_vtop.
From: Don Slutz <dslutz(a)verizon.com>
For eaxmple:
crash> vtop 0xffffffff802ee778
VIRTUAL PHYSICAL
ffffffff802ee778 4ee778
PML4 DIRECTORY: ffffffff80001000
PAGE DIRECTORY: 203067
PUD: 203ff0 => 205063
PMD: 205008 => bee2c163
PTE: bee2c770 => 4ee163
PAGE: 4ee000
PTE PHYSICAL FLAGS
4ee163 4ee000 (PRESENT|RW|ACCESSED|DIRTY|GLOBAL)
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
ffff8100006f7410 4ee000 0 0 1 400
crash> set debug 1
debug: 1
crash> vtop 0xffffffff802ee778
VIRTUAL PHYSICAL
ffffffff802ee778 4ee778
PML4 DIRECTORY: ffffffff80001000
PAGE DIRECTORY: 203067
PUD: 203ff0 => 205063
PMD: 205008 => bee2c163
PTE: bee2c770 => 4ee163
PAGE: 4ee000
PTE PHYSICAL FLAGS
4ee163 4ee000 (PRESENT|RW|ACCESSED|DIRTY|GLOBAL)
PAGE PHYSICAL MAPPING INDEX CNT FLAGS
ffff8100006f7410 4ee000 0 0 1 400
rvtop(ffffffff802ee778)=4ee778
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 1 +
memory.c | 19 +++++++++++++++++++
remote.c | 40 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 60 insertions(+)
diff --git a/defs.h b/defs.h
index e7073c6..94ba495 100755
--- a/defs.h
+++ b/defs.h
@@ -5527,6 +5527,7 @@ void dump_sockets_workhorse(ulong, ulong, struct reference *);
*/
int is_remote_daemon(char *);
physaddr_t get_remote_phys_base(physaddr_t, physaddr_t);
+physaddr_t remote_vtop(int, physaddr_t);
void remote_fd_init(void);
int get_remote_file(struct remote_file *);
uint remote_page_size(void);
diff --git a/memory.c b/memory.c
index 5ec454d..d995f4d 100755
--- a/memory.c
+++ b/memory.c
@@ -2932,6 +2932,13 @@ cmd_vtop(void)
do_vtop(vaddr, tc, vtop_flags | loop_vtop_flags);
+ if (REMOTE() && CRASHDEBUG(1)) {
+ ulong pAddr = remote_vtop(tc->processor, vaddr);
+
+ if (pAddr)
+ fprintf(fp, "rvtop(%lx)=%lx\n", vaddr, pAddr);
+ }
+
optind++;
}
}
@@ -17178,3 +17185,15 @@ get_kmem_cache_by_name(char *request)
return found;
}
#endif /* NOT_USED */
+
+/*
+ * Specify Emacs local variables so the formating
+ * of the code stays the same.
+ *
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 8
+ * indent-tabs-mode: t
+ * End:
+ */
diff --git a/remote.c b/remote.c
index 58588ce..2186af9 100755
--- a/remote.c
+++ b/remote.c
@@ -2342,6 +2342,46 @@ get_remote_phys_base(physaddr_t text_start, physaddr_t
virt_phys_base)
}
/*
+ * Do a remote VTOP if supported.
+ */
+physaddr_t
+remote_vtop(int cpu, physaddr_t virt_addr)
+{
+ char sendbuf[BUFSIZE];
+ char recvbuf[BUFSIZE];
+ char *p1, *p2, *p3, *p4;
+ int errflag;
+ ulong value;
+
+ if (!rc->remoteType[0])
+ return 0; /* Not a special remote. */
+
+ BZERO(sendbuf, BUFSIZE);
+ BZERO(recvbuf, BUFSIZE);
+ sprintf(sendbuf, "VTOP %d %llx", cpu, virt_addr);
+ remote_tcp_write_string(pc->sockfd, sendbuf);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+
+ if (CRASHDEBUG(2))
+ fprintf(fp, "remote_vtop(%d,%llx): [%s]\n",
+ cpu, virt_addr, recvbuf);
+
+ if (strstr(recvbuf, "<FAIL>"))
+ error(FATAL, "remote_vtop for CPU %d\n", cpu);
+ p1 = strtok(recvbuf, " "); /* VTOP */
+ p2 = strtok(NULL, " "); /* cpu */
+ p3 = strtok(NULL, " "); /* vaddr */
+ p4 = strtok(NULL, " "); /* paddr */
+
+ errflag = 0;
+ value = htol(p4, RETURN_ON_ERROR|QUIET, &errflag);
+ if (!errflag) {
+ return value;
+ }
+ return 0;
+}
+
+/*
*
* Set up the file descriptors and file name strings if they haven't
* been set up before:
--
1.8.4
5:46 a.m.
New subject: [PATCH 5/7] bt: get remote live registers if possible.
From: Don Slutz <dslutz(a)verizon.com>
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 1 +
kernel.c | 22 ++++++++++++++++++++--
remote.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 78 insertions(+), 2 deletions(-)
diff --git a/defs.h b/defs.h
index 94ba495..adbfadc 100755
--- a/defs.h
+++ b/defs.h
@@ -5528,6 +5528,7 @@ void dump_sockets_workhorse(ulong, ulong, struct reference *);
int is_remote_daemon(char *);
physaddr_t get_remote_phys_base(physaddr_t, physaddr_t);
physaddr_t remote_vtop(int, physaddr_t);
+int get_remote_regs(struct bt_info *, ulong *, ulong *);
void remote_fd_init(void);
int get_remote_file(struct remote_file *);
uint remote_page_size(void);
diff --git a/kernel.c b/kernel.c
index 7b451c9..ab50361 100755
--- a/kernel.c
+++ b/kernel.c
@@ -1116,6 +1116,9 @@ non_matching_kernel(void)
if (REMOTE_DUMPFILE())
fprintf(fp, "%s@%s (remote dumpfile)\n",
pc->server_memsrc, pc->server);
+ else if (REMOTE_ACTIVE())
+ fprintf(fp, "%s@%s (remote system)\n",
+ pc->server_memsrc, pc->server);
else {
if (dumpfile_is_split())
show_split_dumpfiles();
@@ -2408,7 +2411,7 @@ back_trace(struct bt_info *bt)
(BT_KSTACKP|BT_TEXT_SYMBOLS|BT_TEXT_SYMBOLS_ALL)))
fprintf(fp, "(active)\n");
- if (!(bt->flags & (BT_TEXT_SYMBOLS|BT_TEXT_SYMBOLS_ALL)))
+ if (!(bt->flags & (BT_TEXT_SYMBOLS|BT_TEXT_SYMBOLS_ALL) || REMOTE_ACTIVE()))
return;
}
@@ -2486,7 +2489,10 @@ back_trace(struct bt_info *bt)
get_xendump_regs(bt, &eip, &esp);
else if (SADUMP_DUMPFILE())
get_sadump_regs(bt, &eip, &esp);
- else
+ else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
+ if (get_remote_regs(bt, &eip, &esp))
+ machdep->get_stack_frame(bt, &eip, &esp);
+ } else
machdep->get_stack_frame(bt, &eip, &esp);
if (bt->flags & BT_KSTACKP) {
@@ -9210,3 +9216,15 @@ dump_variable_length_record(void)
hq_close();
}
+
+/*
+ * Specify Emacs local variables so the formating
+ * of the code stays the same.
+ *
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 8
+ * indent-tabs-mode: t
+ * End:
+ */
diff --git a/remote.c b/remote.c
index 2186af9..4b0388d 100755
--- a/remote.c
+++ b/remote.c
@@ -2382,6 +2382,63 @@ remote_vtop(int cpu, physaddr_t virt_addr)
}
/*
+ * Get a copy of the daemon machine cpu regs.
+ */
+int
+get_remote_regs(struct bt_info *bt, ulong *eip, ulong *esp)
+{
+ char sendbuf[BUFSIZE];
+ char recvbuf[BUFSIZE];
+ char *p1, *p2, *p3, *p4, *p5, *p6;
+ int errflag;
+ ulong value;
+
+ if (!rc->remoteType[0])
+ return 1; /* Not a special remote. */
+
+ *eip = 0;
+ *esp = 0;
+
+ BZERO(sendbuf, BUFSIZE);
+ BZERO(recvbuf, BUFSIZE);
+ sprintf(sendbuf, "FETCH_LIVE_IP_SP_BP %d", bt->tc->processor);
+ if (remote_tcp_write_string(pc->sockfd, sendbuf))
+ return 1;
+ errflag = remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ if (errflag <= 0)
+ return 1;
+
+ if (CRASHDEBUG(1))
+ fprintf(fp, "get_remote_regs(cpu=%d): [%s]\n",
+ bt->tc->processor, recvbuf);
+
+ if (strstr(recvbuf, "<FAIL>")) {
+ error(INFO, "get_remote_regs for CPU %d\n", bt->tc->processor);
+ return 1;
+ }
+ p1 = strtok(recvbuf, " "); /* FETCH_LIVE_IP_SP_BP */
+ p1 = strtok(NULL, " "); /* cpu */
+ p1 = strtok(NULL, ":"); /* cs */
+ p2 = strtok(NULL, " "); /* ip */
+ p3 = strtok(NULL, ":"); /* ss */
+ p4 = strtok(NULL, " "); /* sp */
+ p5 = strtok(NULL, " "); /* bp */
+
+ errflag = 0;
+ value = htol(p2, RETURN_ON_ERROR|QUIET, &errflag);
+ if (!errflag) {
+ *eip = value;
+ }
+
+ errflag = 0;
+ value = htol(p4, RETURN_ON_ERROR|QUIET, &errflag);
+ if (!errflag) {
+ *esp = value;
+ }
+ return 0;
+}
+
+/*
*
* Set up the file descriptors and file name strings if they haven't
* been set up before:
--
1.8.4
5:46 a.m.
New subject: [PATCH 6/7] Add get_remote_cr3
From: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 1 +
remote.c | 39 +++++++++++++++++++++++++++++++++++++++
2 files changed, 40 insertions(+)
diff --git a/defs.h b/defs.h
index adbfadc..7a77dab 100755
--- a/defs.h
+++ b/defs.h
@@ -5529,6 +5529,7 @@ int is_remote_daemon(char *);
physaddr_t get_remote_phys_base(physaddr_t, physaddr_t);
physaddr_t remote_vtop(int, physaddr_t);
int get_remote_regs(struct bt_info *, ulong *, ulong *);
+physaddr_t get_remote_cr3(int);
void remote_fd_init(void);
int get_remote_file(struct remote_file *);
uint remote_page_size(void);
diff --git a/remote.c b/remote.c
index 4b0388d..67e5ba8 100755
--- a/remote.c
+++ b/remote.c
@@ -2439,6 +2439,45 @@ get_remote_regs(struct bt_info *bt, ulong *eip, ulong *esp)
}
/*
+ * Get a remote cr3 if supported.
+ */
+physaddr_t
+get_remote_cr3(int cpu)
+{
+ char sendbuf[BUFSIZE];
+ char recvbuf[BUFSIZE];
+ char *p1, *p2, *p3;
+ int errflag;
+ ulong value;
+
+ if (!rc->remoteType[0])
+ return 0; /* Not a special remote. */
+
+ BZERO(sendbuf, BUFSIZE);
+ BZERO(recvbuf, BUFSIZE);
+ sprintf(sendbuf, "FETCH_LIVE_CR3 %d", cpu);
+ if (remote_tcp_write_string(pc->sockfd, sendbuf))
+ return 0;
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+
+ if (CRASHDEBUG(1))
+ fprintf(fp, "get_remote_cr3: [%s]\n", recvbuf);
+
+ if (strstr(recvbuf, "<FAIL>"))
+ error(FATAL, "get_remote_cr3 for CPU %d\n", cpu);
+ p1 = strtok(recvbuf, " "); /* FETCH_LIVE_CR3 */
+ p2 = strtok(NULL, " "); /* cpu */
+ p3 = strtok(NULL, " "); /* cr3 */
+
+ errflag = 0;
+ value = htol(p3, RETURN_ON_ERROR|QUIET, &errflag);
+ if (!errflag)
+ return value;
+ return 0;
+}
+
+
+/*
*
* Set up the file descriptors and file name strings if they haven't
* been set up before:
--
1.8.4
5:46 a.m.
New subject: [PATCH 7/7] Add support for non-live remote.
From: Don Slutz <dslutz(a)verizon.com>
This is accessed via:
crash localhost:5991,/dev/xenmem /usr/lib/debug/lib/modules/2.6.18-128.el5/vmlinux
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 3 ++-
kernel.c | 5 ++++-
memory.c | 2 +-
remote.c | 44 +++++++++++++++++++++++---------------------
4 files changed, 30 insertions(+), 24 deletions(-)
diff --git a/defs.h b/defs.h
index 7a77dab..d964e33 100755
--- a/defs.h
+++ b/defs.h
@@ -504,7 +504,8 @@ struct program_context {
#define GET_LOG (0x200ULL)
#define VMCOREINFO (0x400ULL)
#define ALLOW_FP (0x800ULL)
-#define REMOTE_NIL (0x1000ULL)
+#define REMOTE_NILF (0x1000ULL)
+#define REMOTE_NIL() (pc->flags2 & REMOTE_NILF)
char *cleanup;
char *namelist_orig;
char *namelist_debug_orig;
diff --git a/kernel.c b/kernel.c
index ab50361..02d3ca0 100755
--- a/kernel.c
+++ b/kernel.c
@@ -2489,7 +2489,7 @@ back_trace(struct bt_info *bt)
get_xendump_regs(bt, &eip, &esp);
else if (SADUMP_DUMPFILE())
get_sadump_regs(bt, &eip, &esp);
- else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
+ else if (REMOTE_NIL() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
if (get_remote_regs(bt, &eip, &esp))
machdep->get_stack_frame(bt, &eip, &esp);
} else
@@ -4677,6 +4677,9 @@ display_sys_stats(void)
if (REMOTE_DUMPFILE())
fprintf(fp, "%s@%s (remote dumpfile)",
pc->server_memsrc, pc->server);
+ else if (REMOTE_NIL())
+ fprintf(fp, "%s@%s (remote paused system)\n",
+ pc->server_memsrc, pc->server);
else {
if (dumpfile_is_split())
show_split_dumpfiles();
diff --git a/memory.c b/memory.c
index d995f4d..9b76c9f 100755
--- a/memory.c
+++ b/memory.c
@@ -14903,7 +14903,7 @@ memory_page_size(void)
if (machdep->pagesize)
return machdep->pagesize;
- if (REMOTE_MEMSRC())
+ if (REMOTE_MEMSRC() || REMOTE_NIL())
return remote_page_size();
switch (pc->flags & MEMORY_SOURCES)
diff --git a/remote.c b/remote.c
index 67e5ba8..c75a006 100755
--- a/remote.c
+++ b/remote.c
@@ -1943,7 +1943,7 @@ is_remote_daemon(char *dp)
remote_tcp_write_string(pc->sockfd, sendbuf);
remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, 0);
if (!strstr(recvbuf, "<FAIL>"))
- pc->flags2 |= REMOTE_NIL;
+ pc->flags2 |= REMOTE_NILF;
/*
* Get the remote machine type and verify a match. The daemon pid
* is also used as a live system initial context.
@@ -1952,7 +1952,7 @@ is_remote_daemon(char *dp)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "MACHINE_PID");
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
p1 = strtok(recvbuf, " "); /* MACHINE */
p1 = strtok(NULL, " "); /* machine type */
if (CRASHDEBUG(1))
@@ -2056,7 +2056,7 @@ remote_file_type(char *file)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "TYPE %s", file);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>"))
error(FATAL, "invalid remote file name: %s\n", file);
@@ -2226,7 +2226,7 @@ remote_file_open(struct remote_file *rfp)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "OPEN %s", rfp->filename);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (CRASHDEBUG(1))
fprintf(fp, "remote_file_open: [%s]\n", recvbuf);
@@ -2261,7 +2261,7 @@ remote_file_close(struct remote_file *rfp)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "CLOSE %d", rfp->fd);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
return (strstr(recvbuf, "OK") ? TRUE : FALSE);
}
@@ -2279,7 +2279,7 @@ remote_proc_version(char *buf)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "PROC_VERSION");
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (STREQ(recvbuf, "<FAIL>")) {
buf[0] = 0;
return FALSE;
@@ -2360,7 +2360,7 @@ remote_vtop(int cpu, physaddr_t virt_addr)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "VTOP %d %llx", cpu, virt_addr);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (CRASHDEBUG(2))
fprintf(fp, "remote_vtop(%d,%llx): [%s]\n",
@@ -2404,7 +2404,7 @@ get_remote_regs(struct bt_info *bt, ulong *eip, ulong *esp)
sprintf(sendbuf, "FETCH_LIVE_IP_SP_BP %d", bt->tc->processor);
if (remote_tcp_write_string(pc->sockfd, sendbuf))
return 1;
- errflag = remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ errflag = remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (errflag <= 0)
return 1;
@@ -2458,7 +2458,7 @@ get_remote_cr3(int cpu)
sprintf(sendbuf, "FETCH_LIVE_CR3 %d", cpu);
if (remote_tcp_write_string(pc->sockfd, sendbuf))
return 0;
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (CRASHDEBUG(1))
fprintf(fp, "get_remote_cr3: [%s]\n", recvbuf);
@@ -2652,7 +2652,7 @@ copy_to_local_namelist(struct remote_file *rfp)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "DEBUGGING_SYMBOLS %s", rfp->filename);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "NO_DEBUG")) {
sprintf(readbuf, "%s@%s", rfp->filename, pc->server);
pc->namelist = readbuf;
@@ -2765,7 +2765,7 @@ identical_namelist(char *file, struct remote_file *rfp)
sprintf(sendbuf, "LINUX_VERSION %s", rfp->filename);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -2808,7 +2808,7 @@ remote_file_checksum(struct remote_file *rfp)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "SUM %s", rfp->filename);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>")) {
error(INFO, "%s: does not exist on server %s\n",
rfp->filename, pc->server);
@@ -3078,7 +3078,7 @@ remote_find_booted_kernel(struct remote_file *rfp)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "FIND_BOOTED_KERNEL");
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
strtok(recvbuf, " "); /* FIND_BOOTED_KERNEL */
p1 = strtok(NULL, " "); /* filename */
if (STREQ(p1, "<FAIL>"))
@@ -3099,7 +3099,7 @@ remote_lkcd_dump_init(void)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "LKCD_DUMP_INIT %d %s", pc->rmfd,
pc->server_memsrc);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -3137,7 +3137,7 @@ remote_s390_dump_init(void)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "S390_DUMP_INIT %d %s", pc->rmfd,
pc->server_memsrc);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -3173,7 +3173,7 @@ remote_netdump_init(void)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "NETDUMP_INIT %d %s", pc->rmfd,
pc->server_memsrc);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>"))
return FALSE;
@@ -3205,6 +3205,8 @@ remote_page_size(void)
if (REMOTE_ACTIVE())
sprintf(sendbuf, "PAGESIZE LIVE");
+ else if (REMOTE_NIL())
+ sprintf(sendbuf, "PAGESIZE NIL");
else if (pc->flags & REM_NETDUMP)
sprintf(sendbuf, "PAGESIZE NETDUMP");
else if (pc->flags & REM_MCLXCD)
@@ -3218,7 +3220,7 @@ remote_page_size(void)
"cannot determine remote page size (unknown memory source)\n");
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "FAIL"))
error(FATAL, "cannot determine remote page size\n");
strtok(recvbuf, " "); /* PAGESIZE */
@@ -3472,7 +3474,7 @@ find_remote_module_objfile(struct load_module *lm, char *module,
char *retbuf)
sprintf(sendbuf, "FIND_MODULE %s %s",
kt->utsname.release, module);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
if (strstr(recvbuf, "<FAIL>")) {
fprintf(fp, "find_remote_module_objfile: [%s]\n",
recvbuf);
@@ -3533,7 +3535,7 @@ remote_free_memory(void)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "MEMORY FREE %s", type);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
p1 = strtok(recvbuf, " "); /* MEMORY */
p1 = strtok(NULL, " "); /* FREE */
p1 = strtok(NULL, " "); /* MCLXCD, LKCD etc. */
@@ -3569,7 +3571,7 @@ remote_memory_used(void)
BZERO(recvbuf, BUFSIZE);
sprintf(sendbuf, "MEMORY USED %s", type);
remote_tcp_write_string(pc->sockfd, sendbuf);
- remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, recvbuf, BUFSIZE-1, REMOTE_NIL());
p1 = strtok(recvbuf, " "); /* MEMORY */
p1 = strtok(NULL, " "); /* FREE */
p1 = strtok(NULL, " "); /* MCLXCD, LKCD, etc. */
@@ -3868,7 +3870,7 @@ remote_exit(void)
/*
* Read but ignore the return status -- we don't really care...
*/
- remote_tcp_read_string(pc->sockfd, buf, BUFSIZE-1, pc->flags2 &
REMOTE_NIL);
+ remote_tcp_read_string(pc->sockfd, buf, BUFSIZE-1, REMOTE_NIL());
}
#endif /* !DAEMON */
--
1.8.4
6:51 a.m.
On Tue, Nov 19, 2013 at 06:46:39AM -0500, Don Slutz wrote:
From: Don Slutz <dslutz(a)verizon.com>
Currently crash 4.0 and later using the code:
http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
There is some very limited documention on crashes remote ptotocol in:
http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
and some fixes in the attachment 0001-xen-crashd-Connect-crash-with-domain.patch
from the 1st link above.
How ever there are issues with the current code:
1) The remote protocol has a minor issue (which I have not been able to
happen) based on the fact TCP/IP is stream based protocol. This means
a RECV or a SEND may not do the fully requested size of data. In fact
the current code assumes that the amount of data that a SEND is called
with will all be read with a single RECV.
2) The most common mismatch between crash and older kernels is
phys_base. In the remote case, see if the remote server supports
vitrual memory access, and if so, see if phys_base can be retreived.
3) crash assumes that the remote system is active and can not return
currect IP and SP to do a better back trace.
4) enable a non-active mode of remote access.
This patch set attempts to fix these:
The fix for #1 I have called NIL mode (patch 1).
The fix for #2 uses get_remote_phys_base (patch 3).
The fix for #3 uses get_remote_regs (patch 5).
The fix for #4 uses special file /dev/xenmem (patch 7).
Where is this file created? In guest or in host. Is it done by udev?
Where is driver for this file?
It also REMOTE_NIL() to indicate "remote paused system".
Don Slutz (7):
Add NIL mode to remote.
remote_proc_version: NULL terminate passed buffer on error.
Add get_remote_phys_base.
Add remote_vtop.
bt: get remote live registers if possible.
Add get_remote_cr3
Add support for non-live remote.
defs.h | 8 +-
kernel.c | 25 +++-
memory.c | 25 +++-
remote.c | 462 +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
x86_64.c | 24 ++++
5 files changed, 465 insertions(+), 79 deletions(-)
I have done quick review. It looks quite nice. There are two nitpicks
for me. Please do not use camelCase (I have found one in patch 3 in
struct _remote_context) and do not add Emacs directives at the end
of files (this is Xen specific stuff).
Daniel
9:11 a.m.
On 11/19/13 07:51, Daniel Kiper wrote:
On Tue, Nov 19, 2013 at 06:46:39AM -0500, Don Slutz wrote:
> From: Don Slutz <dslutz(a)verizon.com>
>
> Currently crash 4.0 and later using the code:
>
> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
>
> There is some very limited documention on crashes remote ptotocol in:
>
> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
>
> and some fixes in the attachment 0001-xen-crashd-Connect-crash-with-domain.patch
> from the 1st link above.
>
> How ever there are issues with the current code:
>
> 1) The remote protocol has a minor issue (which I have not been able to
> happen) based on the fact TCP/IP is stream based protocol. This means
> a RECV or a SEND may not do the fully requested size of data. In fact
> the current code assumes that the amount of data that a SEND is called
> with will all be read with a single RECV.
>
> 2) The most common mismatch between crash and older kernels is
> phys_base. In the remote case, see if the remote server supports
> vitrual memory access, and if so, see if phys_base can be retreived.
>
> 3) crash assumes that the remote system is active and can not return
> currect IP and SP to do a better back trace.
>
> 4) enable a non-active mode of remote access.
>
> This patch set attempts to fix these:
>
> The fix for #1 I have called NIL mode (patch 1).
> The fix for #2 uses get_remote_phys_base (patch 3).
> The fix for #3 uses get_remote_regs (patch 5).
> The fix for #4 uses special file /dev/xenmem (patch 7).
Where is this file created? In guest or in host. Is it done by udev?
Where is driver for this file?
This is a fake file, it only exists in xen-crashd.
> It also REMOTE_NIL() to indicate "remote paused
system".
>
> Don Slutz (7):
> Add NIL mode to remote.
> remote_proc_version: NULL terminate passed buffer on error.
> Add get_remote_phys_base.
> Add remote_vtop.
> bt: get remote live registers if possible.
> Add get_remote_cr3
> Add support for non-live remote.
>
> defs.h | 8 +-
> kernel.c | 25 +++-
> memory.c | 25 +++-
> remote.c | 462 +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
> x86_64.c | 24 ++++
> 5 files changed, 465 insertions(+), 79 deletions(-)
I have done quick review. It looks quite nice. There are two nitpicks
for me. Please do not use camelCase (I have found one in patch 3 in
struct _remote_context) and do not add Emacs directives at the end
of files (this is Xen specific stuff).
I will try and remove all camelCase.
I can remove the emacs stuff, but I know it saves me time. This is because of:
* c-basic-offset: 8
* indent-tabs-mode: t
Where as Xen:
* c-basic-offset: 4
* indent-tabs-mode: nil
So my default is the Xen one not the crash one.
-Don Slutz
Daniel
10:08 a.m.
----- Original Message -----
From: Don Slutz <dslutz(a)verizon.com>
Currently crash 4.0 and later using the code:
http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
There is some very limited documention on crashes remote ptotocol in:
http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
and some fixes in the attachment
0001-xen-crashd-Connect-crash-with-domain.patch
from the 1st link above.
How ever there are issues with the current code:
1) The remote protocol has a minor issue (which I have not been able to
happen) based on the fact TCP/IP is stream based protocol. This means
a RECV or a SEND may not do the fully requested size of data. In fact
the current code assumes that the amount of data that a SEND is called
with will all be read with a single RECV.
2) The most common mismatch between crash and older kernels is
phys_base. In the remote case, see if the remote server supports
vitrual memory access, and if so, see if phys_base can be retreived.
3) crash assumes that the remote system is active and can not return
currect IP and SP to do a better back trace.
4) enable a non-active mode of remote access.
This patch set attempts to fix these:
The fix for #1 I have called NIL mode (patch 1).
The fix for #2 uses get_remote_phys_base (patch 3).
The fix for #3 uses get_remote_regs (patch 5).
The fix for #4 uses special file /dev/xenmem (patch 7).
It also REMOTE_NIL() to indicate "remote paused system".
Don Slutz (7):
Add NIL mode to remote.
remote_proc_version: NULL terminate passed buffer on error.
Add get_remote_phys_base.
Add remote_vtop.
bt: get remote live registers if possible.
Add get_remote_cr3
Add support for non-live remote.
defs.h | 8 +-
kernel.c | 25 +++-
memory.c | 25 +++-
remote.c | 462
+++++++++++++++++++++++++++++++++++++++++++++++++++++----------
x86_64.c | 24 ++++
5 files changed, 465 insertions(+), 79 deletions(-)
--
1.8.4
Hi Don,
A few comments...
I don't really understand what entity the code is communicating with,
so for the most part, I'm primarily interested in making sure that your
patches in no way can affect the normal usage of the crash utility.
That being said, is the term "NIL mode" some kind of Xen-ism? It wasn't
until I saw the display_sys_stats() function that I even understood that
REMOTE_NIL mean "(remote paused system)". Anyway, if you made it something
like "REMOTE_PAUSED" or something like that, the code might make some sense
to the untrained eye.
And it's not entirely clear to me what REMOTE_ACTIVE() means now? It used
to imply communicating with a remote crash daeamon that accessed the live
system that the daemon was running on. Has that meaning been changed to mean
that crash is communicating with a daemon that's accessing a "live" Xen
guest
kernel?
I'm confused about these two changes, where you re-patch your own patch:
- else
+ else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
+ if (get_remote_regs(bt, &eip, &esp))
+ machdep->get_stack_frame(bt, &eip, &esp);
+ } else
followed by:
- else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
+ else if (REMOTE_NIL() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
if (get_remote_regs(bt, &eip, &esp))
machdep->get_stack_frame(bt, &eip, &esp);
} else
If I'm not mistaken, it would seem to be impossible for "(bt->task ==
tt->this_task)"
to ever be true in the REMOTE_NIL() case? As I recall tt->this_task used to be the
PID of the remote daemon itself, which would be accessing the memory of its own kernel.
So how could the pid of the remote daemon ever be the target of a back trace of a
paused guest? And so I don't understand why you first made an if clause for
REMOTE_ACTIVE() to begin with -- whatever that means now -- and then changed it to
REMOTE_NIL()?
In any case, since the ultimate change is only interested in REMOTE_NIL(),
can you make it all less of an eyesore by encapulating its if case
something like:
else if (SADUMP_DUMPFILE())
get_sadump_regs(bt, &eip, &esp);
+ else if (REMOTE_NIL()) {
+ if (!((bt->task == tt->this_task) || is_task_active(bt->task))
||
+ !get_remote_regs(bt, &eip, &esp))
+ machdep->get_stack_frame(bt, &eip, &esp);
} else
machdep->get_stack_frame(bt, &eip, &esp);
Note above, for consistency's sake with the rest of the crash utility,
can you make get_remote_regs() return non-zero when it successfully gets
the registers, and zero if it fails?
And lastly, can you fix these?
$ make warn
...
remote.c: In function ‘validate_phys_base’:
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 3 has type ‘physaddr_t’ [-Wformat]
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 4 has type ‘physaddr_t’ [-Wformat]
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 5 has type ‘physaddr_t’ [-Wformat]
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 6 has type ‘physaddr_t’ [-Wformat]
remote.c: In function ‘remote_vtop’:
remote.c:2361:2: warning: format ‘%llx’ expects argument of type ‘long long unsigned
int’, but argument 4 has type ‘physaddr_t’ [-Wformat]
remote.c:2367:4: warning: format ‘%llx’ expects argument of type ‘long long unsigned
int’, but argument 4 has type ‘physaddr_t’ [-Wformat]
remote.c:2352:18: warning: variable ‘p3’ set but not used [-Wunused-but-set-variable]
remote.c:2352:13: warning: variable ‘p2’ set but not used [-Wunused-but-set-variable]
remote.c:2352:8: warning: variable ‘p1’ set but not used [-Wunused-but-set-variable]
remote.c: In function ‘get_remote_regs’:
remote.c:2392:33: warning: unused variable ‘p6’ [-Wunused-variable]
remote.c:2392:28: warning: variable ‘p5’ set but not used [-Wunused-but-set-variable]
remote.c:2392:18: warning: variable ‘p3’ set but not used [-Wunused-but-set-variable]
remote.c:2392:8: warning: variable ‘p1’ set but not used [-Wunused-but-set-variable]
remote.c: In function ‘get_remote_cr3’:
remote.c:2449:13: warning: variable ‘p2’ set but not used [-Wunused-but-set-variable]
remote.c:2449:8: warning: variable ‘p1’ set but not used [-Wunused-but-set-variable]
...
And I'm not interested in appending these to the 4 files that you changed:
+/*
+ * Specify Emacs local variables so the formating
+ * of the code stays the same.
+ *
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 8
+ * indent-tabs-mode: t
+ * End:
+ */
Thanks,
Dave
1:03 p.m.
On 11/19/13 11:08, Dave Anderson wrote:
----- Original Message -----
> From: Don Slutz <dslutz(a)verizon.com>
>
> Currently crash 4.0 and later using the code:
>
> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
>
> There is some very limited documention on crashes remote ptotocol in:
>
> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
>
> and some fixes in the attachment
> 0001-xen-crashd-Connect-crash-with-domain.patch
> from the 1st link above.
>
> How ever there are issues with the current code:
>
> 1) The remote protocol has a minor issue (which I have not been able to
> happen) based on the fact TCP/IP is stream based protocol. This means
> a RECV or a SEND may not do the fully requested size of data. In fact
> the current code assumes that the amount of data that a SEND is called
> with will all be read with a single RECV.
>
> 2) The most common mismatch between crash and older kernels is
> phys_base. In the remote case, see if the remote server supports
> vitrual memory access, and if so, see if phys_base can be retreived.
>
> 3) crash assumes that the remote system is active and can not return
> currect IP and SP to do a better back trace.
>
> 4) enable a non-active mode of remote access.
>
> This patch set attempts to fix these:
>
> The fix for #1 I have called NIL mode (patch 1).
> The fix for #2 uses get_remote_phys_base (patch 3).
> The fix for #3 uses get_remote_regs (patch 5).
> The fix for #4 uses special file /dev/xenmem (patch 7).
> It also REMOTE_NIL() to indicate "remote paused system".
>
> Don Slutz (7):
> Add NIL mode to remote.
> remote_proc_version: NULL terminate passed buffer on error.
> Add get_remote_phys_base.
> Add remote_vtop.
> bt: get remote live registers if possible.
> Add get_remote_cr3
> Add support for non-live remote.
>
> defs.h | 8 +-
> kernel.c | 25 +++-
> memory.c | 25 +++-
> remote.c | 462
> +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
> x86_64.c | 24 ++++
> 5 files changed, 465 insertions(+), 79 deletions(-)
>
> --
> 1.8.4
Hi Don,
A few comments...
I don't really understand what entity the code is communicating with,
so for the most part, I'm primarily interested in making sure that your
patches in no way can affect the normal usage of the crash utility.
That being said, is the term "NIL mode" some kind of Xen-ism?
Nope, it is
term I came up with to describe the protocol change.
It wasn't
until I saw the display_sys_stats() function that I even understood that
REMOTE_NIL mean "(remote paused system)". Anyway, if you made it something
like "REMOTE_PAUSED" or something like that, the code might make some sense
to the untrained eye.
Will do. Looking back at it, I ended up overloading the
term. :(
And it's not entirely clear to me what REMOTE_ACTIVE() means now? It used
to imply communicating with a remote crash daeamon that accessed the live
system that the daemon was running on. Has that meaning been changed to mean
that crash is communicating with a daemon that's accessing a "live" Xen
guest
kernel?
The meaning is not as clear. Since the existing versions of crash already
have REMOTE_ACTIVE() and I wanted the Xen utility to be usable to them, in both cases the
"live" Xen guest kernel is being accessed while it is paused. I added
REMOTE_PAUSED() to have a few changes to crash to correctly understand that the guest is
paused. As part of this I add the fake file "/dev/xenmem".
I'm confused about these two changes, where you re-patch your own patch:
- else
+ else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
+ if (get_remote_regs(bt, &eip, &esp))
+ machdep->get_stack_frame(bt, &eip, &esp);
+ } else
followed by:
- else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
+ else if (REMOTE_NIL() && ((bt->task == tt->this_task) ||
is_task_active(bt->task))) {
if (get_remote_regs(bt, &eip, &esp))
machdep->get_stack_frame(bt, &eip, &esp);
} else
Clearly I need to adjust the patch order.
If I'm not mistaken, it would seem to be impossible for
"(bt->task == tt->this_task)"
to ever be true in the REMOTE_NIL() case? As I recall tt->this_task used to be the
PID of the remote daemon itself, which would be accessing the memory of its own kernel.
So how could the pid of the remote daemon ever be the target of a back trace of a
paused guest? And so I don't understand why you first made an if clause for
REMOTE_ACTIVE() to begin with -- whatever that means now -- and then changed it to
REMOTE_NIL()?
This came about from some old changes from 2012 to a crash version
4.0 for other work. I had not thought of /dev/xenmem at the time, and just copied the
condition from above. I see no reason not to drop "bt->task ==
tt->this_task". Will check that the right actions are still done.
In any case, since the ultimate change is only interested in
REMOTE_NIL(),
can you make it all less of an eyesore by encapulating its if case
something like:
else if (SADUMP_DUMPFILE())
get_sadump_regs(bt, &eip, &esp);
+ else if (REMOTE_NIL()) {
+ if (!((bt->task == tt->this_task) || is_task_active(bt->task))
||
+ !get_remote_regs(bt, &eip, &esp))
+ machdep->get_stack_frame(bt, &eip, &esp);
} else
machdep->get_stack_frame(bt, &eip, &esp);
Note above, for consistency's sake with the rest of the crash utility,
can you make get_remote_regs() return non-zero when it successfully gets
the registers, and zero if it fails?
Sure.
And lastly, can you fix these?
$ make warn
...
remote.c: In function ‘validate_phys_base’:
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 3 has type ‘physaddr_t’ [-Wformat]
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 4 has type ‘physaddr_t’ [-Wformat]
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 5 has type ‘physaddr_t’ [-Wformat]
remote.c:2302:25: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 6 has type ‘physaddr_t’ [-Wformat]
remote.c: In function ‘remote_vtop’:
remote.c:2361:2: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 4 has type ‘physaddr_t’ [-Wformat]
remote.c:2367:4: warning: format ‘%llx’ expects argument of type ‘long long
unsigned int’, but argument 4 has type ‘physaddr_t’ [-Wformat]
remote.c:2352:18: warning: variable ‘p3’ set but not used
[-Wunused-but-set-variable]
remote.c:2352:13: warning: variable ‘p2’ set but not used
[-Wunused-but-set-variable]
remote.c:2352:8: warning: variable ‘p1’ set but not used [-Wunused-but-set-variable]
remote.c: In function ‘get_remote_regs’:
remote.c:2392:33: warning: unused variable ‘p6’ [-Wunused-variable]
remote.c:2392:28: warning: variable ‘p5’ set but not used
[-Wunused-but-set-variable]
remote.c:2392:18: warning: variable ‘p3’ set but not used
[-Wunused-but-set-variable]
remote.c:2392:8: warning: variable ‘p1’ set but not used [-Wunused-but-set-variable]
remote.c: In function ‘get_remote_cr3’:
remote.c:2449:13: warning: variable ‘p2’ set but not used
[-Wunused-but-set-variable]
remote.c:2449:8: warning: variable ‘p1’ set but not used [-Wunused-but-set-variable]
...
Yes.
And I'm not interested in appending these to the 4 files that you
changed:
+/*
+ * Specify Emacs local variables so the formating
+ * of the code stays the same.
+ *
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 8
+ * indent-tabs-mode: t
+ * End:
+ */
Thanks,
Dave
Ok, that is 2 votes to drop this, so I will.
-Don Slutz
1:28 p.m.
Hi Don,
I still need some help understanding this "resurrection" of the remote
access facility, i.e., the new meanings of the REMOTE_xxx flags/macros.
The currently-existing comment for the is_remote_daemon() function
in remote.c describes the (obsolete) usage of the facility:
/*
* Parse, verify and establish a connection with the network daemon
* specified on the crash command line.
*
* The format is: [remote-hostname]:port[,remote-namelist][,remote-dumpfile]
*
* where everything but the port number is optional, and the remote-namelist
* and remote-dumpfile can be reversed.
*
* 1. The default remote host is the local host.
* 2. The default dumpfile is /dev/mem.
* 3. If no remote-namelist and remote-dumpfile are given, the daemon
* is queried for a kernel that matches the remote /proc/version.
* If no local kernel namelist is entered, the remote version will
* be copied locally when fd_init() is called.
* 4. If a remote-dumpfile is given with no remote namelist, it is presumed
* that the kernel namelist will be entered locally.
*/
Back then, the remote "crashd" daemon could be contacted for:
(1) live access to the remote machine's kernel using its /dev/mem, or
(2) accessing a vmlinux/vmcore pair that existed on the remote machine.
Can you explain this new usage? It sounds like there is a remote daemon
running on a remote Dom0 kernel, and that your new functionality can only
be used to:
(1) access a running domU guest hosted by that remote system, or
(2) access a paused domU guest hosted by that remote system.
And that there is also a device that the the remote daemon uses
to access the memory of the guest, be it paused or live.
Am I in the ballpark?
Dave
----- Original Message -----
On 11/19/13 11:08, Dave Anderson wrote:
>
> ----- Original Message -----
>> From: Don Slutz <dslutz(a)verizon.com>
>>
>> Currently crash 4.0 and later using the code:
>>
>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
>>
>> There is some very limited documention on crashes remote ptotocol in:
>>
>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
>>
>> and some fixes in the attachment
>> 0001-xen-crashd-Connect-crash-with-domain.patch
>> from the 1st link above.
>>
>> How ever there are issues with the current code:
>>
>> 1) The remote protocol has a minor issue (which I have not been able to
>> happen) based on the fact TCP/IP is stream based protocol. This means
>> a RECV or a SEND may not do the fully requested size of data. In fact
>> the current code assumes that the amount of data that a SEND is called
>> with will all be read with a single RECV.
>>
>> 2) The most common mismatch between crash and older kernels is
>> phys_base. In the remote case, see if the remote server supports
>> vitrual memory access, and if so, see if phys_base can be retreived.
>>
>> 3) crash assumes that the remote system is active and can not return
>> currect IP and SP to do a better back trace.
>>
>> 4) enable a non-active mode of remote access.
>>
>> This patch set attempts to fix these:
>>
>> The fix for #1 I have called NIL mode (patch 1).
>> The fix for #2 uses get_remote_phys_base (patch 3).
>> The fix for #3 uses get_remote_regs (patch 5).
>> The fix for #4 uses special file /dev/xenmem (patch 7).
>> It also REMOTE_NIL() to indicate "remote paused system".
>>
>> Don Slutz (7):
>> Add NIL mode to remote.
>> remote_proc_version: NULL terminate passed buffer on error.
>> Add get_remote_phys_base.
>> Add remote_vtop.
>> bt: get remote live registers if possible.
>> Add get_remote_cr3
>> Add support for non-live remote.
>>
>> defs.h | 8 +-
>> kernel.c | 25 +++-
>> memory.c | 25 +++-
>> remote.c | 462
>> +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
>> x86_64.c | 24 ++++
>> 5 files changed, 465 insertions(+), 79 deletions(-)
>>
>> --
>> 1.8.4
>
> Hi Don,
>
> A few comments...
>
> I don't really understand what entity the code is communicating with,
> so for the most part, I'm primarily interested in making sure that your
> patches in no way can affect the normal usage of the crash utility.
>
> That being said, is the term "NIL mode" some kind of Xen-ism?
Nope, it is term I came up with to describe the protocol change.
> It wasn't
> until I saw the display_sys_stats() function that I even understood that
> REMOTE_NIL mean "(remote paused system)". Anyway, if you made it
something
> like "REMOTE_PAUSED" or something like that, the code might make some
sense
> to the untrained eye.
Will do. Looking back at it, I ended up overloading the term. :(
>
> And it's not entirely clear to me what REMOTE_ACTIVE() means now? It used
> to imply communicating with a remote crash daeamon that accessed the live
> system that the daemon was running on. Has that meaning been changed to
> mean
> that crash is communicating with a daemon that's accessing a "live"
Xen
> guest
> kernel?
The meaning is not as clear. Since the existing versions of crash already
have REMOTE_ACTIVE() and I wanted the Xen utility to be usable to them, in
both cases the "live" Xen guest kernel is being accessed while it is paused.
I added REMOTE_PAUSED() to have a few changes to crash to correctly
understand that the guest is paused. As part of this I add the fake file
"/dev/xenmem".
>
> I'm confused about these two changes, where you re-patch your own patch:
>
> - else
> + else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
> is_task_active(bt->task))) {
> + if (get_remote_regs(bt, &eip, &esp))
> + machdep->get_stack_frame(bt, &eip, &esp);
> + } else
>
> followed by:
>
> - else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task) ||
> is_task_active(bt->task))) {
> + else if (REMOTE_NIL() && ((bt->task == tt->this_task) ||
> is_task_active(bt->task))) {
> if (get_remote_regs(bt, &eip, &esp))
> machdep->get_stack_frame(bt, &eip, &esp);
> } else
Clearly I need to adjust the patch order.
> If I'm not mistaken, it would seem to be impossible for "(bt->task ==
> tt->this_task)"
> to ever be true in the REMOTE_NIL() case? As I recall tt->this_task used
> to be the
> PID of the remote daemon itself, which would be accessing the memory of its
> own kernel.
> So how could the pid of the remote daemon ever be the target of a back
> trace of a
> paused guest? And so I don't understand why you first made an if clause
> for
> REMOTE_ACTIVE() to begin with -- whatever that means now -- and then
> changed it to
> REMOTE_NIL()?
This came about from some old changes from 2012 to a crash version 4.0 for
other work. I had not thought of /dev/xenmem at the time, and just copied
the condition from above. I see no reason not to drop "bt->task ==
tt->this_task". Will check that the right actions are still done.
> In any case, since the ultimate change is only interested in REMOTE_NIL(),
> can you make it all less of an eyesore by encapulating its if case
> something like:
>
> else if (SADUMP_DUMPFILE())
> get_sadump_regs(bt, &eip, &esp);
> + else if (REMOTE_NIL()) {
> + if (!((bt->task == tt->this_task) ||
> is_task_active(bt->task)) ||
> + !get_remote_regs(bt, &eip, &esp))
> + machdep->get_stack_frame(bt, &eip, &esp);
> } else
> machdep->get_stack_frame(bt, &eip, &esp);
>
> Note above, for consistency's sake with the rest of the crash utility,
> can you make get_remote_regs() return non-zero when it successfully gets
> the registers, and zero if it fails?
Sure.
> And lastly, can you fix these?
>
> $ make warn
> ...
> remote.c: In function ‘validate_phys_base’:
> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
> ‘long long unsigned int’, but argument 3 has type ‘physaddr_t’
> [-Wformat]
> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
> [-Wformat]
> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
> ‘long long unsigned int’, but argument 5 has type ‘physaddr_t’
> [-Wformat]
> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
> ‘long long unsigned int’, but argument 6 has type ‘physaddr_t’
> [-Wformat]
> remote.c: In function ‘remote_vtop’:
> remote.c:2361:2: warning: format ‘%llx’ expects argument of type
> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
> [-Wformat]
> remote.c:2367:4: warning: format ‘%llx’ expects argument of type
> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
> [-Wformat]
> remote.c:2352:18: warning: variable ‘p3’ set but not used
> [-Wunused-but-set-variable]
> remote.c:2352:13: warning: variable ‘p2’ set but not used
> [-Wunused-but-set-variable]
> remote.c:2352:8: warning: variable ‘p1’ set but not used
> [-Wunused-but-set-variable]
> remote.c: In function ‘get_remote_regs’:
> remote.c:2392:33: warning: unused variable ‘p6’ [-Wunused-variable]
> remote.c:2392:28: warning: variable ‘p5’ set but not used
> [-Wunused-but-set-variable]
> remote.c:2392:18: warning: variable ‘p3’ set but not used
> [-Wunused-but-set-variable]
> remote.c:2392:8: warning: variable ‘p1’ set but not used
> [-Wunused-but-set-variable]
> remote.c: In function ‘get_remote_cr3’:
> remote.c:2449:13: warning: variable ‘p2’ set but not used
> [-Wunused-but-set-variable]
> remote.c:2449:8: warning: variable ‘p1’ set but not used
> [-Wunused-but-set-variable]
> ...
Yes.
> And I'm not interested in appending these to the 4 files that you changed:
>
> +/*
> + * Specify Emacs local variables so the formating
> + * of the code stays the same.
> + *
> + * Local variables:
> + * mode: C
> + * c-set-style: "BSD"
> + * c-basic-offset: 8
> + * indent-tabs-mode: t
> + * End:
> + */
>
> Thanks,
> Dave
Ok, that is 2 votes to drop this, so I will.
-Don Slutz
6:20 a.m.
On 11/19/13 14:28, Dave Anderson wrote:
Hi Don,
I still need some help understanding this "resurrection" of the remote
access facility, i.e., the new meanings of the REMOTE_xxx flags/macros.
The currently-existing comment for the is_remote_daemon() function
in remote.c describes the (obsolete) usage of the facility:
/*
* Parse, verify and establish a connection with the network daemon
* specified on the crash command line.
*
* The format is: [remote-hostname]:port[,remote-namelist][,remote-dumpfile]
*
* where everything but the port number is optional, and the remote-namelist
* and remote-dumpfile can be reversed.
*
* 1. The default remote host is the local host.
* 2. The default dumpfile is /dev/mem.
* 3. If no remote-namelist and remote-dumpfile are given, the daemon
* is queried for a kernel that matches the remote /proc/version.
* If no local kernel namelist is entered, the remote version will
* be copied locally when fd_init() is called.
* 4. If a remote-dumpfile is given with no remote namelist, it is presumed
* that the kernel namelist will be entered locally.
*/
Back then, the remote "crashd" daemon could be contacted for:
(1) live access to the remote machine's kernel using its /dev/mem, or
(2) accessing a vmlinux/vmcore pair that existed on the remote machine.
Can you explain this new usage? It sounds like there is a remote daemon
running on a remote Dom0 kernel, and that your new functionality can only
be used to:
(1) access a running domU guest hosted by that remote system, or
(2) access a paused domU guest hosted by that remote system.
And that there is also a device that the the remote daemon uses
to access the memory of the guest, be it paused or live.
Am I in the ballpark?
Yes.
The currently proposed code for Dom0, does not support (1) mostly
because it seems much better to me to be able to poke around and look at
a fixed state of the domU.
There is a (3) in xen: guest has shutdown. A shutdown guest has
subtypes like "requested shutdown", "requested reboot", "detected
crash", etc. Xen will set a guest in shutdown state when a double fault
or triple fault happens. A paused guest is the same as an unpaused
guest in shutdown state. Using a much older Xen, I known that that
version treated a double fault is the same as a reboot request. The bug
I was tracking down turned out to be a extra swapgs which will cause
many linux kernel's to double fault on a page fault.
I was also looking to be able to use old versions of crash so as to
make it easier to use and/or deploy. Since the 7.0.3 or older version of
crash only support "live access to the remote machine's kernel using its
/dev/mem "; I picked to default to be a paused system. Since crash
thinks the remote system is live, it keeps fetching state which is not
changing.
The "device that the the remote daemon uses ", is more a defined
ABI which has 2 layers libxl and libxc (what I all xc_* routines). In
this case I am using libxc calls. None of these require the guest to be
paused.
So it would not be hard to change the Xen code to support new
option (1) if there is need for such a mode. I could see adding some
sort of pause control as an extension to crash.
I have recently learn of another Xen tool: gdbsx. It is designed
to connect gdb to a running guest and pause and resume the guest on
demand. While I have not done very much with it, it looks to be
powerful. Since it is just gdb, it is missing crash's kernel support.
Dave
While working on this email, I found out that it looks better to me to
include REMOTE_PAUSED() in REMOTE_MEMSRC(). I.E. either add the following:
From b4093734b2db8666f155032ffd9637dcfda0fc0b Mon Sep 17 00:00:00 2001
From: Don Slutz <dslutz(a)verizon.com>
Date: Wed, 20 Nov 2013 07:17:51 -0500
Subject: [PATCH] Add REMOTE_PAUSED() to REMOTE_MEMSRC()
And remove the redunent 'REMOTE_MEMSRC() || REMOTE_PAUSED()'.
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 2 +-
memory.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/defs.h b/defs.h
index dae9662..2ec5540 100755
--- a/defs.h
+++ b/defs.h
@@ -254,7 +254,7 @@ struct number_option {
#define REMOTE_ACTIVE() (pc->flags & REM_LIVE_SYSTEM)
#define REMOTE_DUMPFILE() \
(pc->flags & (REM_NETDUMP|REM_MCLXCD|REM_LKCD|REM_S390D))
-#define REMOTE_MEMSRC() (REMOTE_ACTIVE() || REMOTE_DUMPFILE())
+#define REMOTE_MEMSRC() (REMOTE_ACTIVE() || REMOTE_PAUSED() ||
REMOTE_DUMPFILE())
#define LKCD_DUMPFILE() (pc->flags & (LKCD|REM_LKCD))
#define NETDUMP_DUMPFILE() (pc->flags & (NETDUMP|REM_NETDUMP))
#define DISKDUMP_DUMPFILE() (pc->flags & DISKDUMP)
diff --git a/memory.c b/memory.c
index cf8da2c..d17ba3c 100755
--- a/memory.c
+++ b/memory.c
@@ -14903,7 +14903,7 @@ memory_page_size(void)
if (machdep->pagesize)
return machdep->pagesize;
- if (REMOTE_MEMSRC() || REMOTE_PAUSED())
+ if (REMOTE_MEMSRC())
return remote_page_size();
switch (pc->flags & MEMORY_SOURCES)
--
1.7.11.7
or adjust patch v2 7/7 to also do this.
-Don Slutz
----- Original Message -----
> On 11/19/13 11:08, Dave Anderson wrote:
>> ----- Original Message -----
>>> From: Don Slutz <dslutz(a)verizon.com>
>>>
>>> Currently crash 4.0 and later using the code:
>>>
>>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
>>>
>>> There is some very limited documention on crashes remote ptotocol in:
>>>
>>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
>>>
>>> and some fixes in the attachment
>>> 0001-xen-crashd-Connect-crash-with-domain.patch
>>> from the 1st link above.
>>>
>>> How ever there are issues with the current code:
>>>
>>> 1) The remote protocol has a minor issue (which I have not been able to
>>> happen) based on the fact TCP/IP is stream based protocol. This means
>>> a RECV or a SEND may not do the fully requested size of data. In fact
>>> the current code assumes that the amount of data that a SEND is called
>>> with will all be read with a single RECV.
>>>
>>> 2) The most common mismatch between crash and older kernels is
>>> phys_base. In the remote case, see if the remote server supports
>>> vitrual memory access, and if so, see if phys_base can be retreived.
>>>
>>> 3) crash assumes that the remote system is active and can not return
>>> currect IP and SP to do a better back trace.
>>>
>>> 4) enable a non-active mode of remote access.
>>>
>>> This patch set attempts to fix these:
>>>
>>> The fix for #1 I have called NIL mode (patch 1).
>>> The fix for #2 uses get_remote_phys_base (patch 3).
>>> The fix for #3 uses get_remote_regs (patch 5).
>>> The fix for #4 uses special file /dev/xenmem (patch 7).
>>> It also REMOTE_NIL() to indicate "remote paused system".
>>>
>>> Don Slutz (7):
>>> Add NIL mode to remote.
>>> remote_proc_version: NULL terminate passed buffer on error.
>>> Add get_remote_phys_base.
>>> Add remote_vtop.
>>> bt: get remote live registers if possible.
>>> Add get_remote_cr3
>>> Add support for non-live remote.
>>>
>>> defs.h | 8 +-
>>> kernel.c | 25 +++-
>>> memory.c | 25 +++-
>>> remote.c | 462
>>> +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
>>> x86_64.c | 24 ++++
>>> 5 files changed, 465 insertions(+), 79 deletions(-)
>>>
>>> --
>>> 1.8.4
>>
>> Hi Don,
>>
>> A few comments...
>>
>> I don't really understand what entity the code is communicating with,
>> so for the most part, I'm primarily interested in making sure that your
>> patches in no way can affect the normal usage of the crash utility.
>>
>> That being said, is the term "NIL mode" some kind of Xen-ism?
> Nope, it is term I came up with to describe the protocol change.
>> It wasn't
>> until I saw the display_sys_stats() function that I even understood that
>> REMOTE_NIL mean "(remote paused system)". Anyway, if you made it
something
>> like "REMOTE_PAUSED" or something like that, the code might make some
sense
>> to the untrained eye.
> Will do. Looking back at it, I ended up overloading the term. :(
>> And it's not entirely clear to me what REMOTE_ACTIVE() means now? It used
>> to imply communicating with a remote crash daeamon that accessed the live
>> system that the daemon was running on. Has that meaning been changed to
>> mean
>> that crash is communicating with a daemon that's accessing a "live"
Xen
>> guest
>> kernel?
> The meaning is not as clear. Since the existing versions of crash already
> have REMOTE_ACTIVE() and I wanted the Xen utility to be usable to them, in
> both cases the "live" Xen guest kernel is being accessed while it is
paused.
> I added REMOTE_PAUSED() to have a few changes to crash to correctly
> understand that the guest is paused. As part of this I add the fake file
> "/dev/xenmem".
>> I'm confused about these two changes, where you re-patch your own patch:
>>
>> - else
>> + else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task)
||
>> is_task_active(bt->task))) {
>> + if (get_remote_regs(bt, &eip, &esp))
>> + machdep->get_stack_frame(bt, &eip, &esp);
>> + } else
>>
>> followed by:
>>
>> - else if (REMOTE_ACTIVE() && ((bt->task == tt->this_task)
||
>> is_task_active(bt->task))) {
>> + else if (REMOTE_NIL() && ((bt->task == tt->this_task) ||
>> is_task_active(bt->task))) {
>> if (get_remote_regs(bt, &eip, &esp))
>> machdep->get_stack_frame(bt, &eip, &esp);
>> } else
> Clearly I need to adjust the patch order.
>> If I'm not mistaken, it would seem to be impossible for "(bt->task
==
>> tt->this_task)"
>> to ever be true in the REMOTE_NIL() case? As I recall tt->this_task used
>> to be the
>> PID of the remote daemon itself, which would be accessing the memory of its
>> own kernel.
>> So how could the pid of the remote daemon ever be the target of a back
>> trace of a
>> paused guest? And so I don't understand why you first made an if clause
>> for
>> REMOTE_ACTIVE() to begin with -- whatever that means now -- and then
>> changed it to
>> REMOTE_NIL()?
> This came about from some old changes from 2012 to a crash version 4.0 for
> other work. I had not thought of /dev/xenmem at the time, and just copied
> the condition from above. I see no reason not to drop "bt->task ==
> tt->this_task". Will check that the right actions are still done.
>> In any case, since the ultimate change is only interested in REMOTE_NIL(),
>> can you make it all less of an eyesore by encapulating its if case
>> something like:
>>
>> else if (SADUMP_DUMPFILE())
>> get_sadump_regs(bt, &eip, &esp);
>> + else if (REMOTE_NIL()) {
>> + if (!((bt->task == tt->this_task) ||
>> is_task_active(bt->task)) ||
>> + !get_remote_regs(bt, &eip, &esp))
>> + machdep->get_stack_frame(bt, &eip, &esp);
>> } else
>> machdep->get_stack_frame(bt, &eip, &esp);
>>
>> Note above, for consistency's sake with the rest of the crash utility,
>> can you make get_remote_regs() return non-zero when it successfully gets
>> the registers, and zero if it fails?
> Sure.
>> And lastly, can you fix these?
>>
>> $ make warn
>> ...
>> remote.c: In function ‘validate_phys_base’:
>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>> ‘long long unsigned int’, but argument 3 has type ‘physaddr_t’
>> [-Wformat]
>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>> [-Wformat]
>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>> ‘long long unsigned int’, but argument 5 has type ‘physaddr_t’
>> [-Wformat]
>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>> ‘long long unsigned int’, but argument 6 has type ‘physaddr_t’
>> [-Wformat]
>> remote.c: In function ‘remote_vtop’:
>> remote.c:2361:2: warning: format ‘%llx’ expects argument of type
>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>> [-Wformat]
>> remote.c:2367:4: warning: format ‘%llx’ expects argument of type
>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>> [-Wformat]
>> remote.c:2352:18: warning: variable ‘p3’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c:2352:13: warning: variable ‘p2’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c:2352:8: warning: variable ‘p1’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c: In function ‘get_remote_regs’:
>> remote.c:2392:33: warning: unused variable ‘p6’ [-Wunused-variable]
>> remote.c:2392:28: warning: variable ‘p5’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c:2392:18: warning: variable ‘p3’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c:2392:8: warning: variable ‘p1’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c: In function ‘get_remote_cr3’:
>> remote.c:2449:13: warning: variable ‘p2’ set but not used
>> [-Wunused-but-set-variable]
>> remote.c:2449:8: warning: variable ‘p1’ set but not used
>> [-Wunused-but-set-variable]
>> ...
> Yes.
>
>> And I'm not interested in appending these to the 4 files that you changed:
>>
>> +/*
>> + * Specify Emacs local variables so the formating
>> + * of the code stays the same.
>> + *
>> + * Local variables:
>> + * mode: C
>> + * c-set-style: "BSD"
>> + * c-basic-offset: 8
>> + * indent-tabs-mode: t
>> + * End:
>> + */
>>
>> Thanks,
>> Dave
> Ok, that is 2 votes to drop this, so I will.
> -Don Slutz
>
1:58 p.m.
Hi Don,
This v2 version, plus the REMOTE_MEMSRC() patch below, is queued for
crash-7.0.4. How would you like the changelog message to read?
Tentatively I've got this:
- Resurrection of the remote analysis capability for use with the
"xen-crashd" daemon running on a Xen Dom0 host, which communicates
with a paused or shutdown DomU guest kernel. The daemon can be
accessed like so:
$ crash localhost:5991,/dev/xenmem vmlinux
(dslutz(a)verizon.com)
Thanks,
Dave
----- Original Message -----
On 11/19/13 14:28, Dave Anderson wrote:
>
> Hi Don,
>
> I still need some help understanding this "resurrection" of the remote
> access facility, i.e., the new meanings of the REMOTE_xxx flags/macros.
>
> The currently-existing comment for the is_remote_daemon() function
> in remote.c describes the (obsolete) usage of the facility:
>
> /*
> * Parse, verify and establish a connection with the network daemon
> * specified on the crash command line.
> *
> * The format is:
> [remote-hostname]:port[,remote-namelist][,remote-dumpfile]
> *
> * where everything but the port number is optional, and the
> remote-namelist
> * and remote-dumpfile can be reversed.
> *
> * 1. The default remote host is the local host.
> * 2. The default dumpfile is /dev/mem.
> * 3. If no remote-namelist and remote-dumpfile are given, the daemon
> * is queried for a kernel that matches the remote /proc/version.
> * If no local kernel namelist is entered, the remote version will
> * be copied locally when fd_init() is called.
> * 4. If a remote-dumpfile is given with no remote namelist, it is
> presumed
> * that the kernel namelist will be entered locally.
> */
>
> Back then, the remote "crashd" daemon could be contacted for:
>
> (1) live access to the remote machine's kernel using its /dev/mem, or
> (2) accessing a vmlinux/vmcore pair that existed on the remote machine.
>
> Can you explain this new usage? It sounds like there is a remote daemon
> running on a remote Dom0 kernel, and that your new functionality can only
> be used to:
>
> (1) access a running domU guest hosted by that remote system, or
> (2) access a paused domU guest hosted by that remote system.
>
> And that there is also a device that the the remote daemon uses
> to access the memory of the guest, be it paused or live.
>
> Am I in the ballpark?
Yes.
The currently proposed code for Dom0, does not support (1) mostly
because it seems much better to me to be able to poke around and look at
a fixed state of the domU.
There is a (3) in xen: guest has shutdown. A shutdown guest has
subtypes like "requested shutdown", "requested reboot",
"detected
crash", etc. Xen will set a guest in shutdown state when a double fault
or triple fault happens. A paused guest is the same as an unpaused
guest in shutdown state. Using a much older Xen, I known that that
version treated a double fault is the same as a reboot request. The bug
I was tracking down turned out to be a extra swapgs which will cause
many linux kernel's to double fault on a page fault.
I was also looking to be able to use old versions of crash so as to
make it easier to use and/or deploy. Since the 7.0.3 or older version of
crash only support "live access to the remote machine's kernel using its
/dev/mem "; I picked to default to be a paused system. Since crash
thinks the remote system is live, it keeps fetching state which is not
changing.
The "device that the the remote daemon uses ", is more a defined
ABI which has 2 layers libxl and libxc (what I all xc_* routines). In
this case I am using libxc calls. None of these require the guest to be
paused.
So it would not be hard to change the Xen code to support new
option (1) if there is need for such a mode. I could see adding some
sort of pause control as an extension to crash.
I have recently learn of another Xen tool: gdbsx. It is designed
to connect gdb to a running guest and pause and resume the guest on
demand. While I have not done very much with it, it looks to be
powerful. Since it is just gdb, it is missing crash's kernel support.
>
> Dave
While working on this email, I found out that it looks better to me to
include REMOTE_PAUSED() in REMOTE_MEMSRC(). I.E. either add the following:
From b4093734b2db8666f155032ffd9637dcfda0fc0b Mon Sep 17 00:00:00 2001
From: Don Slutz <dslutz(a)verizon.com>
Date: Wed, 20 Nov 2013 07:17:51 -0500
Subject: [PATCH] Add REMOTE_PAUSED() to REMOTE_MEMSRC()
And remove the redunent 'REMOTE_MEMSRC() || REMOTE_PAUSED()'.
Signed-off-by: Don Slutz <dslutz(a)verizon.com>
---
defs.h | 2 +-
memory.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/defs.h b/defs.h
index dae9662..2ec5540 100755
--- a/defs.h
+++ b/defs.h
@@ -254,7 +254,7 @@ struct number_option {
#define REMOTE_ACTIVE() (pc->flags & REM_LIVE_SYSTEM)
#define REMOTE_DUMPFILE() \
(pc->flags & (REM_NETDUMP|REM_MCLXCD|REM_LKCD|REM_S390D))
-#define REMOTE_MEMSRC() (REMOTE_ACTIVE() || REMOTE_DUMPFILE())
+#define REMOTE_MEMSRC() (REMOTE_ACTIVE() || REMOTE_PAUSED() ||
REMOTE_DUMPFILE())
#define LKCD_DUMPFILE() (pc->flags & (LKCD|REM_LKCD))
#define NETDUMP_DUMPFILE() (pc->flags & (NETDUMP|REM_NETDUMP))
#define DISKDUMP_DUMPFILE() (pc->flags & DISKDUMP)
diff --git a/memory.c b/memory.c
index cf8da2c..d17ba3c 100755
--- a/memory.c
+++ b/memory.c
@@ -14903,7 +14903,7 @@ memory_page_size(void)
if (machdep->pagesize)
return machdep->pagesize;
- if (REMOTE_MEMSRC() || REMOTE_PAUSED())
+ if (REMOTE_MEMSRC())
return remote_page_size();
switch (pc->flags & MEMORY_SOURCES)
--
1.7.11.7
or adjust patch v2 7/7 to also do this.
-Don Slutz
>
> ----- Original Message -----
>> On 11/19/13 11:08, Dave Anderson wrote:
>>> ----- Original Message -----
>>>> From: Don Slutz <dslutz(a)verizon.com>
>>>>
>>>> Currently crash 4.0 and later using the code:
>>>>
>>>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
>>>>
>>>> There is some very limited documention on crashes remote ptotocol in:
>>>>
>>>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
>>>>
>>>> and some fixes in the attachment
>>>> 0001-xen-crashd-Connect-crash-with-domain.patch
>>>> from the 1st link above.
>>>>
>>>> How ever there are issues with the current code:
>>>>
>>>> 1) The remote protocol has a minor issue (which I have not been able to
>>>> happen) based on the fact TCP/IP is stream based protocol. This
>>>> means
>>>> a RECV or a SEND may not do the fully requested size of data. In
>>>> fact
>>>> the current code assumes that the amount of data that a SEND is
>>>> called
>>>> with will all be read with a single RECV.
>>>>
>>>> 2) The most common mismatch between crash and older kernels is
>>>> phys_base. In the remote case, see if the remote server supports
>>>> vitrual memory access, and if so, see if phys_base can be
>>>> retreived.
>>>>
>>>> 3) crash assumes that the remote system is active and can not return
>>>> currect IP and SP to do a better back trace.
>>>>
>>>> 4) enable a non-active mode of remote access.
>>>>
>>>> This patch set attempts to fix these:
>>>>
>>>> The fix for #1 I have called NIL mode (patch 1).
>>>> The fix for #2 uses get_remote_phys_base (patch 3).
>>>> The fix for #3 uses get_remote_regs (patch 5).
>>>> The fix for #4 uses special file /dev/xenmem (patch 7).
>>>> It also REMOTE_NIL() to indicate "remote paused system".
>>>>
>>>> Don Slutz (7):
>>>> Add NIL mode to remote.
>>>> remote_proc_version: NULL terminate passed buffer on error.
>>>> Add get_remote_phys_base.
>>>> Add remote_vtop.
>>>> bt: get remote live registers if possible.
>>>> Add get_remote_cr3
>>>> Add support for non-live remote.
>>>>
>>>> defs.h | 8 +-
>>>> kernel.c | 25 +++-
>>>> memory.c | 25 +++-
>>>> remote.c | 462
>>>> +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
>>>> x86_64.c | 24 ++++
>>>> 5 files changed, 465 insertions(+), 79 deletions(-)
>>>>
>>>> --
>>>> 1.8.4
>>>
>>> Hi Don,
>>>
>>> A few comments...
>>>
>>> I don't really understand what entity the code is communicating with,
>>> so for the most part, I'm primarily interested in making sure that your
>>> patches in no way can affect the normal usage of the crash utility.
>>>
>>> That being said, is the term "NIL mode" some kind of Xen-ism?
>> Nope, it is term I came up with to describe the protocol change.
>>> It wasn't
>>> until I saw the display_sys_stats() function that I even understood that
>>> REMOTE_NIL mean "(remote paused system)". Anyway, if you made it
>>> something
>>> like "REMOTE_PAUSED" or something like that, the code might make
some
>>> sense
>>> to the untrained eye.
>> Will do. Looking back at it, I ended up overloading the term. :(
>>> And it's not entirely clear to me what REMOTE_ACTIVE() means now? It
>>> used
>>> to imply communicating with a remote crash daeamon that accessed the live
>>> system that the daemon was running on. Has that meaning been changed to
>>> mean
>>> that crash is communicating with a daemon that's accessing a
"live" Xen
>>> guest
>>> kernel?
>> The meaning is not as clear. Since the existing versions of crash already
>> have REMOTE_ACTIVE() and I wanted the Xen utility to be usable to them, in
>> both cases the "live" Xen guest kernel is being accessed while it is
>> paused.
>> I added REMOTE_PAUSED() to have a few changes to crash to correctly
>> understand that the guest is paused. As part of this I add the fake file
>> "/dev/xenmem".
>>> I'm confused about these two changes, where you re-patch your own
patch:
>>>
>>> - else
>>> + else if (REMOTE_ACTIVE() && ((bt->task ==
tt->this_task) ||
>>> is_task_active(bt->task))) {
>>> + if (get_remote_regs(bt, &eip, &esp))
>>> + machdep->get_stack_frame(bt, &eip,
&esp);
>>> + } else
>>>
>>> followed by:
>>>
>>> - else if (REMOTE_ACTIVE() && ((bt->task ==
tt->this_task) ||
>>> is_task_active(bt->task))) {
>>> + else if (REMOTE_NIL() && ((bt->task == tt->this_task)
||
>>> is_task_active(bt->task))) {
>>> if (get_remote_regs(bt, &eip, &esp))
>>> machdep->get_stack_frame(bt, &eip,
&esp);
>>> } else
>> Clearly I need to adjust the patch order.
>>> If I'm not mistaken, it would seem to be impossible for
"(bt->task ==
>>> tt->this_task)"
>>> to ever be true in the REMOTE_NIL() case? As I recall tt->this_task
used
>>> to be the
>>> PID of the remote daemon itself, which would be accessing the memory of
>>> its
>>> own kernel.
>>> So how could the pid of the remote daemon ever be the target of a back
>>> trace of a
>>> paused guest? And so I don't understand why you first made an if
clause
>>> for
>>> REMOTE_ACTIVE() to begin with -- whatever that means now -- and then
>>> changed it to
>>> REMOTE_NIL()?
>> This came about from some old changes from 2012 to a crash version 4.0 for
>> other work. I had not thought of /dev/xenmem at the time, and just copied
>> the condition from above. I see no reason not to drop "bt->task ==
>> tt->this_task". Will check that the right actions are still done.
>>> In any case, since the ultimate change is only interested in
>>> REMOTE_NIL(),
>>> can you make it all less of an eyesore by encapulating its if case
>>> something like:
>>>
>>> else if (SADUMP_DUMPFILE())
>>> get_sadump_regs(bt, &eip, &esp);
>>> + else if (REMOTE_NIL()) {
>>> + if (!((bt->task == tt->this_task) ||
>>> is_task_active(bt->task)) ||
>>> + !get_remote_regs(bt, &eip, &esp))
>>> + machdep->get_stack_frame(bt, &eip,
&esp);
>>> } else
>>> machdep->get_stack_frame(bt, &eip, &esp);
>>>
>>> Note above, for consistency's sake with the rest of the crash utility,
>>> can you make get_remote_regs() return non-zero when it successfully gets
>>> the registers, and zero if it fails?
>> Sure.
>>> And lastly, can you fix these?
>>>
>>> $ make warn
>>> ...
>>> remote.c: In function ‘validate_phys_base’:
>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>> ‘long long unsigned int’, but argument 3 has type ‘physaddr_t’
>>> [-Wformat]
>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>>> [-Wformat]
>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>> ‘long long unsigned int’, but argument 5 has type ‘physaddr_t’
>>> [-Wformat]
>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>> ‘long long unsigned int’, but argument 6 has type ‘physaddr_t’
>>> [-Wformat]
>>> remote.c: In function ‘remote_vtop’:
>>> remote.c:2361:2: warning: format ‘%llx’ expects argument of type
>>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>>> [-Wformat]
>>> remote.c:2367:4: warning: format ‘%llx’ expects argument of type
>>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>>> [-Wformat]
>>> remote.c:2352:18: warning: variable ‘p3’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c:2352:13: warning: variable ‘p2’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c:2352:8: warning: variable ‘p1’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c: In function ‘get_remote_regs’:
>>> remote.c:2392:33: warning: unused variable ‘p6’ [-Wunused-variable]
>>> remote.c:2392:28: warning: variable ‘p5’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c:2392:18: warning: variable ‘p3’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c:2392:8: warning: variable ‘p1’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c: In function ‘get_remote_cr3’:
>>> remote.c:2449:13: warning: variable ‘p2’ set but not used
>>> [-Wunused-but-set-variable]
>>> remote.c:2449:8: warning: variable ‘p1’ set but not used
>>> [-Wunused-but-set-variable]
>>> ...
>> Yes.
>>
>>> And I'm not interested in appending these to the 4 files that you
>>> changed:
>>>
>>> +/*
>>> + * Specify Emacs local variables so the formating
>>> + * of the code stays the same.
>>> + *
>>> + * Local variables:
>>> + * mode: C
>>> + * c-set-style: "BSD"
>>> + * c-basic-offset: 8
>>> + * indent-tabs-mode: t
>>> + * End:
>>> + */
>>>
>>> Thanks,
>>> Dave
>> Ok, that is 2 votes to drop this, so I will.
>> -Don Slutz
>>
5:07 p.m.
On 11/20/13 14:58, Dave Anderson wrote:
Hi Don,
This v2 version, plus the REMOTE_MEMSRC() patch below, is queued for
crash-7.0.4. How would you like the changelog message to read?
Tentatively I've got this:
- Resurrection of the remote analysis capability for use with the
"xen-crashd" daemon running on a Xen Dom0 host, which communicates
with a paused or shutdown DomU guest kernel. The daemon can be
accessed like so:
$ crash localhost:5991,/dev/xenmem vmlinux
(dslutz(a)verizon.com)
Thanks,
Dave
That changelog message looks good. However the default port that I have posted to
xen-devel is 5001. So it would help to change 5991 to 5001. (I was testing remotely a
lot, and had opened port 5991 in the firewall. So I just got use to using 5991...)
-Don Slutz
----- Original Message -----
> On 11/19/13 14:28, Dave Anderson wrote:
>> Hi Don,
>>
>> I still need some help understanding this "resurrection" of the remote
>> access facility, i.e., the new meanings of the REMOTE_xxx flags/macros.
>>
>> The currently-existing comment for the is_remote_daemon() function
>> in remote.c describes the (obsolete) usage of the facility:
>>
>> /*
>> * Parse, verify and establish a connection with the network daemon
>> * specified on the crash command line.
>> *
>> * The format is:
>> [remote-hostname]:port[,remote-namelist][,remote-dumpfile]
>> *
>> * where everything but the port number is optional, and the
>> remote-namelist
>> * and remote-dumpfile can be reversed.
>> *
>> * 1. The default remote host is the local host.
>> * 2. The default dumpfile is /dev/mem.
>> * 3. If no remote-namelist and remote-dumpfile are given, the daemon
>> * is queried for a kernel that matches the remote /proc/version.
>> * If no local kernel namelist is entered, the remote version will
>> * be copied locally when fd_init() is called.
>> * 4. If a remote-dumpfile is given with no remote namelist, it is
>> presumed
>> * that the kernel namelist will be entered locally.
>> */
>>
>> Back then, the remote "crashd" daemon could be contacted for:
>>
>> (1) live access to the remote machine's kernel using its /dev/mem, or
>> (2) accessing a vmlinux/vmcore pair that existed on the remote machine.
>>
>> Can you explain this new usage? It sounds like there is a remote daemon
>> running on a remote Dom0 kernel, and that your new functionality can only
>> be used to:
>>
>> (1) access a running domU guest hosted by that remote system, or
>> (2) access a paused domU guest hosted by that remote system.
>>
>> And that there is also a device that the the remote daemon uses
>> to access the memory of the guest, be it paused or live.
>>
>> Am I in the ballpark?
> Yes.
>
> The currently proposed code for Dom0, does not support (1) mostly
> because it seems much better to me to be able to poke around and look at
> a fixed state of the domU.
>
> There is a (3) in xen: guest has shutdown. A shutdown guest has
> subtypes like "requested shutdown", "requested reboot",
"detected
> crash", etc. Xen will set a guest in shutdown state when a double fault
> or triple fault happens. A paused guest is the same as an unpaused
> guest in shutdown state. Using a much older Xen, I known that that
> version treated a double fault is the same as a reboot request. The bug
> I was tracking down turned out to be a extra swapgs which will cause
> many linux kernel's to double fault on a page fault.
>
> I was also looking to be able to use old versions of crash so as to
> make it easier to use and/or deploy. Since the 7.0.3 or older version of
> crash only support "live access to the remote machine's kernel using its
> /dev/mem "; I picked to default to be a paused system. Since crash
> thinks the remote system is live, it keeps fetching state which is not
> changing.
>
> The "device that the the remote daemon uses ", is more a defined
> ABI which has 2 layers libxl and libxc (what I all xc_* routines). In
> this case I am using libxc calls. None of these require the guest to be
> paused.
>
> So it would not be hard to change the Xen code to support new
> option (1) if there is need for such a mode. I could see adding some
> sort of pause control as an extension to crash.
>
> I have recently learn of another Xen tool: gdbsx. It is designed
> to connect gdb to a running guest and pause and resume the guest on
> demand. While I have not done very much with it, it looks to be
> powerful. Since it is just gdb, it is missing crash's kernel support.
>
>> Dave
> While working on this email, I found out that it looks better to me to
> include REMOTE_PAUSED() in REMOTE_MEMSRC(). I.E. either add the following:
>
> From b4093734b2db8666f155032ffd9637dcfda0fc0b Mon Sep 17 00:00:00 2001
> From: Don Slutz <dslutz(a)verizon.com>
> Date: Wed, 20 Nov 2013 07:17:51 -0500
> Subject: [PATCH] Add REMOTE_PAUSED() to REMOTE_MEMSRC()
>
> And remove the redunent 'REMOTE_MEMSRC() || REMOTE_PAUSED()'.
>
> Signed-off-by: Don Slutz <dslutz(a)verizon.com>
> ---
> defs.h | 2 +-
> memory.c | 2 +-
> 2 files changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/defs.h b/defs.h
> index dae9662..2ec5540 100755
> --- a/defs.h
> +++ b/defs.h
> @@ -254,7 +254,7 @@ struct number_option {
> #define REMOTE_ACTIVE() (pc->flags & REM_LIVE_SYSTEM)
> #define REMOTE_DUMPFILE() \
> (pc->flags & (REM_NETDUMP|REM_MCLXCD|REM_LKCD|REM_S390D))
> -#define REMOTE_MEMSRC() (REMOTE_ACTIVE() || REMOTE_DUMPFILE())
> +#define REMOTE_MEMSRC() (REMOTE_ACTIVE() || REMOTE_PAUSED() ||
> REMOTE_DUMPFILE())
> #define LKCD_DUMPFILE() (pc->flags & (LKCD|REM_LKCD))
> #define NETDUMP_DUMPFILE() (pc->flags & (NETDUMP|REM_NETDUMP))
> #define DISKDUMP_DUMPFILE() (pc->flags & DISKDUMP)
> diff --git a/memory.c b/memory.c
> index cf8da2c..d17ba3c 100755
> --- a/memory.c
> +++ b/memory.c
> @@ -14903,7 +14903,7 @@ memory_page_size(void)
> if (machdep->pagesize)
> return machdep->pagesize;
>
> - if (REMOTE_MEMSRC() || REMOTE_PAUSED())
> + if (REMOTE_MEMSRC())
> return remote_page_size();
>
> switch (pc->flags & MEMORY_SOURCES)
> --
> 1.7.11.7
>
> or adjust patch v2 7/7 to also do this.
>
>
> -Don Slutz
>> ----- Original Message -----
>>> On 11/19/13 11:08, Dave Anderson wrote:
>>>> ----- Original Message -----
>>>>> From: Don Slutz <dslutz(a)verizon.com>
>>>>>
>>>>> Currently crash 4.0 and later using the code:
>>>>>
>>>>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02569.html
>>>>>
>>>>> There is some very limited documention on crashes remote ptotocol
in:
>>>>>
>>>>> http://lists.xen.org/archives/html/xen-devel/2013-11/msg02352.html
>>>>>
>>>>> and some fixes in the attachment
>>>>> 0001-xen-crashd-Connect-crash-with-domain.patch
>>>>> from the 1st link above.
>>>>>
>>>>> How ever there are issues with the current code:
>>>>>
>>>>> 1) The remote protocol has a minor issue (which I have not been able
to
>>>>> happen) based on the fact TCP/IP is stream based protocol.
This
>>>>> means
>>>>> a RECV or a SEND may not do the fully requested size of data.
In
>>>>> fact
>>>>> the current code assumes that the amount of data that a SEND
is
>>>>> called
>>>>> with will all be read with a single RECV.
>>>>>
>>>>> 2) The most common mismatch between crash and older kernels is
>>>>> phys_base. In the remote case, see if the remote server
supports
>>>>> vitrual memory access, and if so, see if phys_base can be
>>>>> retreived.
>>>>>
>>>>> 3) crash assumes that the remote system is active and can not return
>>>>> currect IP and SP to do a better back trace.
>>>>>
>>>>> 4) enable a non-active mode of remote access.
>>>>>
>>>>> This patch set attempts to fix these:
>>>>>
>>>>> The fix for #1 I have called NIL mode (patch 1).
>>>>> The fix for #2 uses get_remote_phys_base (patch 3).
>>>>> The fix for #3 uses get_remote_regs (patch 5).
>>>>> The fix for #4 uses special file /dev/xenmem (patch 7).
>>>>> It also REMOTE_NIL() to indicate "remote paused
system".
>>>>>
>>>>> Don Slutz (7):
>>>>> Add NIL mode to remote.
>>>>> remote_proc_version: NULL terminate passed buffer on error.
>>>>> Add get_remote_phys_base.
>>>>> Add remote_vtop.
>>>>> bt: get remote live registers if possible.
>>>>> Add get_remote_cr3
>>>>> Add support for non-live remote.
>>>>>
>>>>> defs.h | 8 +-
>>>>> kernel.c | 25 +++-
>>>>> memory.c | 25 +++-
>>>>> remote.c | 462
>>>>> +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
>>>>> x86_64.c | 24 ++++
>>>>> 5 files changed, 465 insertions(+), 79 deletions(-)
>>>>>
>>>>> --
>>>>> 1.8.4
>>>>
>>>> Hi Don,
>>>>
>>>> A few comments...
>>>>
>>>> I don't really understand what entity the code is communicating
with,
>>>> so for the most part, I'm primarily interested in making sure that
your
>>>> patches in no way can affect the normal usage of the crash utility.
>>>>
>>>> That being said, is the term "NIL mode" some kind of Xen-ism?
>>> Nope, it is term I came up with to describe the protocol change.
>>>> It wasn't
>>>> until I saw the display_sys_stats() function that I even understood that
>>>> REMOTE_NIL mean "(remote paused system)". Anyway, if you made
it
>>>> something
>>>> like "REMOTE_PAUSED" or something like that, the code might
make some
>>>> sense
>>>> to the untrained eye.
>>> Will do. Looking back at it, I ended up overloading the term. :(
>>>> And it's not entirely clear to me what REMOTE_ACTIVE() means now?
It
>>>> used
>>>> to imply communicating with a remote crash daeamon that accessed the
live
>>>> system that the daemon was running on. Has that meaning been changed to
>>>> mean
>>>> that crash is communicating with a daemon that's accessing a
"live" Xen
>>>> guest
>>>> kernel?
>>> The meaning is not as clear. Since the existing versions of crash already
>>> have REMOTE_ACTIVE() and I wanted the Xen utility to be usable to them, in
>>> both cases the "live" Xen guest kernel is being accessed while it
is
>>> paused.
>>> I added REMOTE_PAUSED() to have a few changes to crash to correctly
>>> understand that the guest is paused. As part of this I add the fake file
>>> "/dev/xenmem".
>>>> I'm confused about these two changes, where you re-patch your own
patch:
>>>>
>>>> - else
>>>> + else if (REMOTE_ACTIVE() && ((bt->task ==
tt->this_task) ||
>>>> is_task_active(bt->task))) {
>>>> + if (get_remote_regs(bt, &eip, &esp))
>>>> + machdep->get_stack_frame(bt, &eip,
&esp);
>>>> + } else
>>>>
>>>> followed by:
>>>>
>>>> - else if (REMOTE_ACTIVE() && ((bt->task ==
tt->this_task) ||
>>>> is_task_active(bt->task))) {
>>>> + else if (REMOTE_NIL() && ((bt->task ==
tt->this_task) ||
>>>> is_task_active(bt->task))) {
>>>> if (get_remote_regs(bt, &eip, &esp))
>>>> machdep->get_stack_frame(bt, &eip,
&esp);
>>>> } else
>>> Clearly I need to adjust the patch order.
>>>> If I'm not mistaken, it would seem to be impossible for
"(bt->task ==
>>>> tt->this_task)"
>>>> to ever be true in the REMOTE_NIL() case? As I recall tt->this_task
used
>>>> to be the
>>>> PID of the remote daemon itself, which would be accessing the memory of
>>>> its
>>>> own kernel.
>>>> So how could the pid of the remote daemon ever be the target of a back
>>>> trace of a
>>>> paused guest? And so I don't understand why you first made an if
clause
>>>> for
>>>> REMOTE_ACTIVE() to begin with -- whatever that means now -- and then
>>>> changed it to
>>>> REMOTE_NIL()?
>>> This came about from some old changes from 2012 to a crash version 4.0 for
>>> other work. I had not thought of /dev/xenmem at the time, and just copied
>>> the condition from above. I see no reason not to drop "bt->task ==
>>> tt->this_task". Will check that the right actions are still done.
>>>> In any case, since the ultimate change is only interested in
>>>> REMOTE_NIL(),
>>>> can you make it all less of an eyesore by encapulating its if case
>>>> something like:
>>>>
>>>> else if (SADUMP_DUMPFILE())
>>>> get_sadump_regs(bt, &eip, &esp);
>>>> + else if (REMOTE_NIL()) {
>>>> + if (!((bt->task == tt->this_task) ||
>>>> is_task_active(bt->task)) ||
>>>> + !get_remote_regs(bt, &eip, &esp))
>>>> + machdep->get_stack_frame(bt, &eip,
&esp);
>>>> } else
>>>> machdep->get_stack_frame(bt, &eip, &esp);
>>>>
>>>> Note above, for consistency's sake with the rest of the crash
utility,
>>>> can you make get_remote_regs() return non-zero when it successfully gets
>>>> the registers, and zero if it fails?
>>> Sure.
>>>> And lastly, can you fix these?
>>>>
>>>> $ make warn
>>>> ...
>>>> remote.c: In function ‘validate_phys_base’:
>>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>>> ‘long long unsigned int’, but argument 3 has type ‘physaddr_t’
>>>> [-Wformat]
>>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>>>> [-Wformat]
>>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>>> ‘long long unsigned int’, but argument 5 has type ‘physaddr_t’
>>>> [-Wformat]
>>>> remote.c:2302:25: warning: format ‘%llx’ expects argument of type
>>>> ‘long long unsigned int’, but argument 6 has type ‘physaddr_t’
>>>> [-Wformat]
>>>> remote.c: In function ‘remote_vtop’:
>>>> remote.c:2361:2: warning: format ‘%llx’ expects argument of type
>>>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>>>> [-Wformat]
>>>> remote.c:2367:4: warning: format ‘%llx’ expects argument of type
>>>> ‘long long unsigned int’, but argument 4 has type ‘physaddr_t’
>>>> [-Wformat]
>>>> remote.c:2352:18: warning: variable ‘p3’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c:2352:13: warning: variable ‘p2’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c:2352:8: warning: variable ‘p1’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c: In function ‘get_remote_regs’:
>>>> remote.c:2392:33: warning: unused variable ‘p6’ [-Wunused-variable]
>>>> remote.c:2392:28: warning: variable ‘p5’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c:2392:18: warning: variable ‘p3’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c:2392:8: warning: variable ‘p1’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c: In function ‘get_remote_cr3’:
>>>> remote.c:2449:13: warning: variable ‘p2’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> remote.c:2449:8: warning: variable ‘p1’ set but not used
>>>> [-Wunused-but-set-variable]
>>>> ...
>>> Yes.
>>>
>>>> And I'm not interested in appending these to the 4 files that you
>>>> changed:
>>>>
>>>> +/*
>>>> + * Specify Emacs local variables so the formating
>>>> + * of the code stays the same.
>>>> + *
>>>> + * Local variables:
>>>> + * mode: C
>>>> + * c-set-style: "BSD"
>>>> + * c-basic-offset: 8
>>>> + * indent-tabs-mode: t
>>>> + * End:
>>>> + */
>>>>
>>>> Thanks,
>>>> Dave
>>> Ok, that is 2 votes to drop this, so I will.
>>> -Don Slutz
>>>
>
4024
days inactive
4025
days old
devel@lists.crash-utility.osci.io
15 comments
3 participants
participants (3)
-
Daniel Kiper -
Dave Anderson -
Don Slutz