3:10 a.m.
The following series implements LZO compression support to
makedumpfile. LZO is as good as in size but by far better in speed
than ZLIB, readucing down time during generation of crash dump and
refiltering.
The RFC discussion:
http://lists.infradead.org/pipermail/kexec/2011-November/005783.html
http://lists.infradead.org/pipermail/kexec/2011-December/005868.html
How to build:
1. Get lzo libraries: lzo, lzo-devel and lzo-minilzo from either of
the following:
1) Original author's website:
http://www.oberhumer.com/opensource/lzo/
2) yum framework on fedora. Older releases don't have the packages.
2. Apply the patch set to makedumpfile v1.4.2.
3. Do make.
How to use:
Introduce new -l option. If a user specify this, makedumpfile
generates dumpfile compressed by pages with lzo compression.
Example)
$ makedumpfile -l vmcore dumpfile
Performance evaluation:
- Kumagai-san's evaluation simulating actually working servers:
http://lists.infradead.org/pipermail/kexec/2011-December/005868.html
- My evaluation focusing on the worst cases:
http://lists.infradead.org/pipermail/kexec/2011-November/005783.html
LZO Support for crash:
I'll post LZO support patch for crash after makedumpfile merges
these patches.
---
HATAYAMA Daisuke (2):
Add help and manual messages about LZO compression support
Add LZO Support
Makefile | 2 +-
diskdump_mod.h | 3 ++-
makedumpfile.8 | 6 +++---
makedumpfile.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++---------
makedumpfile.h | 2 ++
print_info.c | 16 ++++++++--------
6 files changed, 64 insertions(+), 22 deletions(-)
--
HATAYAMA Daisuke
3:10 a.m.
New subject: [PATCH makedumpfile 1/2] Add LZO Support
Add -l option as one of the command-line options users can specify. If
-l option is specified, then makedumpfile generates dumpfile in
kdump-compressed format with lzo compression by each page.
Signed-off-by: HATAYAMA Daisuke <d.hatayama(a)jp.fujitsu.com>
---
Makefile | 2 +-
diskdump_mod.h | 3 ++-
makedumpfile.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++---------
makedumpfile.h | 2 ++
4 files changed, 53 insertions(+), 11 deletions(-)
diff --git a/Makefile b/Makefile
index e6b7b89..21b3058 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,7 @@ OBJ_PART = print_info.o dwarf_info.o elf_info.o erase_info.o
sadump_info.o
SRC_ARCH = arch/arm.c arch/x86.c arch/x86_64.c arch/ia64.c arch/ppc64.c arch/s390x.c
OBJ_ARCH = arch/arm.o arch/x86.o arch/x86_64.o arch/ia64.o arch/ppc64.o arch/s390x.o
-LIBS = -ldw -lbz2 -lebl -ldl -lelf -lz
+LIBS = -ldw -lbz2 -lebl -ldl -lelf -lz -llzo2
ifneq ($(LINKTYPE), dynamic)
LIBS := -static $(LIBS)
endif
diff --git a/diskdump_mod.h b/diskdump_mod.h
index c1de972..e672485 100644
--- a/diskdump_mod.h
+++ b/diskdump_mod.h
@@ -78,7 +78,8 @@ struct kdump_sub_header {
};
/* page flags */
-#define DUMP_DH_COMPRESSED 0x1 /* page is compressed */
+#define DUMP_DH_COMPRESSED_ZLIB 0x1 /* page is compressed with zlib */
+#define DUMP_DH_COMPRESSED_LZO 0x2 /* paged is compressed with lzo */
/* descriptor of each page for vmcore */
typedef struct page_desc {
diff --git a/makedumpfile.c b/makedumpfile.c
index c51fda3..e0079b8 100644
--- a/makedumpfile.c
+++ b/makedumpfile.c
@@ -257,7 +257,7 @@ readpmem_kdump_compressed(unsigned long long paddr, void *bufptr,
size_t size)
goto error;
}
- if (pd.flags & DUMP_DH_COMPRESSED) {
+ if (pd.flags & DUMP_DH_COMPRESSED_ZLIB) {
retlen = info->page_size;
ret = uncompress((unsigned char *)buf2, &retlen,
(unsigned char *)buf, pd.size);
@@ -266,6 +266,17 @@ readpmem_kdump_compressed(unsigned long long paddr, void *bufptr,
size_t size)
goto error;
}
memcpy(bufptr, buf2 + page_offset, size);
+ } else if (info->flag_lzo_support
+ && (pd.flags & DUMP_DH_COMPRESSED_LZO)) {
+ retlen = info->page_size;
+ ret = lzo1x_decompress_safe((unsigned char *)buf, pd.size,
+ (unsigned char *)buf2, &retlen,
+ LZO1X_MEM_DECOMPRESS);
+ if ((ret != LZO_E_OK) || (retlen != info->page_size)) {
+ ERRMSG("Uncompress failed: %d\n", ret);
+ goto error;
+ }
+ memcpy(bufptr, buf2 + page_offset, size);
} else
memcpy(bufptr, buf + page_offset, size);
@@ -2498,6 +2509,9 @@ initial(void)
unsigned long size;
int debug_info = FALSE;
+ if (lzo_init() == LZO_E_OK)
+ info->flag_lzo_support = TRUE;
+
if (!is_xen_memory() && info->flag_exclude_xen_dom) {
MSG("'-X' option is disable,");
MSG("because %s is not Xen's memory core image.\n",
info->name_memory);
@@ -4665,10 +4679,11 @@ write_kdump_pages(struct cache_data *cd_header, struct cache_data
*cd_page)
off_t offset_data = 0;
struct disk_dump_header *dh = info->dump_header;
unsigned char buf[info->page_size], *buf_out = NULL;
- unsigned long len_buf_out;
+ unsigned long len_buf_out, len_buf_out_zlib, len_buf_out_lzo;
struct dump_bitmap bitmap2;
struct timeval tv_start;
const off_t failed = (off_t)-1;
+ lzo_bytep wrkmem = NULL;
int ret = FALSE;
@@ -4677,7 +4692,16 @@ write_kdump_pages(struct cache_data *cd_header, struct cache_data
*cd_page)
initialize_2nd_bitmap(&bitmap2);
- len_buf_out = compressBound(info->page_size);
+ if ((wrkmem = malloc(LZO1X_1_MEM_COMPRESS)) == NULL) {
+ ERRMSG("Can't allocate memory for the working memory. %s\n",
+ strerror(errno));
+ goto out;
+ }
+
+ len_buf_out_zlib = compressBound(info->page_size);
+ len_buf_out_lzo = info->page_size + info->page_size / 16 + 64 + 3;
+ len_buf_out = MAX(len_buf_out_zlib, len_buf_out_lzo);
+
if ((buf_out = malloc(len_buf_out)) == NULL) {
ERRMSG("Can't allocate memory for the compression buffer. %s\n",
strerror(errno));
@@ -4759,11 +4783,21 @@ write_kdump_pages(struct cache_data *cd_header, struct cache_data
*cd_page)
* Compress the page data.
*/
size_out = len_buf_out;
- if (info->flag_compress
- && (compress2(buf_out, &size_out, buf,
- info->page_size, Z_BEST_SPEED) == Z_OK)
+ if ((info->flag_compress & DUMP_DH_COMPRESSED_ZLIB)
+ && ((size_out = len_buf_out),
+ compress2(buf_out, &size_out, buf, info->page_size,
+ Z_BEST_SPEED) == Z_OK)
&& (size_out < info->page_size)) {
- pd.flags = 1;
+ pd.flags = DUMP_DH_COMPRESSED_ZLIB;
+ pd.size = size_out;
+ memcpy(buf, buf_out, pd.size);
+ } else if (info->flag_lzo_support
+ && (info->flag_compress & DUMP_DH_COMPRESSED_LZO)
+ && ((size_out = info->page_size),
+ lzo1x_1_compress(buf, info->page_size, buf_out,
+ &size_out, wrkmem) == LZO_E_OK)
+ && (size_out < info->page_size)) {
+ pd.flags = DUMP_DH_COMPRESSED_LZO;
pd.size = size_out;
memcpy(buf, buf_out, pd.size);
} else {
@@ -4806,6 +4840,8 @@ write_kdump_pages(struct cache_data *cd_header, struct cache_data
*cd_page)
out:
if (buf_out != NULL)
free(buf_out);
+ if (wrkmem != NULL)
+ free(wrkmem);
return ret;
}
@@ -6896,7 +6932,7 @@ main(int argc, char *argv[])
info->block_order = DEFAULT_ORDER;
message_level = DEFAULT_MSG_LEVEL;
- while ((opt = getopt_long(argc, argv, "b:cDd:EFfg:hi:MRrsvXx:", longopts,
+ while ((opt = getopt_long(argc, argv, "b:cDd:EFfg:hi:lMRrsvXx:", longopts,
NULL)) != -1) {
switch (opt) {
case 'b':
@@ -6906,7 +6942,7 @@ main(int argc, char *argv[])
info->name_filterconfig = optarg;
break;
case 'c':
- info->flag_compress = 1;
+ info->flag_compress = DUMP_DH_COMPRESSED_ZLIB;
break;
case 'D':
flag_debug = TRUE;
@@ -6945,6 +6981,9 @@ main(int argc, char *argv[])
goto out;
info->flag_sadump_diskset = 1;
break;
+ case 'l':
+ info->flag_compress = DUMP_DH_COMPRESSED_LZO;
+ break;
case 'm':
message_level = atoi(optarg);
break;
diff --git a/makedumpfile.h b/makedumpfile.h
index ebb8929..9594ca7 100644
--- a/makedumpfile.h
+++ b/makedumpfile.h
@@ -31,6 +31,7 @@
#include <libelf.h>
#include <byteswap.h>
#include <getopt.h>
+#include <lzo/lzo1x.h>
#include "common.h"
#include "dwarf_info.h"
#include "diskdump_mod.h"
@@ -767,6 +768,7 @@ struct DumpInfo {
int num_dump_level; /* number of dump level */
int array_dump_level[NUM_ARRAY_DUMP_LEVEL];
int flag_compress; /* flag of compression */
+ int flag_lzo_support; /* flag of LZO compression support */
int flag_elf_dumpfile; /* flag of creating ELF dumpfile */
int flag_generate_vmcoreinfo;/* flag of generating vmcoreinfo file */
int flag_read_vmcoreinfo; /* flag of reading vmcoreinfo file */
3:10 a.m.
New subject: [PATCH makedumpfile 2/2] Add help and manual messages about LZO compression support
Signed-off-by: HATAYAMA Daisuke <d.hatayama(a)jp.fujitsu.com>
---
makedumpfile.8 | 6 +++---
print_info.c | 16 ++++++++--------
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/makedumpfile.8 b/makedumpfile.8
index 4733420..f607bf5 100644
--- a/makedumpfile.8
+++ b/makedumpfile.8
@@ -121,8 +121,8 @@ configuration, you need to use --diskset option.
.SH OPTIONS
.TP
-\fB\-c\fR
-Compress dump data by each page.
+\fB\-c,\-l\fR
+Compress dump data by each page using zlib for -c option or lzo for -l option.
.br
A user cannot specify this option with \-E option, because the ELF format does
not support compressed data.
@@ -196,7 +196,7 @@ by dump_level 11, makedumpfile retries it by dump_level 31.
\fB\-E\fR
Create \fIDUMPFILE\fR in the ELF format.
.br
-This option cannot be specified with \-c option, because the ELF format does not
+This option cannot be specified with either of \-c option or \-l option, because the ELF
format does not
support compressed data.
.br
.B Example:
diff --git a/print_info.c b/print_info.c
index 31460b7..237892e 100644
--- a/print_info.c
+++ b/print_info.c
@@ -35,15 +35,15 @@ print_usage(void)
MSG("\n");
MSG("Usage:\n");
MSG(" Creating DUMPFILE:\n");
- MSG(" # makedumpfile [-c|-E] [-d DL] [-x VMLINUX|-i VMCOREINFO] VMCORE
DUMPFILE\n");
+ MSG(" # makedumpfile [-c|-l|-E] [-d DL] [-x VMLINUX|-i VMCOREINFO] VMCORE
DUMPFILE\n");
MSG("\n");
MSG(" Creating DUMPFILE with filtered kernel data specified through filter
config\n");
MSG(" file:\n");
- MSG(" # makedumpfile [-c|-E] [-d DL] -x VMLINUX --config FILTERCONFIGFILE
VMCORE\n");
+ MSG(" # makedumpfile [-c|-l|-E] [-d DL] -x VMLINUX --config FILTERCONFIGFILE
VMCORE\n");
MSG(" DUMPFILE\n");
MSG("\n");
MSG(" Outputting the dump data in the flattened format to the standard
output:\n");
- MSG(" # makedumpfile -F [-c|-E] [-d DL] [-x VMLINUX|-i VMCOREINFO]
VMCORE\n");
+ MSG(" # makedumpfile -F [-c|-l|-E] [-d DL] [-x VMLINUX|-i VMCOREINFO]
VMCORE\n");
MSG("\n");
MSG(" Rearranging the dump data in the flattened format to a readable
DUMPFILE:\n");
MSG(" # makedumpfile -R DUMPFILE\n");
@@ -70,14 +70,14 @@ print_usage(void)
MSG("\n");
MSG("\n");
MSG(" Creating DUMPFILE from multiple VMCOREs generated on sadump diskset
configuration:\n");
- MSG(" # makedumpfile [-c] [-d DL] -x VMLINUX --diskset=VMCORE1
--diskset=VMCORE2\n");
+ MSG(" # makedumpfile [-c|-l] [-d DL] -x VMLINUX --diskset=VMCORE1
--diskset=VMCORE2\n");
MSG(" [--diskset=VMCORE3 ..] DUMPFILE\n");
MSG("\n");
MSG("\n");
MSG("Available options:\n");
- MSG(" [-c]:\n");
- MSG(" Compress dump data by each page.\n");
- MSG(" A user cannot specify this option with -E option, because the ELF
format\n");
+ MSG(" [-c|-l]:\n");
+ MSG(" Compress dump data by each page using zlib for -c option and lzo for -l
option.\n");
+ MSG(" A user cannot specify either of these options with -E option, because
the ELF format\n");
MSG(" does not support compressed data.\n");
MSG(" THIS IS ONLY FOR THE CRASH UTILITY.\n");
MSG("\n");
@@ -103,7 +103,7 @@ print_usage(void)
MSG("\n");
MSG(" [-E]:\n");
MSG(" Create DUMPFILE in the ELF format.\n");
- MSG(" This option cannot be specified with -c option, because the
ELF\n");
+ MSG(" This option cannot be specified with either of -c option or -l option,
because the ELF\n");
MSG(" format does not support compressed data.\n");
MSG("\n");
MSG(" [-x VMLINUX]:\n");
1:50 a.m.
Hello Hatayama-san,
On Mon, 20 Feb 2012 18:10:00 +0900
HATAYAMA Daisuke <d.hatayama(a)jp.fujitsu.com> wrote:
The following series implements LZO compression support to
makedumpfile. LZO is as good as in size but by far better in speed
than ZLIB, readucing down time during generation of crash dump and
refiltering.
The RFC discussion:
http://lists.infradead.org/pipermail/kexec/2011-November/005783.html
http://lists.infradead.org/pipermail/kexec/2011-December/005868.html
How to build:
1. Get lzo libraries: lzo, lzo-devel and lzo-minilzo from either of
the following:
1) Original author's website:
http://www.oberhumer.com/opensource/lzo/
2) yum framework on fedora. Older releases don't have the packages.
2. Apply the patch set to makedumpfile v1.4.2.
3. Do make.
How to use:
Introduce new -l option. If a user specify this, makedumpfile
generates dumpfile compressed by pages with lzo compression.
Example)
$ makedumpfile -l vmcore dumpfile
Performance evaluation:
- Kumagai-san's evaluation simulating actually working servers:
http://lists.infradead.org/pipermail/kexec/2011-December/005868.html
- My evaluation focusing on the worst cases:
http://lists.infradead.org/pipermail/kexec/2011-November/005783.html
LZO Support for crash:
I'll post LZO support patch for crash after makedumpfile merges
these patches.
Thank you for your patches.
I'll review and merge them into v1.4.4 with making LZO support optional.
I'll post the patch to make it optional after v1.4.3 is released.
Please wait for a while.
Thanks
Atsushi Kumagai
8:14 a.m.
Hello Kumagai-san,
From: Atsushi Kumagai <kumagai-atsushi(a)mxc.nes.nec.co.jp>
Subject: Re: [PATCH makedumpfile 0/2] LZO Compression Support
Date: Tue, 21 Feb 2012 16:50:41 +0900
Hello Hatayama-san,
On Mon, 20 Feb 2012 18:10:00 +0900
HATAYAMA Daisuke <d.hatayama(a)jp.fujitsu.com> wrote:
> The following series implements LZO compression support to
> makedumpfile. LZO is as good as in size but by far better in speed
> than ZLIB, readucing down time during generation of crash dump and
> refiltering.
>
> The RFC discussion:
>
> http://lists.infradead.org/pipermail/kexec/2011-November/005783.html
> http://lists.infradead.org/pipermail/kexec/2011-December/005868.html
>
> How to build:
>
> 1. Get lzo libraries: lzo, lzo-devel and lzo-minilzo from either of
> the following:
>
> 1) Original author's website:
> http://www.oberhumer.com/opensource/lzo/
>
> 2) yum framework on fedora. Older releases don't have the packages.
>
> 2. Apply the patch set to makedumpfile v1.4.2.
>
> 3. Do make.
>
> How to use:
>
> Introduce new -l option. If a user specify this, makedumpfile
> generates dumpfile compressed by pages with lzo compression.
>
> Example)
> $ makedumpfile -l vmcore dumpfile
>
> Performance evaluation:
>
> - Kumagai-san's evaluation simulating actually working servers:
> http://lists.infradead.org/pipermail/kexec/2011-December/005868.html
>
> - My evaluation focusing on the worst cases:
> http://lists.infradead.org/pipermail/kexec/2011-November/005783.html
>
> LZO Support for crash:
>
> I'll post LZO support patch for crash after makedumpfile merges
> these patches.
Thank you for your patches.
I'll review and merge them into v1.4.4 with making LZO support optional.
I'll post the patch to make it optional after v1.4.3 is released.
Please wait for a while.
Thanks
Atsushi Kumagai
Thanks. But I'm OK if I'll repost the patch after you have decided how
you change Makefile to support optional building. Or is it more
effective for you to do them by yourself at the same time?
Thanks.
HATAYAMA, Daisuke
8:42 p.m.
Hello Hatayama-san,
On Tue, 21 Feb 2012 17:01:25 +0900 ( )
HATAYAMA Daisuke <d.hatayama(a)jp.fujitsu.com> wrote:
> Thank you for your patches.
>
> I'll review and merge them into v1.4.4 with making LZO support optional.
> I'll post the patch to make it optional after v1.4.3 is released.
>
> Please wait for a while.
>
> Thanks
> Atsushi Kumagai
>
Thanks. But I'm OK if I'll repost the patch after you have decided how
you change Makefile to support optional building. Or is it more
effective for you to do them by yourself at the same time?
I wonder if you could repost the patch with implementing my idea.
My idea is that using build option (like "LINKTYPE" at v1.4.2).
Example)
1. enable lzo support
$ make USELZO=on
2. disable lzo support (default)
$ make
In future, I would like to use automake but don't worry
about it now.
Thanks
Atsushi Kumagai
8:14 a.m.
From: Atsushi Kumagai <kumagai-atsushi(a)mxc.nes.nec.co.jp>
Subject: Re: [PATCH makedumpfile 0/2] LZO Compression Support
Date: Wed, 22 Feb 2012 11:42:09 +0900
Hello Hatayama-san,
On Tue, 21 Feb 2012 17:01:25 +0900 ( )
HATAYAMA Daisuke <d.hatayama(a)jp.fujitsu.com> wrote:
> > Thank you for your patches.
> >
> > I'll review and merge them into v1.4.4 with making LZO support optional.
> > I'll post the patch to make it optional after v1.4.3 is released.
> >
> > Please wait for a while.
> >
> > Thanks
> > Atsushi Kumagai
> >
>
> Thanks. But I'm OK if I'll repost the patch after you have decided how
> you change Makefile to support optional building. Or is it more
> effective for you to do them by yourself at the same time?
I wonder if you could repost the patch with implementing my idea.
My idea is that using build option (like "LINKTYPE" at v1.4.2).
Example)
1. enable lzo support
$ make USELZO=on
2. disable lzo support (default)
$ make
In future, I would like to use automake but don't worry
about it now.
I see. I'll repost the patch soon.
Thanks.
HATAYAMA, Daisuke
229
days inactive
4513
days old
devel@lists.crash-utility.osci.io
6 comments
2 participants
participants (2)
-
Atsushi Kumagai -
HATAYAMA Daisuke