Add support for lz4 write filter.

Makefile.am

@@ -195,6 +195,7 @@ libarchive_la_SOURCES= \
 	libarchive/archive_write_add_filter_grzip.c \
 	libarchive/archive_write_add_filter_gzip.c \
 	libarchive/archive_write_add_filter_lrzip.c \
+	libarchive/archive_write_add_filter_lz4.c \
 	libarchive/archive_write_add_filter_lzop.c \
 	libarchive/archive_write_add_filter_none.c \
 	libarchive/archive_write_add_filter_program.c \
@@ -492,6 +493,7 @@ libarchive_test_SOURCES= \
 	libarchive/test/test_write_filter_gzip.c \
 	libarchive/test/test_write_filter_gzip_timestamp.c \
 	libarchive/test/test_write_filter_lrzip.c \
+	libarchive/test/test_write_filter_lz4.c \
 	libarchive/test/test_write_filter_lzip.c \
 	libarchive/test/test_write_filter_lzma.c \
 	libarchive/test/test_write_filter_lzop.c \
@@ -882,6 +884,7 @@ bsdtar_test_SOURCES= \
 	tar/test/test_option_k.c \
 	tar/test/test_option_keep_newer_files.c \
 	tar/test/test_option_lrzip.c \
+	tar/test/test_option_lz4.c \
 	tar/test/test_option_lzma.c \
 	tar/test/test_option_lzop.c \
 	tar/test/test_option_n.c \
@@ -1028,6 +1031,7 @@ bsdcpio_test_SOURCES= \
 	cpio/test/test_option_help.c \
 	cpio/test/test_option_l.c \
 	cpio/test/test_option_lrzip.c \
+	cpio/test/test_option_lz4.c \
 	cpio/test/test_option_lzma.c \
 	cpio/test/test_option_lzop.c \
 	cpio/test/test_option_m.c \

cpio/bsdcpio.1

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 7, 2012
+.Dd August 14, 2014
 .Dt CPIO 1
 .Os
 .Sh NAME
@@ -181,6 +181,11 @@ instead of copying.
 Compress the resulting archive with
 .Xr lrzip 1 .
 In input mode, this option is ignored.
+.It Fl Fl lz4
+(o mode only)
+Compress the archive with lz4-compatible compression before writing it.
+In input mode, this option is ignored; lz4 compression is recognized
+automatically on input.
 .It Fl Fl lzma
 (o mode only)
 Compress the file with lzma-compatible compression before writing it.

cpio/cmdline.c

@@ -73,6 +73,7 @@ static const struct option {
 	{ "link",			0, 'l' },
 	{ "list",			0, 't' },
 	{ "lrzip",			0, OPTION_LRZIP },
+	{ "lz4",			0, OPTION_LZ4 },
 	{ "lzma",			0, OPTION_LZMA },
 	{ "lzop",			0, OPTION_LZOP },
 	{ "make-directories",		0, 'd' },

cpio/cpio.c

@@ -261,6 +261,7 @@ main(int argc, char *argv[])
 			cpio->option_link = 1;
 			break;
 		case OPTION_LRZIP:
+		case OPTION_LZ4:
 		case OPTION_LZMA: /* GNU tar, others */
 		case OPTION_LZOP: /* GNU tar, others */
 			cpio->compress = opt;
@@ -526,6 +527,9 @@ mode_out(struct cpio *cpio)
 	case OPTION_LRZIP:
 		r = archive_write_add_filter_lrzip(cpio->archive);
 		break;
+	case OPTION_LZ4:
+		r = archive_write_add_filter_lz4(cpio->archive);
+		break;
 	case OPTION_LZMA:
 		r = archive_write_add_filter_lzma(cpio->archive);
 		break;

cpio/cpio.h

@@ -101,6 +101,7 @@ enum {
 	OPTION_GRZIP,
 	OPTION_INSECURE,
 	OPTION_LRZIP,
+	OPTION_LZ4,
 	OPTION_LZMA,
 	OPTION_LZOP,
 	OPTION_NO_PRESERVE_OWNER,

cpio/test/CMakeLists.txt

@@ -39,6 +39,7 @@ IF(ENABLE_CPIO AND ENABLE_TEST)
     test_option_help.c
     test_option_l.c
     test_option_lrzip.c
+    test_option_lz4.c
     test_option_lzma.c
     test_option_lzop.c
     test_option_m.c

cpio/test/test_option_lz4.c

@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2014 Michihiro NAKAJIMA
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_option_lz4)
+{
+	char *p;
+	int r;
+	size_t s;
+
+	/* Create a file. */
+	assertMakeFile("f", 0644, "a");
+
+	/* Archive it with lz4 compression. */
+	r = systemf("echo f | %s -o --lz4 >archive.out 2>archive.err",
+	    testprog);
+	p = slurpfile(&s, "archive.err");
+	p[s] = '\0';
+	if (r != 0) {
+		if (strstr(p, "compression not available") != NULL) {
+			skipping("This version of bsdcpio was compiled "
+			    "without lz4 support");
+			return;
+		}
+		failure("--lz4 option is broken");
+		assertEqualInt(r, 0);
+		return;
+	}
+	/* Check that the archive file has an lz4 signature. */
+	p = slurpfile(&s, "archive.out");
+	assert(s > 2);
+	assertEqualMem(p, "\x04\x22\x4d\x18", 4);
+}

libarchive/CMakeLists.txt

@@ -115,6 +115,7 @@ SET(libarchive_SOURCES
   archive_write_add_filter_grzip.c
   archive_write_add_filter_gzip.c
   archive_write_add_filter_lrzip.c
+  archive_write_add_filter_lz4.c
   archive_write_add_filter_lzop.c
   archive_write_add_filter_none.c
   archive_write_add_filter_program.c

libarchive/archive.h

@@ -715,6 +715,7 @@ __LA_DECL int archive_write_add_filter_compress(struct archive *);
 __LA_DECL int archive_write_add_filter_grzip(struct archive *);
 __LA_DECL int archive_write_add_filter_gzip(struct archive *);
 __LA_DECL int archive_write_add_filter_lrzip(struct archive *);
+__LA_DECL int archive_write_add_filter_lz4(struct archive *);
 __LA_DECL int archive_write_add_filter_lzip(struct archive *);
 __LA_DECL int archive_write_add_filter_lzma(struct archive *);
 __LA_DECL int archive_write_add_filter_lzop(struct archive *);

libarchive/archive_util.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2012 Michihiro NAKAJIMA
+ * Copyright (c) 2009-2012,2014 Michihiro NAKAJIMA
  * Copyright (c) 2003-2007 Tim Kientzle
  * All rights reserved.
  *
@@ -54,6 +54,9 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_util.c 201098 2009-12-28 02:58:1
 #ifdef HAVE_BZLIB_H
 #include <bzlib.h>
 #endif
+#ifdef HAVE_LZ4_H
+#include <lz4.h>
+#endif
 
 #include "archive.h"
 #include "archive_private.h"
@@ -113,7 +116,11 @@ archive_version_details(void)
 			archive_strncat(&str, p, sep - p);
 		}
 #endif
-	    }
+#if defined(HAVE_LZ4_H) && defined(HAVE_LIBLZ4)
+		archive_string_sprintf(&str, " liblz4/%d.%d.%d",
+		    LZ4_VERSION_MAJOR, LZ4_VERSION_MINOR, LZ4_VERSION_RELEASE);
+#endif
+	}
 	return str.s;
 }
 

libarchive/archive_write_add_filter.c

@@ -47,6 +47,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] =
 	{ ARCHIVE_FILTER_COMPRESS,	archive_write_add_filter_compress },
 	{ ARCHIVE_FILTER_GRZIP,		archive_write_add_filter_grzip },
 	{ ARCHIVE_FILTER_LRZIP,		archive_write_add_filter_lrzip },
+	{ ARCHIVE_FILTER_LZ4,		archive_write_add_filter_lz4 },
 	{ ARCHIVE_FILTER_LZIP,		archive_write_add_filter_lzip },
 	{ ARCHIVE_FILTER_LZMA,		archive_write_add_filter_lzma },
 	{ ARCHIVE_FILTER_LZOP,		archive_write_add_filter_lzip },

libarchive/archive_write_add_filter_by_name.c

@@ -51,6 +51,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] =
 	{ "grzip",		archive_write_add_filter_grzip },
 	{ "gzip",		archive_write_add_filter_gzip },
 	{ "lrzip",		archive_write_add_filter_lrzip },
+	{ "lz4",		archive_write_add_filter_lz4 },
 	{ "lzip",		archive_write_add_filter_lzip },
 	{ "lzma",		archive_write_add_filter_lzma },
 	{ "lzop",		archive_write_add_filter_lzop },

libarchive/archive_write_add_filter_lz4.c

@@ -0,0 +1,642 @@
+/*-
+ * Copyright (c) 2014 Michihiro NAKAJIMA
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "archive_platform.h"
+
+__FBSDID("$FreeBSD$");
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+#include <stdio.h>
+#ifdef HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#ifdef HAVE_LZ4_H
+#include <lz4.h>
+#endif
+#ifdef HAVE_LZ4HC_H
+#include <lz4hc.h>
+#endif
+
+#include "archive.h"
+#include "archive_endian.h"
+#include "archive_private.h"
+#include "archive_write_private.h"
+#include "archive_xxhash.h"
+
+#define LZ4_MAGICNUMBER	0x184d2204
+
+struct private_data {
+	int		 compression_level;
+	uint8_t		 header_written:1;
+	uint8_t		 version_number:1;
+	uint8_t		 block_independence:1;
+	uint8_t		 block_checksum:1;
+	uint8_t		 stream_size:1;
+	uint8_t		 stream_checksum:1;
+	uint8_t		 preset_dictionary:1;
+	uint8_t		 block_maximum_size:3;
+#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
+	int64_t		 total_in;
+	char		*out;
+	char		*out_buffer;
+	size_t		 out_buffer_size;
+	size_t		 out_block_size;
+	char		*in;
+	char		*in_buffer_allocated;
+	char		*in_buffer;
+	size_t		 in_buffer_size;
+	size_t		 block_size;
+
+	void		*xxh32_state;
+	void		*lz4_stream;
+#else
+	struct archive_write_program_data *pdata;
+#endif
+};
+
+static int archive_filter_lz4_close(struct archive_write_filter *);
+static int archive_filter_lz4_free(struct archive_write_filter *);
+static int archive_filter_lz4_open(struct archive_write_filter *);
+static int archive_filter_lz4_options(struct archive_write_filter *,
+		    const char *, const char *);
+static int archive_filter_lz4_write(struct archive_write_filter *,
+		    const void *, size_t);
+
+/*
+ * Add a lz4 compression filter to this write handle.
+ */
+int
+archive_write_add_filter_lz4(struct archive *_a)
+{
+	struct archive_write *a = (struct archive_write *)_a;
+	struct archive_write_filter *f = __archive_write_allocate_filter(_a);
+	struct private_data *data;
+
+	archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
+	    ARCHIVE_STATE_NEW, "archive_write_add_filter_lz4");
+
+	data = calloc(1, sizeof(*data));
+	if (data == NULL) {
+		archive_set_error(&a->archive, ENOMEM, "Out of memory");
+		return (ARCHIVE_FATAL);
+	}
+
+	/*
+	 * Setup default settings.
+	 */
+	data->compression_level = 1;
+	data->version_number = 0x01;
+	data->block_independence = 1;
+	data->block_checksum = 0;
+	data->stream_size = 0;
+	data->stream_checksum = 1;
+	data->preset_dictionary = 0;
+	data->block_maximum_size = 7;
+
+	/*
+	 * Setup a filter setting.
+	 */
+	f->data = data;
+	f->options = &archive_filter_lz4_options;
+	f->close = &archive_filter_lz4_close;
+	f->free = &archive_filter_lz4_free;
+	f->open = &archive_filter_lz4_open;
+	f->code = ARCHIVE_FILTER_LZ4;
+	f->name = "lz4";
+#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
+	return (ARCHIVE_OK);
+#else
+	/*
+	 * We don't have lz4 library, and execute external lz4 program
+	 * instead.
+	 */
+	data->pdata = __archive_write_program_allocate();
+	if (data->pdata == NULL) {
+		free(data);
+		archive_set_error(&a->archive, ENOMEM, "Out of memory");
+		return (ARCHIVE_FATAL);
+	}
+	data->compression_level = 0;
+	archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+	    "Using external lz4 program");
+	return (ARCHIVE_WARN);
+#endif
+}
+
+/*
+ * Set write options.
+ */
+static int
+archive_filter_lz4_options(struct archive_write_filter *f,
+    const char *key, const char *value)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	if (strcmp(key, "compression-level") == 0) {
+		if (value == NULL || !(value[0] >= '1' && value[0] <= '9') ||
+		    value[1] != '\0')
+			return (ARCHIVE_WARN);
+		data->compression_level = value[0] - '0';
+		return (ARCHIVE_OK);
+	}
+	if (strcmp(key, "stream-checksum") == 0) {
+		data->stream_checksum = value != NULL;
+		return (ARCHIVE_OK);
+	}
+	if (strcmp(key, "block-checksum") == 0) {
+		data->block_checksum = value != NULL;
+		return (ARCHIVE_OK);
+	}
+	if (strcmp(key, "block-size") == 0) {
+		if (value == NULL || !(value[0] >= '4' && value[0] <= '7') ||
+		    value[1] != '\0')
+			return (ARCHIVE_WARN);
+		data->block_maximum_size = value[0] - '0';
+		return (ARCHIVE_OK);
+	}
+	if (strcmp(key, "block-dependence") == 0) {
+		data->block_independence = value == NULL;
+		return (ARCHIVE_OK);
+	}
+
+	/* Note: The "warn" return is just to inform the options
+	 * supervisor that we didn't handle it.  It will generate
+	 * a suitable error if no one used this option. */
+	return (ARCHIVE_WARN);
+}
+
+#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
+/* Don't compile this if we don't have liblz4. */
+
+static int drive_compressor(struct archive_write_filter *, const char *,
+    size_t);
+static int drive_compressor_independence(struct archive_write_filter *,
+    const char *, size_t);
+static int drive_compressor_dependence(struct archive_write_filter *,
+    const char *, size_t);
+static int lz4_write_stream_descriptor(struct archive_write_filter *);
+static ssize_t lz4_write_one_block(struct archive_write_filter *, const char *,
+    size_t);
+
+
+/*
+ * Setup callback.
+ */
+static int
+archive_filter_lz4_open(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int ret;
+	size_t required_size;
+	static size_t bkmap[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024,
+			   4 * 1024 * 1024 };
+	size_t pre_block_size;
+
+	ret = __archive_write_open_filter(f->next_filter);
+	if (ret != 0)
+		return (ret);
+
+	if (data->block_maximum_size < 4)
+		data->block_size = bkmap[0];
+	else
+		data->block_size = bkmap[data->block_maximum_size - 4];
+
+	required_size = 4 + 15 + 4 + data->block_size + 4 + 4;
+	if (data->out_buffer_size < required_size) {
+		size_t bs = required_size, bpb;
+		free(data->out_buffer);
+		if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
+			/* Buffer size should be a multiple number of
+			 * the of bytes per block for performance. */
+			bpb = archive_write_get_bytes_per_block(f->archive);
+			if (bpb > bs)
+				bs = bpb;
+			else if (bpb != 0) {
+				bs += bpb;
+				bs -= bs % bpb;
+			}
+		}
+		data->out_block_size = bs;
+		bs += required_size;
+		data->out_buffer = malloc(bs);
+		data->out = data->out_buffer;
+		data->out_buffer_size = bs;
+	}
+
+	pre_block_size = (data->block_independence)? 0: 64 * 1024;
+	if (data->in_buffer_size < data->block_size + pre_block_size) {
+		free(data->in_buffer_allocated);
+		data->in_buffer_size = data->block_size;
+		data->in_buffer_allocated =
+		    malloc(data->in_buffer_size + pre_block_size);
+		data->in_buffer = data->in_buffer_allocated + pre_block_size;
+		if (!data->block_independence && data->compression_level >= 3)
+		    data->in_buffer = data->in_buffer_allocated;
+		data->in = data->in_buffer;
+		data->in_buffer_size = data->block_size;
+	}
+
+	if (data->out_buffer == NULL || data->in_buffer_allocated == NULL) {
+		archive_set_error(f->archive, ENOMEM,
+		    "Can't allocate data for compression buffer");
+		return (ARCHIVE_FATAL);
+	}
+
+	f->write = archive_filter_lz4_write;
+
+	return (ARCHIVE_OK);
+}
+
+/*
+ * Write data to the out stream.
+ *
+ * Returns ARCHIVE_OK if all data written, error otherwise.
+ */
+static int
+archive_filter_lz4_write(struct archive_write_filter *f,
+    const void *buff, size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int ret = ARCHIVE_OK;
+	const char *p;
+	size_t remaining;
+	ssize_t size;
+
+	/* If we haven't written a stream descriptor, we have to do it first. */
+	if (!data->header_written) {
+		ret = lz4_write_stream_descriptor(f);
+		if (ret != ARCHIVE_OK)
+			return (ret);
+		data->header_written = 1;
+	}
+
+	/* Update statistics */
+	data->total_in += length;
+
+	p = (const char *)buff;
+	remaining = length;
+	while (remaining) {
+		size_t l;
+		/* Compress input data to output buffer */
+		size = lz4_write_one_block(f, p, remaining);
+		if (size < ARCHIVE_OK)
+			return (ARCHIVE_FATAL);
+		l = data->out - data->out_buffer;
+		if (l >= data->out_block_size) {
+			ret = __archive_write_filter(f->next_filter,
+			    data->out_buffer, data->out_block_size);
+			l -= data->out_block_size;
+			memcpy(data->out_buffer,
+			    data->out_buffer + data->out_block_size, l);
+			data->out = data->out_buffer + l;
+			if (ret < ARCHIVE_WARN)
+				break;
+		}
+		p += size;
+		remaining -= size;
+	}
+
+	return (ret);
+}
+
+/*
+ * Finish the compression.
+ */
+static int
+archive_filter_lz4_close(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int ret, r1;
+
+	/* Finish compression cycle. */
+	ret = (int)lz4_write_one_block(f, NULL, 0);
+	if (ret >= 0) {
+		/*
+		 * Write the last block and the end of the stream data.
+		 */
+
+		/* Write End Of Stream. */
+		memset(data->out, 0, 4); data->out += 4;
+		/* Write Stream checksum if needed. */
+		if (data->stream_checksum) {
+			unsigned int checksum;
+			checksum = __archive_xxhash.XXH32_digest(
+					data->xxh32_state);
+			data->xxh32_state = NULL;
+			archive_le32enc(data->out, checksum);
+			data->out += 4;
+		}
+		ret = __archive_write_filter(f->next_filter,
+			    data->out_buffer, data->out - data->out_buffer);
+	}
+
+	r1 = __archive_write_close_filter(f->next_filter);
+	return (r1 < ret ? r1 : ret);
+}
+
+static int
+archive_filter_lz4_free(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	if (data->lz4_stream != NULL) {
+		if (data->compression_level < 3)
+			LZ4_free(data->lz4_stream);
+		else
+			LZ4_freeHC(data->lz4_stream);
+	}
+	free(data->out_buffer);
+	free(data->in_buffer_allocated);
+	free(data->xxh32_state);
+	free(data);
+	f->data = NULL;
+	return (ARCHIVE_OK);
+}
+
+static int
+lz4_write_stream_descriptor(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	uint8_t *sd;
+
+	sd = (uint8_t *)data->out;
+	/* Write Magic Number. */
+	archive_le32enc(&sd[0], LZ4_MAGICNUMBER);
+	/* FLG */
+	sd[4] = (data->version_number << 6)
+	      | (data->block_independence << 5)
+	      | (data->block_checksum << 4)
+	      | (data->stream_size << 3)
+	      | (data->stream_checksum << 2)
+	      | (data->preset_dictionary << 0);
+	/* BD */
+	sd[5] = (data->block_maximum_size << 4);
+	sd[6] = (__archive_xxhash.XXH32(&sd[4], 2, 0) >> 8) & 0xff;
+	data->out += 7;
+	if (data->stream_checksum)
+		data->xxh32_state = __archive_xxhash.XXH32_init(0);
+	else
+		data->xxh32_state = NULL;
+	return (ARCHIVE_OK);
+}
+
+static ssize_t
+lz4_write_one_block(struct archive_write_filter *f, const char *p,
+    size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	ssize_t r;
+
+	if (p == NULL && length == 0) {
+		/* Compress remaining uncompressed data. */
+		if (data->in_buffer == data->in)
+			return 0;
+		else {
+			size_t l = data->in - data->in_buffer;
+			r = drive_compressor(f, data->in_buffer, l);
+			if (r == ARCHIVE_OK)
+				r = (ssize_t)l;
+		}
+	} else if ((data->block_independence || data->compression_level < 3) &&
+	    data->in_buffer == data->in && length >= data->block_size) {
+		r = drive_compressor(f, p, data->block_size);
+		if (r == ARCHIVE_OK)
+			r = (ssize_t)data->block_size;
+	} else {
+		size_t remaining_size = data->in_buffer_size -
+			(data->in - data->in_buffer);
+		size_t l = (remaining_size > length)? length: remaining_size;
+		memcpy(data->in, p, l);
+		data->in += l;
+		if (l == remaining_size) {
+			r = drive_compressor(f, data->in_buffer,
+			    data->block_size);
+			if (r == ARCHIVE_OK)
+				r = (ssize_t)l;
+			data->in = data->in_buffer;
+		} else
+			r = (ssize_t)l;
+	}
+
+	return (r);
+}
+
+
+/*
+ * Utility function to push input data through compressor, writing
+ * full output blocks as necessary.
+ *
+ * Note that this handles both the regular write case (finishing ==
+ * false) and the end-of-archive case (finishing == true).
+ */
+static int
+drive_compressor(struct archive_write_filter *f, const char *p, size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	if (data->stream_checksum)
+		__archive_xxhash.XXH32_update(data->xxh32_state,
+			p, (int)length);
+	if (data->block_independence)
+		return drive_compressor_independence(f, p, length);
+	else
+		return drive_compressor_dependence(f, p, length);
+}
+
+static int
+drive_compressor_independence(struct archive_write_filter *f, const char *p,
+    size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	unsigned int outsize;
+
+	if (data->compression_level < 4)
+		outsize = LZ4_compress_limitedOutput(p, data->out + 4,
+		    (int)length, (int)data->block_size);
+	else
+		outsize = LZ4_compressHC2_limitedOutput(p, data->out + 4,
+		    (int)length, (int)data->block_size,
+		    data->compression_level);
+
+	if (outsize) {
+		/* The buffer is compressed. */
+		archive_le32enc(data->out, outsize);
+		data->out += 4;
+	} else {
+		/* The buffer is not compressed. The commpressed size was
+		 * bigger than its uncompressed size. */
+		archive_le32enc(data->out, length | 0x80000000);
+		data->out += 4;
+		memcpy(data->out, p, length);
+		outsize = length;
+	}
+	data->out += outsize;
+	if (data->block_checksum) {
+		unsigned int checksum =
+		    __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
+		archive_le32enc(data->out, checksum);
+		data->out += 4;
+	}
+	return (ARCHIVE_OK);
+}
+
+static int
+drive_compressor_dependence(struct archive_write_filter *f, const char *p,
+    size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int outsize;
+
+	if (data->compression_level < 3) {
+		if (data->lz4_stream == NULL) {
+			data->lz4_stream = LZ4_createStream();
+			if (data->lz4_stream == NULL) {
+				archive_set_error(f->archive, ENOMEM,
+				    "Can't allocate data for compression"
+				    " buffer");
+				return (ARCHIVE_FATAL);
+			}
+		}
+		outsize = LZ4_compress_limitedOutput_continue(
+		    data->lz4_stream, p, data->out + 4, (int)length,
+		    (int)data->block_size);
+	} else {
+		if (data->lz4_stream == NULL) {
+			data->lz4_stream =
+			    LZ4_createHC(data->in_buffer_allocated);
+			if (data->lz4_stream == NULL) {
+				archive_set_error(f->archive, ENOMEM,
+				    "Can't allocate data for compression"
+				    " buffer");
+				return (ARCHIVE_FATAL);
+			}
+		}
+		outsize = LZ4_compressHC2_limitedOutput_continue(
+		    data->lz4_stream, p, data->out + 4, (int)length,
+		    (int)data->block_size, data->compression_level);
+	}
+
+	if (outsize) {
+		/* The buffer is compressed. */
+		archive_le32enc(data->out, outsize);
+		data->out += 4;
+	} else {
+		/* The buffer is not compressed. The commpressed size was
+		 * bigger than its uncompressed size. */
+		archive_le32enc(data->out, length | 0x80000000);
+		data->out += 4;
+		memcpy(data->out, p, length);
+		outsize = length;
+	}
+	data->out += outsize;
+	if (data->block_checksum) {
+		unsigned int checksum =
+		    __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
+		archive_le32enc(data->out, checksum);
+		data->out += 4;
+	}
+
+	if (length == data->block_size) {
+#define DICT_SIZE	(64 * 1024)
+		if (data->compression_level < 3)
+			LZ4_saveDict(data->lz4_stream,
+			    data->in_buffer_allocated, DICT_SIZE);
+		else {
+			LZ4_slideInputBufferHC(data->lz4_stream);
+			data->in_buffer = data->in_buffer_allocated + DICT_SIZE;
+		}
+#undef DICT_SIZE
+	}
+	return (ARCHIVE_OK);
+}
+
+#else /* HAVE_LIBLZ4 */
+
+static int
+archive_filter_lz4_open(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	struct archive_string as;
+	int r;
+
+	archive_string_init(&as);
+	archive_strcpy(&as, "lz4 -z -q -q");
+
+	/* Specify a compression level. */
+	if (data->compression_level > 0) {
+		archive_strcat(&as, " -");
+		archive_strappend_char(&as, '0' + data->compression_level);
+	}
+	/* Specify a block size. */
+	archive_strcat(&as, " -B");
+	archive_strappend_char(&as, '0' + data->block_maximum_size);
+
+	if (data->block_checksum)
+		archive_strcat(&as, " -BX");
+	if (data->stream_checksum == 0)
+		archive_strcat(&as, " -Sx");
+	if (data->block_independence == 0)
+		archive_strcat(&as, " -BD");
+
+	f->write = archive_filter_lz4_write;
+
+	r = __archive_write_program_open(f, data->pdata, as.s);
+	archive_string_free(&as);
+	return (r);
+}
+
+static int
+archive_filter_lz4_write(struct archive_write_filter *f, const void *buff,
+    size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	return __archive_write_program_write(f, data->pdata, buff, length);
+}
+
+static int
+archive_filter_lz4_close(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	return __archive_write_program_close(f, data->pdata);
+}
+
+static int
+archive_filter_lz4_free(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	__archive_write_program_free(data->pdata);
+	free(data);
+	return (ARCHIVE_OK);
+}
+
+#endif /* HAVE_LIBLZ4 */

libarchive/test/CMakeLists.txt

@@ -192,6 +192,7 @@ IF(ENABLE_TEST)
     test_write_filter_gzip.c
     test_write_filter_gzip_timestamp.c
     test_write_filter_lrzip.c
+    test_write_filter_lz4.c
     test_write_filter_lzip.c
     test_write_filter_lzma.c
     test_write_filter_lzop.c

libarchive/test/test_write_filter_lz4.c

@@ -0,0 +1,392 @@
+/*-
+ * Copyright (c) 2014 Michihiro NAKAJIMA
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+/*
+ * A basic exercise of lz4 reading and writing.
+ */
+
+DEFINE_TEST(test_write_filter_lz4)
+{
+	struct archive_entry *ae;
+	struct archive* a;
+	char *buff, *data;
+	size_t buffsize, datasize;
+	char path[16];
+	size_t used1, used2;
+	int i, r, use_prog = 0, filecount;
+
+	assert((a = archive_write_new()) != NULL);
+	r = archive_write_add_filter_lz4(a);
+	if (r != ARCHIVE_OK) {
+		if (canLzop() && r == ARCHIVE_WARN)
+			use_prog = 1;
+		else {
+			skipping("lz4 writing not supported on this platform");
+			assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+			return;
+		}
+	}
+
+	buffsize = 2000000;
+	assert(NULL != (buff = (char *)malloc(buffsize)));
+
+	datasize = 10000;
+	assert(NULL != (data = (char *)calloc(1, datasize)));
+	filecount = 10;
+
+	/*
+	 * Write a filecount files and read them all back.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_per_block(a, 1024));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_in_last_block(a, 1024));
+	assertEqualInt(ARCHIVE_FILTER_LZ4, archive_filter_code(a, 0));
+	assertEqualString("lz4", archive_filter_name(a, 0));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_open_memory(a, buff, buffsize, &used1));
+	assert((ae = archive_entry_new()) != NULL);
+	archive_entry_set_filetype(ae, AE_IFREG);
+	archive_entry_set_size(ae, datasize);
+	for (i = 0; i < filecount; i++) {
+		sprintf(path, "file%03d", i);
+		archive_entry_copy_pathname(ae, path);
+		assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
+		assertA(datasize
+		    == (size_t)archive_write_data(a, data, datasize));
+	}
+	archive_entry_free(ae);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	r = archive_read_support_filter_lz4(a);
+	if (r == ARCHIVE_WARN) {
+		skipping("Can't verify lz4 writing by reading back;"
+		    " lz4 reading not fully supported on this platform");
+	} else {
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_open_memory(a, buff, used1));
+		for (i = 0; i < filecount; i++) {
+			sprintf(path, "file%03d", i);
+			if (!assertEqualInt(ARCHIVE_OK,
+				archive_read_next_header(a, &ae)))
+				break;
+			assertEqualString(path, archive_entry_pathname(ae));
+			assertEqualInt((int)datasize, archive_entry_size(ae));
+		}
+		assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	}
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+	/*
+	 * Repeat the cycle again, this time setting some compression
+	 * options.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_per_block(a, 10));
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualIntA(a, ARCHIVE_FAILED,
+	    archive_write_set_options(a, "lz4:nonexistent-option=0"));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_options(a, "lz4:compression-level=1"));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "9"));
+	assertEqualIntA(a, ARCHIVE_FAILED,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "abc"));
+	assertEqualIntA(a, ARCHIVE_FAILED,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "99"));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_options(a, "lz4:compression-level=9"));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_open_memory(a, buff, buffsize, &used2));
+	for (i = 0; i < filecount; i++) {
+		sprintf(path, "file%03d", i);
+		assert((ae = archive_entry_new()) != NULL);
+		archive_entry_copy_pathname(ae, path);
+		archive_entry_set_size(ae, datasize);
+		archive_entry_set_filetype(ae, AE_IFREG);
+		assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
+		assertA(datasize == (size_t)archive_write_data(
+		    a, data, datasize));
+		archive_entry_free(ae);
+	}
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	failure("compression-level=9 wrote %d bytes, default wrote %d bytes",
+	    (int)used2, (int)used1);
+	assert(used2 < used1);
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	r = archive_read_support_filter_lz4(a);
+	if (r != ARCHIVE_OK && !use_prog) {
+		skipping("lz4 reading not fully supported on this platform");
+	} else {
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_support_filter_all(a));
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_open_memory(a, buff, used2));
+		for (i = 0; i < filecount; i++) {
+			sprintf(path, "file%03d", i);
+			if (!assertEqualInt(ARCHIVE_OK,
+				archive_read_next_header(a, &ae)))
+				break;
+			assertEqualString(path, archive_entry_pathname(ae));
+			assertEqualInt((int)datasize, archive_entry_size(ae));
+		}
+		assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	}
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+	/*
+	 * Repeat again, with much lower compression.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_per_block(a, 10));
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "1"));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_open_memory(a, buff, buffsize, &used2));
+	for (i = 0; i < filecount; i++) {
+		sprintf(path, "file%03d", i);
+		assert((ae = archive_entry_new()) != NULL);
+		archive_entry_copy_pathname(ae, path);
+		archive_entry_set_size(ae, datasize);
+		archive_entry_set_filetype(ae, AE_IFREG);
+		assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
+		failure("Writing file %s", path);
+		assertEqualIntA(a, datasize,
+		    (size_t)archive_write_data(a, data, datasize));
+		archive_entry_free(ae);
+	}
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+#if 0
+	failure("Compression-level=1 wrote %d bytes; default wrote %d bytes",
+	    (int)used2, (int)used1);
+	assert(used2 > used1);
+#endif
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
+	r = archive_read_support_filter_lz4(a);
+	if (r == ARCHIVE_WARN) {
+		skipping("lz4 reading not fully supported on this platform");
+	} else {
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_open_memory(a, buff, used2));
+		for (i = 0; i < filecount; i++) {
+			sprintf(path, "file%03d", i);
+			if (!assertEqualInt(ARCHIVE_OK,
+				archive_read_next_header(a, &ae)))
+				break;
+			assertEqualString(path, archive_entry_pathname(ae));
+			assertEqualInt((int)datasize, archive_entry_size(ae));
+		}
+		assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	}
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+	/*
+	 * Test various premature shutdown scenarios to make sure we
+	 * don't crash or leak memory.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_open_memory(a, buff, buffsize, &used2));
+	assertEqualInt(ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	/*
+	 * Clean up.
+	 */
+	free(data);
+	free(buff);
+}
+
+static void
+test_options(const char *options)
+{
+	struct archive_entry *ae;
+	struct archive* a;
+	char *buff, *data;
+	size_t buffsize, datasize;
+	char path[16];
+	size_t used1;
+	int i, r, use_prog = 0, filecount;
+
+	assert((a = archive_write_new()) != NULL);
+	r = archive_write_add_filter_lz4(a);
+	if (r != ARCHIVE_OK) {
+		if (canLzop() && r == ARCHIVE_WARN)
+			use_prog = 1;
+		else {
+			skipping("lz4 writing not supported on this platform");
+			assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+			return;
+		}
+	}
+
+	buffsize = 2000000;
+	assert(NULL != (buff = (char *)malloc(buffsize)));
+
+	datasize = 10000;
+	assert(NULL != (data = (char *)calloc(1, datasize)));
+	filecount = 10;
+
+	/*
+	 * Write a filecount files and read them all back.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, (use_prog)?ARCHIVE_WARN:ARCHIVE_OK,
+	    archive_write_add_filter_lz4(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_options(a, options));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_per_block(a, 1024));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_in_last_block(a, 1024));
+	assertEqualInt(ARCHIVE_FILTER_LZ4, archive_filter_code(a, 0));
+	assertEqualString("lz4", archive_filter_name(a, 0));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_open_memory(a, buff, buffsize, &used1));
+	assert((ae = archive_entry_new()) != NULL);
+	archive_entry_set_filetype(ae, AE_IFREG);
+	archive_entry_set_size(ae, datasize);
+	for (i = 0; i < filecount; i++) {
+		sprintf(path, "file%03d", i);
+		archive_entry_copy_pathname(ae, path);
+		assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
+		assertA(datasize
+		    == (size_t)archive_write_data(a, data, datasize));
+	}
+	archive_entry_free(ae);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	r = archive_read_support_filter_lz4(a);
+	if (r == ARCHIVE_WARN) {
+		skipping("Can't verify lz4 writing by reading back;"
+		    " lz4 reading not fully supported on this platform");
+	} else {
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_open_memory(a, buff, used1));
+		for (i = 0; i < filecount; i++) {
+			sprintf(path, "file%03d", i);
+			if (!assertEqualInt(ARCHIVE_OK,
+				archive_read_next_header(a, &ae)))
+				break;
+			assertEqualString(path, archive_entry_pathname(ae));
+			assertEqualInt((int)datasize, archive_entry_size(ae));
+		}
+		assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	}
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+	/*
+	 * Clean up.
+	 */
+	free(data);
+	free(buff);
+}
+
+DEFINE_TEST(test_write_filter_lz4_disable_stream_checksum)
+{
+	test_options("lz4:!stream-checksum");
+}
+
+DEFINE_TEST(test_write_filter_lz4_enable_block_checksum)
+{
+	test_options("lz4:block-checksum");
+}
+
+DEFINE_TEST(test_write_filter_lz4_block_size_4)
+{
+	test_options("lz4:block-size=4");
+}
+
+DEFINE_TEST(test_write_filter_lz4_block_size_5)
+{
+	test_options("lz4:block-size=5");
+}
+
+DEFINE_TEST(test_write_filter_lz4_block_size_6)
+{
+	test_options("lz4:block-size=6");
+}
+
+DEFINE_TEST(test_write_filter_lz4_block_dependence)
+{
+	test_options("lz4:block-dependence");
+}
+
+DEFINE_TEST(test_write_filter_lz4_block_dependence_hc)
+{
+	test_options("lz4:block-dependence,lz4:compression-level=9");
+}

tar/bsdtar.1

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 1, 2012
+.Dd August 14, 2014
 .Dt TAR 1
 .Os
 .Sh NAME
@@ -318,6 +318,11 @@ Issue a warning message unless all links to each file are archived.
 Compress the resulting archive with
 .Xr lrzip 1 .
 In extract or list modes, this option is ignored.
+.It Fl Fl lz4
+(c mode only)
+Compress the archive with lz4-compatible compression before writing it.
+In input mode, this option is ignored; lz4 compression is recognized
+automatically on input.
 .It Fl Fl lzma
 (c mode only) Compress the resulting archive with the original LZMA algorithm.
 Use of this option is discouraged and new archives should be created with
@@ -498,6 +503,20 @@ Supported values are bzip2, gzip, lzo (ultra fast),
 and zpaq (best, extremely slow).
 .It Cm lrzip:compression-level
 A decimal integer from 1 to 9 specifying the lrzip compression level.
+.It Cm lz4:compression-level
+A decimal integer from 1 to 9 specifying the lzop compression level.
+.It Cm lz4:stream-checksum
+Enalbe stream checksum. This is by default, use
+.Cm lz4:!stream-checksum
+to disable.
+.It Cm lz4:block-checksum
+Enable block checksum (Disabled by default).
+.It Cm lz4:block-size
+A decimal integer from 4 to 7 specifying the lz4 compression block size
+(7 is set by default).
+.It Cm lz4:block-dependence
+Use the previous block of the block being compressed for
+a compression dictionary to improve compression ratio.
 .It Cm lzop:compression-level
 A decimal integer from 1 to 9 specifying the lzop compression level.
 .It Cm xz:compression-level

tar/bsdtar.c

@@ -397,6 +397,7 @@ main(int argc, char **argv)
 			bsdtar->option_warn_links = 1;
 			break;
 		case OPTION_LRZIP:
+		case OPTION_LZ4:
 		case OPTION_LZIP: /* GNU tar beginning with 1.23 */
 		case OPTION_LZMA: /* GNU tar beginning with 1.20 */
 		case OPTION_LZOP: /* GNU tar beginning with 1.21 */
@@ -407,6 +408,7 @@ main(int argc, char **argv)
 			compression = opt;
 			switch (opt) {
 			case OPTION_LRZIP: compression_name = "lrzip"; break;
+			case OPTION_LZ4:  compression_name = "lz4"; break;
 			case OPTION_LZIP: compression_name = "lzip"; break; 
 			case OPTION_LZMA: compression_name = "lzma"; break; 
 			case OPTION_LZOP: compression_name = "lzop"; break; 

tar/bsdtar.h

@@ -129,6 +129,7 @@ enum {
 	OPTION_INCLUDE,
 	OPTION_KEEP_NEWER_FILES,
 	OPTION_LRZIP,
+	OPTION_LZ4,
 	OPTION_LZIP,
 	OPTION_LZMA,
 	OPTION_LZOP,

tar/cmdline.c

@@ -102,6 +102,7 @@ static const struct bsdtar_option {
 	{ "keep-old-files",       0, 'k' },
 	{ "list",                 0, 't' },
 	{ "lrzip",                0, OPTION_LRZIP },
+	{ "lz4",                  0, OPTION_LZ4 },
 	{ "lzip",                 0, OPTION_LZIP },
 	{ "lzma",                 0, OPTION_LZMA },
 	{ "lzop",                 0, OPTION_LZOP },

tar/creation_set.c

@@ -75,6 +75,7 @@ get_filter_code(const char *suffix)
 		{ ".grz",	"grzip" },
 		{ ".lrz",	"lrzip" },
 		{ ".lz",	"lzip" },
+		{ ".lz4",	"lz4" },
 		{ ".lzo",	"lzop" },
 		{ ".lzma",	"lzma" },
 		{ ".uu",	"uuencode" },

tar/test/CMakeLists.txt

@@ -18,6 +18,7 @@ IF(ENABLE_TAR AND ENABLE_TEST)
     test_extract_tar_gz.c
     test_extract_tar_lrz.c
     test_extract_tar_lz.c
+    test_extract_tar_lz4.c
     test_extract_tar_lzma.c
     test_extract_tar_lzo.c
     test_extract_tar_xz.c

tar/test/test_option_lz4.c

@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2014 Michihiro NAKAJIMA
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_option_lz4)
+{
+	char *p;
+	int r;
+	size_t s;
+
+	/* Create a file. */
+	assertMakeFile("f", 0644, "a");
+
+	/* Archive it with lz4 compression. */
+	r = systemf("%s -cf - --lz4 f >archive.out 2>archive.err",
+	    testprog);
+	p = slurpfile(&s, "archive.err");
+	p[s] = '\0';
+	if (r != 0) {
+		if (strstr(p, "Unsupported compression") != NULL) {
+			skipping("This version of bsdtar was compiled "
+			    "without lz4 support");
+			return;
+		}
+		failure("--lz4 option is broken");
+		assertEqualInt(r, 0);
+		return;
+	}
+	/* Check that the archive file has an lz4 signature. */
+	p = slurpfile(&s, "archive.out");
+	assert(s > 2);
+	assertEqualMem(p, "\x04\x22\x4d\x18", 4);
+}