Merge pull request #3114 from stoeckmann/open_oob

Fix OOB in `archive_read_open_filenames_w` on some systems

(cherry picked from commit 7957d6d3b0)

libarchive/archive_read_open_filename.c

@@ -40,6 +40,9 @@
 #ifdef HAVE_IO_H
 #include <io.h>
 #endif
+#ifdef HAVE_LIMITS_H
+#include <limits.h>
+#endif
 #ifdef HAVE_STDLIB_H
 #include <stdlib.h>
 #endif
@@ -181,7 +184,7 @@ archive_read_open_filenames_w(struct archive *a, const wchar_t **wfilenames,
 		if (wfilename == NULL)
 			wfilename = L"";
 		mine = calloc(1,
-			sizeof(*mine) + wcslen(wfilename) * sizeof(wchar_t));
+			sizeof(*mine) + wcslen(wfilename) * MB_LEN_MAX);
 		if (mine == NULL)
 			goto no_memory;
 		mine->block_size = block_size;

libarchive/test/test_open_filename.c

@@ -24,6 +24,8 @@
  */
 #include "test.h"
 
+#include <locale.h>
+
 static void
 test_open_filename_mbs(void)
 {
@@ -197,3 +199,24 @@ DEFINE_TEST(test_open_filename)
 	test_open_filename_mbs();
 	test_open_filename_wcs();
 }
+
+DEFINE_TEST(test_open_filename_wcs_oob)
+{
+	struct archive *a;
+
+	/* Continue even if not available for best test coverage. */
+	setlocale(LC_ALL, "en_US.UTF-8");
+
+	/*
+	 * Try to open file with filename which might use more bytes in
+	 * MBS representation than in wchar_t if platform uses 16 bit wchar_t,
+	 * e.g. Cygwin (3 times a Euro sign, i.e. 3*2+2=8 vs 3*3+1=10 bytes).
+	 */
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
+	assertEqualIntA(a, ARCHIVE_FATAL,
+	    archive_read_open_filename_w(a, L"\u20AC\u20AC\u20AC", 512));
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+}