rar: Support large headers on 32 bit systems (#2596)

Support header sizes larger than 32 bit even on 32 bit systems, since
these normally have large file support. Otherwise an unsigned integer
overflow could occur, leading to erroneous parsing on these systems.

Signed-off-by: Tobias Stoeckmann <tobias@stoeckmann.org>

Makefile.am

@@ -888,6 +888,7 @@ libarchive_test_EXTRA_DIST=\
 	libarchive/test/test_read_format_rar_encryption_data.rar.uu \
 	libarchive/test/test_read_format_rar_encryption_header.rar.uu \
 	libarchive/test/test_read_format_rar_encryption_partially.rar.uu \
+	libarchive/test/test_read_format_rar_endarc_huge.rar.uu \
 	libarchive/test/test_read_format_rar_filter.rar.uu \
 	libarchive/test/test_read_format_rar_invalid1.rar.uu \
 	libarchive/test/test_read_format_rar_multi_lzss_blocks.rar.uu \

libarchive/archive_read_support_format_rar.c

@@ -910,7 +910,7 @@ archive_read_format_rar_read_header(struct archive_read *a,
   const void *h;
   const char *p;
   struct rar *rar;
-  size_t skip;
+  int64_t skip;
   char head_type;
   int ret;
   unsigned flags;
@@ -972,7 +972,7 @@ archive_read_format_rar_read_header(struct archive_read *a,
     case MAIN_HEAD:
       rar->main_flags = archive_le16dec(p + 3);
       skip = archive_le16dec(p + 5);
-      if (skip < 7 + sizeof(rar->reserved1) + sizeof(rar->reserved2)) {
+      if ((size_t)skip < 7 + sizeof(rar->reserved1) + sizeof(rar->reserved2)) {
         archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
           "Invalid header size");
         return (ARCHIVE_FATAL);
@@ -984,7 +984,8 @@ archive_read_format_rar_read_header(struct archive_read *a,
       memcpy(rar->reserved2, p + 7 + sizeof(rar->reserved1),
              sizeof(rar->reserved2));
       if (rar->main_flags & MHD_ENCRYPTVER) {
-        if (skip < 7 + sizeof(rar->reserved1) + sizeof(rar->reserved2)+1) {
+        if ((size_t)skip <
+            7 + sizeof(rar->reserved1) + sizeof(rar->reserved2) + 1) {
           archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
             "Invalid header size");
           return (ARCHIVE_FATAL);
@@ -1053,16 +1054,18 @@ archive_read_format_rar_read_header(struct archive_read *a,
       /* Skim the entire header and compute the CRC. */
       crc32_val = 0;
       while (skip > 0) {
-        size_t to_read = skip;
-        if (to_read > 32 * 1024)
+        unsigned to_read;
+        if (skip > 32 * 1024)
           to_read = 32 * 1024;
+        else
+          to_read = (unsigned)skip;
         if ((h = __archive_read_ahead(a, to_read, NULL)) == NULL) {
           archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
             "Bad RAR file");
           return (ARCHIVE_FATAL);
         }
         p = h;
-        crc32_val = crc32(crc32_val, (const unsigned char *)p, (unsigned int)to_read);
+        crc32_val = crc32(crc32_val, (const unsigned char *)p, to_read);
         __archive_read_consume(a, to_read);
         skip -= to_read;
       }

libarchive/test/test_read_format_rar.c

@@ -3810,6 +3810,26 @@ DEFINE_TEST(test_read_format_rar_multivolume_uncompressed_files)
   assertEqualIntA(a, ARCHIVE_OK, archive_read_free(a));
 }
 
+DEFINE_TEST(test_read_format_rar_endarc_huge)
+{
+  const char* reffile = "test_read_format_rar_endarc_huge.rar";
+
+  struct archive_entry *ae;
+  struct archive *a;
+
+  extract_reference_file(reffile);
+  assert((a = archive_read_new()) != NULL);
+  assertA(0 == archive_read_support_filter_all(a));
+  assertA(0 == archive_read_support_format_all(a));
+  assertA(0 == archive_read_open_filename(a, reffile, 10240));
+
+  /* Test for truncation */
+  assertA(ARCHIVE_FATAL == archive_read_next_header(a, &ae));
+
+  assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+  assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+}
+
 DEFINE_TEST(test_read_format_rar_ppmd_use_after_free)
 {
   uint8_t buf[16];

libarchive/test/test_read_format_rar_endarc_huge.rar.uu

@@ -0,0 +1,4 @@
+begin 644 test_read_format_rar_endarc_huge.rar
+24F%R(1H'````>P"`"P#W____
+`
+end