Merge pull request #3004 from kientzle/kientzle-rar5-bad-tables

[RAR5] FAIL if the decode table is > 2^16

Makefile.am

@@ -542,6 +542,7 @@ libarchive_test_SOURCES= \
 	libarchive/test/test_read_format_rar_invalid1.c \
 	libarchive/test/test_read_format_rar_overflow.c \
 	libarchive/test/test_read_format_rar5.c \
+	libarchive/test/test_read_format_rar5_bad_tables.c \
 	libarchive/test/test_read_format_rar5_loop_bug.c \
 	libarchive/test/test_read_format_raw.c \
 	libarchive/test/test_read_format_tar.c \
@@ -1008,6 +1009,7 @@ libarchive_test_EXTRA_DIST=\
 	libarchive/test/test_read_format_rar5_block_size_is_too_small.rar.uu \
 	libarchive/test/test_read_format_rar5_decode_number_out_of_bounds_read.rar.uu \
 	libarchive/test/test_read_format_rar5_window_buf_and_size_desync.rar.uu \
+	libarchive/test/test_read_format_rar5_bad_tables.rar.uu \
 	libarchive/test/test_read_format_rar5_bad_window_sz_in_mltarc_file.rar.uu \
 	libarchive/test/test_read_format_rar5_data_ready_pointer_leak.rar.uu \
 	libarchive/test/test_read_format_raw.bufr.uu \

libarchive/archive_read_support_format_rar5.c

@@ -2647,6 +2647,15 @@ static int create_decode_tables(uint8_t* bit_length,
 		upper_limit <<= 1;
 	}
 
+	/* Verify the code-length distribution is not over-subscribed.
+	 * After the loop above, upper_limit == sum(lc[i] * 2^(16-i)).
+	 * For a valid prefix-free code this must be <= 2^16 = 65536.
+	 * An over-subscribed table (> 65536) cannot produce a valid
+	 * decode table and must be rejected. */
+	if(upper_limit > 65536) {
+		return ARCHIVE_FAILED;
+	}
+
 	memcpy(decode_pos_clone, table->decode_pos, sizeof(decode_pos_clone));
 
 	for(i = 0; i < size; i++) {

libarchive/test/CMakeLists.txt

@@ -176,6 +176,7 @@ IF(ENABLE_TEST)
     test_read_format_rar_filter.c
     test_read_format_rar_overflow.c
     test_read_format_rar5.c
+    test_read_format_rar5_bad_tables.c
     test_read_format_rar5_loop_bug.c
     test_read_format_raw.c
     test_read_format_tar.c

libarchive/test/test_read_format_rar5_bad_tables.c

@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2026 Tim Kientzle
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+
+/*
+ * The reference archive contains one compressed entry whose HUFF_BC
+ * meta-table has all 20 nibbles set to bit-length 1.  This is an
+ * over-subscribed Huffman table (Kraft sum = 10 >> 1), which
+ * create_decode_tables() must reject.  Reading the entry data must
+ * therefore fail, not succeed.
+ */
+DEFINE_TEST(test_read_format_rar5_bad_tables)
+{
+	const char *reffile = "test_read_format_rar5_bad_tables.rar";
+	struct archive_entry *ae;
+	struct archive *a;
+	const void *buf;
+	size_t size;
+	la_int64_t offset;
+
+	extract_reference_file(reffile);
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_read_open_filename(a, reffile, 10240));
+
+	/* The file header is valid; the bad data is in the compressed block. */
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae));
+
+	/* Reading must fail because the meta-table is over-subscribed.
+	 * TODO: Investigate whether ARCHIVE_FAILED is achievable here
+	 * instead of ARCHIVE_FATAL, which would allow the caller to skip
+	 * to a subsequent valid entry.  That would also require expanding
+	 * the reference archive to include a valid entry after the bad one. */
+	assertEqualIntA(a, ARCHIVE_FATAL,
+	    archive_read_data_block(a, &buf, &size, &offset));
+
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_free(a));
+}

libarchive/test/test_read_format_rar5_bad_tables.rar.uu

@@ -0,0 +1,8 @@
+begin 644 test_read_format_rar5_bad_tables.rar
+M4F%R(1H'`0#%&C,R`P$``(C&P1@9`@*$`0!``(`$`0YB861?=&%B;&5S+G1X
+M=,@2@``1$1$1$1$1$1$1````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M`````````````````````````````````````````````````````````#GY
+%LH$"!0``
+`
+end