xar: Fix for unsupported digests

* In _checksum_init() and _checksum_final(), fail if the requested digest
  is not supported (e.g. archive uses SHA256 but libarchive was compiled
  without SHA256 support)

* In checksum_init(), fail with a meaningful error message if either
  _checksum_init() call fails.

* In checksum_final(), simplify the logic and improve the error message.

* Make the various digest tests conditional on support for the digest
  being tested.

Fixes:		6f10adcd59 ("xar: Add support for SHA256 and SHA512")

libarchive/archive_read_support_format_xar.c

@@ -406,7 +406,7 @@ static int	heap_add_entry(struct archive_read *a,
 static struct xar_file *heap_get_entry(struct heap_queue *);
 static int	add_link(struct archive_read *,
     struct xar *, struct xar_file *);
-static void	checksum_init(struct archive_read *, int, int);
+static int	checksum_init(struct archive_read *, int, int);
 static void	checksum_update(struct archive_read *, const void *,
 		    size_t, const void *, size_t);
 static int	checksum_final(struct archive_read *, const void *,
@@ -1035,8 +1035,7 @@ rd_contents_init(struct archive_read *a, enum enctype encoding,
 	if ((r = decompression_init(a, encoding)) != ARCHIVE_OK)
 		return (r);
 	/* Init checksum library. */
-	checksum_init(a, a_sum_alg, e_sum_alg);
-	return (ARCHIVE_OK);
+	return (checksum_init(a, a_sum_alg, e_sum_alg));
 }
 
 static int
@@ -1342,7 +1341,7 @@ add_link(struct archive_read *a, struct xar *xar, struct xar_file *file)
 	return (ARCHIVE_OK);
 }
 
-static void
+static int
 _checksum_init(struct chksumwork *sumwrk, int sum_alg)
 {
 	sumwrk->alg = sum_alg;
@@ -1365,7 +1364,10 @@ _checksum_init(struct chksumwork *sumwrk, int sum_alg)
 		archive_sha512_init(&(sumwrk->sha512ctx));
 		break;
 #endif
+	default:
+		return (ARCHIVE_FATAL);
 	}
+	return (ARCHIVE_OK);
 }
 
 static void
@@ -1431,17 +1433,24 @@ _checksum_final(struct chksumwork *sumwrk, const void *val, size_t len)
 			r = ARCHIVE_FAILED;
 		break;
 #endif
+	default:
+		r = ARCHIVE_FAILED;
 	}
 	return (r);
 }
 
-static void
+static int
 checksum_init(struct archive_read *a, int a_sum_alg, int e_sum_alg)
 {
 	struct xar *xar = a->format->data;
 
-	_checksum_init(&(xar->a_sumwrk), a_sum_alg);
-	_checksum_init(&(xar->e_sumwrk), e_sum_alg);
+	if (_checksum_init(&(xar->a_sumwrk), a_sum_alg) != ARCHIVE_OK ||
+	    _checksum_init(&(xar->e_sumwrk), e_sum_alg) != ARCHIVE_OK) {
+		archive_set_error(&(a->archive), ARCHIVE_ERRNO_MISC,
+		    "Unsupported checksum");
+		return (ARCHIVE_FATAL);
+	}
+	return (ARCHIVE_OK);
 }
 
 static void
@@ -1459,15 +1468,14 @@ checksum_final(struct archive_read *a, const void *a_sum_val,
     size_t a_sum_len, const void *e_sum_val, size_t e_sum_len)
 {
 	struct xar *xar = a->format->data;
-	int r;
 
-	r = _checksum_final(&(xar->a_sumwrk), a_sum_val, a_sum_len);
-	if (r == ARCHIVE_OK)
-		r = _checksum_final(&(xar->e_sumwrk), e_sum_val, e_sum_len);
-	if (r != ARCHIVE_OK)
+	if (_checksum_final(&(xar->a_sumwrk), a_sum_val, a_sum_len) != ARCHIVE_OK ||
+	    _checksum_final(&(xar->e_sumwrk), e_sum_val, e_sum_len) != ARCHIVE_OK) {
 		archive_set_error(&(a->archive), ARCHIVE_ERRNO_MISC,
-		    "Sumcheck error");
-	return (r);
+		    "Checksum error");
+		return (ARCHIVE_FATAL);
+	}
+	return (ARCHIVE_OK);
 }
 
 static int

libarchive/test/test_write_format_xar.c

@@ -321,26 +321,42 @@ DEFINE_TEST(test_write_format_xar)
 	/* Disable TOC checksum. */
 	test_xar("!toc-checksum");
 	test_xar("toc-checksum=none");
+#ifdef ARCHIVE_HAS_SHA1
 	/* Specify TOC checksum type to sha1. */
 	test_xar("toc-checksum=sha1");
+#endif
+#ifdef ARCHIVE_HAS_MD5
 	/* Specify TOC checksum type to md5. */
 	test_xar("toc-checksum=md5");
+#endif
+#ifdef ARCHIVE_HAS_SHA256
 	/* Specify TOC checksum type to sha256. */
 	test_xar("toc-checksum=sha256");
+#endif
+#ifdef ARCHIVE_HAS_SHA512
 	/* Specify TOC checksum type to sha512. */
 	test_xar("toc-checksum=sha512");
+#endif
 
 	/* Disable file checksum. */
 	test_xar("!checksum");
 	test_xar("checksum=none");
+#ifdef ARCHIVE_HAS_SHA1
 	/* Specify file checksum type to sha1. */
 	test_xar("checksum=sha1");
+#endif
+#ifdef ARCHIVE_HAS_MD5
 	/* Specify file checksum type to md5. */
 	test_xar("checksum=md5");
+#endif
+#ifdef ARCHIVE_HAS_SHA256
 	/* Specify file checksum type to sha256. */
 	test_xar("checksum=sha256");
+#endif
+#ifdef ARCHIVE_HAS_SHA512
 	/* Specify file checksum type to sha512. */
 	test_xar("checksum=sha512");
+#endif
 
 	/* Disable compression. */
 	test_xar("!compression");