Fix memory leaks on open error paths

If open functions fail to store client data, free internally allocated
memory before returning an error code.

Writer open functions must check magic themselves.

Signed-off-by: Tobias Stoeckmann <tobias@stoeckmann.org>

libarchive/archive_read_open_fd.c

@@ -109,8 +109,11 @@ archive_read_open_fd(struct archive *a, int fd, size_t block_size)
 	archive_read_set_seek_callback(a, file_seek);
 	archive_read_set_close_callback(a, file_close);
 	r = archive_read_set_callback_data(a, mine);
-	if (r < 0)
+	if (r < 0) {
+		free(mine->buffer);
+		free(mine);
 		return (r);
+	}
 	return (archive_read_open1(a));
 }
 

libarchive/archive_read_open_file.c

@@ -106,8 +106,11 @@ archive_read_open_FILE(struct archive *a, FILE *f)
 	archive_read_set_seek_callback(a, FILE_seek);
 	archive_read_set_close_callback(a, FILE_close);
 	r = archive_read_set_callback_data(a, mine);
-	if (r < 0)
+	if (r < 0) {
+		free(mine->buffer);
+		free(mine);
 		return (r);
+	}
 	return (archive_read_open1(a));
 }
 

libarchive/archive_read_open_memory.c

@@ -84,8 +84,10 @@ archive_read_open_memory2(struct archive *a, const void *buff,
 	archive_read_set_skip_callback(a, memory_read_skip);
 	archive_read_set_close_callback(a, memory_read_close);
 	r = archive_read_set_callback_data(a, mine);
-	if (r < 0)
+	if (r < 0) {
+		free(mine);
 		return (r);
+	}
 	return (archive_read_open1(a));
 }
 

libarchive/archive_write_open_fd.c

@@ -48,6 +48,7 @@
 #endif
 
 #include "archive.h"
+#include "archive_private.h"
 
 struct write_fd_data {
 	int		fd;
@@ -62,6 +63,9 @@ archive_write_open_fd(struct archive *a, int fd)
 {
 	struct write_fd_data *mine;
 
+	archive_check_magic(a, ARCHIVE_WRITE_MAGIC,
+	    ARCHIVE_STATE_NEW, "archive_write_open_fd");
+
 	mine = malloc(sizeof(*mine));
 	if (mine == NULL) {
 		archive_set_error(a, ENOMEM, "No memory");

libarchive/archive_write_open_file.c

@@ -45,6 +45,7 @@
 #endif
 
 #include "archive.h"
+#include "archive_private.h"
 
 struct write_FILE_data {
 	FILE		*f;
@@ -59,6 +60,9 @@ archive_write_open_FILE(struct archive *a, FILE *f)
 {
 	struct write_FILE_data *mine;
 
+	archive_check_magic(a, ARCHIVE_WRITE_MAGIC,
+	    ARCHIVE_STATE_NEW, "archive_write_open_FILE");
+
 	mine = malloc(sizeof(*mine));
 	if (mine == NULL) {
 		archive_set_error(a, ENOMEM, "No memory");

libarchive/archive_write_open_filename.c

@@ -98,6 +98,9 @@ open_filename(struct archive *a, int mbs_fn, const void *filename)
 	struct write_file_data *mine;
 	int r;
 
+	archive_check_magic(a, ARCHIVE_WRITE_MAGIC,
+	    ARCHIVE_STATE_NEW, "open_filename");
+
 	mine = calloc(1, sizeof(*mine));
 	if (mine == NULL) {
 		archive_set_error(a, ENOMEM, "No memory");

libarchive/archive_write_open_memory.c

@@ -30,6 +30,7 @@
 #include <string.h>
 
 #include "archive.h"
+#include "archive_private.h"
 
 struct write_memory_data {
 	size_t	used;
@@ -52,6 +53,9 @@ archive_write_open_memory(struct archive *a, void *buff, size_t buffSize, size_t
 {
 	struct write_memory_data *mine;
 
+	archive_check_magic(a, ARCHIVE_WRITE_MAGIC,
+	    ARCHIVE_STATE_NEW, "archive_write_open_memory");
+
 	mine = calloc(1, sizeof(*mine));
 	if (mine == NULL) {
 		archive_set_error(a, ENOMEM, "No memory");