Check for integer overflow

Signed-off-by: Sebastian Ramacher <sebastian+dev@ramacher.at>

zathura/checked-integer-arithmetic.c

@@ -0,0 +1,16 @@
+/* See LICENSE file for license and copyright information */
+
+#include "checked-integer-arithmetic.h"
+#include <stdint.h>
+#include <limits.h>
+
+#ifndef HAVE_BUILTIN
+bool
+checked_umul(unsigned int lhs, unsigned int rhs, unsigned int* res)
+{
+  const uint64_t r = (uint64_t) lhs * (uint64_t) rhs;
+  *res = (unsigned int) r;
+
+  return r > UINT_MAX;
+}
+#endif

zathura/checked-integer-arithmetic.h

@@ -0,0 +1,39 @@
+/* See LICENSE file for license and copyright information */
+#ifndef ZATHURA_CHECKED_INTEGER_ARITHMETIC_H
+#define ZATHURA_CHECKED_INTEGER_ARITHMETIC_H
+
+#include "macros.h"
+#include <stdbool.h>
+
+#if __GNUC__ >= 5
+#define HAVE_BUILTIN
+#elif defined(__clang__)
+#if __has_builtin(__builtin_add_overflow)
+#define HAVE_BUILTIN
+#endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef HAVE_BUILTIN
+#define checked_umul(lhs, rhs, res) __builtin_umul_overflow((lhs), (rhs), (res))
+#else
+/**
+ * Helper function for multiplication with overflow detection. This function has
+ * the same semantics as the __builtin_*mul_overflow functions.
+ *
+ * @param[in] lhs first operand
+ * @param[in] rhs second operand
+ * @param[out] res result
+ * @return true if an overflow occurred, false otherwise
+ */
+HIDDEN bool checked_umul(unsigned int lhs, unsigned int rhs, unsigned int* res);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

zathura/types.c

@@ -7,6 +7,7 @@
 #include "types.h"
 #include "links.h"
 #include "internal.h"
+#include "checked-integer-arithmetic.h"
 
 zathura_index_element_t*
 zathura_index_element_new(const char* title)
@@ -40,13 +41,22 @@ zathura_index_element_free(zathura_index_element_t* index)
 zathura_image_buffer_t*
 zathura_image_buffer_create(unsigned int width, unsigned int height)
 {
+  g_return_val_if_fail(width != 0, NULL);
+  g_return_val_if_fail(height != 0, NULL);
+
+  unsigned int size = 0;
+  if (checked_umul(width, height, &size) == false ||
+      checked_umul(size, 3, &size) == false) {
+    return NULL;
+  }
+
   zathura_image_buffer_t* image_buffer = malloc(sizeof(zathura_image_buffer_t));
 
   if (image_buffer == NULL) {
     return NULL;
   }
 
-  image_buffer->data = calloc(width * height * 3, sizeof(unsigned char));
+  image_buffer->data = calloc(size, sizeof(unsigned char));
 
   if (image_buffer->data == NULL) {
     free(image_buffer);