Require allocation size be <= PTRDIFF_MAX

This makese it safe to cast rowBytes to ptrdiff_t. Part of the fix to AOMediaCodec#2354.
wantehchang · Aug 1, 2024 · da4b45d · da4b45d
1 parent 622ec93
commit da4b45d
Showing 1 changed file with 2 additions and 14 deletions.
diff --git a/src/avif.c b/src/avif.c
@@ -357,13 +357,7 @@ avifResult avifImageAllocatePlanes(avifImage * image, avifPlanesFlags planes)
         return AVIF_RESULT_INVALID_ARGUMENT;
     }
     const uint32_t fullRowBytes = channelSize * image->width;
-#if UINT32_MAX > PTRDIFF_MAX
-    // Make sure it is safe to cast image->yuvRowBytes[i] or image->alphaRowBytes to ptrdiff_t.
-    if (fullRowBytes > PTRDIFF_MAX) {
-        return AVIF_RESULT_INVALID_ARGUMENT;
-    }
-#endif
-    if (image->height > SIZE_MAX / fullRowBytes) {
+    if (image->height > PTRDIFF_MAX / fullRowBytes) {
         return AVIF_RESULT_INVALID_ARGUMENT;
     }
     const size_t fullSize = (size_t)fullRowBytes * image->height;
@@ -637,13 +631,7 @@ avifResult avifRGBImageAllocatePixels(avifRGBImage * rgb)
         return AVIF_RESULT_INVALID_ARGUMENT;
     }
     const uint32_t rowBytes = rgb->width * pixelSize;
-#if UINT32_MAX > PTRDIFF_MAX
-    // Make sure it is safe to cast rgb->rowBytes to ptrdiff_t.
-    if (rowBytes > PTRDIFF_MAX) {
-        return AVIF_RESULT_INVALID_ARGUMENT;
-    }
-#endif
-    if (rgb->height > SIZE_MAX / rowBytes) {
+    if (rgb->height > PTRDIFF_MAX / rowBytes) {
         return AVIF_RESULT_INVALID_ARGUMENT;
     }
     rgb->pixels = (uint8_t *)avifAlloc((size_t)rowBytes * rgb->height);