From e5e9a42858ebf1e7c98c698b04570702989617ac Mon Sep 17 00:00:00 2001 From: Wan-Teh Chang Date: Wed, 17 Jul 2024 15:38:35 -0700 Subject: [PATCH] alpha.c: Fix overflows in multiply with rowBytes Replace multiplication with rgb.rowBytes with addition of rgb.rowBytes to a row pointer in a loop. Related to https://github.com/AOMediaCodec/libavif/issues/2271. --- src/alpha.c | 138 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 48 deletions(-) diff --git a/src/alpha.c b/src/alpha.c index 97ed2c42be..f73400ae32 100644 --- a/src/alpha.c +++ b/src/alpha.c @@ -10,22 +10,26 @@ void avifFillAlpha(const avifAlphaParams * params) { if (params->dstDepth > 8) { const uint16_t maxChannel = (uint16_t)((1 << params->dstDepth) - 1); + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - *((uint16_t *)dstRow) = maxChannel; - dstRow += params->dstPixelBytes; + *((uint16_t *)dstPixel) = maxChannel; + dstPixel += params->dstPixelBytes; } + dstRow += params->dstRowBytes; } } else { // In this case, (1 << params->dstDepth) - 1 is always equal to 255. const uint8_t maxChannel = 255; + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - *dstRow = maxChannel; - dstRow += params->dstPixelBytes; + *dstPixel = maxChannel; + dstPixel += params->dstPixelBytes; } + dstRow += params->dstRowBytes; } } } @@ -43,22 +47,34 @@ void avifReformatAlpha(const avifAlphaParams * params) if (params->srcDepth > 8) { // no depth rescale, uint16_t -> uint16_t + const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes]; + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)]; - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + const uint8_t * srcPixel = srcRow; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - *((uint16_t *)&dstRow[i * params->dstPixelBytes]) = *((const uint16_t *)&srcRow[i * params->srcPixelBytes]); + *((uint16_t *)dstPixel) = *((const uint16_t *)srcPixel); + srcPixel += params->srcPixelBytes; + dstPixel += params->dstPixelBytes; } + srcRow += params->srcRowBytes; + dstRow += params->dstRowBytes; } } else { // no depth rescale, uint8_t -> uint8_t + const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes]; + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)]; - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + const uint8_t * srcPixel = srcRow; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - dstRow[i * params->dstPixelBytes] = srcRow[i * params->srcPixelBytes]; + *dstPixel = *srcPixel; + srcPixel += params->srcPixelBytes; + dstPixel += params->dstPixelBytes; } + srcRow += params->srcRowBytes; + dstRow += params->dstRowBytes; } } } else { @@ -68,30 +84,42 @@ void avifReformatAlpha(const avifAlphaParams * params) if (params->dstDepth > 8) { // depth rescale, uint16_t -> uint16_t + const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes]; + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)]; - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + const uint8_t * srcPixel = srcRow; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - int srcAlpha = *((const uint16_t *)&srcRow[i * params->srcPixelBytes]); + int srcAlpha = *((const uint16_t *)srcPixel); float alphaF = (float)srcAlpha / srcMaxChannelF; int dstAlpha = (int)(0.5f + (alphaF * dstMaxChannelF)); dstAlpha = AVIF_CLAMP(dstAlpha, 0, dstMaxChannel); - *((uint16_t *)&dstRow[i * params->dstPixelBytes]) = (uint16_t)dstAlpha; + *((uint16_t *)dstPixel) = (uint16_t)dstAlpha; + srcPixel += params->srcPixelBytes; + dstPixel += params->dstPixelBytes; } + srcRow += params->srcRowBytes; + dstRow += params->dstRowBytes; } } else { // depth rescale, uint16_t -> uint8_t + const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes]; + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)]; - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + const uint8_t * srcPixel = srcRow; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - int srcAlpha = *((const uint16_t *)&srcRow[i * params->srcPixelBytes]); + int srcAlpha = *((const uint16_t *)srcPixel); float alphaF = (float)srcAlpha / srcMaxChannelF; int dstAlpha = (int)(0.5f + (alphaF * dstMaxChannelF)); dstAlpha = AVIF_CLAMP(dstAlpha, 0, dstMaxChannel); - dstRow[i * params->dstPixelBytes] = (uint8_t)dstAlpha; + *dstPixel = (uint8_t)dstAlpha; + srcPixel += params->srcPixelBytes; + dstPixel += params->dstPixelBytes; } + srcRow += params->srcRowBytes; + dstRow += params->dstRowBytes; } } } else { @@ -99,16 +127,22 @@ void avifReformatAlpha(const avifAlphaParams * params) assert(params->dstDepth > 8); // depth rescale, uint8_t -> uint16_t + const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes]; + uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes]; for (uint32_t j = 0; j < params->height; ++j) { - const uint8_t * srcRow = ¶ms->srcPlane[params->srcOffsetBytes + (j * params->srcRowBytes)]; - uint8_t * dstRow = ¶ms->dstPlane[params->dstOffsetBytes + (j * params->dstRowBytes)]; + const uint8_t * srcPixel = srcRow; + uint8_t * dstPixel = dstRow; for (uint32_t i = 0; i < params->width; ++i) { - int srcAlpha = srcRow[i * params->srcPixelBytes]; + int srcAlpha = *srcPixel; float alphaF = (float)srcAlpha / srcMaxChannelF; int dstAlpha = (int)(0.5f + (alphaF * dstMaxChannelF)); dstAlpha = AVIF_CLAMP(dstAlpha, 0, dstMaxChannel); - *((uint16_t *)&dstRow[i * params->dstPixelBytes]) = (uint16_t)dstAlpha; + *((uint16_t *)dstPixel) = (uint16_t)dstAlpha; + srcPixel += params->srcPixelBytes; + dstPixel += params->dstPixelBytes; } + srcRow += params->srcRowBytes; + dstRow += params->dstRowBytes; } } } @@ -138,14 +172,13 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb) if (rgb->depth > 8) { if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint16_t * pixel = (uint16_t *)row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint16_t * pixel = (uint16_t *)&row[i * 8]; uint16_t a = pixel[3]; if (a >= max) { // opaque is no-op - continue; } else if (a == 0) { // result must be zero pixel[0] = 0; @@ -157,16 +190,17 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb) pixel[1] = (uint16_t)avifRoundf((float)pixel[1] * (float)a / maxF); pixel[2] = (uint16_t)avifRoundf((float)pixel[2] * (float)a / maxF); } + pixel += 4; } + row += rgb->rowBytes; } } else { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint16_t * pixel = (uint16_t *)row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint16_t * pixel = (uint16_t *)&row[i * 8]; uint16_t a = pixel[0]; if (a >= max) { - continue; } else if (a == 0) { pixel[1] = 0; pixel[2] = 0; @@ -176,19 +210,20 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb) pixel[2] = (uint16_t)avifRoundf((float)pixel[2] * (float)a / maxF); pixel[3] = (uint16_t)avifRoundf((float)pixel[3] * (float)a / maxF); } + pixel += 4; } + row += rgb->rowBytes; } } } else { if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint8_t * pixel = row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint8_t * pixel = &row[i * 4]; uint8_t a = pixel[3]; // uint8_t can't exceed 255 if (a == max) { - continue; } else if (a == 0) { pixel[0] = 0; pixel[1] = 0; @@ -198,16 +233,17 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb) pixel[1] = (uint8_t)avifRoundf((float)pixel[1] * (float)a / maxF); pixel[2] = (uint8_t)avifRoundf((float)pixel[2] * (float)a / maxF); } + pixel += 4; } + row += rgb->rowBytes; } } else { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint8_t * pixel = row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint8_t * pixel = &row[i * 4]; uint8_t a = pixel[0]; if (a == max) { - continue; } else if (a == 0) { pixel[1] = 0; pixel[2] = 0; @@ -217,7 +253,9 @@ avifResult avifRGBImagePremultiplyAlpha(avifRGBImage * rgb) pixel[2] = (uint8_t)avifRoundf((float)pixel[2] * (float)a / maxF); pixel[3] = (uint8_t)avifRoundf((float)pixel[3] * (float)a / maxF); } + pixel += 4; } + row += rgb->rowBytes; } } } @@ -249,14 +287,13 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb) if (rgb->depth > 8) { if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint16_t * pixel = (uint16_t *)row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint16_t * pixel = (uint16_t *)&row[i * 8]; uint16_t a = pixel[3]; if (a >= max) { // opaque is no-op - continue; } else if (a == 0) { // prevent division by zero pixel[0] = 0; @@ -270,16 +307,17 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb) pixel[1] = (uint16_t)AVIF_MIN(c2, maxF); pixel[2] = (uint16_t)AVIF_MIN(c3, maxF); } + pixel += 4; } + row += rgb->rowBytes; } } else { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint16_t * pixel = (uint16_t *)row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint16_t * pixel = (uint16_t *)&row[i * 8]; uint16_t a = pixel[0]; if (a >= max) { - continue; } else if (a == 0) { pixel[1] = 0; pixel[2] = 0; @@ -292,18 +330,19 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb) pixel[2] = (uint16_t)AVIF_MIN(c2, maxF); pixel[3] = (uint16_t)AVIF_MIN(c3, maxF); } + pixel += 4; } + row += rgb->rowBytes; } } } else { if (rgb->format == AVIF_RGB_FORMAT_RGBA || rgb->format == AVIF_RGB_FORMAT_BGRA) { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint8_t * pixel = row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint8_t * pixel = &row[i * 4]; uint8_t a = pixel[3]; if (a == max) { - continue; } else if (a == 0) { pixel[0] = 0; pixel[1] = 0; @@ -316,16 +355,17 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb) pixel[1] = (uint8_t)AVIF_MIN(c2, maxF); pixel[2] = (uint8_t)AVIF_MIN(c3, maxF); } + pixel += 4; } + row += rgb->rowBytes; } } else { + uint8_t * row = rgb->pixels; for (uint32_t j = 0; j < rgb->height; ++j) { - uint8_t * row = &rgb->pixels[j * rgb->rowBytes]; + uint8_t * pixel = row; for (uint32_t i = 0; i < rgb->width; ++i) { - uint8_t * pixel = &row[i * 4]; uint8_t a = pixel[0]; if (a == max) { - continue; } else if (a == 0) { pixel[1] = 0; pixel[2] = 0; @@ -338,7 +378,9 @@ avifResult avifRGBImageUnpremultiplyAlpha(avifRGBImage * rgb) pixel[2] = (uint8_t)AVIF_MIN(c2, maxF); pixel[3] = (uint8_t)AVIF_MIN(c3, maxF); } + pixel += 4; } + row += rgb->rowBytes; } } }