Detect a libaom v3.6.0 bug with large images

This bug, reported in https://crbug.com/aomedia/2871#c12, causes libaom
v3.6.0 to encode bitstreams for certain large frame sizes that cannot be
decoded by existing AV1 decoders.

Change the EncodingTest.ReasonableValidDimensions test to use dimensions
that avoid this bug.

src/codec_aom.c

@@ -535,6 +535,13 @@ static avifBool avifFindAOMScalingMode(const avifFraction * avifMode, AOM_SCALIN
     return AVIF_FALSE;
 }
 
+static avifBool doesLevelMatch(int width, int height, int levelWidth, int levelHeight, int levelDimMult)
+{
+    const int64_t levelLumaPels = (int64_t)levelWidth * levelHeight;
+    const int64_t lumaPels = (int64_t)width * height;
+    return lumaPels <= levelLumaPels && width <= levelWidth * levelDimMult && height <= levelHeight * levelDimMult;
+}
+
 static avifBool aomCodecEncodeFinish(avifCodec * codec, avifCodecEncodeOutput * output);
 
 static avifResult aomCodecEncodeImage(avifCodec * codec,
@@ -689,6 +696,21 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
         cfg->g_input_bit_depth = image->depth;
         cfg->g_w = image->width;
         cfg->g_h = image->height;
+
+        // Detect the libaom v3.6.0 bug described in
+        // https://crbug.com/aomedia/2871#c12. See the changes to
+        // av1/encoder/encoder.c in
+        // https://aomedia-review.googlesource.com/c/aom/+/174421.
+        static const int aomVersion_3_6_0 = (3 << 16) | (6 << 8);
+        if (aomVersion == aomVersion_3_6_0) {
+            if (!doesLevelMatch(image->width, image->height, 8192, 4352, 2) &&
+                (doesLevelMatch(image->width, image->height, 16384, 8704, 2) ||
+                 doesLevelMatch(image->width, image->height, 32768, 17408, 2))) {
+                avifDiagnosticsPrintf(codec->diag, "Detected libaom v3.6.0 bug with large images. Upgrade to libaom v3.6.1 or later.");
+                return AVIF_RESULT_UNKNOWN_ERROR;
+            }
+        }
+
         if (addImageFlags & AVIF_ADD_IMAGE_FLAG_SINGLE) {
             // Set the maximum number of frames to encode to 1. This instructs
             // libaom to set still_picture and reduced_still_picture_header to

tests/gtest/avifallocationtest.cc

@@ -148,8 +148,10 @@ TEST(EncodingTest, MinimumValidDimensions) {
 }
 
 TEST(EncodingTest, ReasonableValidDimensions) {
+  // 16384 x 8704 is the maximum width x height allowed in the currently
+  // defined levels.
   TestEncoding(16384, 1, 12, AVIF_RESULT_OK);
-  TestEncoding(1, 16384, 12, AVIF_RESULT_OK);
+  TestEncoding(1, 8704, 12, AVIF_RESULT_OK);
 }
 
 // 65536 is the maximum AV1 frame dimension allowed by the AV1 specification.