avcodec/libaomenc: check return value of queue_frames()

Signed-off-by: James Almer <[email protected]>
diff --git a/libavcodec/libaomenc.c b/libavcodec/libaomenc.c
index fb9a6ff..a82b933 100644
--- a/libavcodec/libaomenc.c
+++ b/libavcodec/libaomenc.c
@@ -1307,6 +1307,8 @@
         return AVERROR_INVALIDDATA;
     }
     coded_size = queue_frames(avctx, pkt);
+    if (coded_size < 0)
+        return coded_size;
 
     if (!frame && avctx->flags & AV_CODEC_FLAG_PASS1) {
         size_t b64_size = AV_BASE64_SIZE(ctx->twopass_stats.sz);