Fix UBSan errors (signed integer overflow)

webrtc/modules/audio_coding/codecs/ilbc/smooth.c

 /*
  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
  * find the smoothed output data
  *---------------------------------------------------------------*/
 
 void WebRtcIlbcfix_Smooth(
     int16_t *odata,   /* (o) smoothed output */
     int16_t *current,  /* (i) the un enhanced residual for
                                 this block */
     int16_t *surround  /* (i) The approximation from the
                                 surrounding sequences */
                           ) {
-  int16_t maxtot, scale, scale1, scale2;
+  int16_t scale, scale1, scale2;
   int16_t A, B, C, denomW16;
   int32_t B_W32, denom, num;
   int32_t errs;
   int32_t w00,w10,w11, endiff, crit;
   int32_t w00prim, w10prim, w11_div_w00;
   int16_t w11prim;
   int16_t bitsw00, bitsw10, bitsw11;
   int32_t w11w00, w10w10, w00w00;
-  int16_t max1, max2;
+  uint32_t max1, max2, max12;
 
   /* compute some inner products (ensure no overflow by first calculating proper scale factor) */
 
   w00 = w10 = w11 = 0;
 
-  max1=WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL);
-  max2=WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL);
-  maxtot=WEBRTC_SPL_MAX(max1, max2);
-
-  scale=WebRtcSpl_GetSizeInBits(maxtot);
-  scale = (int16_t)(2 * scale) - 26;
+  // Calculate a right shift that will let us sum ENH_BLOCKL pairwise products
+  // of values from the two sequences without overflowing an int32_t. (The +1
+  // in max1 and max2 are because WebRtcSpl_MaxAbsValueW16 will return 2**15 -
+  // 1 if the input array contains -2**15.)
+  max1 = WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL) + 1;
+  max2 = WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL) + 1;
+  max12 = WEBRTC_SPL_MAX(max1, max2);
+  scale = (64 - 31) -
+          WebRtcSpl_CountLeadingZeros64((max12 * max12) * (uint64_t)ENH_BLOCKL);
   scale=WEBRTC_SPL_MAX(0, scale);
 
   w00=WebRtcSpl_DotProductWithScale(current,current,ENH_BLOCKL,scale);
   w11=WebRtcSpl_DotProductWithScale(surround,surround,ENH_BLOCKL,scale);
   w10=WebRtcSpl_DotProductWithScale(surround,current,ENH_BLOCKL,scale);
 
   if (w00<0) w00 = WEBRTC_SPL_WORD32_MAX;
   if (w11<0) w11 = WEBRTC_SPL_WORD32_MAX;
 
   /* Rescale w00 and w11 to w00prim and w11prim, so that w00prim/w11prim
@@ skipping 133 matching lines @@
     }
 
     /* create smoothed sequence */
 
     WebRtcSpl_ScaleAndAddVectors(surround, A, 9,
                                 current, B, 14,
                                 odata, ENH_BLOCKL);
   }
   return;
 }