Fix several bugs in error stack trace formatting.

src/messages.js

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1064 matching lines @@
   }
   var constructorName = constructor.name;
   if (!constructorName) {
     return requireConstructor ? null :
         %_CallFunction(obj.receiver, ObjectToString);
   }
   return constructorName;
 }
 
 
+// Flag to prevent recursive call of Error.prepareStackTrace.
+var formatting_custom_stack_trace = false;
+
+
 function captureStackTrace(obj, cons_opt) {
   var stackTraceLimit = $Error.stackTraceLimit;
   if (!stackTraceLimit || !IS_NUMBER(stackTraceLimit)) return;
   if (stackTraceLimit < 0 || stackTraceLimit > 10000) {
     stackTraceLimit = 10000;
   }
   var stack = %CollectStackTrace(obj,
                                  cons_opt ? cons_opt : captureStackTrace,
                                  stackTraceLimit);
 
   // Don't be lazy if the error stack formatting is custom (observable).
-  if (IS_FUNCTION($Error.prepareStackTrace)) {
-    var custom_stacktrace_fun = $Error.prepareStackTrace;
-    // Use default error formatting for the case that custom formatting throws.
-    $Error.prepareStackTrace = null;
+  if (IS_FUNCTION($Error.prepareStackTrace) && !formatting_custom_stack_trace) {
     var array = [];
     %MoveArrayContents(GetStackFrames(stack), array);
-    obj.stack = custom_stacktrace_fun(obj, array);
-    $Error.prepareStackTrace = custom_stacktrace_fun;
+    formatting_custom_stack_trace = true;
+    try {
+      obj.stack = $Error.prepareStackTrace(obj, array);
+    } catch (e) {
+      throw e;  // The custom formatting function threw.  Rethrow.
+    } finally {
+      formatting_custom_stack_trace = false;
+    }
     return;
   }
 
   var error_string = FormatErrorString(obj);
   // Note that 'obj' and 'this' maybe different when called on objects that
   // have the error object on its prototype chain.  The getter replaces itself
   // with a data property as soon as the stack trace has been formatted.
   // The getter must not change the object layout as it may be called after GC.
   var getter = function() {
     if (IS_STRING(stack)) return stack;
@@ skipping 179 matching lines @@
     %SetOverflowedStackTrace(this, void 0);
   }
 
   %DefineOrRedefineAccessorProperty(
       boilerplate, 'stack', getter, setter, DONT_ENUM);
 
   return boilerplate;
 }
 
 var kStackOverflowBoilerplate = SetUpStackOverflowBoilerplate();