Cap the record layer version number of TLS ClientHello to TLS 1.0

net/third_party/nss/patches/recordlayerversion.patch

+Index: net/third_party/nss/ssl/sslimpl.h
+===================================================================
+--- net/third_party/nss/ssl/sslimpl.h   (revision 146623)
++++ net/third_party/nss/ssl/sslimpl.h   (working copy)
+@@ -294,6 +294,8 @@
+ #define ssl_SEND_FLAG_NO_BUFFER                0x20000000
+ #define ssl_SEND_FLAG_USE_EPOCH                0x10000000 /* DTLS only */
+ #define ssl_SEND_FLAG_NO_RETRANSMIT    0x08000000 /* DTLS only */
++#define ssl_SEND_FLAG_CAP_RECORD_VERSION \
++                                       0x04000000 /* TLS only */
+ #define ssl_SEND_FLAG_MASK             0x7f000000
+ 
+ /*
+@@ -1414,6 +1416,7 @@
+ ssl3_CompressMACEncryptRecord(ssl3CipherSpec *   cwSpec,
+                              PRBool             isServer,
+                              PRBool             isDTLS,
++                             PRBool             capRecordVersion,
+                               SSL3ContentType    type,
+                              const SSL3Opaque * pIn,
+                              PRUint32           contentLen,
+Index: net/third_party/nss/ssl/ssl3con.c
+===================================================================
+--- net/third_party/nss/ssl/ssl3con.c   (revision 146623)
++++ net/third_party/nss/ssl/ssl3con.c   (working copy)
+@@ -2057,6 +2057,7 @@
+ ssl3_CompressMACEncryptRecord(ssl3CipherSpec *   cwSpec,
+                              PRBool             isServer,
+                              PRBool             isDTLS,
++                             PRBool             capRecordVersion,
+                               SSL3ContentType    type,
+                              const SSL3Opaque * pIn,
+                              PRUint32           contentLen,
+@@ -2216,8 +2217,13 @@
+        wrBuf->buf[11] = MSB(cipherBytes);
+        wrBuf->buf[12] = LSB(cipherBytes);
+     } else {
+-       wrBuf->buf[1] = MSB(cwSpec->version);
+-       wrBuf->buf[2] = LSB(cwSpec->version);
++       SSL3ProtocolVersion version = cwSpec->version;
++
++       if (capRecordVersion) {
++           version = PR_MIN(SSL_LIBRARY_VERSION_TLS_1_0, version);
++       }
++       wrBuf->buf[1] = MSB(version);
++       wrBuf->buf[2] = LSB(version);
+        wrBuf->buf[3] = MSB(cipherBytes);
+        wrBuf->buf[4] = LSB(cipherBytes);
+     }
+@@ -2247,7 +2253,14 @@
+  *    all ciphertext into the pending ciphertext buffer.
+  * ssl_SEND_FLAG_USE_EPOCH (for DTLS)
+  *    Forces the use of the provided epoch
+- *
++ * ssl_SEND_FLAG_CAP_RECORD_VERSION
++ *    Caps the record layer version number of TLS ClientHello to { 3, 1 }
++ *    (TLS 1.0). Some TLS 1.0 servers (which seem to use F5 BIG-IP) ignore 
++ *    ClientHello.client_version and use the record layer version number
++ *    (TLSPlaintext.version) instead when negotiating protocol versions. In
++ *    addition, if the record layer version number of ClientHello is { 3, 2 }
++ *    (TLS 1.1) or higher, these servers reset the TCP connections. Set this
++ *    flag to work around such servers.
+  */
+ PRInt32
+ ssl3_SendRecord(   sslSocket *        ss,
+@@ -2260,6 +2273,7 @@
+     sslBuffer      *          wrBuf      = &ss->sec.writeBuf;
+     SECStatus                 rv;
+     PRInt32                   totalSent   = 0;
++    PRBool                    capRecordVersion;
+ 
+     SSL_TRC(3, ("%d: SSL3[%d] SendRecord type: %s nIn=%d",
+                SSL_GETPID(), ss->fd, ssl3_DecodeContentType(type),
+@@ -2268,6 +2282,16 @@
+ 
+     PORT_Assert( ss->opt.noLocks || ssl_HaveXmitBufLock(ss) );
+ 
++    capRecordVersion = ((flags & ssl_SEND_FLAG_CAP_RECORD_VERSION) != 0);
++
++    if (capRecordVersion) {
++       /* ssl_SEND_FLAG_CAP_RECORD_VERSION can only be used with
++        * TLS ClientHello. */
++       PORT_Assert(!IS_DTLS(ss));
++       PORT_Assert(type == content_handshake);
++       PORT_Assert(ss->ssl3.hs.ws == wait_server_hello);
++    }
++
+     if (ss->ssl3.initialized == PR_FALSE) {
+        /* This can happen on a server if the very first incoming record
+        ** looks like a defective ssl3 record (e.g. too long), and we're
+@@ -2324,7 +2348,8 @@
+ 
+            rv = ssl3_CompressMACEncryptRecord(ss->ssl3.cwSpec,
+                                               ss->sec.isServer, IS_DTLS(ss),
+-                                              type, pIn, 1, wrBuf);
++                                              capRecordVersion, type, pIn,
++                                              1, wrBuf);
+            if (rv != SECSuccess)
+                goto spec_locked_loser;
+ 
+@@ -2337,7 +2362,8 @@
+ 
+            rv = ssl3_CompressMACEncryptRecord(ss->ssl3.cwSpec,
+                                               ss->sec.isServer, IS_DTLS(ss),
+-                                              type, pIn + 1, contentLen - 1,
++                                              capRecordVersion, type,
++                                              pIn + 1, contentLen - 1,
+                                               &secondRecord);
+            if (rv == SECSuccess) {
+                PRINT_BUF(50, (ss, "send (encrypted) record data [2/2]:",
+@@ -2349,6 +2375,7 @@
+                rv = ssl3_CompressMACEncryptRecord(ss->ssl3.cwSpec,
+                                                   ss->sec.isServer,
+                                                   IS_DTLS(ss),
++                                                  capRecordVersion,
+                                                   type, pIn,
+                                                   contentLen, wrBuf);
+            } else {
+@@ -2560,6 +2587,8 @@
+ static SECStatus
+ ssl3_FlushHandshakeMessages(sslSocket *ss, PRInt32 flags)
+ {
++    static const PRInt32 allowedFlags = ssl_SEND_FLAG_FORCE_INTO_BUFFER |
++                                        ssl_SEND_FLAG_CAP_RECORD_VERSION;
+     PRInt32 rv = SECSuccess;
+ 
+     PORT_Assert( ss->opt.noLocks || ssl_HaveSSL3HandshakeLock(ss));
+@@ -2568,9 +2597,9 @@
+     if (!ss->sec.ci.sendBuf.buf || !ss->sec.ci.sendBuf.len)
+        return rv;
+ 
+-    /* only this flag is allowed */
+-    PORT_Assert(!(flags & ~ssl_SEND_FLAG_FORCE_INTO_BUFFER));
+-    if ((flags & ~ssl_SEND_FLAG_FORCE_INTO_BUFFER) != 0) {
++    /* only these flags are allowed */
++    PORT_Assert(!(flags & ~allowedFlags));
++    if ((flags & ~allowedFlags) != 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        rv = SECFailure;
+     } else {
+@@ -3981,8 +4010,10 @@
+     int              num_suites;
+     int              actual_count = 0;
+     PRBool           isTLS = PR_FALSE;
++    PRBool           serverVersionKnown = PR_FALSE;
+     PRInt32          total_exten_len = 0;
+     unsigned         numCompressionMethods;
++    PRInt32          flags;
+ 
+     SSL_TRC(3, ("%d: SSL3[%d]: send client_hello handshake", SSL_GETPID(),
+                ss->fd));
+@@ -4070,6 +4101,7 @@
+     }
+ 
+     if (sid) {
++       serverVersionKnown = PR_TRUE;
+        SSL_AtomicIncrementLong(& ssl3stats.sch_sid_cache_hits );
+ 
+        /* Are we attempting a stateless session resume? */
+@@ -4305,7 +4337,11 @@
+            ssl_renegotiation_info_xtn;
+     }
+ 
+-    rv = ssl3_FlushHandshake(ss, 0);
++    flags = 0;
++    if (!serverVersionKnown && !IS_DTLS(ss)) {
++       flags |= ssl_SEND_FLAG_CAP_RECORD_VERSION;
++    }
++    rv = ssl3_FlushHandshake(ss, flags);
+     if (rv != SECSuccess) {
+        return rv;      /* error code set by ssl3_FlushHandshake */
+     }
+Index: net/third_party/nss/ssl/dtls1con.c
+===================================================================
+--- net/third_party/nss/ssl/dtls1con.c  (revision 146623)
++++ net/third_party/nss/ssl/dtls1con.c  (working copy)
+@@ -834,7 +834,8 @@
+ 
+     if (cwSpec) {
+         rv = ssl3_CompressMACEncryptRecord(cwSpec, ss->sec.isServer, PR_TRUE,
+-                                          type, pIn, contentLen, wrBuf);
++                                          PR_FALSE, type, pIn, contentLen,
++                                          wrBuf);
+     } else {
+         PR_NOT_REACHED("Couldn't find a cipher spec matching epoch");
+        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);