Index: third_party/WebKit/Source/devtools/scripts/convert-3pas-product-registry.js |
diff --git a/third_party/WebKit/Source/devtools/scripts/convert-3pas-product-registry.js b/third_party/WebKit/Source/devtools/scripts/convert-3pas-product-registry.js |
new file mode 100644 |
index 0000000000000000000000000000000000000000..a187664c35d595242210ebdb4051b426de77a34b |
--- /dev/null |
+++ b/third_party/WebKit/Source/devtools/scripts/convert-3pas-product-registry.js |
@@ -0,0 +1,429 @@ |
+const fs = require('fs'); |
+ |
+/* |
+How to use: |
+1) Get dump of data as CSV format and name it 3pas.csv same directory as this script. |
+2) Header fields in the CSV will be used as keys when destructing into JSON objects [ie: top row data should not have spaces or special chars] |
+3) The two important column names are: 'name_legal_product' and 'domain'. |
+4) There may not be a header named 'prefix'. |
+5) 'name_legal_product' Will have it's data cleaned up a bit, so be prepared for it to change. |
+6) This script tries to de-duplicate any data, so be prepared for many entries to go away if it finds a shorter one. |
+7) This script will output a javascript file in the product_registry's data format. |
+*/ |
+ |
+/* |
+ * Configurable variables. You may need to tweak these to be compatible with |
+ * the server-side, but the defaults work in most cases. |
+ */ |
+const hexcase = 0; /* hex output format. 0 - lowercase; 1 - uppercase */ |
+const b64pad = '='; /* base-64 pad character. "=" for strict RFC compliance */ |
+const chrsz = 8; /* bits per input character. 8 - ASCII; 16 - Unicode */ |
+ |
+var data = fs.readFileSync('3pas.csv', 'utf8'); |
+var headerLine = data.split('\n', 1)[0]; |
+data = data.substr(headerLine.length); |
+var headerLineOrigLength = headerLine.length; |
+ |
+var columnNames = Array.from(csvUnmarshaller(headerLine)).map(v => v[0]); |
+var lineObjs = []; |
+ |
+var marshaller = csvUnmarshaller(data, 2); |
+var lineObj = {}; |
+var colIndex = 0; |
+for (var [colData, isEnding] of marshaller) { |
+ if (!(columnNames[colIndex] in lineObj)) |
+ lineObj[columnNames[colIndex]] = colData; |
+ colIndex++; |
+ if (isEnding) { |
+ lineObj = {}; |
+ lineObjs.push(lineObj); |
+ colIndex = 0; |
+ } |
+} |
+ |
+var map = new Map(); |
+for (var lineObj of lineObjs) { |
+ if (lineObj.domain === null || lineObj.domain === undefined || |
+ (lineObj.status_allowed !== 'allowed' && lineObj.status_allowed !== 'disallowed')) |
+ continue; |
+ lineObj.domain = |
+ lineObj.domain.trim().toLowerCase().replace(/[^a-z0-9_\-*.]/g, '').replace(/^www\.(?=[^.]+\.[^.]+$)/, ''); |
+ |
+ lineObj.name_legal_product = lineObj.name_legal_product.trim() |
+ .replace(/\s\s/g, ' ') |
+ .replace(/[\x00-\x1F]/g, '') |
+ .replace(/"/g, '"') |
+ // The following two lines are to keep input data from currupting output data. |
+ .replace(/","/g, '') |
+ .replace(/},{/g, '') |
+ .replace(/“|”/g, '"') |
+ .replace(/,$/g, '') |
+ .replace(/&/g, '&') |
+ // This is how csv escapes double quotes. |
+ .replace(/""/g, '"'); |
+ if (!map.has(lineObj.domain)) |
+ map.set(lineObj.domain, lineObj); |
+} |
+ |
+lineObjs = Array.from(map.values()); |
+ |
+var map = new Map(); |
+for (var lineObj of lineObjs) { |
+ if (!lineObj) |
+ continue; |
+ var domain = lineObj.domain.trim(); |
+ if (!domain.length) |
+ continue; |
+ var prefixSuffix = domain.split('*'); |
+ if (prefixSuffix.length > 2) |
+ throw 'We do not support multiple * in domains'; |
+ var prefix = ''; |
+ var suffixDomain = ''; |
+ if (prefixSuffix.length === 1) { |
+ suffixDomain = prefixSuffix[0]; |
+ } else { |
+ prefix = prefixSuffix[0]; |
+ if (prefix === '') |
+ prefix = '*'; |
+ suffixDomain = prefixSuffix[1]; |
+ } |
+ |
+ var domainParts = suffixDomain.split('.'); |
+ if (domainParts.length < 2) |
+ throw 'Invalid domain'; |
+ var baseDomain = domainParts[domainParts.length - 2] + '.' + domainParts[domainParts.length - 1]; |
+ while (domainParts[0] === '') |
+ domainParts.shift(); |
+ lineObj.domain = domainParts.join('.'); |
+ lineObj.prefix = prefix; |
+ |
+ var mapOfSubdomains = map.get(baseDomain); |
+ if (!mapOfSubdomains) { |
+ mapOfSubdomains = new Map(); |
+ map.set(baseDomain, mapOfSubdomains); |
+ } |
+ |
+ var prefixMap = mapOfSubdomains.get(lineObj.domain); |
+ if (!prefixMap) { |
+ prefixMap = new Map(); |
+ mapOfSubdomains.set(lineObj.domain, prefixMap); |
+ } |
+ if (prefixMap.has(prefix)) |
+ console.log('Problem with: ', domain, lineObj.domain); |
+ prefixMap.set(prefix, lineObj); |
+} |
+ |
+var outputProducts = []; |
+var outputObj = new Map(); |
+for (var [baseDomain, subdomains] of map) { |
+ for (var prefixes of subdomains.values()) { |
+ SKIP_ENTRY: for (var lineObj of prefixes.values()) { |
+ var prefix = lineObj.prefix; |
+ var wildLineObj = prefixes.get('*'); |
+ if (wildLineObj && prefix !== '*') { |
+ if (wildLineObj.name_legal_product === lineObj.name_legal_product) { |
+ // Skip entry, since wild card is there and already in table. |
+ continue SKIP_ENTRY; |
+ } |
+ } |
+ var fullSubdomain = lineObj.domain; |
+ var domainParts = lineObj.domain.split('.'); |
+ // Ignore fist one since we are on it now. |
+ var previousDomainPart = domainParts.shift(); |
+ var ignoreEntry = false; |
+ |
+ while (domainParts.length > 1) { |
+ var subdomain = domainParts.join('.'); |
+ var subdomainPrefixes = subdomains.get(subdomain); |
+ if (subdomainPrefixes) { |
+ for (var innerLineObj of subdomainPrefixes.values()) { |
+ if (innerLineObj.prefix === '' || innerLineObj.name_legal_product !== lineObj.name_legal_product) |
+ continue; |
+ if (innerLineObj.prefix === '*') |
+ continue SKIP_ENTRY; |
+ // Per chat with 3pas team. We need to check prefix on subdomain not top level domain. |
+ // ie: f*.foo.bar -> [b.f00.foo.bar, true], [f00.foo.bar, true], [f00.b.foo.bar, false] |
+ if (previousDomainPart.substr(0, innerLineObj.prefix.length) === innerLineObj.prefix) |
+ continue SKIP_ENTRY; |
+ } |
+ } |
+ previousDomainPart = domainParts.shift(); |
+ } |
+ var outputPart = outputObj.get(fullSubdomain); |
+ if (!outputPart) { |
+ outputPart = {hash: hex_sha1(fullSubdomain).substr(0, 16), prefixes: {}}; |
+ outputObj.set(fullSubdomain, outputPart); |
+ } |
+ outputPart.prefixes[lineObj.prefix] = registerOutputProduct(lineObj.name_legal_product); |
+ } |
+ } |
+} |
+ |
+console.log( |
+ '// Copyright 2017 The Chromium Authors. All rights reserved.\n' + |
+ '// Use of this source code is governed by a BSD-style license that can be\n' + |
+ '// found in the LICENSE file.\n' + |
+ '// clang-format off\n' + |
+ '/* eslint-disable */\n' + |
+ 'ProductRegistry.register(['); |
+var data = JSON.stringify(outputProducts).replace(/","/g, '",\n "'); |
+console.log(' ' + data.substring(1, data.length - 1)); |
+console.log('],'); |
+console.log('['); |
+var outputObjArray = Array.from(outputObj.values()); |
+for (var i = 0; i < outputObjArray.length; i++) { |
+ var obj = outputObjArray[i]; |
+ var lineEnding = (i === outputObjArray.length - 1) ? '' : ','; |
+ var comments = []; |
+ for (var prefix in obj.prefixes) |
+ comments.push('[' + outputProducts[obj.prefixes[prefix]] + ']'); |
+ console.log(' ' + JSON.stringify(obj) + lineEnding + ' // ' + comments.join(' ')); |
+} |
+console.log(']);'); |
+ |
+ |
+// items.forEach(lineObj => console.log(lineObj.name_legal_product.padStart(50), lineObj.domain.padStart(30))); |
+// console.log("With *: ", items.filter(v => v.domain.indexOf('*') !== -1).length); |
+// console.log("Total: ", items.length); |
+ |
+ |
+ |
+// Linear but meh. |
+function registerOutputProduct(name) { |
+ var index = outputProducts.indexOf(name); |
+ if (index === -1) { |
+ outputProducts.push(name); |
+ return outputProducts.length - 1; |
+ } |
+ return index; |
+} |
+ |
+function* csvUnmarshaller(data, lineOffset) { |
+ var origLen = data.length; |
+ var colLength = 0; |
+ var lineNo = lineOffset || 1; |
+ while (data.length) { |
+ var colData; |
+ var match; |
+ if (data[0] === '"') { |
+ match = data.match(/^"((?:[^"]|"")*)"(,|\n|$)/m); |
+ if (!match) |
+ throw 'Bad data at line ' + lineNo + ' col: ' + colLength + ' ' + data.substr(0, 15); |
+ } else if (data[0] === '\'') { |
+ match = data.match(/^'((?:[^']|'')*)'(,|\n|$)/m); |
+ if (!match) |
+ throw 'Bad data at line ' + lineNo + ' col: ' + colLength + ' ' + data.substr(0, 15); |
+ } else { |
+ match = data.match(/^([^,\n]*)(,|\n|$)/); |
+ if (!match) |
+ throw 'Bad data at line ' + lineNo + ' col: ' + colLength + ' ' + data.substr(0, 15); |
+ match[1] = match[1] === 'NULL' ? null : match[1]; |
+ } |
+ colLength += match[0].length; |
+ if (match[2] === '\n') { |
+ lineNo++; |
+ colLength = 0; |
+ } |
+ yield [match[1], match[2] === '\n']; |
+ data = data.substr(match[0].length); |
+ } |
+} |
+ |
+ |
+// All sha1 helpers from here down. |
+ |
+ |
+/* |
+ * A JavaScript implementation of the Secure Hash Algorithm, SHA-1, as defined |
+ * in FIPS PUB 180-1 |
+ * Version 2.1a Copyright Paul Johnston 2000 - 2002. |
+ * Other contributors: Greg Holt, Andrew Kepert, Ydnar, Lostinet |
+ * Distributed under the BSD License |
+ * See http://pajhome.org.uk/crypt/md5 for details. |
+ */ |
+ |
+/* |
+ * These are the functions you'll usually want to call |
+ * They take string arguments and return either hex or base-64 encoded strings |
+ */ |
+function hex_sha1(s) { |
+ return binb2hex(core_sha1(str2binb(s), s.length * chrsz)); |
+} |
+function b64_sha1(s) { |
+ return binb2b64(core_sha1(str2binb(s), s.length * chrsz)); |
+} |
+function str_sha1(s) { |
+ return binb2str(core_sha1(str2binb(s), s.length * chrsz)); |
+} |
+function hex_hmac_sha1(key, data) { |
+ return binb2hex(core_hmac_sha1(key, data)); |
+} |
+function b64_hmac_sha1(key, data) { |
+ return binb2b64(core_hmac_sha1(key, data)); |
+} |
+function str_hmac_sha1(key, data) { |
+ return binb2str(core_hmac_sha1(key, data)); |
+} |
+ |
+/* |
+ * Perform a simple self-test to see if the VM is working |
+ */ |
+function sha1_vm_test() { |
+ return hex_sha1('abc') == 'a9993e364706816aba3e25717850c26c9cd0d89d'; |
+} |
+ |
+/* |
+ * Calculate the SHA-1 of an array of big-endian words, and a bit length |
+ */ |
+function core_sha1(x, len) { |
+ /* append padding */ |
+ x[len >> 5] |= 0x80 << (24 - len % 32); |
+ x[((len + 64 >> 9) << 4) + 15] = len; |
+ |
+ var w = Array(80); |
+ var a = 1732584193; |
+ var b = -271733879; |
+ var c = -1732584194; |
+ var d = 271733878; |
+ var e = -1009589776; |
+ |
+ for (var i = 0; i < x.length; i += 16) { |
+ var olda = a; |
+ var oldb = b; |
+ var oldc = c; |
+ var oldd = d; |
+ var olde = e; |
+ |
+ for (var j = 0; j < 80; j++) { |
+ if (j < 16) |
+ w[j] = x[i + j]; |
+ else |
+ w[j] = rol(w[j - 3] ^ w[j - 8] ^ w[j - 14] ^ w[j - 16], 1); |
+ var t = safe_add(safe_add(rol(a, 5), sha1_ft(j, b, c, d)), safe_add(safe_add(e, w[j]), sha1_kt(j))); |
+ e = d; |
+ d = c; |
+ c = rol(b, 30); |
+ b = a; |
+ a = t; |
+ } |
+ |
+ a = safe_add(a, olda); |
+ b = safe_add(b, oldb); |
+ c = safe_add(c, oldc); |
+ d = safe_add(d, oldd); |
+ e = safe_add(e, olde); |
+ } |
+ return Array(a, b, c, d, e); |
+} |
+ |
+/* |
+ * Perform the appropriate triplet combination function for the current |
+ * iteration |
+ */ |
+function sha1_ft(t, b, c, d) { |
+ if (t < 20) |
+ return (b & c) | ((~b) & d); |
+ if (t < 40) |
+ return b ^ c ^ d; |
+ if (t < 60) |
+ return (b & c) | (b & d) | (c & d); |
+ return b ^ c ^ d; |
+} |
+ |
+/* |
+ * Determine the appropriate additive constant for the current iteration |
+ */ |
+function sha1_kt(t) { |
+ return (t < 20) ? 1518500249 : (t < 40) ? 1859775393 : (t < 60) ? -1894007588 : -899497514; |
+} |
+ |
+/* |
+ * Calculate the HMAC-SHA1 of a key and some data |
+ */ |
+function core_hmac_sha1(key, data) { |
+ var bkey = str2binb(key); |
+ if (bkey.length > 16) |
+ bkey = core_sha1(bkey, key.length * chrsz); |
+ |
+ var ipad = Array(16), opad = Array(16); |
+ for (var i = 0; i < 16; i++) { |
+ ipad[i] = bkey[i] ^ 0x36363636; |
+ opad[i] = bkey[i] ^ 0x5C5C5C5C; |
+ } |
+ |
+ var hash = core_sha1(ipad.concat(str2binb(data)), 512 + data.length * chrsz); |
+ return core_sha1(opad.concat(hash), 512 + 160); |
+} |
+ |
+/* |
+ * Add integers, wrapping at 2^32. This uses 16-bit operations internally |
+ * to work around bugs in some JS interpreters. |
+ */ |
+function safe_add(x, y) { |
+ var lsw = (x & 0xFFFF) + (y & 0xFFFF); |
+ var msw = (x >> 16) + (y >> 16) + (lsw >> 16); |
+ return (msw << 16) | (lsw & 0xFFFF); |
+} |
+ |
+/* |
+ * Bitwise rotate a 32-bit number to the left. |
+ */ |
+function rol(num, cnt) { |
+ return (num << cnt) | (num >>> (32 - cnt)); |
+} |
+ |
+/* |
+ * Convert an 8-bit or 16-bit string to an array of big-endian words |
+ * In 8-bit function, characters >255 have their hi-byte silently ignored. |
+ */ |
+function str2binb(str) { |
+ var bin = Array(); |
+ var mask = (1 << chrsz) - 1; |
+ for (var i = 0; i < str.length * chrsz; i += chrsz) |
+ bin[i >> 5] |= (str.charCodeAt(i / chrsz) & mask) << (32 - chrsz - i % 32); |
+ return bin; |
+} |
+ |
+/* |
+ * Convert an array of big-endian words to a string |
+ */ |
+function binb2str(bin) { |
+ var str = ''; |
+ var mask = (1 << chrsz) - 1; |
+ for (var i = 0; i < bin.length * 32; i += chrsz) |
+ str += String.fromCharCode((bin[i >> 5] >>> (32 - chrsz - i % 32)) & mask); |
+ return str; |
+} |
+ |
+/* |
+ * Convert an array of big-endian words to a hex string. |
+ */ |
+function binb2hex(binarray) { |
+ var hex_tab = hexcase ? '0123456789ABCDEF' : '0123456789abcdef'; |
+ var str = ''; |
+ for (var i = 0; i < binarray.length * 4; i++) { |
+ str += hex_tab.charAt((binarray[i >> 2] >> ((3 - i % 4) * 8 + 4)) & 0xF) + |
+ hex_tab.charAt((binarray[i >> 2] >> ((3 - i % 4) * 8)) & 0xF); |
+ } |
+ return str; |
+} |
+ |
+/* |
+ * Convert an array of big-endian words to a base-64 string |
+ */ |
+function binb2b64(binarray) { |
+ var tab = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; |
+ var str = ''; |
+ for (var i = 0; i < binarray.length * 4; i += 3) { |
+ var triplet = (((binarray[i >> 2] >> 8 * (3 - i % 4)) & 0xFF) << 16) | |
+ (((binarray[i + 1 >> 2] >> 8 * (3 - (i + 1) % 4)) & 0xFF) << 8) | |
+ ((binarray[i + 2 >> 2] >> 8 * (3 - (i + 2) % 4)) & 0xFF); |
+ for (var j = 0; j < 4; j++) { |
+ if (i * 8 + j * 6 > binarray.length * 32) |
+ str += b64pad; |
+ else |
+ str += tab.charAt((triplet >> 6 * (3 - j)) & 0x3F); |
+ } |
+ } |
+ return str; |
+} |