| OLD | NEW |
| 1 // TODO(jacobr): convert this file to Dart once Dart supports all of the | 1 // TODO(jacobr): convert this file to Dart once Dart supports all of the |
| 2 // nodejs functionality used here. For example, search for all occurences of | 2 // nodejs functionality used here. For example, search for all occurences of |
| 3 // "http." and "fs." | 3 // "http." and "fs." |
| 4 var http = require('http'); | 4 var http = require('http'); |
| 5 var fs = require('fs'); | 5 var fs = require('fs'); |
| 6 | 6 |
| 7 try { | 7 try { |
| 8 fs.mkdirSync('output/crawl'); | 8 fs.mkdirSync('output/crawl'); |
| 9 } catch (e) { | 9 } catch (e) { |
| 10 // It doesn't matter if the directories already exist. | 10 // It doesn't matter if the directories already exist. |
| 11 } | 11 } |
| 12 | 12 |
| 13 var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', 'utf8')); | 13 var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', 'utf8')); |
| 14 | 14 |
| 15 var cacheData = {}; | 15 var cacheData = {}; |
| 16 | 16 |
| 17 function scrape(filename, link) { | 17 function scrape(filename, link) { |
| 18 console.log(link); | 18 console.log(link); |
| 19 var httpsPrefix = "https://"; | 19 var httpsPrefix = "https://"; |
| 20 var prefix = 'https://developer.mozilla.org/'; | 20 var prefix = 'https://developer.mozilla.org/'; |
| 21 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri='; | 21 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri='; |
| 22 if (link.indexOf(prefix) != 0 ) { | 22 if (link.indexOf(prefix) != 0 ) { |
| 23 throw "Unexpected url: " + link; | 23 throw "Unexpected url: " + link; |
| 24 } | 24 } |
| 25 var scrapePath = "/search?q=cache:" + link; | 25 var scrapePath = "/search?q=cache:" + link; |
| 26 // We crawl content from googleusercontent.com so we don't have to worry about | 26 // We crawl content from googleusercontent.com so we don't have to worry about |
| 27 // crawler politeness like we would have to if scraping developer.mozilla.org | 27 // crawler politeness like we would have to if scraping developer.mozilla.org |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 93 } else if (type == 'Notification') { | 93 } else if (type == 'Notification') { |
| 94 link = 'https://developer.mozilla.org/en/DOM/notification'; | 94 link = 'https://developer.mozilla.org/en/DOM/notification'; |
| 95 } else if (type == 'IDBDatabase') { | 95 } else if (type == 'IDBDatabase') { |
| 96 link = 'https://developer.mozilla.org/en/IndexedDB/IDBDatabase' | 96 link = 'https://developer.mozilla.org/en/IndexedDB/IDBDatabase' |
| 97 } | 97 } |
| 98 if (link != null) { | 98 if (link != null) { |
| 99 entry.push({index: 0, link: link, title: type}); | 99 entry.push({index: 0, link: link, title: type}); |
| 100 scrape(type + 0, link); | 100 scrape(type + 0, link); |
| 101 continue; | 101 continue; |
| 102 } | 102 } |
| 103 | 103 |
| 104 for (j = 0; j < items.length; j++) { | 104 for (j = 0; j < items.length; j++) { |
| 105 var item = items[j]; | 105 var item = items[j]; |
| 106 var prefix = 'https://developer.mozilla.org/'; | 106 var prefix = 'https://developer.mozilla.org/'; |
| 107 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri='; | 107 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri='; |
| 108 // Be optimistic and replace article not found links with links to where the | 108 // Be optimistic and replace article not found links with links to where the |
| 109 // article should be. | 109 // article should be. |
| 110 link = item['link']; | 110 link = item['link']; |
| 111 if (link.indexOf(notFoundPrefix) == 0) { | 111 if (link.indexOf(notFoundPrefix) == 0) { |
| 112 link = prefix + link.substr(notFoundPrefix.length); | 112 link = prefix + link.substr(notFoundPrefix.length); |
| 113 } | 113 } |
| 114 | 114 |
| 115 entry.push({index: j, link: link, title: item['title']}); | 115 entry.push({index: j, link: link, title: item['title']}); |
| 116 scrape(type + j, link); | 116 scrape(type + j, link); |
| 117 } | 117 } |
| 118 } | 118 } |
| 119 | 119 |
| 120 fs.writeFileSync('output/crawl/cache.json', JSON.stringify(cacheData, null, ' ')
, 'utf8'); | 120 fs.writeFileSync('output/crawl/cache.json', JSON.stringify(cacheData, null, ' ')
, 'utf8'); |
| OLD | NEW |