Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(76)

Side by Side Diff: utils/apidoc/mdn/crawl.js

Issue 10889017: Repair bitrot in the utils/apidoc/mdn/ code. It now runs all the way through, (Closed) Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/
Patch Set: Created 8 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 // TODO(jacobr): convert this file to Dart once Dart supports all of the 1 // TODO(jacobr): convert this file to Dart once Dart supports all of the
2 // nodejs functionality used here. For example, search for all occurences of 2 // nodejs functionality used here. For example, search for all occurences of
3 // "http." and "fs." 3 // "http." and "fs."
4 var http = require('http'); 4 var http = require('http');
5 var fs = require('fs'); 5 var fs = require('fs');
6 6
7 try { 7 try {
8 fs.mkdirSync('output/crawl'); 8 fs.mkdirSync('output/crawl');
9 } catch (e) { 9 } catch (e) {
10 // It doesn't matter if the directories already exist. 10 // It doesn't matter if the directories already exist.
11 } 11 }
12 12
13 var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', 'utf8')); 13 var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', 'utf8'));
14 14
15 var cacheData = {}; 15 var cacheData = {};
16 16
17 function scrape(filename, link) { 17 function scrape(filename, link) {
18 console.log(link); 18 console.log(link);
19 var httpsPrefix = "https://"; 19 var httpsPrefix = "https://";
20 var prefix = 'https://developer.mozilla.org/'; 20 var prefix = 'https://developer.mozilla.org/';
21 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri='; 21 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri=';
22 if (link.indexOf(prefix) != 0 ) { 22 if (link.indexOf(prefix) != 0 ) {
23 throw "Unexpected url: " + link; 23 throw "Unexpected url: " + link;
24 } 24 }
25 var scrapePath = "/search?q=cache:" + link; 25 var scrapePath = "/search?q=cache:" + link;
26 // We crawl content from googleusercontent.com so we don't have to worry about 26 // We crawl content from googleusercontent.com so we don't have to worry about
27 // crawler politeness like we would have to if scraping developer.mozilla.org 27 // crawler politeness like we would have to if scraping developer.mozilla.org
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
93 } else if (type == 'Notification') { 93 } else if (type == 'Notification') {
94 link = 'https://developer.mozilla.org/en/DOM/notification'; 94 link = 'https://developer.mozilla.org/en/DOM/notification';
95 } else if (type == 'IDBDatabase') { 95 } else if (type == 'IDBDatabase') {
96 link = 'https://developer.mozilla.org/en/IndexedDB/IDBDatabase' 96 link = 'https://developer.mozilla.org/en/IndexedDB/IDBDatabase'
97 } 97 }
98 if (link != null) { 98 if (link != null) {
99 entry.push({index: 0, link: link, title: type}); 99 entry.push({index: 0, link: link, title: type});
100 scrape(type + 0, link); 100 scrape(type + 0, link);
101 continue; 101 continue;
102 } 102 }
103 103
104 for (j = 0; j < items.length; j++) { 104 for (j = 0; j < items.length; j++) {
105 var item = items[j]; 105 var item = items[j];
106 var prefix = 'https://developer.mozilla.org/'; 106 var prefix = 'https://developer.mozilla.org/';
107 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri='; 107 var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri=';
108 // Be optimistic and replace article not found links with links to where the 108 // Be optimistic and replace article not found links with links to where the
109 // article should be. 109 // article should be.
110 link = item['link']; 110 link = item['link'];
111 if (link.indexOf(notFoundPrefix) == 0) { 111 if (link.indexOf(notFoundPrefix) == 0) {
112 link = prefix + link.substr(notFoundPrefix.length); 112 link = prefix + link.substr(notFoundPrefix.length);
113 } 113 }
114 114
115 entry.push({index: j, link: link, title: item['title']}); 115 entry.push({index: j, link: link, title: item['title']});
116 scrape(type + j, link); 116 scrape(type + j, link);
117 } 117 }
118 } 118 }
119 119
120 fs.writeFileSync('output/crawl/cache.json', JSON.stringify(cacheData, null, ' ') , 'utf8'); 120 fs.writeFileSync('output/crawl/cache.json', JSON.stringify(cacheData, null, ' ') , 'utf8');
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698