Integrate MDN content into API documentation.

utils/apidoc/mdn/crawl.js

+var http = require('http');

[nweiz, 2012/02/01 00:10:39]

Why is this file in JS?

[Jacob, 2012/02/01 07:48:26]

Because the dart server libraries currently lack the functionality we need. 
When they do this script should be ported.

[nweiz, 2012/02/01 21:23:02]

It would be good to document exactly what's keeping all the JS scripts as JS.

[Jacob, 2012/02/02 05:26:38]

that seems low value.  Just look at each call to a Node method (e.g. http. and
fs.)
and you have something that is missing in Dart currently.  However I have added
a TODO to convert to dart when all these methods are available.

[nweiz, 2012/02/02 19:54:34]

It was non-obvious to me. It's important that someone coming across this in
three months is able to understand whether or not it should be converted to
Dart.

[Jacob, 2012/02/02 22:03:14]

The trouble is if I made a list of which methods still need to be implemented by
the dart server side support then that list will be obsolete in 3 months so
there isn't value in explicitly listing that here.

+var fs = require('fs');
+
+try {
+  fs.mkdirSync('output/crawl');
+} catch (e) {
+  // It doesn't matter if the directories already exist.    
+}
+
+var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', 'utf8'));
+
+var cacheData = {};
+
+function scrape(filename, link) { 
+  console.log(link);
+  var httpsPrefix = "https://";
+  var prefix = 'https://developer.mozilla.org/';
+  var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri=';
+  if (link.indexOf(prefix) != 0 ) {
+    throw "Unexpected url:" + link;

[nweiz, 2012/02/01 00:10:39]

Space after ":"

[Jacob, 2012/02/01 07:48:26]

Done.

+  }
+  var scrapePath = "/search?q=cache:" + link;
+  // We crawl content from googleusercontent.com so we don't have to worry about
+  // crawler politeness like we would have to if scraping developer.mozilla.org
+  // directly.
+  var options = {
+    host: 'webcache.googleusercontent.com',
+    path: scrapePath,
+    port: 80,
+    method: 'GET'
+  };
+
+  var req = http.request(options, function(res) {
+    res.setEncoding('utf8');
+    var data='';
+
+    res.on('data', function(d) {
+      data += d;
+    });
+    var onClose = function(e) {
+      console.log("Writing crawl result for " + link);
+      fs.writeFileSync("output/crawl/" + filename + ".html", data, 'utf8');
+    }
+    res.on('close', onClose);
+    res.on('end', onClose);
+  });
+  req.end();
+
+  req.on('error', function(e) {
+    throw "Error " + e + " scraping " + link;
+  });
+}
+
+for (var i = 0; i < domTypes.length; i++) {
+  var type = domTypes[i];
+
+  // Json containing the search results for the current type.
+  var data = fs.readFileSync("output/search/" + type + ".json");
+  json = JSON.parse(data);
+  if (!('items' in json)) {
+    console.warn("No search results for " + type);
+    continue;
+  }
+  var items = json['items'];
+
+  var entry = [];
+  cacheData[type] = entry;
+
+  // Hardcode the correct matching url for a few types where the search engine
+  // gets the wrong answer.
+  var link = null;
+  if (type == 'Screen') {
+    link = 'https://developer.mozilla.org/en/DOM/window.screen';
+  } else if (type == 'Text') {
+    link = 'https://developer.mozilla.org/en/DOM/Text';
+  } else if (type == 'Touch') {
+    link = 'https://developer.mozilla.org/en/DOM/Touch';
+  } else if (type == 'TouchEvent' || type == 'webkitTouchEvent' || type == 'WebkitTouchEvent' || type == 'WebKitTouchEvent') {
+    link = 'https://developer.mozilla.org/en/DOM/TouchEvent';
+  } else if (type == 'HTMLSpanElement') {
+    link = 'https://developer.mozilla.org/en/HTML/Element/span';
+  } else if (type == 'HTMLPreElement') {
+    link = 'https://developer.mozilla.org/en/HTML/Element/pre';
+  } else if (type == 'HTMLFrameElement') {
+    link = 'https://developer.mozilla.org/en/HTML/Element/frame';
+  } else if (type == 'HTMLFrameSetElement') {
+    link = 'https://developer.mozilla.org/en/HTML/Element/frameset';
+  } else if (type == 'Geolocation') {
+    link = 'https://developer.mozilla.org/en/nsIDOMGeolocation;'
+  } else if (type == 'Notification') {
+    link = 'https://developer.mozilla.org/en/DOM/notification';
+  } else if (type == 'IDBDatabase') {
+    link = 'https://developer.mozilla.org/en/IndexedDB/IDBDatabase'
+  }
+  if (link != null) {
+    entry.push({index: 0, link: link, title: type});
+    scrape(type + 0, link);
+    continue;
+  }
+  
+  for (j = 0; j < items.length; j++) {
+    var item = items[j];
+    var prefix = 'https://developer.mozilla.org/';
+    var notFoundPrefix = 'https://developer.mozilla.org/Article_not_found?uri=';
+    // Be optimistic and replace article not found links with links to where the
+    // article should be.
+    link = item['link'];
+    if (link.indexOf(notFoundPrefix) == 0) {
+      link = prefix + link.substr(notFoundPrefix.length);
+    }
+
+    entry.push({index: j, link: link, title: item['title']});
+    scrape(type + j, link);
+  }
+}
+
+fs.writeFileSync('output/crawl/cache.json', JSON.stringify(cacheData, null, ' '), 'utf8');