[filemanager] Content metadata moved to the cache.

chrome/browser/resources/file_manager/js/metadata/metadata_cache.js

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * MetadataCache is a map from url to an object containing properties.
  * Properties are divided by types, and all properties of one type are accessed
  * at once.
  * Some of the properties:
  * {
- *   filesystem: size, modificationTime, icon
+ *   filesystem: size, modificationTime
  *   internal: presence
  *   gdata: pinned, present, hosted, editUrl, contentUrl, availableOffline
+ *   fetchedMedia: width, height, etc.
+ *   streaming: url
+ *
+ *   Following are not fetched for non-present gdata files.
+ *   media: artist, album, title, width, height, imageTransform, etc.
  *   thumbnail: url, transform
- *   media: artist, album, title
  * }
  *
  * Typical usages:
  * {
  *   cache.get([entry1, entry2], 'gdata', function(gdata) {
  *     if (gdata[0].pinned && gdata[1].pinned) alert("They are both pinned!");
  *   });
  *
  *   cache.set(entry, 'internal', {presence: 'deleted'});
  *
@@ skipping 29 matching lines @@
    *   re - regexp of urls;
    *   type - metadata type;
    *   callback - the callback.
    * TODO(dgozman): pass entries to observer if present.
    * @private
    */
   this.observers_ = [];
   this.observerId_ = 0;
 
   this.batchCount_ = 0;
+  this.totalCount_ = 0;
+
+  /**
+   * Time of first get query of the current batch. Items updated later than this
+   * will not be evicted.
+   * @private
+   */
+  this.lastBatchStart_ = new Date();
 }
 
 /**
  * Observer type: it will be notified if the url changed is exactlt the same
  * as the url passed.
  */
 MetadataCache.EXACT = 0;
 
 /**
  * Observer type: it will be notified if the url changed is an immediate child
  * of the url passed.
  */
 MetadataCache.CHILDREN = 1;
 
 /**
  * Observer type: it will be notified if the url changed is any descendant
  * of the url passed.
  */
 MetadataCache.DESCENDANTS = 2;
 
 /**
+ * Minimum number of items in cache to start eviction.
+ */
+MetadataCache.EVICTION_NUMBER = 1000;
+
+/**
  * @return {MetadataCache!} The cache with all providers.
  */
 MetadataCache.createFull = function() {
   var cache = new MetadataCache();
   cache.providers_.push(new FilesystemProvider());
   cache.providers_.push(new GDataProvider());
+  cache.providers_.push(new ContentProvider());
   return cache;
 };
 
 /**
+ * @return {boolean} Whether all providers are ready.
+ */
+MetadataCache.prototype.isInitialized = function() {
+  for (var index = 0; index < this.providers_.length; index++) {
+    if (!this.providers_[index].isInitialized()) return false;
+  }
+  return true;
+};
+
+/**
  * Fetches the metadata, puts it in the cache, and passes to callback.
  * If required metadata is already in the cache, does not fetch it again.
  * @param {string|Entry|Array.<string|Entry>} items The list of entries or
  *     file urls. May be just a single item.
  * @param {string} type The metadata type.
  * @param {Function(Object)} callback The metadata is passed to callback.
  */
 MetadataCache.prototype.get = function(items, type, callback) {
   if (!(items instanceof Array)) {
     this.getOne(items, type, callback);
@@ skipping 24 matching lines @@
   }
 };
 
 /**
  * Fetches the metadata for one Entry/FileUrl. See comments to |get|.
  * @param {Entry|string} item The entry or url.
  * @param {string} type Metadata type.
  * @param {Function(Object)} callback The callback.
  */
 MetadataCache.prototype.getOne = function(item, type, callback) {
+  if (type.indexOf('|') != -1) {
+    var types = type.split('|');
+    var result = {};
+    var typesLeft = types.length;
+
+    function onOneType(requestedType, metadata) {
+      result[requestedType] = metadata;
+      typesLeft--;
+      if (typesLeft == 0) callback(result);
+    }
+
+    for (var index = 0; index < types.length; index++) {
+      this.getOne(item, types[index], onOneType.bind(null, types[index]));
+    }
+    return;
+  }
+
   var url = this.itemToUrl_(item);
 
   // Passing entry to fetchers may save one round-trip to APIs.
   var fsEntry = item === url ? null : item;
   callback = callback || function() {};
 
-  if (!(url in this.cache_))
+  if (!(url in this.cache_)) {
     this.cache_[url] = this.createEmptyEntry_();
+    this.totalCount_++;
+  }
 
   var entry = this.cache_[url];
 
   if (type in entry.properties) {
     callback(entry.properties[type]);
     return;
   }
 
   this.startBatchUpdates();
   var providers = this.providers_.slice();
   var currentProvider;
   var self = this;
 
   function onFetched() {
     if (type in entry.properties) {
       self.endBatchUpdates();
       // Got properties from provider.
       callback(entry.properties[type]);
     } else {
       tryNextProvider();
     }
   }
 
   function onProviderProperties(properties) {
     var id = currentProvider.getId();
     var fetchedCallbacks = entry[id].callbacks;
     delete entry[id].callbacks;
-    entry[id].time = new Date();
+    entry.time = new Date();
     self.mergeProperties_(url, properties);
 
     for (var index = 0; index < fetchedCallbacks.length; index++) {
       fetchedCallbacks[index]();
     }
   }
 
   function queryProvider() {
     var id = currentProvider.getId();
     if ('callbacks' in entry[id]) {
@@ skipping 55 matching lines @@
  * @param {string} type The metadata type.
  * @param {Array.<Object>} values List of corresponding metadata values.
  */
 MetadataCache.prototype.set = function(items, type, values) {
   if (!(items instanceof Array))
     items = [items];
 
   this.startBatchUpdates();
   for (var index = 0; index < items.length; index++) {
     var url = this.itemToUrl_(items[index]);
-    if (!(url in this.cache_))
+    if (!(url in this.cache_)) {
       this.cache_[url] = this.createEmptyEntry_();
+      this.totalCount_++;
+    }
     this.cache_[url].properties[type] = values[index];
     this.notifyObservers_(url, type);
   }
   this.endBatchUpdates();
 };
 
 /**
  * Clears the cached metadata values.
  * @param {string|Entry|Array.<string|Entry>} items The list of entries or
  *     file urls. May be just a single item.
@@ skipping 52 matching lines @@
     }
   }
   return false;
 };
 
 /**
  * Start batch updates.
  */
 MetadataCache.prototype.startBatchUpdates = function() {
   this.batchCount_++;
+  if (this.batchCount_ == 1)
+    this.lastBatchStart_ = new Date();
 };
 
 /**
  * End batch updates. Notifies observers if all nested updates are finished.
  */
 MetadataCache.prototype.endBatchUpdates = function() {
   this.batchCount_--;
   if (this.batchCount_ != 0) return;
+  if (this.totalCount_ > MetadataCache.EVICTION_NUMBER)
+    this.evict_();
   for (var index = 0; index < this.observers_.length; index++) {
     var observer = this.observers_[index];
     var urls = [];
     var properties = [];
     for (var url in observer.pending) {
       if (observer.pending.hasOwnProperty(url) && url in this.cache_) {
         urls.push(url);
         properties.push(this.cache_[url].properties[observer.type] || null);
       }
     }
@@ skipping 18 matching lines @@
         // Observer expects array of urls and array of properties.
         observer.callback([url], [this.cache_[url].properties[type] || null]);
       } else {
         observer.pending[url] = true;
       }
     }
   }
 };
 
 /**
+ * Removes the oldest items from the cache.
+ * This method never removes the items from last batch.
+ * @private
+ */
+MetadataCache.prototype.evict_ = function() {
+  var toRemove = [];
+  var removeCount = this.totalCount_ -
+      Math.round(MetadataCache.EVICTION_NUMBER / 2);

[Vladislav Kaznacheev, 2012/05/16 12:46:28]

This computations needs a comment

[dgozman, 2012/05/16 15:12:11]

Done.

+  for (var url in this.cache_) {
+    if (this.cache_.hasOwnProperty(url) &&
+        this.cache_[url].time < this.lastBatchStart_) {
+      toRemove.push(url);
+    }
+  }
+
+  toRemove.sort(function(a, b) {
+    var aTime = this.cache_[a].time;
+    var bTime = this.cache_[b].time;
+    return aTime < bTime ? -1 : aTime > bTime ? 1 : 0;
+  });
+
+  removeCount = Math.min(removeCount, toRemove.length);
+  this.totalCount_ -= removeCount;
+  for (var index = 0; index < removeCount; index++) {
+    delete this.cache_[toRemove[index]];
+  }
+};
+
+/**
  * Converts Entry or file url to url.
  * @param {string|Entry} item Item to convert.
  * @return {string} File url.
  * @private
  */
 MetadataCache.prototype.itemToUrl_ = function(item) {
   if (typeof(item) == 'string')
     return item;
   else
     return item.toURL();
@@ skipping 49 matching lines @@
  * @return {boolean} Whether this provider provides this metadata.
  */
 MetadataProvider2.prototype.providesType = function(type) { return false; };
 
 /**
  * @return {string} Unique provider id.
  */
 MetadataProvider2.prototype.getId = function() { return ''; };
 
 /**
+ * @return {boolean} Whether provider is ready.
+ */
+MetadataProvider2.prototype.isInitialized = function() { return true; };
+
+/**
  * Fetches the metadata. It's suggested to return all the metadata this provider
  * can fetch at once.
  * @param {string} url File url.
  * @param {string} type Requested metadata type.
  * @param {Function(Object)} callback Callback expects a map from metadata type
  *     to metadata value.
  * @param {Entry=} opt_entry The file entry if present.
  */
 MetadataProvider2.prototype.fetch = function(url, type, callback, opt_entry) {
   throw new Error('Default metadata provider cannot fetch.');
 };
 
 
 /**
  * Provider of filesystem metadata.
  * This provider returns the following objects:
- * filesystem: {
- *   size;
- *   modificationTime;
- *   icon - string describing icon type;
- * }
+ * filesystem: { size, modificationTime }
  * @constructor
  */
 function FilesystemProvider() {
-  MetadataProvider2.call(this, 'filesystem');
+  MetadataProvider2.call(this);
 }
 
 FilesystemProvider.prototype = {
   __proto__: MetadataProvider2.prototype
 };
 
 /**
  * @param {string} url The url.
  * @return {boolean} Whether this provider supports the url.
  */
@@ skipping 44 matching lines @@
     onEntry(opt_entry);
   else
     webkitResolveLocalFileSystemURL(url, onEntry, onError);
 };
 
 /**
  * Provider of gdata metadata.
  * This provider returns the following objects:
  *     gdata: { pinned, hosted, present, dirty, editUrl, contentUrl }
  *     thumbnail: { url, transform }
+ *     streaming: { url }
  * @constructor
  */
 function GDataProvider() {
-  MetadataProvider2.call(this, 'gdata');
+  MetadataProvider2.call(this);
 
   // We batch metadata fetches into single API call.
   this.urls_ = [];
   this.callbacks_ = [];
   this.scheduled_ = false;
 
   this.callApiBound_ = this.callApi_.bind(this);
 }
 
 GDataProvider.prototype = {
@@ skipping 12 matching lines @@
  */
 GDataProvider.prototype.supportsUrl = function(url) {
   return GDataProvider.URL_PATTERN.test(url);
 };
 
 /**
  * @param {string} type The metadata type.
  * @return {boolean} Whether this provider provides this metadata.
  */
 GDataProvider.prototype.providesType = function(type) {
-  return type == 'gdata' || type == 'thumbnail';
+  return type == 'gdata' || type == 'thumbnail' ||
+      type == 'streaming' || type == 'media';
 };
 
 /**
  * @return {string} Unique provider id.
  */
 GDataProvider.prototype.getId = function() { return 'gdata'; };
 
 /**
  * Fetches the metadata.
  * @param {string} url File url.
@@ skipping 65 matching lines @@
   result.gdata = {
     present: data.isPresent,
     pinned: data.isPinned,
     hosted: data.isHosted,
     dirty: data.isDirty,
     availableOffline: GDataProvider.isAvailableOffline(data),
     availableWhenMetered: GDataProvider.isAvailableWhenMetered(data),
     contentUrl: (data.contentUrl || '').replace(/\?.*$/gi, ''),
     editUrl: data.editUrl || ''
   };
+
+  if (!data.isPresent) {
+    // Block the local fetch for gdata files, which require downloading.
+    result.thumbnail = { url: '', transform: null };
+    result.media = {};
+  }
+
   if ('thumbnailUrl' in data) {
     result.thumbnail = {
       url: data.thumbnailUrl,
-      transform: ''
+      transform: null
     };
   }
+  if (data.isPresent && ('contentUrl' in data)) {
+    result.streaming = {
+      url: data.contentUrl.replace(/\?.*$/gi, '')
+    };
+  }
   return result;
 };
+
+
+/**
+ * Provider of content metadata.
+ * This provider returns the following objects:
+ * thumbnail: { url, transform }
+ * media: { artist, album, title, width, height, imageTransform, etc. }
+ * fetchedMedia: { same fields here }

[Vladislav Kaznacheev, 2012/05/16 12:46:28]

Please explain why we need fetchedMedia

[dgozman, 2012/05/16 15:12:11]

Explained in comment at the start of file.

+ * @constructor
+ */
+function ContentProvider() {
+  MetadataProvider2.call(this);
+
+  // Pass all URLs to the metadata reader until we have a correct filter.
+  this.urlFilter_ = /.*/;
+
+  var path = document.location.pathname;
+  var workerPath = document.location.origin +
+      path.substring(0, path.lastIndexOf('/') + 1) +
+      'js/metadata/metadata_dispatcher.js';
+
+  this.dispatcher_ = new Worker(workerPath);
+  this.dispatcher_.onmessage = this.onMessage_.bind(this);
+  this.dispatcher_.postMessage({verb: 'init'});
+
+  // Initialization is not complete until the Worker sends back the
+  // 'initialized' message.  See below.
+  this.initialized_ = false;
+
+  // Map from url to callback.
+  // Note that simultaneous requests for same url are handled in MetadataCache.
+  this.callbacks_ = {};
+}
+
+ContentProvider.prototype = {
+  __proto__: MetadataProvider2.prototype
+};
+
+/**
+ * @param {string} url The url.
+ * @return {boolean} Whether this provider supports the url.
+ */
+ContentProvider.prototype.supportsUrl = function(url) {
+  return url.match(this.urlFilter_);
+};
+
+/**
+ * @param {string} type The metadata type.
+ * @return {boolean} Whether this provider provides this metadata.
+ */
+ContentProvider.prototype.providesType = function(type) {
+  return type == 'thumbnail' || type == 'fetchedMedia' || type == 'media';
+};
+
+/**
+ * @return {string} Unique provider id.
+ */
+ContentProvider.prototype.getId = function() { return 'content'; };
+
+/**
+ * Fetches the metadata.
+ * @param {string} url File url.
+ * @param {string} type Requested metadata type.
+ * @param {Function(Object)} callback Callback expects a map from metadata type
+ *     to metadata value.
+ * @param {Entry=} opt_entry The file entry if present.
+ */
+ContentProvider.prototype.fetch = function(url, type, callback, opt_entry) {
+  if (opt_entry && opt_entry.isDirectory) {
+    callback({});
+    return;
+  }
+  this.callbacks_[url] = callback;
+  this.dispatcher_.postMessage({verb: 'request', arguments: [url]});
+};
+
+/**
+ * Dispatch a message from a metadata reader to the appropriate on* method.
+ * @param {Object} event The event.
+ * @private
+ */
+ContentProvider.prototype.onMessage_ = function(event) {
+  var data = event.data;
+
+  var methodName =
+      'on' + data.verb.substr(0, 1).toUpperCase() + data.verb.substr(1) + '_';
+
+  if (!(methodName in this)) {
+    console.log('Unknown message from metadata reader: ' + data.verb, data);
+    return;
+  }
+
+  this[methodName].apply(this, data.arguments);
+};
+
+/**
+ * @return {boolean} Whether provider is ready.
+ */
+ContentProvider.prototype.isInitialized = function() {
+  return this.initialized_;
+};
+
+/**
+ * Handles the 'initialized' message from the metadata reader Worker.
+ * @param {Object} regexp Regexp of supported urls.
+ * @private
+ */
+ContentProvider.prototype.onInitialized_ = function(regexp) {
+  this.urlFilter_ = regexp;
+
+  // Tests can monitor for this state with
+  // ExtensionTestMessageListener listener("worker-initialized");
+  // ASSERT_TRUE(listener.WaitUntilSatisfied());
+  // Automated tests need to wait for this, otherwise we crash in
+  // browser_test cleanup because the worker process still has
+  // URL requests in-flight.
+  chrome.test.sendMessage('worker-initialized');

[Vladislav Kaznacheev, 2012/05/16 12:46:28]

This will be the second place where this message is sent. I suggest removing it
from the old metadata provider.

[dgozman, 2012/05/16 15:12:11]

Done, as discussed offline.

+  this.initialized_ = true;
+};
+
+/**
+ * Handles the 'result' message from the worker.
+ * @param {string} url File url.
+ * @param {Object} metadata The metadata.
+ * @private
+ */
+ContentProvider.prototype.onResult_ = function(url, metadata) {
+  var callback = this.callbacks_[url];
+  delete this.callbacks_[url];
+
+  var result = {};
+
+  if ('thumbnailURL' in metadata) {
+    metadata.thumbnailTransform = metadata.thumbnailTransform || null;
+    result.thumbnail = {
+      url: metadata.thumbnailURL,
+      transform: metadata.thumbnailTransform
+    };
+    delete metadata.thumbnailURL;
+    delete metadata.thumbnailTransform;
+  }
+
+  for (var key in metadata) {
+    if (metadata.hasOwnProperty(key)) {
+      if (!('media' in result)) result.media = {};
+      result.media[key] = metadata[key];
+    }
+  }
+
+  if ('media' in result) {
+    result.fetchedMedia = result.media;
+  }
+
+  callback(result);
+};
+
+/**
+ * Handles the 'error' message from the worker.
+ * @param {string} url File url.
+ * @param {string} step Step failed.
+ * @param {string} error Error description.
+ * @param {Object?} metadata The metadata, if available.
+ * @private
+ */
+ContentProvider.prototype.onError_ = function(url, step, error, metadata) {
+  console.warn('metadata: ' + url + ': ' + step + ': ' + error);
+  metadata = metadata || {};
+  // Prevent asking for thumbnail again.
+  metadata.thumbnailURL = '';
+  this.onResult_(url, metadata);
+};
+
+/**
+ * Handles the 'log' message from the worker.
+ * @param {Array.<*>} arglist Log arguments.
+ * @private
+ */
+ContentProvider.prototype.onLog_ = function(arglist) {
+  console.log.apply(console, ['metadata:'].concat(arglist));
+};