Add support for multipage distillation.

components/dom_distiller/core/distiller.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef COMPONENTS_DOM_DISTILLER_CORE_DISTILLER_H_
 #define COMPONENTS_DOM_DISTILLER_CORE_DISTILLER_H_
 
 #include <map>
+#include <set>
 #include <string>
 
 #include "base/callback.h"
 #include "base/gtest_prod_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/values.h"
 #include "components/dom_distiller/core/distiller_page.h"
 #include "components/dom_distiller/core/distiller_url_fetcher.h"
-#include "components/dom_distiller/core/proto/distilled_page.pb.h"
+#include "components/dom_distiller/core/proto/distilled_article.pb.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "url/gurl.h"
 
 namespace dom_distiller {
 
 class DistillerImpl;
 
 class Distiller {
  public:
-  typedef base::Callback<void(
-      scoped_ptr<DistilledPageProto>)> DistillerCallback;
+  typedef base::Callback<void(scoped_ptr<DistilledArticleProto>)>
+      DistillerCallback;
   virtual ~Distiller() {}
 
   // Distills a page, and asynchrounously returns the article HTML to the
   // supplied callback.
   virtual void DistillPage(const GURL& url,
                            const DistillerCallback& callback) = 0;
 };
 
 class DistillerFactory {
  public:
   virtual scoped_ptr<Distiller> CreateDistiller() = 0;
   virtual ~DistillerFactory() {}
 };
 
 // Factory for creating a Distiller.
 class DistillerFactoryImpl : public DistillerFactory {
  public:
   DistillerFactoryImpl(
       scoped_ptr<DistillerPageFactory> distiller_page_factory,
       scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory);
   virtual ~DistillerFactoryImpl();
   virtual scoped_ptr<Distiller> CreateDistiller() OVERRIDE;
 
  private:
   scoped_ptr<DistillerPageFactory> distiller_page_factory_;
   scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_;
 };
 
 // Distills a article from a page and associated pages.

[cjhopman, 2014/01/29 21:44:13]

This class is starting to handle a lot of things:
loading a page and injecting javascript to distill it
creating and tracking image fetchers
managing the process of distilling multiple pages

I think we should split this up into two classes:
1. handles loading a URL with a DistillerPage and executing javascript on that
page to extract content/title/image_urls
and
2. manages multi-page + creation/tracking of image fetchers

wdyt?

[shashi, 2014/01/29 22:51:37]

Makes sense, but I will prefer to do it in a separate CL, this CL is already a
bit big, plus I will be able to unit_tests for the new classes.
On 2014/01/29 21:44:13, cjhopman wrote:

 class DistillerImpl : public Distiller,
                       public DistillerPage::Delegate {
  public:
   DistillerImpl(
       const DistillerPageFactory& distiller_page_factory,
       const DistillerURLFetcherFactory& distiller_url_fetcher_factory);
   virtual ~DistillerImpl();
 
   // Creates an execution context. This must be called once before any calls are
   // made to distill the page.
   virtual void Init();
 
   virtual void DistillPage(const GURL& url,
                            const DistillerCallback& callback) OVERRIDE;
 
   // PageDistillerContext::Delegate
   virtual void OnLoadURLDone() OVERRIDE;
   virtual void OnExecuteJavaScriptDone(const base::Value* value) OVERRIDE;
 
-  void OnFetchImageDone(const std::string& id, const std::string& response);
+  void OnFetchImageDone(DistilledPageProto* distilled_page_proto,
+                        const std::string& id,
+                        const std::string& response);
 
  private:
   virtual void LoadURL(const GURL& url);
-  virtual void FetchImage(const std::string& image_id, const std::string& item);
+  virtual void FetchImage(DistilledPageProto* distilled_page_proto,
+                          const std::string& image_id,
+                          const std::string& item);
 
   // Injects JavaScript to distill a loaded page down to its important content,
   // e.g., extracting a news article from its surrounding boilerplate.
   void GetDistilledContent();
 
+  // Adds url as a new page to the |article_proto_| and triggers distillation
+  // for the newly added page.
+  void AddAndDistillPage(const GURL& url);
+
+  // Distills the page.
+  void DistillNextPage(const GURL& url);
+
+  // Runs distillation callback when all distillation tasks are finished.
+  void DistillationTaskComplete();
+
+  // Returns the last page of |article_proto_|.
+  DistilledPageProto* GetLastPage() const;
+
   const DistillerPageFactory& distiller_page_factory_;
   const DistillerURLFetcherFactory& distiller_url_fetcher_factory_;
   scoped_ptr<DistillerPage> distiller_page_;
   DistillerCallback distillation_cb_;
 
   std::map<std::string, DistillerURLFetcher* > image_fetchers_;
-
-  scoped_ptr<DistilledPageProto> proto_;
+  scoped_ptr<DistilledArticleProto> article_proto_;
+  bool distillation_in_progress_;
+  // Set to keep track of which urls are already seen by the distiller.
+  std::set<std::string> processed_urls_;
 
   DISALLOW_COPY_AND_ASSIGN(DistillerImpl);
 };
 
 }  // namespace dom_distiller
 
 #endif  // COMPONENTS_DOM_DISTILLER_CORE_DISTILLER_H_