[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_storage_context.h

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_
 #define STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback_forward.h"
 #include "base/gtest_prod_util.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "storage/browser/blob/blob_data_handle.h"
+#include "storage/browser/blob/blob_memory_controller.h"
 #include "storage/browser/blob/blob_storage_registry.h"
 #include "storage/browser/blob/internal_blob_data.h"
 #include "storage/browser/storage_browser_export.h"
 #include "storage/common/blob_storage/blob_storage_constants.h"
-#include "storage/common/data_element.h"
 
 class GURL;
 
 namespace base {
 class FilePath;
 class Time;
+class TaskRunner;
 }
 
 namespace content {
 class BlobDispatcherHost;
 class BlobDispatcherHostTest;
 }
 
 namespace storage {
-
 class BlobDataBuilder;
+class BlobDataHandle;
 class BlobDataItem;
 class BlobDataSnapshot;
 class ShareableBlobDataItem;
 
-// This class handles the logistics of blob Storage within the browser process,
-// and maintains a mapping from blob uuid to the data. The class is single
-// threaded and should only be used on the IO thread.
-// In chromium, there is one instance per profile.
-class STORAGE_EXPORT BlobStorageContext
-    : public base::SupportsWeakPtr<BlobStorageContext> {
+// This class handles the logistics of blob storage within the browser process.
+// We are single threaded and should only be used on the IO thread. In Chromium
+// there is one instance per profile.
+class STORAGE_EXPORT BlobStorageContext {
  public:
+  using PopulatationAllowedCallback =
+      InternalBlobData::PopulatationAllowedCallback;
+
+  // Initializes the context without disk support.
   BlobStorageContext();
+  // Initialized the context with disk support if the file_runner is populated.
+  BlobStorageContext(const base::FilePath& storage_directory,
+                     scoped_refptr<base::TaskRunner> file_runner);
   ~BlobStorageContext();
 
   std::unique_ptr<BlobDataHandle> GetBlobDataFromUUID(const std::string& uuid);
   std::unique_ptr<BlobDataHandle> GetBlobDataFromPublicURL(const GURL& url);
 
-  // Useful for coining blobs from within the browser process. If the
-  // blob cannot be added due to memory consumption, returns NULL.
-  // A builder should not be used twice to create blobs, as the internal
-  // resources are refcounted instead of copied for memory efficiency.
-  // To cleanly use a builder multiple times, please call Clone() on the
-  // builder, or even better for memory savings, clear the builder and append
-  // the previously constructed blob.
   std::unique_ptr<BlobDataHandle> AddFinishedBlob(
       const BlobDataBuilder& builder);
 
-  // Deprecated, use const ref version above.
+  // Deprecated, use const ref version above or BuildBlob below.
   std::unique_ptr<BlobDataHandle> AddFinishedBlob(
       const BlobDataBuilder* builder);
 
+  std::unique_ptr<BlobDataHandle> AddBrokenBlob(
+      const std::string& uuid,
+      const std::string& content_type,
+      const std::string& content_disposition,
+      BlobStatus reason);
+
   // Useful for coining blob urls from within the browser process.
   bool RegisterPublicBlobURL(const GURL& url, const std::string& uuid);
   void RevokePublicBlobURL(const GURL& url);
 
-  size_t memory_usage() const { return memory_usage_; }
   size_t blob_count() const { return registry_.blob_count(); }
-  size_t memory_available() const {
-    return kBlobStorageMaxMemoryUsage - memory_usage_;
+
+  const BlobStorageRegistry& registry() { return registry_; }
+
+  // This builds a blob with the given |input_builder| and returns a handle to
+  // the constructed Blob. Blob metadata and data should be accessed through
+  // this handle.
+  // If there is data present that needs further population then we will call
+  // |can_populate_memory| when we're ready for the user data to be populated
+  // with the PENDING_DATA_POPULATION status. This can happen synchronously or
+  // asynchronously. Otherwise |can_populate_memory| should be null.
+  // In the further population case, the caller must call either
+  // FinishedPopulatingPendingBlob or BreakAndFinishPendingBlob after
+  // |can_populate_memory| is called to signify the data is finished populating
+  // or an error occurred (respectively).
+  // If the returned handle is broken, then the possible error cases are:
+  // * OUT_OF_MEMORY if we don't have enough memory to store the blob,
+  // * REFERENCED_BLOB_BROKEN if a referenced blob is broken or we're
+  //   referencing ourself.
+  std::unique_ptr<BlobDataHandle> BuildBlob(
+      const BlobDataBuilder& input_builder,
+      const PopulatationAllowedCallback& population_allowed_callback);
+
+  // This breaks a blob that is currently being build by using the BuildBlob
+  // method above. Any callbacks waiting on this blob, including the
+  // |population_allowed_callback| callback given to BuildBlob, will be called
+  // with this status code.
+  void BreakAndFinishPendingBlob(const std::string& uuid, BlobStatus code);
+
+  // After calling BuildBlob above, the user should call this to notify the
+  // construction system that the unpopulated data in the given blob has been
+  // populated. Caller must have all pending items populated in the original
+  // builder |content| given in BuildBlob or we'll check-fail. If there isn't
+  // any pending data in the |input_builder| for the BuildBlob call, then this
+  // doesn't need to be called.
+  void FinishedPopulatingPendingBlob(const std::string& uuid);
+
+  const BlobMemoryController& memory_controller() { return memory_controller_; }
+
+  base::WeakPtr<BlobStorageContext> AsWeakPtr() {
+    return ptr_factory_.GetWeakPtr();
   }
 
-  const BlobStorageRegistry& registry() { return registry_; }
-
- private:
-  using BlobRegistryEntry = BlobStorageRegistry::Entry;
-  using BlobConstructedCallback = BlobStorageRegistry::BlobConstructedCallback;
+ protected:
   friend class content::BlobDispatcherHost;
+  friend class content::BlobDispatcherHostTest;
   friend class BlobAsyncBuilderHost;
   friend class BlobAsyncBuilderHostTest;
   friend class BlobDataHandle;
   friend class BlobDataHandle::BlobDataHandleShared;
-  friend class BlobReaderTest;
-  FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, HandleBeforeAsyncCancel);
-  FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, ReadFromIncompleteBlob);
+  friend class BlobFlattenerTest;
+  friend class BlobSliceTest;
   friend class BlobStorageContextTest;
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, IncrementDecrementRef);
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, OnCancelBuildingBlob);
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, PublicBlobUrls);
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest,
-                           TestUnknownBrokenAndBuildingBlobReference);
-  friend class ViewBlobInternalsJob;
-
-  // CompletePendingBlob or CancelPendingBlob should be called after this.
-  void CreatePendingBlob(const std::string& uuid,
-                         const std::string& content_type,
-                         const std::string& content_disposition);
-
-  // This includes resolving blob references in the builder. This will run the
-  // callbacks given in RunOnConstructionComplete.
-  void CompletePendingBlob(const BlobDataBuilder& external_builder);
-
-  // This will run the callbacks given in RunOnConstructionComplete.
-  void CancelPendingBlob(const std::string& uuid,
-                         IPCBlobCreationCancelCode reason);
+
+  // Transforms a BlobDataBuilder into a InternalBlobData with no blob
+  // references. We use BlobSlice to flatten out these references. We record
+  // the total size and items for memory and file quota requests.
+  // Visible for testing.
+  struct STORAGE_EXPORT BlobFlattener {
+    BlobFlattener(const BlobDataBuilder& input_builder,
+                  InternalBlobData* output_blob,
+                  BlobStorageRegistry* registry);
+    ~BlobFlattener();
+
+    // This can be:
+    // * PENDING_INTERNALS if we're populated and don't need quota.
+    // * PENDING if there's pending data that the user needs to
+    //   populate. This also means we need to request quota.
+    // * INVALID_CONSTRUCTION_ARGUMENTS if we have invalid input.
+    // * REFERENCED_BLOB_BROKEN if one of the referenced blobs is broken or we
+    //   reference ourself.
+    BlobStatus status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+
+    // This is the total size of the blob, including all memory, files, etc.
+    uint64_t total_size = 0;
+
+    std::vector<std::pair<std::string, InternalBlobData*>> dependent_blobs;
+
+    uint64_t memory_quota_needed = 0;
+    std::vector<scoped_refptr<ShareableBlobDataItem>> pending_memory_items;
+
+    uint64_t file_quota_needed = 0;
+    std::vector<scoped_refptr<ShareableBlobDataItem>> pending_file_items;
+
+    std::vector<ShareableBlobDataItem*> user_items;
+
+    // These record all future copies we'll need to do from referenced blobs.
+    // This
+    // happens when we do a partial slice from a pending data or file item.
+    std::vector<InternalBlobData::ItemCopyEntry> copies;
+  };
+
+  // Used when a blob reference has a size and offset. Records the source items
+  // and memory we need to copy if either side of slice intersects an item.
+  // Visible for testing.
+  struct STORAGE_EXPORT BlobSlice {
+    BlobSlice(const InternalBlobData& source,
+              uint64_t slice_offset,
+              uint64_t slice_size);
+    ~BlobSlice();
+
+    // Size of memory copying from the source blob.
+    size_t copying_memory_size = 0;
+
+    size_t first_item_slice_offset = 0;
+    // Populated if our first slice item is a temporary item that we'll copy to
+    // later from this |first_source_item|, at offset |first_item_slice_offset|.
+    scoped_refptr<ShareableBlobDataItem> first_source_item;
+    // Populated if our last slice item is a temporary item that we'll copy to
+    // later from this |last_source_item|.
+    scoped_refptr<ShareableBlobDataItem> last_source_item;
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> dest_items;
+  };
 
   void IncrementBlobRefCount(const std::string& uuid);
   void DecrementBlobRefCount(const std::string& uuid);
 
-  // Methods called by BlobDataHandle:
   // This will return an empty snapshot until the blob is complete.
   // TODO(dmurph): After we make the snapshot method in BlobHandle private, then
   // make this DCHECK on the blob not being complete.
   std::unique_ptr<BlobDataSnapshot> CreateSnapshot(const std::string& uuid);
-  bool IsBroken(const std::string& uuid) const;
-  bool IsBeingBuilt(const std::string& uuid) const;
-  // Runs |done| when construction completes, with true if it was successful,
-  // and false if there was an error, which is reported in the second argument
-  // of the callback.
+
+  BlobStatus GetBlobStatus(const std::string& uuid) const;
+
+  // Runs |done| when construction completes with the final status of the blob.
   void RunOnConstructionComplete(const std::string& uuid,
-                                 const BlobConstructedCallback& done);
-
-  // Appends the given blob item to the blob builder.  The new blob
-  // retains ownership of data_item if applicable, and returns false if there
-  // was an error and pouplates the error_code.  We can either have an error of:
-  // OUT_OF_MEMORY: We are out of memory to store this blob.
-  // REFERENCED_BLOB_BROKEN: One of the referenced blobs is broken.
-  bool AppendAllocatedBlobItem(const std::string& target_blob_uuid,
-                               scoped_refptr<BlobDataItem> data_item,
-                               InternalBlobData::Builder* target_blob_data,
-                               IPCBlobCreationCancelCode* error_code);
-
-  // Allocates a shareable blob data item, with blob references resolved.  If
-  // there isn't enough memory, then a nullptr is returned.
-  scoped_refptr<ShareableBlobDataItem> AllocateShareableBlobDataItem(
-      const std::string& target_blob_uuid,
-      scoped_refptr<BlobDataItem> data_item);
-
-  // Deconstructs the blob and appends it's contents to the target blob.  Items
-  // are shared if possible, and copied if the given offset and length
-  // have to split an item.
-  bool AppendBlob(const std::string& target_blob_uuid,
-                  const InternalBlobData& blob,
-                  uint64_t offset,
-                  uint64_t length,
-                  InternalBlobData::Builder* target_blob_data);
+                                 const BlobStatusCallback& done_callback);
+
+  BlobStorageRegistry* mutable_registry() { return &registry_; }
+
+  BlobMemoryController* mutable_memory_controller() {
+    return &memory_controller_;
+  }
+
+ private:
+  std::unique_ptr<BlobDataHandle> CreateHandle(const std::string& uuid,
+                                               InternalBlobData* entry);
+
+  void FinishedPopulatingPendingBlob(InternalBlobData* entry);
+
+  void FinishBuilding(InternalBlobData* entry);
+
+  void RequestUserPopulation(
+      InternalBlobData* entry,
+      std::vector<BlobMemoryController::FileCreationInfo> files);
+
+  void OnEnoughSizeForMemory(const std::string& uuid, bool can_fit);
+
+  void OnEnoughSizeForTransportFiles(
+      const std::string& uuid,
+      bool can_fit,
+      std::vector<BlobMemoryController::FileCreationInfo> files);
+
+  void OnDependentBlobFinished(const std::string& owning_blob_uuid,
+                               BlobStatus reason);
+
+  void ClearAndFreeMemory(const std::string& uuid, InternalBlobData* entry);
+
+  // Shortcut method to set the status of the given items POPULATED_WITH_QUOTA.
+  // We expect the previous state to be QUOTA_GRANTED.
+  void SetItemStateToPopulated(std::vector<ShareableBlobDataItem*>* items);
 
   BlobStorageRegistry registry_;
-
-  // Used to keep track of how much memory is being utilized for blob data,
-  // we count only the items of TYPE_DATA which are held in memory and not
-  // items of TYPE_FILE.
-  size_t memory_usage_;
+  BlobMemoryController memory_controller_;
+  base::WeakPtrFactory<BlobStorageContext> ptr_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(BlobStorageContext);
 };
 
 }  // namespace storage
 
 #endif  // STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_