OLD | NEW |
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "storage/browser/blob/blob_storage_context.h" | 5 #include "storage/browser/blob/blob_storage_context.h" |
6 | 6 |
7 #include <stddef.h> | |
8 #include <stdint.h> | |
9 | |
10 #include <algorithm> | 7 #include <algorithm> |
11 #include <limits> | 8 #include <limits> |
12 #include <memory> | 9 #include <memory> |
| 10 #include <set> |
13 #include <utility> | 11 #include <utility> |
14 | 12 |
15 #include "base/bind.h" | 13 #include "base/bind.h" |
16 #include "base/callback.h" | 14 #include "base/callback.h" |
17 #include "base/location.h" | 15 #include "base/location.h" |
18 #include "base/logging.h" | 16 #include "base/logging.h" |
19 #include "base/memory/ptr_util.h" | 17 #include "base/memory/ptr_util.h" |
20 #include "base/message_loop/message_loop.h" | 18 #include "base/message_loop/message_loop.h" |
21 #include "base/metrics/histogram.h" | 19 #include "base/metrics/histogram.h" |
| 20 #include "base/numerics/safe_conversions.h" |
| 21 #include "base/numerics/safe_math.h" |
| 22 #include "base/task_runner.h" |
22 #include "base/threading/thread_task_runner_handle.h" | 23 #include "base/threading/thread_task_runner_handle.h" |
23 #include "base/trace_event/trace_event.h" | 24 #include "base/trace_event/trace_event.h" |
| 25 #include "storage/browser/blob/blob_async_transport_request_builder.h" |
24 #include "storage/browser/blob/blob_data_builder.h" | 26 #include "storage/browser/blob/blob_data_builder.h" |
25 #include "storage/browser/blob/blob_data_handle.h" | |
26 #include "storage/browser/blob/blob_data_item.h" | 27 #include "storage/browser/blob/blob_data_item.h" |
27 #include "storage/browser/blob/blob_data_snapshot.h" | 28 #include "storage/browser/blob/blob_data_snapshot.h" |
28 #include "storage/browser/blob/shareable_blob_data_item.h" | 29 #include "storage/browser/blob/shareable_blob_data_item.h" |
| 30 #include "storage/common/data_element.h" |
29 #include "url/gurl.h" | 31 #include "url/gurl.h" |
30 | 32 |
31 namespace storage { | 33 namespace storage { |
32 using BlobRegistryEntry = BlobStorageRegistry::Entry; | 34 namespace { |
33 using BlobState = BlobStorageRegistry::BlobState; | 35 using TransportState = InternalBlobData::TransportState; |
34 | 36 using ItemCopyEntry = InternalBlobData::ItemCopyEntry; |
35 BlobStorageContext::BlobStorageContext() : memory_usage_(0) {} | 37 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask; |
| 38 |
| 39 bool IsBytes(DataElement::Type type) { |
| 40 return type == DataElement::TYPE_BYTES || |
| 41 type == DataElement::TYPE_BYTES_DESCRIPTION; |
| 42 } |
| 43 } // namespace |
| 44 |
| 45 BlobStorageContext::BlobFlattener::BlobFlattener( |
| 46 const BlobDataBuilder& input_builder, |
| 47 InternalBlobData* output_blob, |
| 48 BlobStorageRegistry* registry) { |
| 49 const std::string& uuid = input_builder.uuid_; |
| 50 std::set<std::string> dependent_blob_uuids; |
| 51 bool contains_pending_content = false; |
| 52 |
| 53 size_t num_files_with_unknown_size = 0; |
| 54 size_t num_building_dependent_blobs = 0; |
| 55 |
| 56 base::CheckedNumeric<uint64_t> checked_total_size = 0; |
| 57 base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0; |
| 58 base::CheckedNumeric<uint64_t> checked_file_quota_needed = 0; |
| 59 |
| 60 for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) { |
| 61 const DataElement& input_element = input_item->data_element(); |
| 62 DataElement::Type type = input_element.type(); |
| 63 uint64_t length = input_element.length(); |
| 64 |
| 65 if (IsBytes(type)) { |
| 66 DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length()); |
| 67 contains_pending_content |= type == DataElement::TYPE_BYTES_DESCRIPTION; |
| 68 checked_memory_quota_needed += length; |
| 69 checked_total_size += length; |
| 70 scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem( |
| 71 std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED); |
| 72 pending_memory_items.push_back(item); |
| 73 user_items.push_back(item.get()); |
| 74 output_blob->AppendSharedBlobItem(uuid, std::move(item)); |
| 75 continue; |
| 76 } |
| 77 if (type == DataElement::TYPE_BLOB) { |
| 78 InternalBlobData* ref_entry = |
| 79 registry->GetEntry(input_element.blob_uuid()); |
| 80 |
| 81 if (!ref_entry || input_element.blob_uuid() == uuid) { |
| 82 status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
| 83 return; |
| 84 } |
| 85 |
| 86 if (BlobStatusIsError(ref_entry->status())) { |
| 87 status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN; |
| 88 return; |
| 89 } |
| 90 |
| 91 if (ref_entry->total_size() == DataElement::kUnknownSize) { |
| 92 // We can't reference a blob with unknown size. |
| 93 status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
| 94 return; |
| 95 } |
| 96 |
| 97 if (dependent_blob_uuids.find(input_element.blob_uuid()) == |
| 98 dependent_blob_uuids.end()) { |
| 99 dependent_blobs.push_back( |
| 100 std::make_pair(input_element.blob_uuid(), ref_entry)); |
| 101 dependent_blob_uuids.insert(input_element.blob_uuid()); |
| 102 if (BlobStatusIsPending(ref_entry->status())) { |
| 103 num_building_dependent_blobs++; |
| 104 } |
| 105 } |
| 106 |
| 107 length = length == DataElement::kUnknownSize ? ref_entry->total_size() |
| 108 : input_element.length(); |
| 109 checked_total_size += length; |
| 110 |
| 111 // If we're referencing the whole blob, then we don't need to slice. |
| 112 if (input_element.offset() == 0 && length == ref_entry->total_size()) { |
| 113 for (const auto& shareable_item : ref_entry->items()) { |
| 114 output_blob->AppendSharedBlobItem(uuid, shareable_item); |
| 115 } |
| 116 continue; |
| 117 } |
| 118 |
| 119 // Validate our reference has good offset & length. |
| 120 if (input_element.offset() + length > ref_entry->total_size()) { |
| 121 status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
| 122 return; |
| 123 } |
| 124 |
| 125 BlobSlice slice(*ref_entry, input_element.offset(), length); |
| 126 |
| 127 if (slice.first_source_item) { |
| 128 copies.push_back(ItemCopyEntry(slice.first_source_item, |
| 129 slice.first_item_slice_offset, |
| 130 slice.dest_items.front())); |
| 131 pending_memory_items.push_back(slice.dest_items.front()); |
| 132 } |
| 133 if (slice.last_source_item) { |
| 134 copies.push_back( |
| 135 ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back())); |
| 136 pending_memory_items.push_back(slice.dest_items.back()); |
| 137 } |
| 138 checked_memory_quota_needed += slice.copying_memory_size; |
| 139 |
| 140 for (auto& shareable_item : slice.dest_items) { |
| 141 output_blob->AppendSharedBlobItem(uuid, std::move(shareable_item)); |
| 142 } |
| 143 continue; |
| 144 } |
| 145 |
| 146 // If the source item is a temporary file item, then we need to keep track |
| 147 // of that and mark is as needing quota. |
| 148 scoped_refptr<ShareableBlobDataItem> item; |
| 149 if (type == DataElement::TYPE_FILE && |
| 150 BlobDataBuilder::IsFutureFileItem(input_element)) { |
| 151 item = new ShareableBlobDataItem(std::move(input_item), |
| 152 ShareableBlobDataItem::QUOTA_NEEDED); |
| 153 contains_pending_content = true; |
| 154 pending_file_items.push_back(item); |
| 155 user_items.push_back(item.get()); |
| 156 checked_file_quota_needed += length; |
| 157 } else { |
| 158 item = new ShareableBlobDataItem( |
| 159 std::move(input_item), |
| 160 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA); |
| 161 } |
| 162 if (length == DataElement::kUnknownSize) |
| 163 num_files_with_unknown_size++; |
| 164 |
| 165 checked_total_size += length; |
| 166 output_blob->AppendSharedBlobItem(uuid, std::move(item)); |
| 167 } |
| 168 |
| 169 if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) { |
| 170 LOG(ERROR) << "We only allow an unknown size when it's a single file item"; |
| 171 status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
| 172 return; |
| 173 } |
| 174 if (!checked_total_size.IsValid() || !checked_memory_quota_needed.IsValid() || |
| 175 !checked_file_quota_needed.IsValid()) { |
| 176 LOG(ERROR) << "Blob sizes aren't valid."; |
| 177 status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
| 178 return; |
| 179 } |
| 180 total_size = checked_total_size.ValueOrDie(); |
| 181 memory_quota_needed = checked_memory_quota_needed.ValueOrDie(); |
| 182 file_quota_needed = checked_file_quota_needed.ValueOrDie(); |
| 183 if (memory_quota_needed || file_quota_needed) { |
| 184 status = BlobStatus::PENDING_QUOTA; |
| 185 } else if (contains_pending_content) { |
| 186 status = BlobStatus::PENDING_TRANSPORT; |
| 187 } else { |
| 188 // We either have dependent blobs or we're done. Either way, we're still |
| 189 // pending final construction. |
| 190 status = BlobStatus::PENDING_INTERNALS; |
| 191 } |
| 192 } |
| 193 |
| 194 BlobStorageContext::BlobFlattener::~BlobFlattener() {} |
| 195 |
| 196 BlobStorageContext::BlobSlice::BlobSlice(const InternalBlobData& source, |
| 197 uint64_t slice_offset, |
| 198 uint64_t slice_size) { |
| 199 const auto& source_items = source.items(); |
| 200 const auto& offsets = source.offsets(); |
| 201 LOG(ERROR) << "doing a slice at " << slice_offset << " with size " |
| 202 << slice_size; |
| 203 DCHECK_LE(slice_offset + slice_size, source.total_size()); |
| 204 size_t item_index = |
| 205 std::upper_bound(offsets.begin(), offsets.end(), slice_offset) - |
| 206 offsets.begin(); |
| 207 uint64_t item_offset = |
| 208 item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1]; |
| 209 size_t num_items = source_items.size(); |
| 210 |
| 211 size_t first_item_index = item_index; |
| 212 copying_memory_size = 0; |
| 213 |
| 214 // Read starting from 'first_item_index' and 'item_offset'. |
| 215 for (uint64_t total_sliced = 0; |
| 216 item_index < num_items && total_sliced < slice_size; item_index++) { |
| 217 const scoped_refptr<BlobDataItem>& source_item = |
| 218 source_items[item_index]->item(); |
| 219 uint64_t source_length = source_item->length(); |
| 220 DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max()); |
| 221 DCHECK_NE(source_length, 0ull); |
| 222 |
| 223 uint64_t read_size = |
| 224 std::min(source_length - item_offset, slice_size - total_sliced); |
| 225 total_sliced += read_size; |
| 226 |
| 227 if (read_size == source_length) { |
| 228 // We can share the entire item. |
| 229 LOG(ERROR) << "we can share"; |
| 230 dest_items.push_back(source_items[item_index]); |
| 231 continue; |
| 232 } |
| 233 |
| 234 scoped_refptr<BlobDataItem> data_item; |
| 235 ShareableBlobDataItem::State state = |
| 236 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA; |
| 237 switch (source_item->type()) { |
| 238 case DataElement::TYPE_BYTES_DESCRIPTION: |
| 239 case DataElement::TYPE_BYTES: { |
| 240 if (item_index == first_item_index) { |
| 241 first_item_slice_offset = item_offset; |
| 242 first_source_item = source_items[item_index]; |
| 243 } else { |
| 244 last_source_item = source_items[item_index]; |
| 245 } |
| 246 copying_memory_size += read_size; |
| 247 // Since we don't have quota yet for memory, we create temporary items |
| 248 // for this data. When our blob is finished constructing, all dependent |
| 249 // blobs are done, and we have enough memory quota, we'll copy the data |
| 250 // over. |
| 251 std::unique_ptr<DataElement> element(new DataElement()); |
| 252 element->SetToBytesDescription(base::checked_cast<size_t>(read_size)); |
| 253 data_item = new BlobDataItem(std::move(element)); |
| 254 state = ShareableBlobDataItem::QUOTA_NEEDED; |
| 255 break; |
| 256 } |
| 257 case DataElement::TYPE_FILE: { |
| 258 std::unique_ptr<DataElement> element(new DataElement()); |
| 259 element->SetToFilePathRange( |
| 260 source_item->path(), source_item->offset() + item_offset, read_size, |
| 261 source_item->expected_modification_time()); |
| 262 data_item = |
| 263 new BlobDataItem(std::move(element), source_item->data_handle_); |
| 264 |
| 265 if (BlobDataBuilder::IsFutureFileItem(source_item->data_element())) { |
| 266 // Since we don't have file path / reference for our future file, we |
| 267 // create another file item with this temporary file name. When our |
| 268 // blob is finished constructing, all dependent blobs are done, and we |
| 269 // can copy the handle over. |
| 270 LOG(ERROR) << "we're slicing a temp file item!"; |
| 271 if (item_index == first_item_index) { |
| 272 first_item_slice_offset = item_offset; |
| 273 first_source_item = source_items[item_index]; |
| 274 } else { |
| 275 last_source_item = source_items[item_index]; |
| 276 } |
| 277 state = ShareableBlobDataItem::QUOTA_NEEDED; |
| 278 } |
| 279 break; |
| 280 } |
| 281 case DataElement::TYPE_FILE_FILESYSTEM: { |
| 282 std::unique_ptr<DataElement> element(new DataElement()); |
| 283 element->SetToFileSystemUrlRange( |
| 284 source_item->filesystem_url(), source_item->offset() + item_offset, |
| 285 read_size, source_item->expected_modification_time()); |
| 286 data_item = new BlobDataItem(std::move(element)); |
| 287 break; |
| 288 } |
| 289 case DataElement::TYPE_DISK_CACHE_ENTRY: { |
| 290 std::unique_ptr<DataElement> element(new DataElement()); |
| 291 element->SetToDiskCacheEntryRange(source_item->offset() + item_offset, |
| 292 read_size); |
| 293 data_item = |
| 294 new BlobDataItem(std::move(element), source_item->data_handle_, |
| 295 source_item->disk_cache_entry(), |
| 296 source_item->disk_cache_stream_index(), |
| 297 source_item->disk_cache_side_stream_index()); |
| 298 break; |
| 299 } |
| 300 case DataElement::TYPE_BLOB: |
| 301 case DataElement::TYPE_UNKNOWN: |
| 302 CHECK(false) << "Illegal blob item type: " << source_item->type(); |
| 303 } |
| 304 dest_items.push_back( |
| 305 new ShareableBlobDataItem(std::move(data_item), state)); |
| 306 item_offset = 0; |
| 307 } |
| 308 } |
| 309 |
| 310 BlobStorageContext::BlobSlice::~BlobSlice() {} |
| 311 |
| 312 BlobStorageContext::BlobStorageContext() |
| 313 : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()), |
| 314 ptr_factory_(this) {} |
| 315 BlobStorageContext::BlobStorageContext( |
| 316 const base::FilePath& storage_directory, |
| 317 scoped_refptr<base::TaskRunner> file_runner) |
| 318 : memory_controller_(storage_directory, file_runner), ptr_factory_(this) {} |
36 | 319 |
37 BlobStorageContext::~BlobStorageContext() { | 320 BlobStorageContext::~BlobStorageContext() { |
38 } | 321 } |
39 | 322 |
40 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID( | 323 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID( |
41 const std::string& uuid) { | 324 const std::string& uuid) { |
42 BlobRegistryEntry* entry = registry_.GetEntry(uuid); | 325 InternalBlobData* entry = registry_.GetEntry(uuid); |
43 if (!entry) { | 326 if (!entry) |
44 return nullptr; | 327 return nullptr; |
45 } | 328 return CreateHandle(uuid, entry); |
46 return base::WrapUnique( | |
47 new BlobDataHandle(uuid, entry->content_type, entry->content_disposition, | |
48 this, base::ThreadTaskRunnerHandle::Get().get())); | |
49 } | 329 } |
50 | 330 |
51 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL( | 331 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL( |
52 const GURL& url) { | 332 const GURL& url) { |
53 std::string uuid; | 333 std::string uuid; |
54 BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid); | 334 InternalBlobData* entry = registry_.GetEntryFromURL(url, &uuid); |
55 if (!entry) { | 335 if (!entry) |
56 return nullptr; | 336 return nullptr; |
57 } | 337 return CreateHandle(uuid, entry); |
58 return base::WrapUnique( | |
59 new BlobDataHandle(uuid, entry->content_type, entry->content_disposition, | |
60 this, base::ThreadTaskRunnerHandle::Get().get())); | |
61 } | 338 } |
62 | 339 |
63 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( | 340 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( |
64 const BlobDataBuilder& external_builder) { | 341 const BlobDataBuilder& external_builder) { |
65 TRACE_EVENT0("Blob", "Context::AddFinishedBlob"); | 342 TRACE_EVENT0("Blob", "Context::AddFinishedBlob"); |
66 CreatePendingBlob(external_builder.uuid(), external_builder.content_type_, | 343 return BuildBlob(external_builder, PopulatationAllowedCallback()); |
67 external_builder.content_disposition_); | |
68 CompletePendingBlob(external_builder); | |
69 std::unique_ptr<BlobDataHandle> handle = | |
70 GetBlobDataFromUUID(external_builder.uuid_); | |
71 DecrementBlobRefCount(external_builder.uuid_); | |
72 return handle; | |
73 } | 344 } |
74 | 345 |
75 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( | 346 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob( |
76 const BlobDataBuilder* builder) { | 347 const BlobDataBuilder* builder) { |
77 DCHECK(builder); | 348 DCHECK(builder); |
78 return AddFinishedBlob(*builder); | 349 return AddFinishedBlob(*builder); |
79 } | 350 } |
80 | 351 |
| 352 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob( |
| 353 const std::string& uuid, |
| 354 const std::string& content_type, |
| 355 const std::string& content_disposition, |
| 356 BlobStatus reason) { |
| 357 DCHECK(!registry_.HasEntry(uuid)); |
| 358 DCHECK(BlobStatusIsError(reason)); |
| 359 InternalBlobData* entry = |
| 360 registry_.CreateEntry(uuid, content_type, content_disposition); |
| 361 entry->status_ = reason; |
| 362 FinishBuilding(entry); |
| 363 return CreateHandle(uuid, entry); |
| 364 } |
| 365 |
81 bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url, | 366 bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url, |
82 const std::string& uuid) { | 367 const std::string& uuid) { |
83 if (!registry_.CreateUrlMapping(blob_url, uuid)) { | 368 if (!registry_.CreateUrlMapping(blob_url, uuid)) |
84 return false; | 369 return false; |
85 } | |
86 IncrementBlobRefCount(uuid); | 370 IncrementBlobRefCount(uuid); |
87 return true; | 371 return true; |
88 } | 372 } |
89 | 373 |
90 void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) { | 374 void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) { |
91 std::string uuid; | 375 std::string uuid; |
92 if (!registry_.DeleteURLMapping(blob_url, &uuid)) { | 376 if (!registry_.DeleteURLMapping(blob_url, &uuid)) |
93 return; | 377 return; |
94 } | |
95 DecrementBlobRefCount(uuid); | 378 DecrementBlobRefCount(uuid); |
96 } | 379 } |
97 | 380 |
98 void BlobStorageContext::CreatePendingBlob( | 381 std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob( |
99 const std::string& uuid, | 382 const BlobDataBuilder& content, |
100 const std::string& content_type, | 383 const PopulatationAllowedCallback& can_populate_memory) { |
101 const std::string& content_disposition) { | 384 DCHECK(!registry_.HasEntry(content.uuid_)); |
102 DCHECK(!registry_.GetEntry(uuid) && !uuid.empty()); | 385 |
103 registry_.CreateEntry(uuid, content_type, content_disposition); | 386 InternalBlobData* entry = registry_.CreateEntry( |
104 } | 387 content.uuid_, content.content_type_, content.content_disposition_); |
105 | 388 |
106 void BlobStorageContext::CompletePendingBlob( | 389 // This flattens all blob references in the transportion content out and |
107 const BlobDataBuilder& external_builder) { | 390 // stores the complete item representation in the internal data. |
108 BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid()); | 391 BlobFlattener flattener(content, entry, ®istry_); |
| 392 |
| 393 // If we contain invalid references we're invalid input. |
| 394 if (flattener.status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS) { |
| 395 LOG(ERROR) << "invalid construction args"; |
| 396 BreakAndFinishPendingBlob(content.uuid_, |
| 397 BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS); |
| 398 return CreateHandle(content.uuid_, entry); |
| 399 } |
| 400 |
| 401 // If we contain broken references (or we reference ourself) we want to fail. |
| 402 if (flattener.status == BlobStatus::ERR_REFERENCED_BLOB_BROKEN) { |
| 403 LOG(ERROR) << "references don't exist"; |
| 404 BreakAndFinishPendingBlob(content.uuid_, |
| 405 BlobStatus::ERR_REFERENCED_BLOB_BROKEN); |
| 406 return CreateHandle(content.uuid_, entry); |
| 407 } |
| 408 |
| 409 LOG(ERROR) << "We need " << flattener.memory_quota_needed |
| 410 << " memory quota, and " << flattener.file_quota_needed |
| 411 << " file quota."; |
| 412 |
| 413 if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed + |
| 414 flattener.file_quota_needed)) { |
| 415 LOG(ERROR) << "we can't fit."; |
| 416 BreakAndFinishPendingBlob(content.uuid_, BlobStatus::ERR_OUT_OF_MEMORY); |
| 417 return CreateHandle(content.uuid_, entry); |
| 418 } |
| 419 |
| 420 entry->status_ = flattener.status; |
| 421 entry->size_ = flattener.total_size; |
| 422 std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry); |
| 423 |
| 424 size_t num_building_dependent_blobs = 0; |
| 425 std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs; |
| 426 // We hold a handle to all blobs we're using. This is important, as our memory |
| 427 // accounting can be delayed until OnEnoughSizeForBlobData is called, and we |
| 428 // only free memory on canceling when we've done this accounting. If a |
| 429 // dependent blob is dereferenced, then we're the last blob holding onto that |
| 430 // data item, and we need to account for that. So we prevent that case by |
| 431 // holding onto all blobs. |
| 432 for (const std::pair<std::string, InternalBlobData*>& pending_blob : |
| 433 flattener.dependent_blobs) { |
| 434 LOG(ERROR) << "adding dependent blob"; |
| 435 dependent_blobs.push_back( |
| 436 CreateHandle(pending_blob.first, pending_blob.second)); |
| 437 if (BlobStatusIsPending(pending_blob.second->status())) { |
| 438 pending_blob.second->building_state_->build_completion_callbacks |
| 439 .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished, |
| 440 ptr_factory_.GetWeakPtr(), content.uuid_)); |
| 441 num_building_dependent_blobs++; |
| 442 } |
| 443 } |
| 444 |
| 445 DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT || |
| 446 !can_populate_memory) |
| 447 << "There is no pending content for the user to populate, so the " |
| 448 "callback should be null."; |
| 449 DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT || |
| 450 can_populate_memory) |
| 451 << "If we have pending content then there needs to be a callback " |
| 452 "present."; |
| 453 |
| 454 TransportState transport_state = TransportState::DONE; |
| 455 if (!flattener.user_items.empty()) { |
| 456 transport_state = flattener.file_quota_needed > 0 |
| 457 ? TransportState::PENDING_FILE_QUOTA |
| 458 : TransportState::PENDING_MEMORY_QUOTA; |
| 459 } |
| 460 |
| 461 entry->building_state_.reset(new InternalBlobData::BuildingState( |
| 462 transport_state, |
| 463 /* user_data_population_callback */ can_populate_memory, |
| 464 num_building_dependent_blobs, |
| 465 /* memory_quota_needed */ flattener.memory_quota_needed > 0, |
| 466 /* file_quota_needed */ flattener.file_quota_needed > 0)); |
| 467 |
| 468 InternalBlobData::BuildingState* building_state = |
| 469 entry->building_state_.get(); |
| 470 std::swap(building_state->copies, flattener.copies); |
| 471 std::swap(building_state->dependent_blobs, dependent_blobs); |
| 472 std::swap(building_state->user_items, flattener.user_items); |
| 473 |
| 474 if (flattener.memory_quota_needed) { |
| 475 // We can finish our blob before returning from this method. |
| 476 base::WeakPtr<QuotaAllocationTask> pending_request = |
| 477 memory_controller_.ReserveMemoryQuota( |
| 478 std::move(flattener.pending_memory_items), |
| 479 base::Bind(&BlobStorageContext::OnEnoughSizeForMemory, |
| 480 ptr_factory_.GetWeakPtr(), content.uuid_)); |
| 481 if (entry->building_state_) |
| 482 entry->building_state_->memory_quota_request = std::move(pending_request); |
| 483 } |
| 484 |
| 485 if (flattener.file_quota_needed) { |
| 486 // We can finish our blob before returning from this method. |
| 487 base::WeakPtr<QuotaAllocationTask> pending_request = |
| 488 memory_controller_.ReserveFileQuota( |
| 489 std::move(flattener.pending_file_items), |
| 490 base::Bind(&BlobStorageContext::OnEnoughSizeForTransportFiles, |
| 491 ptr_factory_.GetWeakPtr(), content.uuid_)); |
| 492 if (entry->building_state_) |
| 493 entry->building_state_->file_quota_request = std::move(pending_request); |
| 494 } |
| 495 |
| 496 if (entry->CanFinishBuilding()) |
| 497 FinishBuilding(entry); |
| 498 |
| 499 return handle; |
| 500 } |
| 501 |
| 502 void BlobStorageContext::BreakAndFinishPendingBlob(const std::string& uuid, |
| 503 BlobStatus reason) { |
| 504 InternalBlobData* entry = registry_.GetEntry(uuid); |
109 DCHECK(entry); | 505 DCHECK(entry); |
110 DCHECK(!entry->data.get()) << "Blob already constructed: " | 506 DCHECK(BlobStatusIsError(reason)); |
111 << external_builder.uuid(); | 507 PopulatationAllowedCallback user_data_population_callback; |
112 // We want to handle storing our broken blob as well. | 508 if (entry->building_state_ && |
113 switch (entry->state) { | 509 entry->building_state_->user_data_population_callback) { |
114 case BlobState::PENDING: { | 510 user_data_population_callback = |
115 entry->data_builder.reset(new InternalBlobData::Builder()); | 511 entry->building_state_->user_data_population_callback; |
116 InternalBlobData::Builder* internal_data_builder = | 512 entry->building_state_->user_data_population_callback.Reset(); |
117 entry->data_builder.get(); | 513 } |
118 | 514 ClearAndFreeMemory(uuid, entry); |
119 bool broken = false; | 515 entry->status_ = reason; |
120 for (const auto& blob_item : external_builder.items_) { | 516 if (user_data_population_callback) { |
121 IPCBlobCreationCancelCode error_code; | 517 user_data_population_callback.Run( |
122 if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item, | 518 reason, std::vector<BlobMemoryController::FileCreationInfo>()); |
123 internal_data_builder, &error_code)) { | 519 } |
124 broken = true; | 520 entry->items_.clear(); |
125 memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage(); | 521 entry->offsets_.clear(); |
126 entry->state = BlobState::BROKEN; | 522 entry->size_ = 0; |
127 entry->broken_reason = error_code; | 523 FinishBuilding(entry); |
128 entry->data_builder.reset(new InternalBlobData::Builder()); | 524 } |
129 break; | 525 |
130 } | 526 void BlobStorageContext::FinishedPopulatingPendingBlob( |
131 } | 527 const std::string& uuid) { |
132 entry->data = entry->data_builder->Build(); | 528 InternalBlobData* entry = registry_.GetEntry(uuid); |
133 entry->data_builder.reset(); | 529 CHECK(entry) << "There is no blob entry with uuid " << uuid; |
134 entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE; | 530 DCHECK(BlobStatusIsPending(entry->status())); |
135 break; | 531 FinishedPopulatingPendingBlob(entry); |
136 } | |
137 case BlobState::BROKEN: { | |
138 InternalBlobData::Builder builder; | |
139 entry->data = builder.Build(); | |
140 break; | |
141 } | |
142 case BlobState::COMPLETE: | |
143 DCHECK(false) << "Blob already constructed: " << external_builder.uuid(); | |
144 return; | |
145 } | |
146 | |
147 UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size()); | |
148 UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken", | |
149 entry->state == BlobState::BROKEN); | |
150 if (entry->state == BlobState::BROKEN) { | |
151 UMA_HISTOGRAM_ENUMERATION( | |
152 "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason), | |
153 (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1)); | |
154 } | |
155 size_t total_memory = 0, nonshared_memory = 0; | |
156 entry->data->GetMemoryUsage(&total_memory, &nonshared_memory); | |
157 UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024); | |
158 UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize", | |
159 nonshared_memory / 1024); | |
160 TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_); | |
161 | |
162 auto runner = base::ThreadTaskRunnerHandle::Get(); | |
163 for (const auto& callback : entry->build_completion_callbacks) { | |
164 runner->PostTask(FROM_HERE, | |
165 base::Bind(callback, entry->state == BlobState::COMPLETE, | |
166 entry->broken_reason)); | |
167 } | |
168 entry->build_completion_callbacks.clear(); | |
169 } | |
170 | |
171 void BlobStorageContext::CancelPendingBlob(const std::string& uuid, | |
172 IPCBlobCreationCancelCode reason) { | |
173 BlobRegistryEntry* entry = registry_.GetEntry(uuid); | |
174 DCHECK(entry && entry->state == BlobState::PENDING); | |
175 entry->state = BlobState::BROKEN; | |
176 entry->broken_reason = reason; | |
177 CompletePendingBlob(BlobDataBuilder(uuid)); | |
178 } | 532 } |
179 | 533 |
180 void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) { | 534 void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) { |
181 BlobRegistryEntry* entry = registry_.GetEntry(uuid); | 535 InternalBlobData* entry = registry_.GetEntry(uuid); |
182 DCHECK(entry); | 536 DCHECK(entry); |
183 ++(entry->refcount); | 537 ++(entry->refcount_); |
184 } | 538 } |
185 | 539 |
186 void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) { | 540 void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) { |
187 BlobRegistryEntry* entry = registry_.GetEntry(uuid); | 541 InternalBlobData* entry = registry_.GetEntry(uuid); |
188 DCHECK(entry); | 542 DCHECK(entry); |
189 DCHECK_GT(entry->refcount, 0u); | 543 DCHECK_GT(entry->refcount_, 0u); |
190 if (--(entry->refcount) == 0) { | 544 if (--(entry->refcount_) == 0) { |
191 size_t memory_freeing = 0; | 545 LOG(ERROR) << "Clearing blob " << uuid; |
192 if (entry->state == BlobState::COMPLETE) { | 546 ClearAndFreeMemory(uuid, entry); |
193 memory_freeing = entry->data->GetUnsharedMemoryUsage(); | |
194 entry->data->RemoveBlobFromShareableItems(uuid); | |
195 } | |
196 DCHECK_LE(memory_freeing, memory_usage_); | |
197 memory_usage_ -= memory_freeing; | |
198 registry_.DeleteEntry(uuid); | 547 registry_.DeleteEntry(uuid); |
199 } | 548 } |
200 } | 549 } |
201 | 550 |
202 std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot( | 551 std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot( |
203 const std::string& uuid) { | 552 const std::string& uuid) { |
204 std::unique_ptr<BlobDataSnapshot> result; | 553 std::unique_ptr<BlobDataSnapshot> result; |
205 BlobRegistryEntry* entry = registry_.GetEntry(uuid); | 554 InternalBlobData* entry = registry_.GetEntry(uuid); |
206 if (entry->state != BlobState::COMPLETE) { | 555 if (entry->status() != BlobStatus::DONE) |
207 return result; | 556 return result; |
208 } | 557 |
209 | |
210 const InternalBlobData& data = *entry->data; | |
211 std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot( | 558 std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot( |
212 uuid, entry->content_type, entry->content_disposition)); | 559 uuid, entry->content_type(), entry->content_disposition())); |
213 snapshot->items_.reserve(data.items().size()); | 560 snapshot->items_.reserve(entry->items().size()); |
214 for (const auto& shareable_item : data.items()) { | 561 for (const auto& shareable_item : entry->items()) { |
215 snapshot->items_.push_back(shareable_item->item()); | 562 snapshot->items_.push_back(shareable_item->item()); |
216 } | 563 } |
| 564 memory_controller_.NotifyMemoryItemsUsed(entry->items()); |
217 return snapshot; | 565 return snapshot; |
218 } | 566 } |
219 | 567 |
220 bool BlobStorageContext::IsBroken(const std::string& uuid) const { | 568 BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const { |
221 const BlobRegistryEntry* entry = registry_.GetEntry(uuid); | 569 const InternalBlobData* entry = registry_.GetEntry(uuid); |
222 if (!entry) { | 570 if (!entry) |
223 return true; | 571 return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
224 } | 572 return entry->status(); |
225 return entry->state == BlobState::BROKEN; | |
226 } | |
227 | |
228 bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const { | |
229 const BlobRegistryEntry* entry = registry_.GetEntry(uuid); | |
230 if (!entry) { | |
231 return false; | |
232 } | |
233 return entry->state == BlobState::PENDING; | |
234 } | 573 } |
235 | 574 |
236 void BlobStorageContext::RunOnConstructionComplete( | 575 void BlobStorageContext::RunOnConstructionComplete( |
237 const std::string& uuid, | 576 const std::string& uuid, |
238 const BlobConstructedCallback& done) { | 577 const BlobStatusCallback& done) { |
239 BlobRegistryEntry* entry = registry_.GetEntry(uuid); | 578 InternalBlobData* entry = registry_.GetEntry(uuid); |
240 DCHECK(entry); | 579 DCHECK(entry); |
241 switch (entry->state) { | 580 if (BlobStatusIsPending(entry->status())) { |
242 case BlobState::COMPLETE: | 581 entry->building_state_->build_completion_callbacks.push_back(done); |
243 done.Run(true, IPCBlobCreationCancelCode::UNKNOWN); | 582 return; |
244 return; | 583 } |
245 case BlobState::BROKEN: | 584 done.Run(entry->status()); |
246 done.Run(false, entry->broken_reason); | 585 } |
247 return; | 586 |
248 case BlobState::PENDING: | 587 std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle( |
249 entry->build_completion_callbacks.push_back(done); | 588 const std::string& uuid, |
250 return; | 589 InternalBlobData* entry) { |
251 } | 590 return base::WrapUnique(new BlobDataHandle( |
252 NOTREACHED(); | 591 uuid, entry->content_type_, entry->content_disposition_, entry->size_, |
253 } | 592 this, base::ThreadTaskRunnerHandle::Get().get())); |
254 | 593 } |
255 bool BlobStorageContext::AppendAllocatedBlobItem( | 594 |
256 const std::string& target_blob_uuid, | 595 void BlobStorageContext::FinishedPopulatingPendingBlob(InternalBlobData* entry)
{ |
257 scoped_refptr<BlobDataItem> blob_item, | 596 |
258 InternalBlobData::Builder* target_blob_builder, | 597 SetItemStateToPopulated(&entry->building_state_->user_items); |
259 IPCBlobCreationCancelCode* error_code) { | 598 entry->building_state_->transport_state = TransportState::DONE; |
260 DCHECK(error_code); | 599 entry->status_ = BlobStatus::PENDING_INTERNALS; |
261 *error_code = IPCBlobCreationCancelCode::UNKNOWN; | 600 if (entry->CanFinishBuilding()) |
262 bool error = false; | 601 FinishBuilding(entry); |
263 | 602 } |
264 // The blob data is stored in the canonical way which only contains a | 603 |
265 // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items | 604 void BlobStorageContext::FinishBuilding(InternalBlobData* entry) { |
266 // are expanded into the primitive constituent types and reused if possible. | 605 DCHECK(entry); |
267 // 1) The Data item is denoted by the raw data and length. | 606 |
268 // 2) The File item is denoted by the file path, the range and the expected | 607 if (BlobStatusIsPending(entry->status_)) { |
269 // modification time. | 608 for (const ItemCopyEntry& copy : entry->building_state_->copies) { |
270 // 3) The FileSystem File item is denoted by the FileSystem URL, the range | 609 // Our source item can be a file if it was a slice of an unpopulated file, |
271 // and the expected modification time. | 610 // or a slice of data that was then paged to disk. |
272 // 4) The Blob item is denoted by the source blob and an offset and size. | 611 size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length()); |
273 // Internal items that are fully used by the new blob (not cut by the | 612 DataElement::Type dest_type = copy.dest_item->item()->type(); |
274 // offset or size) are shared between the blobs. Otherwise, the relevant | 613 switch (copy.source_item->item()->type()) { |
275 // portion of the item is copied. | 614 case DataElement::TYPE_BYTES: { |
276 | 615 DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION); |
277 DCHECK(blob_item->data_element_ptr()); | 616 const char* src_data = |
278 const DataElement& data_element = blob_item->data_element(); | 617 copy.source_item->item()->bytes() + copy.source_item_offset; |
279 uint64_t length = data_element.length(); | 618 copy.dest_item->item()->item_->SetToBytes(src_data, dest_size); |
280 uint64_t offset = data_element.offset(); | 619 } break; |
281 UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend", | 620 case DataElement::TYPE_FILE: { |
282 memory_usage_ / 1024); | 621 LOG(ERROR) << "Source item has been paged or we started as a file, " |
283 switch (data_element.type()) { | 622 "so we're grabbing file ref " |
284 case DataElement::TYPE_BYTES: | 623 << dest_type; |
285 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024); | 624 |
286 DCHECK(!offset); | 625 // If we expected a memory item (and our source was paged to disk) we |
287 if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) { | 626 // free that memory. |
288 error = true; | 627 if (dest_type == DataElement::TYPE_BYTES_DESCRIPTION) |
289 *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY; | 628 copy.dest_item->set_memory_allocation(nullptr); |
290 break; | 629 |
| 630 const DataElement& source_element = |
| 631 copy.source_item->item()->data_element(); |
| 632 std::unique_ptr<DataElement> new_element(new DataElement()); |
| 633 new_element->SetToFilePathRange( |
| 634 source_element.path(), |
| 635 source_element.offset() + copy.source_item_offset, dest_size, |
| 636 source_element.expected_modification_time()); |
| 637 scoped_refptr<BlobDataItem> new_item(new BlobDataItem( |
| 638 std::move(new_element), copy.source_item->item()->data_handle_)); |
| 639 copy.dest_item->set_item(std::move(new_item)); |
| 640 } break; |
| 641 case DataElement::TYPE_UNKNOWN: |
| 642 case DataElement::TYPE_BLOB: |
| 643 case DataElement::TYPE_BYTES_DESCRIPTION: |
| 644 case DataElement::TYPE_FILE_FILESYSTEM: |
| 645 case DataElement::TYPE_DISK_CACHE_ENTRY: |
| 646 NOTREACHED(); |
| 647 break; |
291 } | 648 } |
292 memory_usage_ += length; | 649 copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA); |
293 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 650 } |
294 target_blob_uuid, blob_item, | 651 entry->status_ = BlobStatus::DONE; |
295 ShareableBlobDataItem::POPULATED_WITH_QUOTA)); | 652 } |
296 break; | 653 |
297 case DataElement::TYPE_FILE: { | 654 LOG(ERROR) << "Finished blob."; |
298 bool full_file = (length == std::numeric_limits<uint64_t>::max()); | 655 |
299 UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file); | 656 std::vector<BlobStatusCallback> callbacks; |
300 if (!full_file) { | 657 if (entry->building_state_.get()) { |
301 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File", | 658 std::swap(callbacks, entry->building_state_->build_completion_callbacks); |
302 (length - offset) / 1024); | 659 entry->building_state_.reset(); |
303 } | 660 } |
304 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 661 |
305 target_blob_uuid, blob_item, | 662 memory_controller_.NotifyMemoryItemsUsed(entry->items()); |
306 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA)); | 663 |
307 break; | 664 auto runner = base::ThreadTaskRunnerHandle::Get(); |
308 } | 665 for (const auto& callback : callbacks) |
309 case DataElement::TYPE_FILE_FILESYSTEM: { | 666 runner->PostTask(FROM_HERE, base::Bind(callback, entry->status())); |
310 bool full_file = (length == std::numeric_limits<uint64_t>::max()); | 667 |
311 UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown", | 668 for (const auto& shareable_item : entry->items()) { |
312 full_file); | 669 DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION, |
313 if (!full_file) { | 670 shareable_item->item()->type()); |
314 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem", | 671 if (!shareable_item->IsPopulated()) { |
315 (length - offset) / 1024); | 672 PrintTo(*shareable_item, &LOG_STREAM(ERROR)); |
316 } | 673 DCHECK(shareable_item->IsPopulated()) << shareable_item->state(); |
317 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 674 } |
318 target_blob_uuid, blob_item, | 675 } |
319 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA)); | 676 } |
320 break; | 677 |
321 } | 678 void BlobStorageContext::RequestUserPopulation( |
322 case DataElement::TYPE_BLOB: { | 679 InternalBlobData* entry, |
323 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob", | 680 std::vector<BlobMemoryController::FileCreationInfo> files) { |
324 (length - offset) / 1024); | 681 LOG(ERROR) << "Checking if we need user population"; |
325 // We grab the handle to ensure it stays around while we copy it. | 682 |
326 std::unique_ptr<BlobDataHandle> src = | 683 InternalBlobData::BuildingState* building_state = |
327 GetBlobDataFromUUID(data_element.blob_uuid()); | 684 entry->building_state_.get(); |
328 if (!src || src->IsBroken() || src->IsBeingBuilt()) { | 685 if (building_state->user_data_population_callback) { |
329 error = true; | 686 LOG(ERROR) << "yes, asking user for population"; |
330 *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN; | 687 PopulatationAllowedCallback user_data_population_callback = |
331 break; | 688 building_state->user_data_population_callback; |
332 } | 689 building_state->user_data_population_callback.Reset(); |
333 BlobRegistryEntry* other_entry = | 690 user_data_population_callback.Run(BlobStatus::PENDING_TRANSPORT, |
334 registry_.GetEntry(data_element.blob_uuid()); | 691 std::move(files)); |
335 DCHECK(other_entry->data); | 692 return; |
336 if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length, | 693 } |
337 target_blob_builder)) { | 694 LOG(ERROR) << "no populatio gneed"; |
338 error = true; | 695 DCHECK(files.empty()); |
339 *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY; | 696 FinishedPopulatingPendingBlob(entry); |
340 } | 697 } |
341 break; | 698 |
342 } | 699 void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid, |
343 case DataElement::TYPE_DISK_CACHE_ENTRY: { | 700 bool success) { |
344 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry", | 701 if (!success) { |
345 (length - offset) / 1024); | 702 BreakAndFinishPendingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY); |
346 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 703 return; |
347 target_blob_uuid, blob_item, | 704 } |
348 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA)); | 705 InternalBlobData* entry = registry_.GetEntry(uuid); |
349 break; | 706 if (!entry || !entry->building_state_.get()) |
350 } | 707 return; |
351 case DataElement::TYPE_BYTES_DESCRIPTION: | 708 InternalBlobData::BuildingState& building_state = *entry->building_state_; |
352 case DataElement::TYPE_UNKNOWN: | 709 building_state.memory_quota_granted = true; |
353 NOTREACHED(); | 710 DCHECK(!building_state.memory_quota_request); |
354 break; | 711 |
355 } | 712 // We can crequest memory quota for both blob copies and transport memory. |
356 UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend", | 713 if (building_state.transport_state == TransportState::PENDING_MEMORY_QUOTA) { |
357 memory_usage_ / 1024); | 714 entry->status_ = BlobStatus::PENDING_TRANSPORT; |
358 return !error; | 715 building_state.transport_state = TransportState::PENDING_TRANSPORT; |
359 } | 716 RequestUserPopulation( |
360 | 717 entry, std::vector<BlobMemoryController::FileCreationInfo>()); |
361 bool BlobStorageContext::AppendBlob( | 718 } else if (building_state.transport_state == TransportState::DONE) { |
362 const std::string& target_blob_uuid, | 719 // Transport data could have been already transported by files. |
363 const InternalBlobData& blob, | 720 entry->status_ = BlobStatus::PENDING_INTERNALS; |
364 uint64_t offset, | 721 } |
365 uint64_t length, | 722 |
366 InternalBlobData::Builder* target_blob_builder) { | 723 if (entry->CanFinishBuilding()) |
367 DCHECK_GT(length, 0ull); | 724 FinishBuilding(entry); |
368 | 725 } |
369 const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items(); | 726 |
370 auto iter = items.begin(); | 727 void BlobStorageContext::OnEnoughSizeForTransportFiles( |
371 if (offset) { | 728 const std::string& uuid, |
372 for (; iter != items.end(); ++iter) { | 729 bool success, |
373 const BlobDataItem& item = *(iter->get()->item()); | 730 std::vector<BlobMemoryController::FileCreationInfo> files) { |
374 if (offset >= item.length()) | 731 if (!success) { |
375 offset -= item.length(); | 732 BreakAndFinishPendingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY); |
376 else | 733 return; |
377 break; | 734 } |
378 } | 735 InternalBlobData* entry = registry_.GetEntry(uuid); |
379 } | 736 if (!entry || !entry->building_state_.get()) |
380 | 737 return; |
381 for (; iter != items.end() && length > 0; ++iter) { | 738 InternalBlobData::BuildingState& building_state = *entry->building_state_; |
382 scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get(); | 739 building_state.file_quota_granted = true; |
383 const BlobDataItem& item = *(shareable_item->item()); | 740 DCHECK(building_state.transport_state == TransportState::PENDING_FILE_QUOTA); |
384 uint64_t item_length = item.length(); | 741 DCHECK(!building_state.file_quota_request); |
385 DCHECK_GT(item_length, offset); | 742 |
386 uint64_t current_length = item_length - offset; | 743 entry->status_ = BlobStatus::PENDING_TRANSPORT; |
387 uint64_t new_length = current_length > length ? length : current_length; | 744 building_state.transport_state = TransportState::PENDING_TRANSPORT; |
388 | 745 RequestUserPopulation(entry, std::move(files)); |
389 bool reusing_blob_item = offset == 0 && new_length == item.length(); | 746 |
390 UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item); | 747 if (entry->CanFinishBuilding()) |
391 if (reusing_blob_item) { | 748 FinishBuilding(entry); |
392 shareable_item->referencing_blobs_mutable()->insert(target_blob_uuid); | 749 } |
393 target_blob_builder->AppendSharedBlobItem(shareable_item); | 750 |
394 length -= new_length; | 751 void BlobStorageContext::OnDependentBlobFinished( |
395 continue; | 752 const std::string& owning_blob_uuid, |
396 } | 753 BlobStatus status) { |
397 | 754 InternalBlobData* entry = registry_.GetEntry(owning_blob_uuid); |
398 // We need to do copying of the items when we have a different offset or | 755 if (!entry || !entry->building_state_) |
399 // length | 756 return; |
400 switch (item.type()) { | 757 |
401 case DataElement::TYPE_BYTES: { | 758 if (BlobStatusIsError(status)) { |
402 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes", | 759 DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status) |
403 new_length / 1024); | 760 << "Referenced blob should never be dereferenced while we " |
404 if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) { | 761 << "are depending on it, as our system holds a handle."; |
405 return false; | 762 BreakAndFinishPendingBlob(owning_blob_uuid, |
406 } | 763 BlobStatus::ERR_REFERENCED_BLOB_BROKEN); |
407 DCHECK(!item.offset()); | 764 return; |
408 std::unique_ptr<DataElement> element(new DataElement()); | 765 } |
409 element->SetToBytes(item.bytes() + offset, | 766 DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u); |
410 static_cast<int64_t>(new_length)); | 767 --entry->building_state_->num_building_dependent_blobs; |
411 memory_usage_ += new_length; | 768 |
412 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 769 if (entry->CanFinishBuilding()) |
413 target_blob_uuid, new BlobDataItem(std::move(element)), | 770 FinishBuilding(entry); |
414 ShareableBlobDataItem::POPULATED_WITH_QUOTA)); | 771 } |
415 } break; | 772 |
416 case DataElement::TYPE_FILE: { | 773 void BlobStorageContext::ClearAndFreeMemory(const std::string& uuid, |
417 DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max()) | 774 InternalBlobData* entry) { |
418 << "We cannot use a section of a file with an unknown length"; | 775 if (entry->building_state_) { |
419 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File", | 776 InternalBlobData::BuildingState* building_state = |
420 new_length / 1024); | 777 entry->building_state_.get(); |
421 std::unique_ptr<DataElement> element(new DataElement()); | 778 if (building_state->memory_quota_request) { |
422 element->SetToFilePathRange(item.path(), item.offset() + offset, | 779 building_state->memory_quota_request->Cancel(); |
423 new_length, | 780 } |
424 item.expected_modification_time()); | 781 if (building_state->file_quota_request) { |
425 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 782 building_state->file_quota_request->Cancel(); |
426 target_blob_uuid, | 783 } |
427 new BlobDataItem(std::move(element), item.data_handle_), | 784 } |
428 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA)); | 785 entry->RemoveBlobFromShareableItems(uuid); |
429 } break; | 786 |
430 case DataElement::TYPE_FILE_FILESYSTEM: { | 787 entry->items_.clear(); |
431 UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem", | 788 entry->offsets_.clear(); |
432 new_length / 1024); | 789 } |
433 std::unique_ptr<DataElement> element(new DataElement()); | 790 |
434 element->SetToFileSystemUrlRange(item.filesystem_url(), | 791 void BlobStorageContext::SetItemStateToPopulated( |
435 item.offset() + offset, new_length, | 792 std::vector<ShareableBlobDataItem*>* items) { |
436 item.expected_modification_time()); | 793 for (ShareableBlobDataItem* shareable_item : *items) { |
437 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | 794 DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED); |
438 target_blob_uuid, new BlobDataItem(std::move(element)), | 795 shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA); |
439 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA)); | 796 } |
440 } break; | |
441 case DataElement::TYPE_DISK_CACHE_ENTRY: { | |
442 std::unique_ptr<DataElement> element(new DataElement()); | |
443 element->SetToDiskCacheEntryRange(item.offset() + offset, | |
444 new_length); | |
445 target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem( | |
446 target_blob_uuid, | |
447 new BlobDataItem(std::move(element), item.data_handle_, | |
448 item.disk_cache_entry(), | |
449 item.disk_cache_stream_index(), | |
450 item.disk_cache_side_stream_index()), | |
451 ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA)); | |
452 } break; | |
453 case DataElement::TYPE_BYTES_DESCRIPTION: | |
454 case DataElement::TYPE_BLOB: | |
455 case DataElement::TYPE_UNKNOWN: | |
456 CHECK(false) << "Illegal blob item type: " << item.type(); | |
457 } | |
458 length -= new_length; | |
459 offset = 0; | |
460 } | |
461 return true; | |
462 } | 797 } |
463 | 798 |
464 } // namespace storage | 799 } // namespace storage |
OLD | NEW |