OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "storage/browser/blob/blob_memory_controller.h" | |
6 | |
7 #include <algorithm> | |
8 | |
9 #include "base/callback.h" | |
10 #include "base/callback_helpers.h" | |
11 #include "base/containers/small_map.h" | |
12 #include "base/files/file_util.h" | |
13 #include "base/guid.h" | |
14 #include "base/location.h" | |
15 #include "base/memory/ptr_util.h" | |
16 #include "base/metrics/histogram_macros.h" | |
17 #include "base/numerics/safe_conversions.h" | |
18 #include "base/numerics/safe_math.h" | |
19 #include "base/single_thread_task_runner.h" | |
20 #include "base/stl_util.h" | |
21 #include "base/strings/string_number_conversions.h" | |
22 #include "base/task_runner.h" | |
23 #include "base/task_runner_util.h" | |
24 #include "base/threading/thread_restrictions.h" | |
25 #include "base/time/time.h" | |
26 #include "base/trace_event/trace_event.h" | |
27 #include "base/tuple.h" | |
28 #include "storage/browser/blob/blob_data_builder.h" | |
29 #include "storage/browser/blob/blob_data_item.h" | |
30 #include "storage/browser/blob/shareable_blob_data_item.h" | |
31 #include "storage/browser/blob/shareable_file_reference.h" | |
32 #include "storage/common/data_element.h" | |
33 | |
34 using base::File; | |
35 using base::FilePath; | |
36 | |
37 namespace storage { | |
38 namespace { | |
39 using FileCreationInfo = BlobMemoryController::FileCreationInfo; | |
40 using MemoryAllocation = BlobMemoryController::MemoryAllocation; | |
41 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask; | |
42 | |
43 File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) { | |
44 File::Error error = File::FILE_OK; | |
45 base::CreateDirectoryAndGetError(blob_storage_dir, &error); | |
46 UMA_HISTOGRAM_ENUMERATION("Storage.Blob.CreateDirectoryResult", -error, | |
47 -File::FILE_ERROR_MAX); | |
48 return error; | |
49 } | |
50 | |
51 void DestructFile(File infos_without_references) {} | |
52 | |
53 // Used for new unpopulated file items. Caller must populate file reference in | |
54 // returned FileCreationInfos. | |
55 std::pair<std::vector<FileCreationInfo>, File::Error> CreateEmptyFiles( | |
56 const FilePath& blob_storage_dir, | |
57 scoped_refptr<base::TaskRunner> file_task_runner, | |
58 std::vector<base::FilePath> file_paths) { | |
59 base::ThreadRestrictions::AssertIOAllowed(); | |
60 | |
61 File::Error dir_create_status = CreateBlobDirectory(blob_storage_dir); | |
62 if (dir_create_status != File::FILE_OK) | |
63 return std::make_pair(std::vector<FileCreationInfo>(), dir_create_status); | |
kinuko
2016/10/20 16:15:34
nit: {} for consistency (or remove {} for other si
dmurph
2016/10/20 21:05:23
Removing {} from single line body blocks, this was
| |
64 | |
65 std::vector<FileCreationInfo> result; | |
66 for (const base::FilePath& file_path : file_paths) { | |
67 FileCreationInfo creation_info; | |
68 // Try to open our file. | |
69 File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE); | |
70 creation_info.path = std::move(file_path); | |
71 creation_info.file_deletion_runner = file_task_runner; | |
72 creation_info.error = file.error_details(); | |
73 if (creation_info.error != File::FILE_OK) | |
74 return std::make_pair(std::vector<FileCreationInfo>(), | |
75 creation_info.error); | |
kinuko
2016/10/20 16:15:34
nit: {} for multi-line body
dmurph
2016/10/20 21:05:23
Done.
| |
76 | |
77 // Grab the file info to get the "last modified" time and store the file. | |
78 File::Info file_info; | |
79 bool success = file.GetInfo(&file_info); | |
80 creation_info.error = success ? File::FILE_OK : File::FILE_ERROR_FAILED; | |
81 if (!success) | |
82 return std::make_pair(std::vector<FileCreationInfo>(), | |
83 creation_info.error); | |
kinuko
2016/10/20 16:15:34
ditto
dmurph
2016/10/20 21:05:22
Done.
| |
84 creation_info.file = std::move(file); | |
85 | |
86 result.push_back(std::move(creation_info)); | |
87 } | |
88 return std::make_pair(std::move(result), File::FILE_OK); | |
89 } | |
90 | |
91 // Used to evict multiple memory items out to a single file. Caller must | |
92 // populate file reference in returned FileCreationInfo. | |
93 FileCreationInfo CreateFileAndWriteItems( | |
94 const FilePath& blob_storage_dir, | |
95 const FilePath& file_path, | |
96 scoped_refptr<base::TaskRunner> file_task_runner, | |
97 std::vector<DataElement*> items, | |
98 size_t total_size_bytes) { | |
99 DCHECK_NE(0u, total_size_bytes); | |
100 UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.PageFileSize", total_size_bytes / 1024); | |
101 base::ThreadRestrictions::AssertIOAllowed(); | |
102 | |
103 FileCreationInfo creation_info; | |
104 creation_info.file_deletion_runner = std::move(file_task_runner); | |
105 creation_info.error = CreateBlobDirectory(blob_storage_dir); | |
106 if (creation_info.error != File::FILE_OK) | |
107 return creation_info; | |
kinuko
2016/10/20 16:15:34
nit: {} for consistency (here and below... looks l
dmurph
2016/10/20 21:05:23
Done.
| |
108 | |
109 // Create the page file. | |
110 File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE); | |
111 creation_info.path = file_path; | |
112 creation_info.error = file.error_details(); | |
113 if (creation_info.error != File::FILE_OK) | |
114 return creation_info; | |
115 | |
116 // Write data. | |
117 file.SetLength(total_size_bytes); | |
118 int bytes_written = 0; | |
119 for (DataElement* element : items) { | |
120 DCHECK_EQ(DataElement::TYPE_BYTES, element->type()); | |
121 size_t length = base::checked_cast<size_t>(element->length()); | |
122 size_t bytes_left = length; | |
123 while (bytes_left > 0) { | |
124 bytes_written = | |
125 file.WriteAtCurrentPos(element->bytes() + (length - bytes_left), | |
126 base::saturated_cast<int>(bytes_left)); | |
127 if (bytes_written < 0) | |
128 break; | |
129 DCHECK_LE(static_cast<size_t>(bytes_written), bytes_left); | |
130 bytes_left -= bytes_written; | |
131 } | |
132 if (bytes_written < 0) | |
133 break; | |
134 } | |
135 | |
136 File::Info info; | |
137 bool success = file.GetInfo(&info); | |
138 creation_info.error = | |
139 bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK; | |
140 creation_info.last_modified = info.last_modified; | |
141 return creation_info; | |
142 } | |
143 | |
144 uint64_t GetTotalSizeAndFileSizes( | |
145 const std::vector<scoped_refptr<ShareableBlobDataItem>>& | |
146 unreserved_file_items, | |
147 std::vector<uint64_t>* file_sizes_output) { | |
148 uint64_t total_size_output = 0; | |
149 base::SmallMap<std::map<uint64_t, uint64_t>> file_id_to_sizes; | |
150 for (const auto& item : unreserved_file_items) { | |
151 const DataElement& element = item->item()->data_element(); | |
152 uint64_t file_id = BlobDataBuilder::GetFutureFileID(element); | |
153 auto it = file_id_to_sizes.find(file_id); | |
154 if (it != file_id_to_sizes.end()) { | |
155 it->second = std::max(it->second, element.offset() + element.length()); | |
156 } else { | |
157 file_id_to_sizes[file_id] = element.offset() + element.length(); | |
158 } | |
159 total_size_output += element.length(); | |
160 } | |
161 for (const auto& size_pair : file_id_to_sizes) { | |
162 file_sizes_output->push_back(size_pair.second); | |
163 } | |
164 return total_size_output; | |
165 } | |
166 | |
167 } // namespace | |
168 | |
169 FileCreationInfo::FileCreationInfo() {} | |
170 FileCreationInfo::~FileCreationInfo() { | |
171 if (file.IsValid()) { | |
172 DCHECK(file_deletion_runner); | |
173 file_deletion_runner->PostTask( | |
174 FROM_HERE, base::Bind(&DestructFile, base::Passed(&file))); | |
175 } | |
176 } | |
177 FileCreationInfo::FileCreationInfo(FileCreationInfo&&) = default; | |
178 FileCreationInfo& FileCreationInfo::operator=(FileCreationInfo&&) = default; | |
179 | |
180 MemoryAllocation::MemoryAllocation( | |
181 base::WeakPtr<BlobMemoryController> controller, | |
182 uint64_t item_id, | |
183 size_t length) | |
184 : controller(controller), item_id(item_id), length(length) {} | |
kinuko
2016/10/20 16:15:34
nit: can we put an empty line between ctor and dto
dmurph
2016/10/20 21:05:23
Done.
| |
185 MemoryAllocation::~MemoryAllocation() { | |
186 if (controller) | |
187 controller->RevokeMemoryAllocation(item_id, length); | |
188 } | |
189 | |
190 BlobMemoryController::QuotaAllocationTask::~QuotaAllocationTask() {} | |
191 | |
192 // The my_list_position_ iterator is stored so that we can remove ourself from | |
kinuko
2016/10/20 16:15:34
nit: this comment might make better sense if it's
dmurph
2016/10/20 21:05:23
Done.
| |
193 // the task list when it is cancelled. | |
194 class BlobMemoryController::MemoryQuotaAllocationTask | |
195 : public BlobMemoryController::QuotaAllocationTask { | |
196 public: | |
197 MemoryQuotaAllocationTask( | |
198 BlobMemoryController* controller, | |
199 size_t quota_request_size, | |
200 std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items, | |
201 MemoryQuotaRequestCallback done_callback) | |
202 : controller_(controller), | |
203 pending_items_(std::move(pending_items)), | |
204 done_callback_(std::move(done_callback)), | |
205 allocation_size_(quota_request_size), | |
206 weak_factory_(this) {} | |
207 | |
208 ~MemoryQuotaAllocationTask() override = default; | |
209 | |
210 void RunDoneCallback(bool success) { | |
211 // Make sure we clear the weak pointers we gave to the caller beforehand. | |
212 weak_factory_.InvalidateWeakPtrs(); | |
213 if (success) | |
214 controller_->GrantMemoryAllocations(&pending_items_, allocation_size_); | |
215 done_callback_.Run(success); | |
216 } | |
217 | |
218 base::WeakPtr<QuotaAllocationTask> GetWeakPtr() { | |
219 return weak_factory_.GetWeakPtr(); | |
220 } | |
221 | |
222 void Cancel() override { | |
223 DCHECK_GE(controller_->pending_memory_quota_total_size_, allocation_size_); | |
224 controller_->pending_memory_quota_total_size_ -= allocation_size_; | |
225 // This call destroys this object. | |
226 controller_->pending_memory_quota_tasks_.erase(my_list_position_); | |
227 } | |
228 | |
229 void set_my_list_position( | |
230 PendingMemoryQuotaTaskList::iterator my_list_position) { | |
231 my_list_position_ = my_list_position; | |
232 } | |
233 | |
234 size_t allocation_size() const { return allocation_size_; } | |
235 | |
236 private: | |
237 BlobMemoryController* controller_; | |
238 std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_; | |
239 MemoryQuotaRequestCallback done_callback_; | |
240 | |
241 size_t allocation_size_; | |
242 PendingMemoryQuotaTaskList::iterator my_list_position_; | |
243 | |
244 base::WeakPtrFactory<MemoryQuotaAllocationTask> weak_factory_; | |
245 DISALLOW_COPY_AND_ASSIGN(MemoryQuotaAllocationTask); | |
246 }; | |
247 | |
248 // The my_list_position_ iterator is stored so that we can remove ourself from | |
249 // the task list when it is cancelled. | |
kinuko
2016/10/20 16:15:34
ditto.
dmurph
2016/10/20 21:05:23
Done.
| |
250 class BlobMemoryController::FileQuotaAllocationTask | |
251 : public BlobMemoryController::QuotaAllocationTask { | |
252 public: | |
253 FileQuotaAllocationTask( | |
kinuko
2016/10/20 16:15:34
It might be beneficial to note that it posts a tas
dmurph
2016/10/20 21:05:23
Done.
| |
254 BlobMemoryController* memory_controller, | |
255 std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items, | |
256 const FileQuotaRequestCallback& done_callback) | |
257 : controller_(memory_controller), | |
258 done_callback_(done_callback), | |
259 weak_factory_(this) { | |
260 // Get the file sizes and total size. | |
261 std::vector<uint64_t> file_sizes; | |
262 uint64_t total_size = | |
263 GetTotalSizeAndFileSizes(unreserved_file_items, &file_sizes); | |
264 DCHECK_LE(total_size, controller_->GetAvailableFileSpaceForBlobs()); | |
265 allocation_size_ = total_size; | |
266 | |
267 // Check & set our item states. | |
268 for (auto& shareable_item : unreserved_file_items) { | |
269 DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, shareable_item->state()); | |
270 DCHECK_EQ(DataElement::TYPE_FILE, shareable_item->item()->type()); | |
271 shareable_item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED); | |
272 } | |
273 pending_items_ = std::move(unreserved_file_items); | |
274 | |
275 // Increment disk usage and create our file references. | |
276 controller_->disk_used_ += allocation_size_; | |
277 std::vector<base::FilePath> file_paths; | |
278 std::vector<scoped_refptr<ShareableFileReference>> references; | |
279 for (size_t i = 0; i < file_sizes.size(); i++) { | |
280 file_paths.push_back(controller_->GenerateNextPageFileName()); | |
281 references.push_back(ShareableFileReference::GetOrCreate( | |
282 file_paths.back(), ShareableFileReference::DELETE_ON_FINAL_RELEASE, | |
283 controller_->file_runner_.get())); | |
284 references.back()->AddFinalReleaseCallback( | |
285 base::Bind(&BlobMemoryController::OnBlobFileDelete, | |
286 controller_->ptr_factory_.GetWeakPtr(), file_sizes[i])); | |
287 } | |
288 | |
289 // Send file creation task to file thread. | |
290 base::PostTaskAndReplyWithResult( | |
291 controller_->file_runner_.get(), FROM_HERE, | |
292 base::Bind(&CreateEmptyFiles, controller_->blob_storage_dir_, | |
293 controller_->file_runner_, base::Passed(&file_paths)), | |
294 base::Bind(&FileQuotaAllocationTask::OnCreateEmptyFiles, | |
295 weak_factory_.GetWeakPtr(), base::Passed(&references))); | |
296 controller_->RecordTracingCounters(); | |
297 } | |
298 ~FileQuotaAllocationTask() override {} | |
299 | |
300 void RunDoneCallback(bool success, std::vector<FileCreationInfo> file_info) { | |
301 // Make sure we clear the weak pointers we gave to the caller beforehand. | |
302 weak_factory_.InvalidateWeakPtrs(); | |
303 | |
304 // We want to destroy this object on the exit of this method if we were | |
305 // successful. | |
306 std::unique_ptr<FileQuotaAllocationTask> this_object; | |
307 if (success) { | |
308 for (auto& item : pending_items_) { | |
309 item->set_state(ShareableBlobDataItem::QUOTA_GRANTED); | |
310 } | |
311 this_object = std::move(*my_list_position_); | |
312 controller_->pending_file_quota_tasks_.erase(my_list_position_); | |
313 } | |
314 | |
315 done_callback_.Run(success, std::move(file_info)); | |
316 } | |
317 | |
318 base::WeakPtr<QuotaAllocationTask> GetWeakPtr() { | |
319 return weak_factory_.GetWeakPtr(); | |
320 } | |
321 | |
322 void Cancel() override { | |
323 // This call destroys this object. We rely on ShareableFileReference's | |
324 // final release callback for disk_usage_ accounting. | |
325 controller_->pending_file_quota_tasks_.erase(my_list_position_); | |
326 } | |
327 | |
328 void OnCreateEmptyFiles( | |
329 std::vector<scoped_refptr<ShareableFileReference>> references, | |
330 std::pair<std::vector<FileCreationInfo>, File::Error> files_and_error) { | |
331 auto& files = files_and_error.first; | |
332 if (files.empty()) { | |
333 DCHECK_GE(controller_->disk_used_, allocation_size_); | |
334 controller_->disk_used_ -= allocation_size_; | |
335 // This will call our callback and delete the object correctly. | |
336 controller_->DisableFilePaging(files_and_error.second); | |
337 return; | |
338 } | |
339 DCHECK_EQ(files.size(), references.size()); | |
340 for (size_t i = 0; i < files.size(); i++) { | |
341 files[i].file_reference = std::move(references[i]); | |
342 } | |
343 RunDoneCallback(true, std::move(files)); | |
344 } | |
345 | |
346 void set_my_list_position( | |
347 PendingFileQuotaTaskList::iterator my_list_position) { | |
348 my_list_position_ = my_list_position; | |
349 } | |
350 | |
351 private: | |
352 BlobMemoryController* controller_; | |
353 std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_; | |
354 scoped_refptr<base::TaskRunner> file_runner_; | |
355 FileQuotaRequestCallback done_callback_; | |
356 | |
357 uint64_t allocation_size_; | |
358 PendingFileQuotaTaskList::iterator my_list_position_; | |
359 | |
360 base::WeakPtrFactory<FileQuotaAllocationTask> weak_factory_; | |
361 DISALLOW_COPY_AND_ASSIGN(FileQuotaAllocationTask); | |
362 }; | |
363 | |
364 BlobMemoryController::BlobMemoryController( | |
365 const base::FilePath& storage_directory, | |
366 scoped_refptr<base::TaskRunner> file_runner) | |
367 : file_paging_enabled_(file_runner.get() != nullptr), | |
368 blob_storage_dir_(storage_directory), | |
369 file_runner_(std::move(file_runner)), | |
370 populated_memory_items_( | |
371 base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT), | |
372 ptr_factory_(this) {} | |
373 | |
374 BlobMemoryController::~BlobMemoryController() {} | |
375 | |
376 void BlobMemoryController::DisableFilePaging(base::File::Error reason) { | |
377 UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason, | |
378 -File::FILE_ERROR_MAX); | |
379 file_paging_enabled_ = false; | |
380 in_flight_memory_used_ = 0; | |
381 items_paging_to_file_.clear(); | |
382 pending_evictions_ = 0; | |
383 pending_memory_quota_total_size_ = 0; | |
384 populated_memory_items_.Clear(); | |
385 populated_memory_items_bytes_ = 0; | |
386 file_runner_ = nullptr; | |
387 | |
388 PendingMemoryQuotaTaskList old_memory_tasks; | |
389 PendingFileQuotaTaskList old_file_tasks; | |
390 std::swap(old_memory_tasks, pending_memory_quota_tasks_); | |
391 std::swap(old_file_tasks, pending_file_quota_tasks_); | |
392 | |
393 // Don't call the callbacks until we have a consistent state. | |
394 for (auto& memory_request : old_memory_tasks) { | |
395 memory_request->RunDoneCallback(false); | |
396 } | |
397 for (auto& file_request : old_file_tasks) { | |
398 file_request->RunDoneCallback(false, std::vector<FileCreationInfo>()); | |
399 } | |
400 } | |
401 | |
402 BlobMemoryController::Strategy BlobMemoryController::DetermineStrategy( | |
403 size_t preemptive_transported_bytes, | |
404 uint64_t total_transportation_bytes) const { | |
405 if (total_transportation_bytes == 0) { | |
406 return Strategy::NONE_NEEDED; | |
407 } | |
408 if (!CanReserveQuota(total_transportation_bytes)) { | |
409 return Strategy::TOO_LARGE; | |
410 } | |
411 // Handle the case where we have all the bytes preemptively transported, and | |
412 // we can also fit them. | |
413 if (preemptive_transported_bytes == total_transportation_bytes && | |
414 pending_memory_quota_tasks_.empty() && | |
415 preemptive_transported_bytes < GetAvailableMemoryForBlobs()) { | |
416 return Strategy::NONE_NEEDED; | |
417 } | |
418 if (file_paging_enabled_ && | |
419 (total_transportation_bytes > limits_.memory_limit_before_paging())) { | |
420 return Strategy::FILE; | |
421 } | |
422 if (total_transportation_bytes > limits_.max_ipc_memory_size) { | |
423 return Strategy::SHARED_MEMORY; | |
424 } | |
425 return Strategy::IPC; | |
426 } | |
427 | |
428 bool BlobMemoryController::CanReserveQuota(uint64_t size) const { | |
429 // We check each size independently as a blob can't be constructed in both | |
430 // disk and memory. | |
431 return size <= GetAvailableMemoryForBlobs() || | |
432 size <= GetAvailableFileSpaceForBlobs(); | |
433 } | |
434 | |
435 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveMemoryQuota( | |
436 std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items, | |
437 const MemoryQuotaRequestCallback& done_callback) { | |
438 if (unreserved_memory_items.empty()) { | |
439 done_callback.Run(true); | |
440 return base::WeakPtr<QuotaAllocationTask>(); | |
441 } | |
442 | |
443 base::CheckedNumeric<uint64_t> unsafe_total_bytes_needed = 0; | |
444 for (auto& item : unreserved_memory_items) { | |
445 DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, item->state()); | |
446 DCHECK(item->item()->type() == DataElement::TYPE_BYTES_DESCRIPTION || | |
447 item->item()->type() == DataElement::TYPE_BYTES); | |
448 DCHECK(item->item()->length() > 0); | |
449 unsafe_total_bytes_needed += item->item()->length(); | |
450 item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED); | |
451 } | |
452 | |
453 uint64_t total_bytes_needed = unsafe_total_bytes_needed.ValueOrDie(); | |
454 DCHECK_GT(total_bytes_needed, 0ull); | |
455 | |
456 // If we're currently waiting for blobs to page already, then we add | |
457 // ourselves to the end of the queue. Once paging is complete, we'll schedule | |
458 // more paging for any more pending blobs. | |
459 if (!pending_memory_quota_tasks_.empty()) { | |
460 return AppendMemoryTask(total_bytes_needed, | |
461 std::move(unreserved_memory_items), done_callback); | |
462 } | |
463 | |
464 // Store right away if we can. | |
465 if (total_bytes_needed <= GetAvailableMemoryForBlobs()) { | |
466 GrantMemoryAllocations(&unreserved_memory_items, | |
467 static_cast<size_t>(total_bytes_needed)); | |
468 MaybeScheduleEvictionUntilSystemHealthy(); | |
469 done_callback.Run(true); | |
470 return base::WeakPtr<QuotaAllocationTask>(); | |
471 } | |
472 | |
473 // Size is larger than available memory. | |
474 DCHECK(pending_memory_quota_tasks_.empty()); | |
475 DCHECK_EQ(0u, pending_memory_quota_total_size_); | |
476 | |
477 auto weak_ptr = AppendMemoryTask( | |
478 total_bytes_needed, std::move(unreserved_memory_items), done_callback); | |
479 MaybeScheduleEvictionUntilSystemHealthy(); | |
480 return weak_ptr; | |
481 } | |
482 | |
483 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota( | |
484 std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items, | |
485 const FileQuotaRequestCallback& done_callback) { | |
486 pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>( | |
487 this, std::move(unreserved_file_items), done_callback)); | |
488 pending_file_quota_tasks_.back()->set_my_list_position( | |
489 --pending_file_quota_tasks_.end()); | |
490 return pending_file_quota_tasks_.back()->GetWeakPtr(); | |
491 } | |
492 | |
493 void BlobMemoryController::NotifyMemoryItemsUsed( | |
494 std::vector<scoped_refptr<ShareableBlobDataItem>>& items) { | |
495 for (const auto& item : items) { | |
496 DCHECK_EQ(DataElement::TYPE_BYTES, item->item()->type()); | |
497 DCHECK_EQ(ShareableBlobDataItem::POPULATED_WITH_QUOTA, item->state()); | |
498 // We don't want to re-add the item if we're currently paging it to disk. | |
499 if (items_paging_to_file_.find(item->item_id()) != | |
500 items_paging_to_file_.end()) | |
501 return; | |
502 auto iterator = populated_memory_items_.Get(item->item_id()); | |
503 if (iterator == populated_memory_items_.end()) { | |
504 populated_memory_items_bytes_ += | |
505 static_cast<size_t>(item->item()->length()); | |
506 populated_memory_items_.Put(item->item_id(), item.get()); | |
507 } | |
508 } | |
509 MaybeScheduleEvictionUntilSystemHealthy(); | |
510 } | |
511 | |
512 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask( | |
513 uint64_t total_bytes_needed, | |
514 std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items, | |
515 const MemoryQuotaRequestCallback& done_callback) { | |
516 DCHECK(file_paging_enabled_) | |
517 << "Caller tried to reserve memory when CanReserveQuota(" | |
518 << total_bytes_needed << ") would have returned false."; | |
519 | |
520 pending_memory_quota_total_size_ += total_bytes_needed; | |
521 pending_memory_quota_tasks_.push_back( | |
522 base::MakeUnique<MemoryQuotaAllocationTask>( | |
523 this, total_bytes_needed, std::move(unreserved_memory_items), | |
524 std::move(done_callback))); | |
525 pending_memory_quota_tasks_.back()->set_my_list_position( | |
526 --pending_memory_quota_tasks_.end()); | |
527 | |
528 return pending_memory_quota_tasks_.back()->GetWeakPtr(); | |
529 } | |
530 | |
531 void BlobMemoryController::MaybeGrantPendingMemoryRequests() { | |
532 while (!pending_memory_quota_tasks_.empty() && | |
533 limits_.max_blob_in_memory_space - blob_memory_used_ >= | |
534 pending_memory_quota_tasks_.front()->allocation_size()) { | |
535 std::unique_ptr<MemoryQuotaAllocationTask> memory_task = | |
536 std::move(pending_memory_quota_tasks_.front()); | |
537 pending_memory_quota_tasks_.pop_front(); | |
538 pending_memory_quota_total_size_ -= memory_task->allocation_size(); | |
539 memory_task->RunDoneCallback(true); | |
540 } | |
541 RecordTracingCounters(); | |
542 } | |
543 | |
544 size_t BlobMemoryController::CollectItemsForEviction( | |
545 std::vector<scoped_refptr<ShareableBlobDataItem>>* output) { | |
546 base::CheckedNumeric<size_t> total_items_size = 0; | |
547 // Process the recent item list and remove items until we have at least a | |
548 // minimum file size or we're at the end of our items to page to disk. | |
549 while (total_items_size.ValueOrDie() < limits_.min_page_file_size && | |
550 !populated_memory_items_.empty()) { | |
551 auto iterator = --populated_memory_items_.end(); | |
552 ShareableBlobDataItem* item = iterator->second; | |
553 DCHECK_EQ(item->item()->type(), DataElement::TYPE_BYTES); | |
554 populated_memory_items_.Erase(iterator); | |
555 size_t size = base::checked_cast<size_t>(item->item()->length()); | |
556 populated_memory_items_bytes_ -= size; | |
557 total_items_size += size; | |
558 output->push_back(make_scoped_refptr(item)); | |
559 } | |
560 return total_items_size.ValueOrDie(); | |
561 } | |
562 | |
563 void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() { | |
564 // Don't do eviction when others are happening, as we don't change our | |
565 // pending_memory_quota_total_size_ value until after the paging files have | |
566 // been written. | |
567 if (pending_evictions_ != 0 || !file_paging_enabled_) | |
568 return; | |
569 | |
570 // We try to page items to disk until our current system size + requested | |
571 // memory is below our size limit. | |
572 while (pending_memory_quota_total_size_ + blob_memory_used_ > | |
573 limits_.memory_limit_before_paging()) { | |
574 // We only page when we have enough items to fill a whole page file. | |
575 if (populated_memory_items_bytes_ < limits_.min_page_file_size) | |
576 break; | |
577 DCHECK_LE(limits_.min_page_file_size, | |
578 static_cast<uint64_t>(blob_memory_used_)); | |
579 | |
580 std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap; | |
581 size_t total_items_size = CollectItemsForEviction(&items_to_swap); | |
582 if (total_items_size == 0) | |
583 break; | |
584 | |
585 std::vector<DataElement*> items_for_paging; | |
586 for (auto& shared_blob_item : items_to_swap) { | |
587 items_paging_to_file_.insert(shared_blob_item->item_id()); | |
588 items_for_paging.push_back(shared_blob_item->item()->data_element_ptr()); | |
589 } | |
590 | |
591 // Update our bookkeeping. | |
592 pending_evictions_++; | |
593 disk_used_ += total_items_size; | |
594 in_flight_memory_used_ += total_items_size; | |
595 | |
596 // Create our file reference. | |
597 scoped_refptr<ShareableFileReference> file_reference = | |
598 ShareableFileReference::GetOrCreate( | |
599 GenerateNextPageFileName(), | |
600 ShareableFileReference::DELETE_ON_FINAL_RELEASE, | |
601 file_runner_.get()); | |
602 // Add the release callback so we decrement our disk usage on file deletion. | |
603 file_reference->AddFinalReleaseCallback( | |
604 base::Bind(&BlobMemoryController::OnBlobFileDelete, | |
605 ptr_factory_.GetWeakPtr(), total_items_size)); | |
606 | |
607 // Post the file writing task. | |
608 base::PostTaskAndReplyWithResult( | |
609 file_runner_.get(), FROM_HERE, | |
610 base::Bind(&CreateFileAndWriteItems, blob_storage_dir_, | |
611 file_reference->path(), file_runner_, | |
612 base::Passed(&items_for_paging), total_items_size), | |
613 base::Bind(&BlobMemoryController::OnEvictionComplete, | |
614 ptr_factory_.GetWeakPtr(), base::Passed(&file_reference), | |
615 base::Passed(&items_to_swap), total_items_size)); | |
616 } | |
617 RecordTracingCounters(); | |
618 } | |
619 | |
620 void BlobMemoryController::OnEvictionComplete( | |
621 scoped_refptr<ShareableFileReference> file_reference, | |
622 std::vector<scoped_refptr<ShareableBlobDataItem>> items, | |
623 size_t total_items_size, | |
624 FileCreationInfo result) { | |
625 if (!file_paging_enabled_) | |
626 return; | |
627 | |
628 if (result.error != File::FILE_OK) { | |
629 DisableFilePaging(result.error); | |
630 return; | |
631 } | |
632 | |
633 DCHECK_LT(0, pending_evictions_); | |
634 pending_evictions_--; | |
635 | |
636 // Switch item from memory to the new file. | |
637 uint64_t offset = 0; | |
638 for (const scoped_refptr<ShareableBlobDataItem>& shareable_item : items) { | |
639 scoped_refptr<BlobDataItem> new_item( | |
640 new BlobDataItem(base::WrapUnique(new DataElement()), file_reference)); | |
641 new_item->data_element_ptr()->SetToFilePathRange( | |
642 file_reference->path(), offset, shareable_item->item()->length(), | |
643 result.last_modified); | |
644 DCHECK(shareable_item->memory_allocation_); | |
645 shareable_item->set_memory_allocation(nullptr); | |
646 shareable_item->set_item(new_item); | |
647 items_paging_to_file_.erase(shareable_item->item_id()); | |
648 offset += shareable_item->item()->length(); | |
649 } | |
650 in_flight_memory_used_ -= total_items_size; | |
651 | |
652 // We want callback on blobs up to the amount we've freed. | |
653 MaybeGrantPendingMemoryRequests(); | |
654 | |
655 // If we still have more blobs waiting and we're not waiting on more paging | |
656 // operations, schedule more. | |
657 MaybeScheduleEvictionUntilSystemHealthy(); | |
658 } | |
659 | |
660 FilePath BlobMemoryController::GenerateNextPageFileName() { | |
661 std::string file_name = base::Uint64ToString(current_file_num_++); | |
662 return blob_storage_dir_.AppendASCII(file_name); | |
663 } | |
664 | |
665 void BlobMemoryController::RecordTracingCounters() const { | |
666 TRACE_COUNTER2("Blob", "MemoryUsage", "TotalStorage", blob_memory_used_, | |
667 "InFlightToDisk", in_flight_memory_used_); | |
668 TRACE_COUNTER1("Blob", "DiskUsage", disk_used_); | |
669 TRACE_COUNTER1("Blob", "TranfersPendingOnDisk", | |
670 pending_memory_quota_tasks_.size()); | |
671 TRACE_COUNTER1("Blob", "TranfersBytesPendingOnDisk", | |
672 pending_memory_quota_total_size_); | |
673 } | |
674 | |
675 size_t BlobMemoryController::GetAvailableMemoryForBlobs() const { | |
676 if (limits_.max_blob_in_memory_space < memory_usage()) | |
677 return 0; | |
678 return limits_.max_blob_in_memory_space - memory_usage(); | |
679 } | |
680 | |
681 uint64_t BlobMemoryController::GetAvailableFileSpaceForBlobs() const { | |
682 if (!file_paging_enabled_) | |
683 return 0; | |
684 // Sometimes we're only paging part of what we need for the new blob, so add | |
685 // the rest of the size we need into our disk usage if this is the case. | |
686 uint64_t total_disk_used = disk_used_; | |
687 if (in_flight_memory_used_ < pending_memory_quota_total_size_) { | |
688 total_disk_used += | |
689 pending_memory_quota_total_size_ - in_flight_memory_used_; | |
690 } | |
691 if (limits_.max_blob_disk_space < total_disk_used) | |
692 return 0; | |
693 return limits_.max_blob_disk_space - total_disk_used; | |
694 } | |
695 | |
696 void BlobMemoryController::GrantMemoryAllocations( | |
697 std::vector<scoped_refptr<ShareableBlobDataItem>>* items, | |
698 size_t total_bytes) { | |
699 blob_memory_used_ += total_bytes; | |
700 for (auto& item : *items) { | |
701 item->set_state(ShareableBlobDataItem::QUOTA_GRANTED); | |
702 item->set_memory_allocation(base::MakeUnique<MemoryAllocation>( | |
703 ptr_factory_.GetWeakPtr(), item->item_id(), | |
704 base::checked_cast<size_t>(item->item()->length()))); | |
705 } | |
706 } | |
707 | |
708 void BlobMemoryController::RevokeMemoryAllocation(uint64_t item_id, | |
709 size_t length) { | |
710 DCHECK_LE(length, blob_memory_used_); | |
711 blob_memory_used_ -= length; | |
712 auto iterator = populated_memory_items_.Get(item_id); | |
713 if (iterator != populated_memory_items_.end()) { | |
714 DCHECK_GE(populated_memory_items_bytes_, length); | |
715 populated_memory_items_bytes_ -= length; | |
716 populated_memory_items_.Erase(iterator); | |
717 } | |
718 MaybeGrantPendingMemoryRequests(); | |
719 } | |
720 | |
721 void BlobMemoryController::OnBlobFileDelete(uint64_t size, | |
722 const FilePath& path) { | |
723 DCHECK_LE(size, disk_used_); | |
724 disk_used_ -= size; | |
725 } | |
726 | |
727 } // namespace storage | |
OLD | NEW |