Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(3)

Side by Side Diff: sync/internal_api/attachments/on_disk_attachment_store.cc

Issue 2130453004: [Sync] Move //sync to //components/sync. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Rebase. Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "sync/internal_api/public/attachments/on_disk_attachment_store.h"
6
7 #include <stdint.h>
8
9 #include <memory>
10 #include <string>
11 #include <utility>
12
13 #include "base/bind.h"
14 #include "base/callback.h"
15 #include "base/location.h"
16 #include "base/metrics/histogram.h"
17 #include "base/sequenced_task_runner.h"
18 #include "sync/internal_api/attachments/proto/attachment_store.pb.h"
19 #include "sync/internal_api/public/attachments/attachment_util.h"
20 #include "sync/protocol/attachments.pb.h"
21 #include "third_party/leveldatabase/env_chromium.h"
22 #include "third_party/leveldatabase/src/include/leveldb/db.h"
23 #include "third_party/leveldatabase/src/include/leveldb/options.h"
24 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
25 #include "third_party/leveldatabase/src/include/leveldb/status.h"
26 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
27
28 namespace syncer {
29
30 namespace {
31
32 // Prefix for records containing attachment data.
33 const char kDataPrefix[] = "data-";
34
35 // Prefix for records containing attachment metadata.
36 const char kMetadataPrefix[] = "metadata-";
37
38 const char kDatabaseMetadataKey[] = "database-metadata";
39
40 const int32_t kCurrentSchemaVersion = 1;
41
42 const base::FilePath::CharType kLeveldbDirectory[] =
43 FILE_PATH_LITERAL("leveldb");
44
45 // Converts syncer::AttachmentStore::Component values into
46 // attachment_store_pb::RecordMetadata::Component.
47 attachment_store_pb::RecordMetadata::Component ComponentToProto(
48 syncer::AttachmentStore::Component component) {
49 switch (component) {
50 case AttachmentStore::MODEL_TYPE:
51 return attachment_store_pb::RecordMetadata::MODEL_TYPE;
52 case AttachmentStore::SYNC:
53 return attachment_store_pb::RecordMetadata::SYNC;
54 }
55 NOTREACHED();
56 return attachment_store_pb::RecordMetadata::UNKNOWN;
57 }
58
59 leveldb::WriteOptions MakeWriteOptions() {
60 leveldb::WriteOptions write_options;
61 write_options.sync = true;
62 return write_options;
63 }
64
65 leveldb::ReadOptions MakeNonCachingReadOptions() {
66 leveldb::ReadOptions read_options;
67 read_options.fill_cache = false;
68 read_options.verify_checksums = true;
69 return read_options;
70 }
71
72 leveldb::ReadOptions MakeCachingReadOptions() {
73 leveldb::ReadOptions read_options;
74 read_options.fill_cache = true;
75 read_options.verify_checksums = true;
76 return read_options;
77 }
78
79 leveldb::Status ReadStoreMetadata(
80 leveldb::DB* db,
81 attachment_store_pb::StoreMetadata* metadata) {
82 std::string data_str;
83
84 leveldb::Status status =
85 db->Get(MakeCachingReadOptions(), kDatabaseMetadataKey, &data_str);
86 if (!status.ok())
87 return status;
88 if (!metadata->ParseFromString(data_str))
89 return leveldb::Status::Corruption("Metadata record corruption");
90 return leveldb::Status::OK();
91 }
92
93 leveldb::Status WriteStoreMetadata(
94 leveldb::DB* db,
95 const attachment_store_pb::StoreMetadata& metadata) {
96 std::string data_str;
97
98 metadata.SerializeToString(&data_str);
99 return db->Put(MakeWriteOptions(), kDatabaseMetadataKey, data_str);
100 }
101
102 // Adds reference to component into RecordMetadata::component set.
103 // Returns true if record_metadata was modified and needs to be written to disk.
104 bool SetReferenceInRecordMetadata(
105 attachment_store_pb::RecordMetadata* record_metadata,
106 attachment_store_pb::RecordMetadata::Component proto_component) {
107 DCHECK(record_metadata);
108 for (const int component : record_metadata->component()) {
109 if (component == proto_component)
110 return false;
111 }
112 record_metadata->add_component(proto_component);
113 return true;
114 }
115
116 // Drops reference to component from RecordMetadata::component set.
117 // Returns true if record_metadata was modified and needs to be written to disk.
118 bool DropReferenceInRecordMetadata(
119 attachment_store_pb::RecordMetadata* record_metadata,
120 attachment_store_pb::RecordMetadata::Component proto_component) {
121 DCHECK(record_metadata);
122 bool component_removed = false;
123 ::google::protobuf::RepeatedField<int>* mutable_components =
124 record_metadata->mutable_component();
125 for (int i = 0; i < mutable_components->size();) {
126 if (mutable_components->Get(i) == proto_component) {
127 if (i < mutable_components->size() - 1) {
128 // Don't swap last element with itself.
129 mutable_components->SwapElements(i, mutable_components->size() - 1);
130 }
131 mutable_components->RemoveLast();
132 component_removed = true;
133 } else {
134 ++i;
135 }
136 }
137 return component_removed;
138 }
139
140 bool AttachmentHasReferenceFromComponent(
141 const attachment_store_pb::RecordMetadata& record_metadata,
142 attachment_store_pb::RecordMetadata::Component proto_component) {
143 for (const auto& reference_component : record_metadata.component()) {
144 if (reference_component == proto_component) {
145 return true;
146 }
147 }
148 return false;
149 }
150
151 } // namespace
152
153 OnDiskAttachmentStore::OnDiskAttachmentStore(
154 const scoped_refptr<base::SequencedTaskRunner>& callback_task_runner,
155 const base::FilePath& path)
156 : AttachmentStoreBackend(callback_task_runner), path_(path) {
157 }
158
159 OnDiskAttachmentStore::~OnDiskAttachmentStore() {
160 }
161
162 void OnDiskAttachmentStore::Init(
163 const AttachmentStore::InitCallback& callback) {
164 DCHECK(CalledOnValidThread());
165 AttachmentStore::Result result_code = OpenOrCreate(path_);
166 UMA_HISTOGRAM_ENUMERATION("Sync.Attachments.StoreInitResult", result_code,
167 AttachmentStore::RESULT_SIZE);
168 PostCallback(base::Bind(callback, result_code));
169 }
170
171 void OnDiskAttachmentStore::Read(
172 AttachmentStore::Component component,
173 const AttachmentIdList& ids,
174 const AttachmentStore::ReadCallback& callback) {
175 DCHECK(CalledOnValidThread());
176 std::unique_ptr<AttachmentMap> result_map(new AttachmentMap());
177 std::unique_ptr<AttachmentIdList> unavailable_attachments(
178 new AttachmentIdList());
179
180 AttachmentStore::Result result_code =
181 AttachmentStore::STORE_INITIALIZATION_FAILED;
182
183 if (db_) {
184 result_code = AttachmentStore::SUCCESS;
185 for (const auto& id : ids) {
186 std::unique_ptr<Attachment> attachment;
187 attachment = ReadSingleAttachment(id, component);
188 if (attachment) {
189 result_map->insert(std::make_pair(id, *attachment));
190 } else {
191 unavailable_attachments->push_back(id);
192 }
193 }
194 result_code = unavailable_attachments->empty()
195 ? AttachmentStore::SUCCESS
196 : AttachmentStore::UNSPECIFIED_ERROR;
197 } else {
198 *unavailable_attachments = ids;
199 }
200
201 PostCallback(base::Bind(callback, result_code, base::Passed(&result_map),
202 base::Passed(&unavailable_attachments)));
203 }
204
205 void OnDiskAttachmentStore::Write(
206 AttachmentStore::Component component,
207 const AttachmentList& attachments,
208 const AttachmentStore::WriteCallback& callback) {
209 DCHECK(CalledOnValidThread());
210 AttachmentStore::Result result_code =
211 AttachmentStore::STORE_INITIALIZATION_FAILED;
212
213 if (db_) {
214 result_code = AttachmentStore::SUCCESS;
215 AttachmentList::const_iterator iter = attachments.begin();
216 const AttachmentList::const_iterator end = attachments.end();
217 for (; iter != end; ++iter) {
218 if (!WriteSingleAttachment(*iter, component))
219 result_code = AttachmentStore::UNSPECIFIED_ERROR;
220 }
221 }
222 PostCallback(base::Bind(callback, result_code));
223 }
224
225 void OnDiskAttachmentStore::SetReference(AttachmentStore::Component component,
226 const AttachmentIdList& ids) {
227 DCHECK(CalledOnValidThread());
228 if (!db_)
229 return;
230 attachment_store_pb::RecordMetadata::Component proto_component =
231 ComponentToProto(component);
232 for (const auto& id : ids) {
233 attachment_store_pb::RecordMetadata record_metadata;
234 if (!ReadSingleRecordMetadata(id, &record_metadata))
235 continue;
236 if (SetReferenceInRecordMetadata(&record_metadata, proto_component))
237 WriteSingleRecordMetadata(id, record_metadata);
238 }
239 }
240
241 void OnDiskAttachmentStore::DropReference(
242 AttachmentStore::Component component,
243 const AttachmentIdList& ids,
244 const AttachmentStore::DropCallback& callback) {
245 DCHECK(CalledOnValidThread());
246 AttachmentStore::Result result_code =
247 AttachmentStore::STORE_INITIALIZATION_FAILED;
248 if (db_) {
249 attachment_store_pb::RecordMetadata::Component proto_component =
250 ComponentToProto(component);
251 result_code = AttachmentStore::SUCCESS;
252 leveldb::WriteOptions write_options = MakeWriteOptions();
253 for (const auto& id : ids) {
254 attachment_store_pb::RecordMetadata record_metadata;
255 if (!ReadSingleRecordMetadata(id, &record_metadata))
256 continue; // Record not found.
257 if (!DropReferenceInRecordMetadata(&record_metadata, proto_component))
258 continue; // Component is not in components set. Metadata was not
259 // updated.
260 if (record_metadata.component_size() == 0) {
261 // Last reference dropped. Need to delete attachment.
262 leveldb::WriteBatch write_batch;
263 write_batch.Delete(MakeDataKeyFromAttachmentId(id));
264 write_batch.Delete(MakeMetadataKeyFromAttachmentId(id));
265
266 leveldb::Status status = db_->Write(write_options, &write_batch);
267 if (!status.ok()) {
268 // DB::Delete doesn't check if record exists, it returns ok just like
269 // AttachmentStore::Drop should.
270 DVLOG(1) << "DB::Write failed: status=" << status.ToString();
271 result_code = AttachmentStore::UNSPECIFIED_ERROR;
272 }
273 } else {
274 WriteSingleRecordMetadata(id, record_metadata);
275 }
276 }
277 }
278 PostCallback(base::Bind(callback, result_code));
279 }
280
281 void OnDiskAttachmentStore::ReadMetadataById(
282 AttachmentStore::Component component,
283 const AttachmentIdList& ids,
284 const AttachmentStore::ReadMetadataCallback& callback) {
285 DCHECK(CalledOnValidThread());
286 AttachmentStore::Result result_code =
287 AttachmentStore::STORE_INITIALIZATION_FAILED;
288 std::unique_ptr<AttachmentMetadataList> metadata_list(
289 new AttachmentMetadataList());
290 if (db_) {
291 result_code = AttachmentStore::SUCCESS;
292 for (const auto& id : ids) {
293 attachment_store_pb::RecordMetadata record_metadata;
294 if (!ReadSingleRecordMetadata(id, &record_metadata)) {
295 result_code = AttachmentStore::UNSPECIFIED_ERROR;
296 continue;
297 }
298 if (!AttachmentHasReferenceFromComponent(record_metadata,
299 ComponentToProto(component))) {
300 result_code = AttachmentStore::UNSPECIFIED_ERROR;
301 continue;
302 }
303 metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
304 }
305 }
306 PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
307 }
308
309 void OnDiskAttachmentStore::ReadMetadata(
310 AttachmentStore::Component component,
311 const AttachmentStore::ReadMetadataCallback& callback) {
312 DCHECK(CalledOnValidThread());
313 AttachmentStore::Result result_code =
314 AttachmentStore::STORE_INITIALIZATION_FAILED;
315 std::unique_ptr<AttachmentMetadataList> metadata_list(
316 new AttachmentMetadataList());
317
318 if (db_) {
319 attachment_store_pb::RecordMetadata::Component proto_component =
320 ComponentToProto(component);
321 result_code = AttachmentStore::SUCCESS;
322 std::unique_ptr<leveldb::Iterator> db_iterator(
323 db_->NewIterator(MakeNonCachingReadOptions()));
324 DCHECK(db_iterator);
325 for (db_iterator->Seek(kMetadataPrefix); db_iterator->Valid();
326 db_iterator->Next()) {
327 leveldb::Slice key = db_iterator->key();
328 if (!key.starts_with(kMetadataPrefix)) {
329 break;
330 }
331 // Make AttachmentId from levelDB key.
332 key.remove_prefix(strlen(kMetadataPrefix));
333 sync_pb::AttachmentIdProto id_proto;
334 id_proto.set_unique_id(key.ToString());
335 AttachmentId id = AttachmentId::CreateFromProto(id_proto);
336 // Parse metadata record.
337 attachment_store_pb::RecordMetadata record_metadata;
338 if (!record_metadata.ParseFromString(db_iterator->value().ToString())) {
339 DVLOG(1) << "RecordMetadata::ParseFromString failed";
340 result_code = AttachmentStore::UNSPECIFIED_ERROR;
341 continue;
342 }
343 DCHECK_GT(record_metadata.component_size(), 0);
344 if (AttachmentHasReferenceFromComponent(record_metadata, proto_component))
345 metadata_list->push_back(MakeAttachmentMetadata(id, record_metadata));
346 }
347
348 if (!db_iterator->status().ok()) {
349 DVLOG(1) << "DB Iterator failed: status="
350 << db_iterator->status().ToString();
351 result_code = AttachmentStore::UNSPECIFIED_ERROR;
352 }
353 }
354
355 PostCallback(base::Bind(callback, result_code, base::Passed(&metadata_list)));
356 }
357
358 AttachmentStore::Result OnDiskAttachmentStore::OpenOrCreate(
359 const base::FilePath& path) {
360 DCHECK(!db_);
361 base::FilePath leveldb_path = path.Append(kLeveldbDirectory);
362
363 leveldb::DB* db_raw;
364 std::unique_ptr<leveldb::DB> db;
365 leveldb::Options options;
366 options.create_if_missing = true;
367 options.reuse_logs = leveldb_env::kDefaultLogReuseOptionValue;
368 // TODO(pavely): crbug/424287 Consider adding info_log, block_cache and
369 // filter_policy to options.
370 leveldb::Status status =
371 leveldb::DB::Open(options, leveldb_path.AsUTF8Unsafe(), &db_raw);
372 if (!status.ok()) {
373 DVLOG(1) << "DB::Open failed: status=" << status.ToString()
374 << ", path=" << path.AsUTF8Unsafe();
375 return AttachmentStore::UNSPECIFIED_ERROR;
376 }
377
378 db.reset(db_raw);
379
380 attachment_store_pb::StoreMetadata metadata;
381 status = ReadStoreMetadata(db.get(), &metadata);
382 if (!status.ok() && !status.IsNotFound()) {
383 DVLOG(1) << "ReadStoreMetadata failed: status=" << status.ToString();
384 return AttachmentStore::UNSPECIFIED_ERROR;
385 }
386 if (status.IsNotFound()) {
387 // Brand new database.
388 metadata.set_schema_version(kCurrentSchemaVersion);
389 status = WriteStoreMetadata(db.get(), metadata);
390 if (!status.ok()) {
391 DVLOG(1) << "WriteStoreMetadata failed: status=" << status.ToString();
392 return AttachmentStore::UNSPECIFIED_ERROR;
393 }
394 }
395 DCHECK(status.ok());
396
397 // Upgrade code goes here.
398
399 if (metadata.schema_version() != kCurrentSchemaVersion) {
400 DVLOG(1) << "Unknown schema version: " << metadata.schema_version();
401 return AttachmentStore::UNSPECIFIED_ERROR;
402 }
403
404 db_ = std::move(db);
405 return AttachmentStore::SUCCESS;
406 }
407
408 std::unique_ptr<Attachment> OnDiskAttachmentStore::ReadSingleAttachment(
409 const AttachmentId& attachment_id,
410 AttachmentStore::Component component) {
411 std::unique_ptr<Attachment> attachment;
412 attachment_store_pb::RecordMetadata record_metadata;
413 if (!ReadSingleRecordMetadata(attachment_id, &record_metadata)) {
414 return attachment;
415 }
416 if (!AttachmentHasReferenceFromComponent(record_metadata,
417 ComponentToProto(component)))
418 return attachment;
419
420 const std::string key = MakeDataKeyFromAttachmentId(attachment_id);
421 std::string data_str;
422 leveldb::Status status = db_->Get(
423 MakeNonCachingReadOptions(), key, &data_str);
424 if (!status.ok()) {
425 DVLOG(1) << "DB::Get for data failed: status=" << status.ToString();
426 return attachment;
427 }
428 scoped_refptr<base::RefCountedMemory> data =
429 base::RefCountedString::TakeString(&data_str);
430 uint32_t crc32c = ComputeCrc32c(data);
431 if (record_metadata.has_crc32c()) {
432 if (record_metadata.crc32c() != crc32c) {
433 DVLOG(1) << "Attachment crc32c does not match value read from store";
434 return attachment;
435 }
436 if (record_metadata.crc32c() != attachment_id.GetCrc32c()) {
437 DVLOG(1) << "Attachment crc32c does not match value in AttachmentId";
438 return attachment;
439 }
440 }
441 attachment.reset(
442 new Attachment(Attachment::CreateFromParts(attachment_id, data)));
443 return attachment;
444 }
445
446 bool OnDiskAttachmentStore::WriteSingleAttachment(
447 const Attachment& attachment,
448 AttachmentStore::Component component) {
449 const std::string metadata_key =
450 MakeMetadataKeyFromAttachmentId(attachment.GetId());
451 const std::string data_key = MakeDataKeyFromAttachmentId(attachment.GetId());
452
453 std::string metadata_str;
454 leveldb::Status status =
455 db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
456 if (status.ok()) {
457 // Entry exists, don't overwrite.
458 return true;
459 } else if (!status.IsNotFound()) {
460 // Entry exists but failed to read.
461 DVLOG(1) << "DB::Get failed: status=" << status.ToString();
462 return false;
463 }
464 DCHECK(status.IsNotFound());
465
466 leveldb::WriteBatch write_batch;
467 // Write metadata.
468 attachment_store_pb::RecordMetadata metadata;
469 metadata.set_attachment_size(attachment.GetData()->size());
470 metadata.set_crc32c(attachment.GetCrc32c());
471 SetReferenceInRecordMetadata(&metadata, ComponentToProto(component));
472 metadata_str = metadata.SerializeAsString();
473 write_batch.Put(metadata_key, metadata_str);
474 // Write data.
475 scoped_refptr<base::RefCountedMemory> data = attachment.GetData();
476 leveldb::Slice data_slice(data->front_as<char>(), data->size());
477 write_batch.Put(data_key, data_slice);
478
479 status = db_->Write(MakeWriteOptions(), &write_batch);
480 if (!status.ok()) {
481 // Failed to write.
482 DVLOG(1) << "DB::Write failed: status=" << status.ToString();
483 return false;
484 }
485 return true;
486 }
487
488 bool OnDiskAttachmentStore::ReadSingleRecordMetadata(
489 const AttachmentId& attachment_id,
490 attachment_store_pb::RecordMetadata* record_metadata) {
491 DCHECK(record_metadata);
492 const std::string metadata_key =
493 MakeMetadataKeyFromAttachmentId(attachment_id);
494 std::string metadata_str;
495 leveldb::Status status =
496 db_->Get(MakeCachingReadOptions(), metadata_key, &metadata_str);
497 if (!status.ok()) {
498 DVLOG(1) << "DB::Get for metadata failed: status=" << status.ToString();
499 return false;
500 }
501 if (!record_metadata->ParseFromString(metadata_str)) {
502 DVLOG(1) << "RecordMetadata::ParseFromString failed";
503 return false;
504 }
505 DCHECK_GT(record_metadata->component_size(), 0);
506 return true;
507 }
508
509 bool OnDiskAttachmentStore::WriteSingleRecordMetadata(
510 const AttachmentId& attachment_id,
511 const attachment_store_pb::RecordMetadata& record_metadata) {
512 const std::string metadata_key =
513 MakeMetadataKeyFromAttachmentId(attachment_id);
514 std::string metadata_str;
515 metadata_str = record_metadata.SerializeAsString();
516 leveldb::Status status =
517 db_->Put(MakeWriteOptions(), metadata_key, metadata_str);
518 if (!status.ok()) {
519 DVLOG(1) << "DB::Put failed: status=" << status.ToString();
520 return false;
521 }
522 return true;
523 }
524
525 std::string OnDiskAttachmentStore::MakeDataKeyFromAttachmentId(
526 const AttachmentId& attachment_id) {
527 std::string key = kDataPrefix + attachment_id.GetProto().unique_id();
528 return key;
529 }
530
531 std::string OnDiskAttachmentStore::MakeMetadataKeyFromAttachmentId(
532 const AttachmentId& attachment_id) {
533 std::string key = kMetadataPrefix + attachment_id.GetProto().unique_id();
534 return key;
535 }
536
537 AttachmentMetadata OnDiskAttachmentStore::MakeAttachmentMetadata(
538 const AttachmentId& attachment_id,
539 const attachment_store_pb::RecordMetadata& record_metadata) {
540 return AttachmentMetadata(attachment_id, record_metadata.attachment_size());
541 }
542
543 } // namespace syncer
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698