| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include <limits> | |
| 6 #include <set> | |
| 7 #include <string> | |
| 8 | |
| 9 #include "chrome/browser/history/text_database.h" | |
| 10 | |
| 11 #include "base/file_util.h" | |
| 12 #include "base/logging.h" | |
| 13 #include "base/metrics/histogram.h" | |
| 14 #include "base/strings/string_number_conversions.h" | |
| 15 #include "base/strings/stringprintf.h" | |
| 16 #include "base/strings/utf_string_conversions.h" | |
| 17 #include "sql/statement.h" | |
| 18 #include "sql/transaction.h" | |
| 19 | |
| 20 // There are two tables in each database, one full-text search (FTS) table which | |
| 21 // indexes the contents and title of the pages. The other is a regular SQLITE | |
| 22 // table which contains non-indexed information about the page. All columns of | |
| 23 // a FTS table are indexed using the text search algorithm, which isn't what we | |
| 24 // want for things like times. If this were in the FTS table, there would be | |
| 25 // different words in the index for each time number. | |
| 26 // | |
| 27 // "pages" FTS table: | |
| 28 // url URL of the page so searches will match the URL. | |
| 29 // title Title of the page. | |
| 30 // body Body of the page. | |
| 31 // | |
| 32 // "info" regular table: | |
| 33 // time Time the corresponding FTS entry was visited. | |
| 34 // | |
| 35 // We do joins across these two tables by using their internal rowids, which we | |
| 36 // keep in sync between the two tables. The internal rowid is the only part of | |
| 37 // an FTS table that is indexed like a normal table, and the index over it is | |
| 38 // free since sqlite always indexes the internal rowid. | |
| 39 | |
| 40 namespace history { | |
| 41 | |
| 42 namespace { | |
| 43 | |
| 44 // Version 1 uses FTS2 for index files. | |
| 45 // Version 2 uses FTS3. | |
| 46 static const int kCurrentVersionNumber = 2; | |
| 47 static const int kCompatibleVersionNumber = 2; | |
| 48 | |
| 49 // Snippet computation relies on the index of the columns in the original | |
| 50 // create statement. These are the 0-based indices (as strings) of the | |
| 51 // corresponding columns. | |
| 52 const char kTitleColumnIndex[] = "1"; | |
| 53 const char kBodyColumnIndex[] = "2"; | |
| 54 | |
| 55 // The string prepended to the database identifier to generate the filename. | |
| 56 const base::FilePath::CharType kFilePrefix[] = | |
| 57 FILE_PATH_LITERAL("History Index "); | |
| 58 | |
| 59 } // namespace | |
| 60 | |
| 61 TextDatabase::Match::Match() {} | |
| 62 | |
| 63 TextDatabase::Match::~Match() {} | |
| 64 | |
| 65 TextDatabase::TextDatabase(const base::FilePath& path, | |
| 66 DBIdent id, | |
| 67 bool allow_create) | |
| 68 : path_(path), | |
| 69 ident_(id), | |
| 70 allow_create_(allow_create) { | |
| 71 // Compute the file name. | |
| 72 file_name_ = path_.Append(IDToFileName(ident_)); | |
| 73 } | |
| 74 | |
| 75 TextDatabase::~TextDatabase() { | |
| 76 } | |
| 77 | |
| 78 // static | |
| 79 const base::FilePath::CharType* TextDatabase::file_base() { | |
| 80 return kFilePrefix; | |
| 81 } | |
| 82 | |
| 83 // static | |
| 84 base::FilePath TextDatabase::IDToFileName(DBIdent id) { | |
| 85 // Identifiers are intended to be a combination of the year and month, for | |
| 86 // example, 200801 for January 2008. We convert this to | |
| 87 // "History Index 2008-01". However, we don't make assumptions about this | |
| 88 // scheme: the caller should assign IDs as it feels fit with the knowledge | |
| 89 // that they will apppear on disk in this form. | |
| 90 base::FilePath::StringType filename(file_base()); | |
| 91 base::StringAppendF(&filename, FILE_PATH_LITERAL("%d-%02d"), | |
| 92 id / 100, id % 100); | |
| 93 return base::FilePath(filename); | |
| 94 } | |
| 95 | |
| 96 // static | |
| 97 TextDatabase::DBIdent TextDatabase::FileNameToID( | |
| 98 const base::FilePath& file_path) { | |
| 99 base::FilePath::StringType file_name = file_path.BaseName().value(); | |
| 100 | |
| 101 // We don't actually check the prefix here. Since the file system could | |
| 102 // be case insensitive in ways we can't predict (NTFS), checking could | |
| 103 // potentially be the wrong thing to do. Instead, we just look for a suffix. | |
| 104 static const size_t kIDStringLength = 7; // Room for "xxxx-xx". | |
| 105 if (file_name.length() < kIDStringLength) | |
| 106 return 0; | |
| 107 const base::FilePath::StringType suffix( | |
| 108 &file_name[file_name.length() - kIDStringLength]); | |
| 109 | |
| 110 if (suffix.length() != kIDStringLength || | |
| 111 suffix[4] != FILE_PATH_LITERAL('-')) { | |
| 112 return 0; | |
| 113 } | |
| 114 | |
| 115 // TODO: Once StringPiece supports a templated interface over the | |
| 116 // underlying string type, use it here instead of substr, since that | |
| 117 // will avoid needless string copies. StringPiece cannot be used | |
| 118 // right now because base::FilePath::StringType could use either 8 or 16 bit | |
| 119 // characters, depending on the OS. | |
| 120 int year, month; | |
| 121 base::StringToInt(suffix.substr(0, 4), &year); | |
| 122 base::StringToInt(suffix.substr(5, 2), &month); | |
| 123 | |
| 124 return year * 100 + month; | |
| 125 } | |
| 126 | |
| 127 bool TextDatabase::Init() { | |
| 128 // Make sure, if we're not allowed to create the file, that it exists. | |
| 129 if (!allow_create_) { | |
| 130 if (!base::PathExists(file_name_)) | |
| 131 return false; | |
| 132 } | |
| 133 | |
| 134 db_.set_histogram_tag("Text"); | |
| 135 | |
| 136 // Set the database page size to something a little larger to give us | |
| 137 // better performance (we're typically seek rather than bandwidth limited). | |
| 138 // This only has an effect before any tables have been created, otherwise | |
| 139 // this is a NOP. Must be a power of 2 and a max of 8192. | |
| 140 db_.set_page_size(4096); | |
| 141 | |
| 142 // The default cache size is 2000 which give >8MB of data. Since we will often | |
| 143 // have 2-3 of these objects, each with their own 8MB, this adds up very fast. | |
| 144 // We therefore reduce the size so when there are multiple objects, we're not | |
| 145 // too big. | |
| 146 db_.set_cache_size(512); | |
| 147 | |
| 148 // Run the database in exclusive mode. Nobody else should be accessing the | |
| 149 // database while we're running, and this will give somewhat improved perf. | |
| 150 db_.set_exclusive_locking(); | |
| 151 | |
| 152 // Attach the database to our index file. | |
| 153 if (!db_.Open(file_name_)) | |
| 154 return false; | |
| 155 | |
| 156 // Meta table tracking version information. | |
| 157 if (!meta_table_.Init(&db_, kCurrentVersionNumber, kCompatibleVersionNumber)) | |
| 158 return false; | |
| 159 if (meta_table_.GetCompatibleVersionNumber() > kCurrentVersionNumber) { | |
| 160 // This version is too new. We don't bother notifying the user on this | |
| 161 // error, and just fail to use the file. Normally if they have version skew, | |
| 162 // they will get it for the main history file and it won't be necessary | |
| 163 // here. If that's not the case, since this is only indexed data, it's | |
| 164 // probably better to just not give FTS results than strange errors when | |
| 165 // everything else is working OK. | |
| 166 LOG(WARNING) << "Text database is too new."; | |
| 167 return false; | |
| 168 } | |
| 169 | |
| 170 return CreateTables(); | |
| 171 } | |
| 172 | |
| 173 void TextDatabase::BeginTransaction() { | |
| 174 db_.BeginTransaction(); | |
| 175 } | |
| 176 | |
| 177 void TextDatabase::CommitTransaction() { | |
| 178 db_.CommitTransaction(); | |
| 179 } | |
| 180 | |
| 181 bool TextDatabase::CreateTables() { | |
| 182 // FTS table of page contents. | |
| 183 if (!db_.DoesTableExist("pages")) { | |
| 184 if (!db_.Execute("CREATE VIRTUAL TABLE pages USING fts3(" | |
| 185 "TOKENIZE icu," | |
| 186 "url LONGVARCHAR," | |
| 187 "title LONGVARCHAR," | |
| 188 "body LONGVARCHAR)")) | |
| 189 return false; | |
| 190 } | |
| 191 | |
| 192 // Non-FTS table containing URLs and times so we can efficiently find them | |
| 193 // using a regular index (all FTS columns are special and are treated as | |
| 194 // full-text-search, which is not what we want when retrieving this data). | |
| 195 if (!db_.DoesTableExist("info")) { | |
| 196 // Note that there is no point in creating an index over time. Since | |
| 197 // we must always query the entire FTS table (it can not efficiently do | |
| 198 // subsets), we will always end up doing that first, and joining the info | |
| 199 // table off of that. | |
| 200 if (!db_.Execute("CREATE TABLE info(time INTEGER NOT NULL)")) | |
| 201 return false; | |
| 202 } | |
| 203 | |
| 204 // Create the index. | |
| 205 return db_.Execute("CREATE INDEX IF NOT EXISTS info_time ON info(time)"); | |
| 206 } | |
| 207 | |
| 208 bool TextDatabase::AddPageData(base::Time time, | |
| 209 const std::string& url, | |
| 210 const std::string& title, | |
| 211 const std::string& contents) { | |
| 212 sql::Transaction committer(&db_); | |
| 213 if (!committer.Begin()) | |
| 214 return false; | |
| 215 | |
| 216 // Add to the pages table. | |
| 217 sql::Statement add_to_pages(db_.GetCachedStatement(SQL_FROM_HERE, | |
| 218 "INSERT INTO pages (url, title, body) VALUES (?,?,?)")); | |
| 219 add_to_pages.BindString(0, url); | |
| 220 add_to_pages.BindString(1, title); | |
| 221 add_to_pages.BindString(2, contents); | |
| 222 if (!add_to_pages.Run()) | |
| 223 return false; | |
| 224 | |
| 225 int64 rowid = db_.GetLastInsertRowId(); | |
| 226 | |
| 227 // Add to the info table with the same rowid. | |
| 228 sql::Statement add_to_info(db_.GetCachedStatement(SQL_FROM_HERE, | |
| 229 "INSERT INTO info (rowid, time) VALUES (?,?)")); | |
| 230 add_to_info.BindInt64(0, rowid); | |
| 231 add_to_info.BindInt64(1, time.ToInternalValue()); | |
| 232 | |
| 233 if (!add_to_info.Run()) | |
| 234 return false; | |
| 235 | |
| 236 return committer.Commit(); | |
| 237 } | |
| 238 | |
| 239 void TextDatabase::DeletePageData(base::Time time, const std::string& url) { | |
| 240 // First get all rows that match. Selecing on time (which has an index) allows | |
| 241 // us to avoid brute-force searches on the full-text-index table (there will | |
| 242 // generally be only one match per time). | |
| 243 sql::Statement select_ids(db_.GetCachedStatement(SQL_FROM_HERE, | |
| 244 "SELECT info.rowid " | |
| 245 "FROM info JOIN pages ON info.rowid = pages.rowid " | |
| 246 "WHERE info.time=? AND pages.url=?")); | |
| 247 select_ids.BindInt64(0, time.ToInternalValue()); | |
| 248 select_ids.BindString(1, url); | |
| 249 | |
| 250 std::set<int64> rows_to_delete; | |
| 251 while (select_ids.Step()) | |
| 252 rows_to_delete.insert(select_ids.ColumnInt64(0)); | |
| 253 | |
| 254 // Delete from the pages table. | |
| 255 sql::Statement delete_page(db_.GetCachedStatement(SQL_FROM_HERE, | |
| 256 "DELETE FROM pages WHERE rowid=?")); | |
| 257 | |
| 258 for (std::set<int64>::const_iterator i = rows_to_delete.begin(); | |
| 259 i != rows_to_delete.end(); ++i) { | |
| 260 delete_page.BindInt64(0, *i); | |
| 261 if (!delete_page.Run()) | |
| 262 return; | |
| 263 delete_page.Reset(true); | |
| 264 } | |
| 265 | |
| 266 // Delete from the info table. | |
| 267 sql::Statement delete_info(db_.GetCachedStatement(SQL_FROM_HERE, | |
| 268 "DELETE FROM info WHERE rowid=?")); | |
| 269 | |
| 270 for (std::set<int64>::const_iterator i = rows_to_delete.begin(); | |
| 271 i != rows_to_delete.end(); ++i) { | |
| 272 delete_info.BindInt64(0, *i); | |
| 273 if (!delete_info.Run()) | |
| 274 return; | |
| 275 delete_info.Reset(true); | |
| 276 } | |
| 277 } | |
| 278 | |
| 279 void TextDatabase::Optimize() { | |
| 280 sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, | |
| 281 "SELECT OPTIMIZE(pages) FROM pages LIMIT 1")); | |
| 282 statement.Run(); | |
| 283 } | |
| 284 | |
| 285 bool TextDatabase::GetTextMatches(const std::string& query, | |
| 286 const QueryOptions& options, | |
| 287 std::vector<Match>* results, | |
| 288 URLSet* found_urls) { | |
| 289 std::string sql = "SELECT url, title, time, offsets(pages), body FROM pages " | |
| 290 "LEFT OUTER JOIN info ON pages.rowid = info.rowid WHERE "; | |
| 291 sql += options.body_only ? "body " : "pages "; | |
| 292 sql += "MATCH ? AND time >= ? AND time < ? "; | |
| 293 // Times may not be unique, so also sort by rowid to ensure a stable order. | |
| 294 sql += "ORDER BY time DESC, info.rowid DESC"; | |
| 295 | |
| 296 // Generate unique IDs for the two possible variations of the statement, | |
| 297 // so they don't share the same cached prepared statement. | |
| 298 sql::StatementID body_only_id = SQL_FROM_HERE; | |
| 299 sql::StatementID pages_id = SQL_FROM_HERE; | |
| 300 | |
| 301 sql::Statement statement(db_.GetCachedStatement( | |
| 302 (options.body_only ? body_only_id : pages_id), sql.c_str())); | |
| 303 | |
| 304 statement.BindString(0, query); | |
| 305 statement.BindInt64(1, options.EffectiveBeginTime()); | |
| 306 statement.BindInt64(2, options.EffectiveEndTime()); | |
| 307 | |
| 308 // |results| may not be initially empty, so keep track of how many were added | |
| 309 // by this call. | |
| 310 int result_count = 0; | |
| 311 | |
| 312 while (statement.Step()) { | |
| 313 // TODO(brettw) allow canceling the query in the middle. | |
| 314 // if (canceled_or_something) | |
| 315 // break; | |
| 316 | |
| 317 GURL url(statement.ColumnString(0)); | |
| 318 URLSet::const_iterator found_url = found_urls->find(url); | |
| 319 if (found_url != found_urls->end()) | |
| 320 continue; // Don't add this duplicate. | |
| 321 | |
| 322 if (++result_count > options.EffectiveMaxCount()) | |
| 323 break; | |
| 324 | |
| 325 // Fill the results into the vector (avoid copying the URL with Swap()). | |
| 326 results->resize(results->size() + 1); | |
| 327 Match& match = results->at(results->size() - 1); | |
| 328 match.url.Swap(&url); | |
| 329 | |
| 330 match.title = statement.ColumnString16(1); | |
| 331 match.time = base::Time::FromInternalValue(statement.ColumnInt64(2)); | |
| 332 | |
| 333 // Extract any matches in the title. | |
| 334 std::string offsets_str = statement.ColumnString(3); | |
| 335 Snippet::ExtractMatchPositions(offsets_str, kTitleColumnIndex, | |
| 336 &match.title_match_positions); | |
| 337 Snippet::ConvertMatchPositionsToWide(statement.ColumnString(1), | |
| 338 &match.title_match_positions); | |
| 339 | |
| 340 // Extract the matches in the body. | |
| 341 Snippet::MatchPositions match_positions; | |
| 342 Snippet::ExtractMatchPositions(offsets_str, kBodyColumnIndex, | |
| 343 &match_positions); | |
| 344 | |
| 345 // Compute the snippet based on those matches. | |
| 346 std::string body = statement.ColumnString(4); | |
| 347 match.snippet.ComputeSnippet(match_positions, body); | |
| 348 } | |
| 349 statement.Reset(true); | |
| 350 return result_count > options.EffectiveMaxCount(); | |
| 351 } | |
| 352 | |
| 353 } // namespace history | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 16951015:
Remove TextDatabase from the history service. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@replace_fts