· 9 years ago · Nov 11, 2016, 11:40 AM
1/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2/* vim: set ts=8 sts=2 et sw=2 tw=80: */
3/* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7#include "mozStorageConnectionUtils.h"
8
9namespace mozilla {
10namespace storage {
11namespace utils {
12
13namespace {
14
15class MOZ_RAII AutoDisableForeignKeyChecking
16{
17public:
18 explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn)
19 : mConn(aConn)
20 , mForeignKeyCheckingDisabled(false)
21 {
22 nsCOMPtr<mozIStorageStatement> state;
23 nsresult rv = mConn->CreateStatement(NS_LITERAL_CSTRING(
24 "PRAGMA foreign_keys;"
25 ), getter_AddRefs(state));
26 if (NS_WARN_IF(NS_FAILED(rv))) { return; }
27
28 bool hasMoreData = false;
29 rv = state->ExecuteStep(&hasMoreData);
30 if (NS_WARN_IF(NS_FAILED(rv))) { return; }
31
32 int32_t mode;
33 rv = state->GetInt32(0, &mode);
34 if (NS_WARN_IF(NS_FAILED(rv))) { return; }
35
36 if (mode) {
37 nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
38 "PRAGMA foreign_keys = OFF;"
39 ));
40 if (NS_WARN_IF(NS_FAILED(rv))) { return; }
41 mForeignKeyCheckingDisabled = true;
42 }
43 }
44
45 ~AutoDisableForeignKeyChecking()
46 {
47 if (mForeignKeyCheckingDisabled) {
48 nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
49 "PRAGMA foreign_keys = ON;"
50 ));
51 if (NS_WARN_IF(NS_FAILED(rv))) { return; }
52 }
53 }
54
55private:
56 nsCOMPtr<mozIStorageConnection> mConn;
57 bool mForeignKeyCheckingDisabled;
58};
59
60} // anonymous namespace
61
62nsresult
63Validate(mozIStorageConnection* aConn,
64 const int32_t aLatestSchemaVersion,
65 const nsTArray<Expect>& aExpectedSchema)
66{
67 int32_t schemaVersion;
68 nsresult rv = aConn->GetSchemaVersion(&schemaVersion);
69 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
70
71 if (NS_WARN_IF(schemaVersion != aLatestSchemaVersion)) {
72 return NS_ERROR_FAILURE;
73 }
74
75#ifdef DEBUG
76 // Read the schema from the sqlite_master table and compare.
77 nsCOMPtr<mozIStorageStatement> state;
78 rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
79 "SELECT name, type, sql FROM sqlite_master;"
80 ), getter_AddRefs(state));
81 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
82
83 bool hasMoreData = false;
84 while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {
85 nsAutoCString name;
86 rv = state->GetUTF8String(0, name);
87 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
88
89 nsAutoCString type;
90 rv = state->GetUTF8String(1, type);
91 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
92
93 nsAutoCString sql;
94 rv = state->GetUTF8String(2, sql);
95 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
96
97 bool foundMatch = false;
98 for (uint32_t i = 0; i < aExpectedSchema.Length(); ++i) {
99 if (name == aExpectedSchema[i].mName) {
100 if (type != aExpectedSchema[i].mType) {
101 NS_WARNING(nsPrintfCString("Unexpected type for schema entry %s",
102 name.get()).get());
103 return NS_ERROR_FAILURE;
104 }
105
106 if (!aExpectedSchema[i].mIgnoreSql && sql != aExpectedSchema[i].mSql) {
107 NS_WARNING(nsPrintfCString("Unexpected SQL for schema entry %s",
108 name.get()).get());
109 return NS_ERROR_FAILURE;
110 }
111
112 foundMatch = true;
113 break;
114 }
115 }
116
117 if (NS_WARN_IF(!foundMatch)) {
118 NS_WARNING(nsPrintfCString("Unexpected schema entry %s in database",
119 name.get()).get());
120 return NS_ERROR_FAILURE;
121 }
122 }
123#endif
124
125 return rv;
126}
127
128nsresult
129RewriteSchema(mozIStorageConnection* aConn,
130 SchemaRewrite* aSchemaRewrite)
131{
132 nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
133 "PRAGMA writable_schema = ON"
134 ));
135 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
136
137 nsCOMPtr<mozIStorageStatement> state;
138 rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
139 "UPDATE sqlite_master SET sql=:sql WHERE name=:name"
140 ), getter_AddRefs(state));
141 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
142
143 rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("sql"),
144 aSchemaRewrite->mTableSql);
145 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
146
147 rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("name"),
148 aSchemaRewrite->mTableName);
149 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
150
151 rv = state->Execute();
152 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
153
154 rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
155 "PRAGMA writable_schema = OFF"
156 ));
157 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
158
159 return rv;
160}
161
162nsresult
163Migrate(mozIStorageConnection* aConn,
164 const int32_t aFirstShippedSchemaVersion,
165 const int32_t aLatestSchemaVersion,
166 const nsTArray<Migration>& aMigrationList)
167{
168 printf_stderr("Migrate\n");
169 MOZ_ASSERT(!NS_IsMainThread());
170 MOZ_ASSERT(aConn);
171
172 int32_t currentVersion = 0;
173 nsresult rv = aConn->GetSchemaVersion(¤tVersion);
174 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
175
176 nsClassHashtable<nsCStringHashKey, nsCString> schemaRewrites;
177
178 printf_stderr("currentVersion < aLatestSchemaVersion %d < %d\n",
179 currentVersion, aLatestSchemaVersion);
180 while (currentVersion < aLatestSchemaVersion) {
181 printf_stderr("currentVersion %d\n", currentVersion);
182 // Wiping old database should be handled elsewhere because it requires
183 // making a whole new mozIStorageConnection. Make sure we don't
184 // accidentally get here for one of those old databases.
185 MOZ_ASSERT(currentVersion >= aFirstShippedSchemaVersion);
186
187 for (uint32_t i = 0; i < aMigrationList.Length(); ++i) {
188 if (aMigrationList[i].mFromVersion == currentVersion) {
189 RefPtr<SchemaRewrite> schemaRewrite = new SchemaRewrite();
190 rv = aMigrationList[i].mFunc(aConn, getter_AddRefs(schemaRewrite));
191 /*if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
192 if (schemaRewrite != nullptr &&
193 !schemaRewrites.Contains(schemaRewrite->mTableName)) {
194 printf_stderr("Adding schemarewrite\n");
195 schemaRewrites.Put(schemaRewrite->mTableName, &schemaRewrite->mTableSql);
196 }*/
197 break;
198 }
199 }
200
201 DebugOnly<int32_t> lastVersion = currentVersion;
202 rv = aConn->GetSchemaVersion(¤tVersion);
203 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
204 MOZ_ASSERT(currentVersion > lastVersion);
205 }
206
207 printf_stderr("currentVersion == aLatestSchemaVersion %d == %d\n", currentVersion, aLatestSchemaVersion);
208 MOZ_ASSERT(currentVersion == aLatestSchemaVersion);
209
210/* // Now overwrite the master SQL for each table that requires so.
211 for (auto iter = schemaRewrites.Iter(); !iter.Done(); iter.Next()) {
212 printf_stderr("Schema rewrite\n");
213 RefPtr<SchemaRewrite> rewrite;
214 rewrite->mTableName = iter.Key();
215 rewrite->mTableSql = *schemaRewrites.Get(iter.Key());
216 rv = RewriteSchema(aConn, rewrite);
217 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
218 }
219
220 schemaRewrites.Clear();*/
221
222 printf_stderr("return\n");
223 return rv;
224}
225
226nsresult
227CreateOrMigrateSchema(mozIStorageConnection* aConn,
228 const int32_t aFirstShippedSchemaVersion,
229 const int32_t aLatestSchemaVersion,
230 const nsTArray<nsCString>& aTablesSql,
231 const nsTArray<Expect>& aExpectedSchema,
232 const nsTArray<Migration>& aMigrationList)
233{
234 MOZ_ASSERT(!NS_IsMainThread());
235 MOZ_ASSERT(aConn);
236
237 int32_t schemaVersion;
238 nsresult rv = aConn->GetSchemaVersion(&schemaVersion);
239 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
240
241 printf_stderr("SchemaVersion %d == aLatestSchemaVersion %d\n", schemaVersion, aLatestSchemaVersion);
242
243 if (schemaVersion == aLatestSchemaVersion) {
244 // We already have the correct schema version. Validate it matches
245 // our expected schema and then proceed.
246 rv = Validate(aConn, aLatestSchemaVersion, aExpectedSchema);
247 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
248
249 return rv;
250 }
251
252 // Turn off checking foreign keys before starting a transaction, and restore
253 // it once we're done.
254 AutoDisableForeignKeyChecking restoreForeignKeyChecking(aConn);
255 mozStorageTransaction trans(aConn, false,
256 mozIStorageConnection::TRANSACTION_IMMEDIATE);
257 bool needVacuum = false;
258
259 if (schemaVersion) {
260 printf_stderr("Calling Migrate\n");
261 // A schema exists, but its not the current version. Attempt to
262 // migrate it to our new schema.
263 rv = Migrate(aConn, aFirstShippedSchemaVersion, aLatestSchemaVersion,
264 aMigrationList);
265 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
266
267 // Migrations happen infrequently and reflect a chance in DB structure.
268 // This is a good time to rebuild the database. It also helps catch
269 // if a new migration is incorrect by fast failing on the corruption.
270 needVacuum = true;
271 } else {
272 // There is no schema installed. Create the database from scratch.
273 for (uint32_t i = 0; i < aTablesSql.Length(); i++) {
274 rv = aConn->ExecuteSimpleSQL(aTablesSql[i]);
275 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
276 }
277
278 rv = aConn->SetSchemaVersion(aLatestSchemaVersion);
279 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
280
281 rv = aConn->GetSchemaVersion(&schemaVersion);
282 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
283 }
284
285 rv = Validate(aConn, aLatestSchemaVersion, aExpectedSchema);
286 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
287
288 rv = trans.Commit();
289 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
290
291 if (needVacuum) {
292 // Unfortunately, this must be performed outside of the transaction.
293 aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM"));
294 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
295 }
296
297 return rv;
298}
299
300nsresult
301InitializeConnection(mozIStorageConnection* aConn,
302 const uint32_t aPageSize,
303 const uint32_t aGrowthSize,
304 const uint32_t aWalAutoCheckpointPages,
305 const uint32_t aWalAutoCheckpointSize)
306{
307 MOZ_ASSERT(!NS_IsMainThread());
308 MOZ_ASSERT(aConn);
309
310 // This function needs to perform per-connection initialization tasks that
311 // need to happen regardless of the schema.
312
313 nsPrintfCString pragmas(
314 // Use a smaller page size to improve perf/footprint; default is too large
315 "PRAGMA page_size = %u; "
316 // Enable auto_vacuum; this must happen after page_size and before WAL
317 "PRAGMA auto_vacuum = INCREMENTAL; "
318 "PRAGMA foreign_keys = ON; ",
319 aPageSize
320 );
321
322 // Note, the default encoding of UTF-8 is preferred. mozStorage does all
323 // the work necessary to convert UTF-16 nsString values for us. We don't
324 // need ordering and the binary equality operations are correct. So, do
325 // NOT set PRAGMA encoding to UTF-16.
326
327 nsresult rv = aConn->ExecuteSimpleSQL(pragmas);
328 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
329
330 // Limit fragmentation by growing the database by many pages at once.
331 rv = aConn->SetGrowthIncrement(aGrowthSize, EmptyCString());
332 if (rv == NS_ERROR_FILE_TOO_BIG) {
333 NS_WARNING("Not enough disk space to set sqlite growth increment.");
334 rv = NS_OK;
335 }
336 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
337
338 // Enable WAL journaling. This must be performed in a separate transaction
339 // after changing the page_size and enabling auto_vacuum.
340 nsPrintfCString wal(
341 // WAL journal can grow to given number of *pages*
342 "PRAGMA wal_autocheckpoint = %u; "
343 // Always truncate the journal back to given number of *bytes*
344 "PRAGMA journal_size_limit = %u; "
345 // WAL must be enabled at the end to allow page size to be changed, etc.
346 "PRAGMA journal_mode = WAL; ",
347 aWalAutoCheckpointPages,
348 aWalAutoCheckpointSize
349 );
350
351 rv = aConn->ExecuteSimpleSQL(wal);
352 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
353
354 // Verify that we successfully set the vacuum mode to incremental. It
355 // is very easy to put the database in a state where the auto_vacuum
356 // pragma above fails silently.
357#ifdef DEBUG
358 nsCOMPtr<mozIStorageStatement> state;
359 rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
360 "PRAGMA auto_vacuum;"
361 ), getter_AddRefs(state));
362 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
363
364 bool hasMoreData = false;
365 rv = state->ExecuteStep(&hasMoreData);
366 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
367
368 int32_t mode;
369 rv = state->GetInt32(0, &mode);
370 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
371
372 // integer value 2 is incremental mode
373 if (NS_WARN_IF(mode != 2)) { return NS_ERROR_UNEXPECTED; }
374#endif
375
376 return NS_OK;
377}
378
379nsresult
380IncrementalVacuum(mozIStorageConnection* aConn,
381 const int32_t aMaxFreePages)
382{
383 // Determine how much free space is in the database.
384 nsCOMPtr<mozIStorageStatement> state;
385 nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
386 "PRAGMA freelist_count;"
387 ), getter_AddRefs(state));
388 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
389
390 bool hasMoreData = false;
391 rv = state->ExecuteStep(&hasMoreData);
392 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
393
394 int32_t freePages = 0;
395 rv = state->GetInt32(0, &freePages);
396 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
397
398 // We have a relatively small page size, so we want to be careful to avoid
399 // fragmentation. We already use a growth incremental which will cause
400 // sqlite to allocate and release multiple pages at the same time. We can
401 // further reduce fragmentation by making our allocated chunks a bit
402 // "sticky". This is done by creating some hysteresis where we allocate
403 // pages/chunks as soon as we need them, but we only release pages/chunks
404 // when we have a large amount of free space. This helps with the case
405 // where a page is adding and remove resources causing it to dip back and
406 // forth across a chunk boundary.
407 //
408 // So only proceed with releasing pages if we have more than our constant
409 // threshold.
410 if (freePages <= aMaxFreePages) {
411 return NS_OK;
412 }
413
414 // Release the excess pages back to the sqlite VFS. This may also release
415 // chunks of multiple pages back to the OS.
416 int32_t pagesToRelease = freePages - aMaxFreePages;
417
418 rv = aConn->ExecuteSimpleSQL(nsPrintfCString(
419 "PRAGMA incremental_vacuum(%d);", pagesToRelease
420 ));
421 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
422
423 // Verify that our incremental vacuum actually did something
424#ifdef DEBUG
425 rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
426 "PRAGMA freelist_count;"
427 ), getter_AddRefs(state));
428 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
429
430 hasMoreData = false;
431 rv = state->ExecuteStep(&hasMoreData);
432 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
433
434 freePages = 0;
435 rv = state->GetInt32(0, &freePages);
436 if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
437
438 MOZ_ASSERT(freePages <= aMaxFreePages);
439#endif
440
441 return NS_OK;
442}
443
444} // namespace utils
445} // namespace storage
446} // namespace mozilla