--- a/toolkit/components/places/PlacesDBUtils.jsm
+++ b/toolkit/components/places/PlacesDBUtils.jsm
@@ -262,16 +262,48 @@ this.PlacesDBUtils = {
throw new Error("Unable to check database coherence");
}
return logs;
},
async _getBoundCoherenceStatements() {
let cleanupStatements = [];
+ // Create triggers for updating Sync metadata. The "sync change" trigger
+ // bumps the parent's change counter when we update a GUID or move an item
+ // to a different folder, since Sync stores the list of child GUIDs on the
+ // parent. The "sync tombstone" trigger inserts tombstones for deleted
+ // synced bookmarks.
+ cleanupStatements.push({
+ query:
+ `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_change_temp_trigger
+ AFTER UPDATE of guid, parent, position ON moz_bookmarks
+ FOR EACH ROW
+ BEGIN
+ UPDATE moz_bookmarks
+ SET syncChangeCounter = syncChangeCounter + 1
+ WHERE id = NEW.parent OR
+ (OLD.parent <> NEW.parent AND
+ id = OLD.parent);
+ END`,
+ });
+ cleanupStatements.push({
+ query:
+ `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_tombstone_temp_trigger
+ AFTER DELETE ON moz_bookmarks
+ FOR EACH ROW WHEN OLD.guid NOT NULL AND
+ OLD.syncStatus <> 1
+ BEGIN
+ INSERT INTO moz_bookmarks_deleted(guid, dateRemoved)
+ VALUES(OLD.guid, STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000);
+ END`,
+ });
+
+ // MAINTENANCE STATEMENTS SHOULD GO BELOW THIS POINT!
+
// MOZ_ANNO_ATTRIBUTES
// A.1 remove obsolete annotations from moz_annos.
// The 'weave0' idiom exploits character ordering (0 follows /) to
// efficiently select all annos with a 'weave/' prefix.
let deleteObsoleteAnnos = {
query:
`DELETE FROM moz_annos
WHERE type = 4
@@ -486,17 +518,20 @@ this.PlacesDBUtils = {
deleteEmptyTags.params.toolbarGuid = PlacesUtils.bookmarks.toolbarGuid;
deleteEmptyTags.params.unfiledGuid = PlacesUtils.bookmarks.unfiledGuid;
deleteEmptyTags.params.tagsGuid = PlacesUtils.bookmarks.tagsGuid;
cleanupStatements.push(deleteEmptyTags);
// D.4 move orphan items to unsorted folder
let fixOrphanItems = {
query:
- `UPDATE moz_bookmarks SET parent = :unsorted_folder WHERE guid NOT IN (
+ `UPDATE moz_bookmarks SET
+ parent = :unsorted_folder,
+ syncChangeCounter = syncChangeCounter + 1
+ WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT b.id FROM moz_bookmarks b
WHERE NOT EXISTS
(SELECT id FROM moz_bookmarks WHERE id = b.parent LIMIT 1)
)`,
params: {}
};
@@ -509,17 +544,20 @@ this.PlacesDBUtils = {
cleanupStatements.push(fixOrphanItems);
// D.6 fix wrong item types
// Folders and separators should not have an fk.
// If they have a valid fk convert them to bookmarks. Later in D.9 we
// will move eventual children to unsorted bookmarks.
let fixBookmarksAsFolders = {
query:
- `UPDATE moz_bookmarks SET type = :bookmark_type WHERE guid NOT IN (
+ `UPDATE moz_bookmarks
+ SET type = :bookmark_type,
+ syncChangeCounter = syncChangeCounter + 1
+ WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT id FROM moz_bookmarks b
WHERE type IN (:folder_type, :separator_type)
AND fk NOTNULL
)`,
params: {}
};
@@ -533,17 +571,20 @@ this.PlacesDBUtils = {
fixBookmarksAsFolders.params.tagsGuid = PlacesUtils.bookmarks.tagsGuid;
cleanupStatements.push(fixBookmarksAsFolders);
// D.7 fix wrong item types
// Bookmarks should have an fk, if they don't have any, convert them to
// folders.
let fixFoldersAsBookmarks = {
query:
- `UPDATE moz_bookmarks SET type = :folder_type WHERE guid NOT IN (
+ `UPDATE moz_bookmarks
+ SET type = :folder_type,
+ syncChangeCounter = syncChangeCounter + 1
+ WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT id FROM moz_bookmarks b
WHERE type = :bookmark_type
AND fk IS NULL
)`,
params: {}
};
@@ -556,17 +597,20 @@ this.PlacesDBUtils = {
fixFoldersAsBookmarks.params.tagsGuid = PlacesUtils.bookmarks.tagsGuid;
cleanupStatements.push(fixFoldersAsBookmarks);
// D.9 fix wrong parents
// Items cannot have separators or other bookmarks
// as parent, if they have bad parent move them to unsorted bookmarks.
let fixInvalidParents = {
query:
- `UPDATE moz_bookmarks SET parent = :unsorted_folder WHERE guid NOT IN (
+ `UPDATE moz_bookmarks SET
+ parent = :unsorted_folder,
+ syncChangeCounter = syncChangeCounter + 1
+ WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT id FROM moz_bookmarks b
WHERE EXISTS
(SELECT id FROM moz_bookmarks WHERE id = b.parent
AND type IN (:bookmark_type, :separator_type)
LIMIT 1)
)`,
@@ -805,50 +849,43 @@ this.PlacesDBUtils = {
NOT IS_VALID_GUID(guid)`,
params: {
dateRemoved: PlacesUtils.toPRTime(new Date()),
syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
},
});
cleanupStatements.push({
query:
- `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_fix_guids_temp_trigger
- AFTER UPDATE of guid ON moz_bookmarks
- FOR EACH ROW
- BEGIN
- UPDATE moz_bookmarks
- SET syncChangeCounter = syncChangeCounter + 1
- WHERE id = NEW.parent;
- END`,
- });
- cleanupStatements.push({
- query:
`UPDATE moz_bookmarks
SET guid = GENERATE_GUID(),
syncChangeCounter = syncChangeCounter + 1,
syncStatus = :syncStatus
WHERE guid IS NULL OR
NOT IS_VALID_GUID(guid)`,
params: {
syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
},
});
- cleanupStatements.push({
- query: "DROP TRIGGER moz_bm_fix_guids_temp_trigger",
- });
// S.2 drop tombstones for bookmarks that aren't deleted.
cleanupStatements.push({
query:
`DELETE FROM moz_bookmarks_deleted
WHERE guid IN (SELECT guid FROM moz_bookmarks)`,
});
// MAINTENANCE STATEMENTS SHOULD GO ABOVE THIS POINT!
+ cleanupStatements.push({
+ query: "DROP TRIGGER moz_bm_sync_change_temp_trigger",
+ });
+ cleanupStatements.push({
+ query: "DROP TRIGGER moz_bm_sync_tombstone_temp_trigger",
+ });
+
return cleanupStatements;
},
/**
* Tries to vacuum the database.
*
* Note: although this function isn't actually async, we keep it async to
* allow us to maintain a simple, consistent API for the tasks within this object.
--- a/toolkit/components/places/tests/unit/test_preventive_maintenance.js
+++ b/toolkit/components/places/tests/unit/test_preventive_maintenance.js
@@ -32,16 +32,17 @@ function cleanDatabase() {
mDBConn.executeSimpleSQL("DELETE FROM moz_anno_attributes");
mDBConn.executeSimpleSQL("DELETE FROM moz_annos");
mDBConn.executeSimpleSQL("DELETE FROM moz_items_annos");
mDBConn.executeSimpleSQL("DELETE FROM moz_inputhistory");
mDBConn.executeSimpleSQL("DELETE FROM moz_keywords");
mDBConn.executeSimpleSQL("DELETE FROM moz_icons");
mDBConn.executeSimpleSQL("DELETE FROM moz_pages_w_icons");
mDBConn.executeSimpleSQL("DELETE FROM moz_bookmarks WHERE id > " + defaultBookmarksMaxId);
+ mDBConn.executeSimpleSQL("DELETE FROM moz_bookmarks_deleted");
}
function addPlace(aUrl, aFavicon) {
let href = new URL(aUrl || "http://www.mozilla.org").href;
let stmt = mDBConn.createStatement(
"INSERT INTO moz_places (url, url_hash) VALUES (:url, hash(:url))");
stmt.params.url = href;
stmt.execute();
@@ -398,37 +399,48 @@ tests.push({
// ------------------------------------------------------------------------------
tests.push({
name: "D.1",
desc: "Remove items without a valid place",
_validItemId: null,
_invalidItemId: null,
- _placeId: null,
+ _invalidSyncedItemId: null,
+ placeId: null,
setup() {
// Add a place to ensure place_id = 1 is valid
this.placeId = addPlace();
// Insert a valid bookmark
this._validItemId = addBookmark(this.placeId);
// Insert a bookmark with an invalid place
this._invalidItemId = addBookmark(1337);
+ // Insert a synced bookmark with an invalid place. We should write a
+ // tombstone when we remove it.
+ this._invalidSyncedItemId = addBookmark(1337, null, null, null, null, null,
+ "bookmarkAAAA", PlacesUtils.bookmarks.SYNC_STATUS.NORMAL);
},
- check() {
+ async check() {
// Check that valid bookmark is still there
let stmt = mDBConn.createStatement("SELECT id FROM moz_bookmarks WHERE id = :item_id");
stmt.params.item_id = this._validItemId;
do_check_true(stmt.executeStep());
stmt.reset();
// Check that invalid bookmark has been removed
stmt.params.item_id = this._invalidItemId;
do_check_false(stmt.executeStep());
+ stmt.reset();
+ stmt.params.item_id = this._invalidSyncedItemId;
+ do_check_false(stmt.executeStep());
stmt.finalize();
+
+ let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+ do_check_matches(tombstones.map(info => info.guid), ["bookmarkAAAA"]);
}
});
// ------------------------------------------------------------------------------
tests.push({
name: "D.2",
desc: "Remove items that are not uri bookmarks from tag containers",
@@ -522,48 +534,64 @@ tests.push({
desc: "Move orphan items to unsorted folder",
_orphanBookmarkId: null,
_orphanSeparatorId: null,
_orphanFolderId: null,
_bookmarkId: null,
_placeId: null,
- setup() {
+ async setup() {
// Add a place to ensure place_id = 1 is valid
this._placeId = addPlace();
// Insert an orphan bookmark
this._orphanBookmarkId = addBookmark(this._placeId, bs.TYPE_BOOKMARK, 8888);
// Insert an orphan separator
this._orphanSeparatorId = addBookmark(null, bs.TYPE_SEPARATOR, 8888);
// Insert a orphan folder
this._orphanFolderId = addBookmark(null, bs.TYPE_FOLDER, 8888);
// Create a child of the last created folder
this._bookmarkId = addBookmark(this._placeId, bs.TYPE_BOOKMARK, this._orphanFolderId);
},
- check() {
- // Check that bookmarks are now children of a real folder (unsorted)
- let stmt = mDBConn.createStatement("SELECT id FROM moz_bookmarks WHERE id = :item_id AND parent = :parent");
- stmt.params.item_id = this._orphanBookmarkId;
- stmt.params.parent = bs.unfiledBookmarksFolder;
- do_check_true(stmt.executeStep());
- stmt.reset();
- stmt.params.item_id = this._orphanSeparatorId;
- stmt.params.parent = bs.unfiledBookmarksFolder;
- do_check_true(stmt.executeStep());
- stmt.reset();
- stmt.params.item_id = this._orphanFolderId;
- stmt.params.parent = bs.unfiledBookmarksFolder;
- do_check_true(stmt.executeStep());
- stmt.reset();
- stmt.params.item_id = this._bookmarkId;
- stmt.params.parent = this._orphanFolderId;
- do_check_true(stmt.executeStep());
- stmt.finalize();
+ async check() {
+ // Check that bookmarks are now children of a real folder (unfiled)
+ let expectedInfos = [{
+ id: this._orphanBookmarkId,
+ parent: bs.unfiledBookmarksFolder,
+ syncChangeCounter: 1,
+ }, {
+ id: this._orphanSeparatorId,
+ parent: bs.unfiledBookmarksFolder,
+ syncChangeCounter: 1,
+ }, {
+ id: this._orphanFolderId,
+ parent: bs.unfiledBookmarksFolder,
+ syncChangeCounter: 1,
+ }, {
+ id: this._bookmarkId,
+ parent: this._orphanFolderId,
+ syncChangeCounter: 0,
+ }, {
+ id: bs.unfiledBookmarksFolder,
+ parent: bs.placesRoot,
+ syncChangeCounter: 3,
+ }];
+ let db = await PlacesUtils.promiseDBConnection();
+ for (let { id, parent, syncChangeCounter } of expectedInfos) {
+ let rows = await db.executeCached(`
+ SELECT id, syncChangeCounter
+ FROM moz_bookmarks
+ WHERE id = :item_id AND parent = :parent`,
+ { item_id: id, parent });
+ do_check_eq(rows.length, 1);
+
+ let actualChangeCounter = rows[0].getResultByName("syncChangeCounter");
+ do_check_eq(actualChangeCounter, syncChangeCounter);
+ }
}
});
// ------------------------------------------------------------------------------
tests.push({
name: "D.6",
desc: "Fix wrong item types | bookmarks",
@@ -611,90 +639,113 @@ tests.push({
// Add a bookmark with a valid place id
this._validBookmarkId = addBookmark(this._placeId, bs.TYPE_BOOKMARK);
// Add a bookmark with a null place id
this._invalidBookmarkId = addBookmark(null, bs.TYPE_BOOKMARK);
},
check() {
// Check valid bookmark
- let stmt = mDBConn.createStatement("SELECT id FROM moz_bookmarks WHERE id = :item_id AND type = :type");
+ let stmt = mDBConn.createStatement(`
+ SELECT id, syncChangeCounter
+ FROM moz_bookmarks
+ WHERE id = :item_id AND type = :type`);
stmt.params.item_id = this._validBookmarkId;
stmt.params.type = bs.TYPE_BOOKMARK;
do_check_true(stmt.executeStep());
+ do_check_eq(stmt.row.syncChangeCounter, 0);
stmt.reset();
// Check invalid bookmark has been converted to a folder
stmt.params.item_id = this._invalidBookmarkId;
stmt.params.type = bs.TYPE_FOLDER;
do_check_true(stmt.executeStep());
+ do_check_eq(stmt.row.syncChangeCounter, 1);
stmt.finalize();
}
});
// ------------------------------------------------------------------------------
tests.push({
name: "D.9",
desc: "Fix wrong parents",
_bookmarkId: null,
_separatorId: null,
_bookmarkId1: null,
_bookmarkId2: null,
_placeId: null,
- setup() {
+ async setup() {
// Add a place to ensure place_id = 1 is valid
this._placeId = addPlace();
// Insert a bookmark
this._bookmarkId = addBookmark(this._placeId, bs.TYPE_BOOKMARK);
// Insert a separator
this._separatorId = addBookmark(null, bs.TYPE_SEPARATOR);
// Create 3 children of these items
this._bookmarkId1 = addBookmark(this._placeId, bs.TYPE_BOOKMARK, this._bookmarkId);
this._bookmarkId2 = addBookmark(this._placeId, bs.TYPE_BOOKMARK, this._separatorId);
},
- check() {
- // Check that bookmarks are now children of a real folder (unsorted)
- let stmt = mDBConn.createStatement("SELECT id FROM moz_bookmarks WHERE id = :item_id AND parent = :parent");
- stmt.params.item_id = this._bookmarkId1;
- stmt.params.parent = bs.unfiledBookmarksFolder;
- do_check_true(stmt.executeStep());
- stmt.reset();
- stmt.params.item_id = this._bookmarkId2;
- stmt.params.parent = bs.unfiledBookmarksFolder;
- do_check_true(stmt.executeStep());
- stmt.finalize();
+ async check() {
+ // Check that bookmarks are now children of a real folder (unfiled)
+ let expectedInfos = [{
+ id: this._bookmarkId1,
+ parent: bs.unfiledBookmarksFolder,
+ syncChangeCounter: 1,
+ }, {
+ id: this._bookmarkId2,
+ parent: bs.unfiledBookmarksFolder,
+ syncChangeCounter: 1,
+ }, {
+ id: bs.unfiledBookmarksFolder,
+ parent: bs.placesRoot,
+ syncChangeCounter: 2,
+ }];
+ let db = await PlacesUtils.promiseDBConnection();
+ for (let { id, parent, syncChangeCounter } of expectedInfos) {
+ let rows = await db.executeCached(`
+ SELECT id, syncChangeCounter
+ FROM moz_bookmarks
+ WHERE id = :item_id AND parent = :parent`,
+ { item_id: id, parent });
+ do_check_eq(rows.length, 1);
+
+ let actualChangeCounter = rows[0].getResultByName("syncChangeCounter");
+ do_check_eq(actualChangeCounter, syncChangeCounter);
+ }
}
});
// ------------------------------------------------------------------------------
tests.push({
name: "D.10",
desc: "Recalculate positions",
_unfiledBookmarks: [],
_toolbarBookmarks: [],
- setup() {
+ async setup() {
const NUM_BOOKMARKS = 20;
bs.runInBatchMode({
runBatched(aUserData) {
// Add bookmarks to two folders to better perturbate the table.
for (let i = 0; i < NUM_BOOKMARKS; i++) {
bs.insertBookmark(PlacesUtils.unfiledBookmarksFolderId,
NetUtil.newURI("http://example.com/"),
- bs.DEFAULT_INDEX, "testbookmark");
+ bs.DEFAULT_INDEX, "testbookmark", null,
+ PlacesUtils.bookmarks.SOURCES.SYNC);
}
for (let i = 0; i < NUM_BOOKMARKS; i++) {
bs.insertBookmark(PlacesUtils.toolbarFolderId,
NetUtil.newURI("http://example.com/"),
- bs.DEFAULT_INDEX, "testbookmark");
+ bs.DEFAULT_INDEX, "testbookmark", null,
+ PlacesUtils.bookmarks.SOURCES.SYNC);
}
}
}, null);
function randomize_positions(aParent, aResultArray) {
let stmt = mDBConn.createStatement(
`UPDATE moz_bookmarks SET position = :rand
WHERE id IN (
@@ -724,42 +775,54 @@ tests.push({
}
stmt.finalize();
}
// Set random positions for the added bookmarks.
randomize_positions(PlacesUtils.unfiledBookmarksFolderId,
this._unfiledBookmarks);
randomize_positions(PlacesUtils.toolbarFolderId, this._toolbarBookmarks);
+
+ let syncInfos = await PlacesTestUtils.fetchBookmarkSyncFields(
+ PlacesUtils.bookmarks.unfiledGuid, PlacesUtils.bookmarks.toolbarGuid);
+ do_check_true(syncInfos.every(info => info.syncChangeCounter === 0));
},
- check() {
- function check_order(aParent, aResultArray) {
+ async check() {
+ let db = await PlacesUtils.promiseDBConnection();
+
+ async function check_order(aParent, aResultArray) {
// Build the expected ordered list of bookmarks.
- let stmt = mDBConn.createStatement(
- `SELECT id, position FROM moz_bookmarks WHERE parent = :parent
- ORDER BY position ASC`
+ let childRows = await db.executeCached(
+ `SELECT id, position, syncChangeCounter FROM moz_bookmarks
+ WHERE parent = :parent
+ ORDER BY position ASC`,
+ { parent: aParent }
);
- stmt.params.parent = aParent;
- let pass = true;
- while (stmt.executeStep()) {
- print(stmt.row.id + "\t" + stmt.row.position);
- if (aResultArray.indexOf(stmt.row.id) != stmt.row.position) {
- pass = false;
+ for (let row of childRows) {
+ let id = row.getResultByName("id");
+ let position = row.getResultByName("position");
+ if (aResultArray.indexOf(id) != position) {
+ dump_table("moz_bookmarks");
+ do_throw("Unexpected unfiled bookmarks order.");
}
}
- stmt.finalize();
- if (!pass) {
- dump_table("moz_bookmarks");
- do_throw("Unexpected unfiled bookmarks order.");
+
+ let parentRows = await db.executeCached(
+ `SELECT syncChangeCounter FROM moz_bookmarks
+ WHERE id = :parent`,
+ { parent: aParent });
+ for (let row of parentRows) {
+ let actualChangeCounter = row.getResultByName("syncChangeCounter");
+ do_check_true(actualChangeCounter > 0);
}
}
- check_order(PlacesUtils.unfiledBookmarksFolderId, this._unfiledBookmarks);
- check_order(PlacesUtils.toolbarFolderId, this._toolbarBookmarks);
+ await check_order(PlacesUtils.unfiledBookmarksFolderId, this._unfiledBookmarks);
+ await check_order(PlacesUtils.toolbarFolderId, this._toolbarBookmarks);
}
});
// ------------------------------------------------------------------------------
tests.push({
name: "D.12",
desc: "Fix empty-named tags",
@@ -1241,18 +1304,16 @@ tests.push({
// ------------------------------------------------------------------------------
tests.push({
name: "S.1",
desc: "fix invalid GUIDs for synced bookmarks",
_bookmarkInfos: [],
async setup() {
- await PlacesTestUtils.markBookmarksAsSynced();
-
let folderWithInvalidGuid = addBookmark(
null, PlacesUtils.bookmarks.TYPE_FOLDER,
PlacesUtils.bookmarks.bookmarksMenuFolder, /* aKeywordId */ null,
/* aFolderType */ null, "NORMAL folder with invalid GUID",
"{123456}", PlacesUtils.bookmarks.SYNC_STATUS.NORMAL);
let placeIdForBookmarkWithoutGuid = addPlace();
let bookmarkWithoutGuid = addBookmark(
@@ -1321,18 +1382,16 @@ tests.push({
let syncStatus = row.getResultByName("syncStatus");
do_check_eq(syncStatus, expectedInfo.syncStatus);
}
let tombstones = await PlacesTestUtils.fetchSyncTombstones();
do_check_matches(tombstones.map(info => info.guid),
["bookmarkAAAA\n", "{123456}"]);
-
- await PlacesSyncUtils.bookmarks.reset();
},
});
tests.push({
name: "S.2",
desc: "drop tombstones for bookmarks that aren't deleted",
async setup() {
@@ -1441,16 +1500,18 @@ add_task(async function test_preventive_
// Get current bookmarks max ID for cleanup
let stmt = mDBConn.createStatement("SELECT MAX(id) FROM moz_bookmarks");
stmt.executeStep();
defaultBookmarksMaxId = stmt.getInt32(0);
stmt.finalize();
do_check_true(defaultBookmarksMaxId > 0);
for (let test of tests) {
+ await PlacesTestUtils.markBookmarksAsSynced();
+
dump("\nExecuting test: " + test.name + "\n*** " + test.desc + "\n");
await test.setup();
Services.prefs.clearUserPref("places.database.lastMaintenance");
await PlacesDBUtils.maintenanceOnIdle();
// Check the lastMaintenance time has been saved.
do_check_neq(Services.prefs.getIntPref("places.database.lastMaintenance"), null);