Clean up extra files from Mendeley imports

Follow-up to cdee741a6

chrome/content/zotero/fileInterface.js

@@ -306,7 +306,10 @@ var Zotero_File_Interface = new function() {
 		}
 		if (typeof options == 'string' || options instanceof Components.interfaces.nsIFile) {
 			Zotero.debug("WARNING: importFile() now takes a single options object -- update your code");
-			options = { file: options };
+			options = {
+				file: options,
+				createNewCollection: arguments[1]
+			};
 		}
 		
 		var file = options.file ? Zotero.File.pathToFile(options.file) : null;

chrome/content/zotero/import/mendeley/mendeleyImport.js

@@ -806,52 +806,51 @@ Zotero_Import_Mendeley.prototype._convertNote = function (note) {
 
 Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) {
 	var idMap = new Map();
-	await Zotero.DB.executeTransaction(async function () {
+	
-		var lastExistingParentItem;
+	var lastExistingParentItem;
-		for (let i = 0; i < json.length; i++) {
+	for (let i = 0; i < json.length; i++) {
-			let itemJSON = json[i];
+		let itemJSON = json[i];
-			
+		
-			// Check if the item has been previously imported
+		// Check if the item has been previously imported
-			let item = this._findExistingItem(libraryID, itemJSON, lastExistingParentItem);
+		let item = this._findExistingItem(libraryID, itemJSON, lastExistingParentItem);
-			if (item) {
+		if (item) {
-				if (item.isRegularItem()) {
+			if (item.isRegularItem()) {
-					lastExistingParentItem = item;
+				lastExistingParentItem = item;
-					
-					// Update any child items to point to the existing item's key instead of the
-					// new generated one
-					this._updateParentKeys('item', json, i + 1, itemJSON.key, item.key);
-					
-					// Leave item in any collections it's in
-					itemJSON.collections = item.getCollections()
-						.map(id => Zotero.Collections.getLibraryAndKeyFromID(id).key)
-						.concat(itemJSON.collections || []);
-				}
-			}
-			else {
-				lastExistingParentItem = null;
 				
-				item = new Zotero.Item;
+				// Update any child items to point to the existing item's key instead of the
-				item.libraryID = libraryID;
+				// new generated one
-				if (itemJSON.key) {
+				this._updateParentKeys('item', json, i + 1, itemJSON.key, item.key);
-					item.key = itemJSON.key;
+				
-					await item.loadPrimaryData();
+				// Leave item in any collections it's in
-				}
+				itemJSON.collections = item.getCollections()
-			}
+					.map(id => Zotero.Collections.getLibraryAndKeyFromID(id).key)
-			
+					.concat(itemJSON.collections || []);
-			// Remove external id before save
-			let toSave = Object.assign({}, itemJSON);
-			delete toSave.documentID;
-			
-			item.fromJSON(toSave);
-			await item.save({
-				skipSelect: true,
-				skipDateModifiedUpdate: true
-			});
-			if (itemJSON.documentID) {
-				idMap.set(itemJSON.documentID, item.id);
 			}
 		}
-	}.bind(this));
+		else {
+			lastExistingParentItem = null;
+			
+			item = new Zotero.Item;
+			item.libraryID = libraryID;
+			if (itemJSON.key) {
+				item.key = itemJSON.key;
+				await item.loadPrimaryData();
+			}
+		}
+		
+		// Remove external id before save
+		let toSave = Object.assign({}, itemJSON);
+		delete toSave.documentID;
+		
+		item.fromJSON(toSave);
+		await item.saveTx({
+			skipSelect: true,
+			skipDateModifiedUpdate: true
+		});
+		if (itemJSON.documentID) {
+			idMap.set(itemJSON.documentID, item.id);
+		}
+	}
 	return idMap;
 };
 
@@ -984,6 +983,7 @@ Zotero_Import_Mendeley.prototype._saveFilesAndAnnotations = async function (file
 						options.title = file.title;
 						options.url = file.url;
 						options.contentType = file.contentType;
+						options.singleFile = true;
 						attachment = await Zotero.Attachments.importSnapshotFromFile(options);
 					}
 					else {
@@ -1156,3 +1156,55 @@ Zotero_Import_Mendeley.prototype._updateItemCollectionKeys = function (json, old
 		}
 	}
 }
+
+
+//
+// Clean up extra files created <5.0.51
+//
+Zotero_Import_Mendeley.prototype.hasImportedFiles = async function () {
+	return !!(await Zotero.DB.valueQueryAsync(
+		"SELECT itemID FROM itemRelations JOIN relationPredicates USING (predicateID) "
+			+ "WHERE predicate='mendeleyDB:fileHash' LIMIT 1"
+	));
+};
+
+Zotero_Import_Mendeley.prototype.queueFileCleanup = async function () {
+	await Zotero.DB.queryAsync("INSERT INTO settings VALUES ('mImport', 'cleanup', 1)");
+};
+
+Zotero_Import_Mendeley.prototype.deleteNonPrimaryFiles = async function () {
+	var rows = await Zotero.DB.queryAsync(
+		"SELECT key, path FROM itemRelations "
+			+ "JOIN relationPredicates USING (predicateID) "
+			+ "JOIN items USING (itemID) "
+			+ "JOIN itemAttachments USING (itemID) "
+			+ "WHERE predicate='mendeleyDB:fileHash' AND linkMode=1" // imported_url
+	);
+	for (let row of rows) {
+		let dir = (Zotero.Attachments.getStorageDirectoryByLibraryAndKey(1, row.key)).path;
+		if (!row.path.startsWith('storage:')) {
+			Zotero.logError(row.path + " does not start with 'storage:'");
+			continue;
+		}
+		let filename = row.path.substr(8);
+		
+		Zotero.debug(`Checking for extra files in ${dir}`);
+		await Zotero.File.iterateDirectory(dir, function* (iterator) {
+			while (true) {
+				let entry = yield iterator.next();
+				if (entry.name.startsWith('.zotero') || entry.name == filename) {
+					continue;
+				}
+				Zotero.debug(`Deleting ${entry.path}`);
+				try {
+					yield OS.File.remove(entry.path);
+				}
+				catch (e) {
+					Zotero.logError(e);
+				}
+			}
+		});
+	}
+	
+	await Zotero.DB.queryAsync("DELETE FROM settings WHERE setting='mImport' AND key='cleanup'");
+};

chrome/content/zotero/xpcom/attachments.js

@@ -176,14 +176,18 @@ Zotero.Attachments = new function(){
 	
 	
 	/**
-	 * @param {Object} options - 'file', 'url', 'title', 'contentType', 'charset', 'parentItemID'
+	 * @param {Object} options - 'file', 'url', 'title', 'contentType', 'charset', 'parentItemID', 'singleFile'
 	 * @return {Promise<Zotero.Item>}
 	 */
 	this.importSnapshotFromFile = Zotero.Promise.coroutine(function* (options) {
 		Zotero.debug('Importing snapshot from file');
 		
 		var file = Zotero.File.pathToFile(options.file);
-		var fileName = file.leafName;
+		// TODO: Fix main filename when copying directory, though in that case it's probably
+		// from our own export and already clean
+		var fileName = options.singleFile
+			? Zotero.File.getValidFileName(file.leafName)
+			: file.leafName;
 		var url = options.url;
 		var title = options.title;
 		var contentType = options.contentType;
@@ -194,7 +198,7 @@ Zotero.Attachments = new function(){
 			throw new Error("parentItemID not provided");
 		}
 		
-		var attachmentItem, itemID, destDir, newFile;
+		var attachmentItem, itemID, destDir, newPath;
 		try {
 			yield Zotero.DB.executeTransaction(function* () {
 				// Create a new attachment
@@ -217,13 +221,23 @@ Zotero.Attachments = new function(){
 				var storageDir = Zotero.getStorageDirectory();
 				destDir = this.getStorageDirectory(attachmentItem);
 				yield OS.File.removeDir(destDir.path);
-				file.parent.copyTo(storageDir, destDir.leafName);
+				newPath = OS.Path.join(destDir.path, fileName);
-				
+				// Copy single file to new directory
-				// Point to copied file
+				if (options.singleFile) {
-				newFile = destDir.clone();
+					yield this.createDirectoryForItem(attachmentItem);
-				newFile.append(file.leafName);
+					yield OS.File.copy(file.path, newPath);
+				}
+				// Copy entire parent directory (for HTML snapshots)
+				else {
+					file.parent.copyTo(storageDir, destDir.leafName);
+				}
 			}.bind(this));
-			yield _postProcessFile(attachmentItem, newFile, contentType, charset);
+			yield _postProcessFile(
+				attachmentItem,
+				Zotero.File.pathToFile(newPath),
+				contentType,
+				charset
+			);
 		}
 		catch (e) {
 			Zotero.logError(e);

chrome/content/zotero/xpcom/schema.js

@@ -2433,6 +2433,14 @@ Zotero.Schema = new function(){
 				}
 			}
 			
+			else if (i == 101) {
+				Components.utils.import("chrome://zotero/content/import/mendeley/mendeleyImport.js");
+				let importer = new Zotero_Import_Mendeley();
+				if (yield importer.hasImportedFiles()) {
+					yield importer.queueFileCleanup();
+				}
+			}
+			
 			// If breaking compatibility or doing anything dangerous, clear minorUpdateFrom
 		}
 		

chrome/content/zotero/xpcom/sync/syncAPIClient.js

@@ -820,7 +820,7 @@ Zotero.Sync.APIClient.prototype = {
 	
 	_checkBackoff: function (xmlhttp) {
 		var backoff = xmlhttp.getResponseHeader("Backoff");
-		if (backoff && Number.isInteger(backoff)) {
+		if (backoff && parseInt(backoff) == backoff) {
 			// TODO: Update status?
 			this.caller.pause(backoff * 1000);
 		}
@@ -831,7 +831,7 @@ Zotero.Sync.APIClient.prototype = {
 		var retryAfter = xmlhttp.getResponseHeader("Retry-After");
 		var delay;
 		if (!retryAfter) return false;
-		if (!Number.isInteger(retryAfter)) {
+		if (parseInt(retryAfter) != retryAfter) {
 			Zotero.logError(`Invalid Retry-After delay ${retryAfter}`);
 			return false;
 		}

chrome/content/zotero/zoteroPane.js

@@ -260,6 +260,18 @@ var ZoteroPane = new function()
 				ZoteroPane_Local.show();
 			}, 0);
 		}
+		
+		// TEMP: Clean up extra files from Mendeley imports <5.0.51
+		setTimeout(async function () {
+			var needsCleanup = await Zotero.DB.valueQueryAsync(
+				"SELECT COUNT(*) FROM settings WHERE setting='mImport' AND key='cleanup'"
+			)
+			if (!needsCleanup) return;
+			
+			Components.utils.import("chrome://zotero/content/import/mendeley/mendeleyImport.js");
+			var importer = new Zotero_Import_Mendeley();
+			importer.deleteNonPrimaryFiles();
+		}, 10000)
 	}
 	
 	

components/zotero-service.js

@@ -636,7 +636,10 @@ ZoteroCommandLineHandler.prototype = {
 								var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
 								   .getService(Components.interfaces.nsIWindowMediator);
 								var browserWindow = wm.getMostRecentWindow("navigator:browser");
-								browserWindow.Zotero_File_Interface.importFile(file, checkState.value);
+								browserWindow.Zotero_File_Interface.importFile({
+									file,
+									createNewCollection: checkState.value
+								});
 							}
 						}
 					});

install.rdf

@@ -6,7 +6,7 @@
   
     <em:id>zotero@chnm.gmu.edu</em:id>
     <em:name>Zotero</em:name>
-    <em:version>5.0.49.SOURCE</em:version>
+    <em:version>5.0.51.SOURCE</em:version>
     <em:creator>Center for History and New Media<br/>George Mason University</em:creator>
     <em:contributor>Dan Cohen</em:contributor>
     <em:contributor>Sean Takats</em:contributor>

resource/schema/userdata.sql

@@ -1,4 +1,4 @@
--- 100
+-- 101
 
 -- Copyright (c) 2009 Center for History and New Media
 --                    George Mason University, Fairfax, Virginia, USA

test/tests/syncAPIClientTest.js

@@ -174,7 +174,7 @@ describe("Zotero.Sync.APIClient", function () {
 							req.respond(
 								503,
 								{
-									"Retry-After": 5
+									"Retry-After": "5"
 								},
 								""
 							);
@@ -183,7 +183,7 @@ describe("Zotero.Sync.APIClient", function () {
 							req.respond(
 								503,
 								{
-									"Retry-After": 10
+									"Retry-After": "10"
 								},
 								""
 							);