Closes #1036, Migrate storage directory folders to secondary keys

Also moves orphaned directories into orphaned-files folder in data dir

Safety first: Keeps track of moved files, moving them back if there's an error before the end of the upgrade process (since the keys are generated randomly and would be different if recreated), and creates a zotero.moved-files.36.bak file with a list of id/key pairs

chrome/content/zotero/xpcom/attachments.js

@@ -36,7 +36,9 @@ Zotero.Attachments = new function(){
 	this.createMissingAttachment = createMissingAttachment;
 	this.getFileBaseNameFromItem = getFileBaseNameFromItem;
 	this.createDirectoryForItem = createDirectoryForItem;
+	this.createDirectoryForMissingItem = createDirectoryForMissingItem;
 	this.getStorageDirectory = getStorageDirectory;
+	this.getMissingStorageDirectory = getMissingStorageDirectory;
 	this.getPath = getPath;
 	
 	var self = this;
@@ -56,6 +58,7 @@ Zotero.Attachments = new function(){
 			attachmentItem.setSource(sourceItemID);
 			attachmentItem.attachmentLinkMode = this.LINK_MODE_IMPORTED_FILE;
 			var itemID = attachmentItem.save();
+			attachmentItem = Zotero.Items.get(itemID);
 			
 			// Create directory for attachment files within storage directory
 			var destDir = this.createDirectoryForItem(itemID);
@@ -85,10 +88,9 @@ Zotero.Attachments = new function(){
 			
 			try {
 				// Clean up
-				if (itemID){
+				if (itemID) {
-					var itemDir = Zotero.getStorageDirectory();
+					var itemDir = this.getStorageDirectory(itemID);
-					itemDir.append(itemID);
+					if (itemDir.exists()) {
-					if (itemDir.exists()){
 						itemDir.remove(true);
 					}
 				}
@@ -138,15 +140,17 @@ Zotero.Attachments = new function(){
 			// create a proper item, but at the moment this is only called by
 			// translate.js, which sets the metadata fields itself
 			var itemID = attachmentItem.save();
+			attachmentItem = Zotero.Items.get(itemID)
+			var attachmentKey = attachmentItem.key;
 			
 			var storageDir = Zotero.getStorageDirectory();
-			file.parent.copyTo(storageDir, itemID);
+			file.parent.copyTo(storageDir, attachmentKey);
 			
 			// Point to copied file
 			var newFile = Components.classes["@mozilla.org/file/local;1"]
 							.createInstance(Components.interfaces.nsILocalFile);
 			newFile.initWithFile(storageDir);
-			newFile.append(itemID);
+			newFile.append(attachmentKey);
 			newFile.append(file.leafName);
 			
 			attachmentItem.path = this.getPath(newFile, this.LINK_MODE_IMPORTED_URL);
@@ -162,10 +166,9 @@ Zotero.Attachments = new function(){
 			
 			try {
 				// Clean up
-				if (itemID){
+				if (itemID) {
-					var itemDir = Zotero.getStorageDirectory();
+					var itemDir = this.getStorageDirectory(itemID);
-					itemDir.append(itemID);
+					if (itemDir.exists()) {
-					if (itemDir.exists()){
 						itemDir.remove(true);
 					}
 				}
@@ -260,6 +263,7 @@ Zotero.Attachments = new function(){
 					attachmentItem.attachmentLinkMode = Zotero.Attachments.LINK_MODE_IMPORTED_URL;
 					attachmentItem.attachmentMIMEType = mimeType;
 					var itemID = attachmentItem.save();
+					attachmentItem = Zotero.Items.get(itemID);
 					
 					// Add to collections
 					if (parentCollectionIDs){
@@ -280,8 +284,6 @@ Zotero.Attachments = new function(){
 					
 					wbp.progressListener = new Zotero.WebProgressFinishListener(function(){
 						try {
-							var attachmentItem = Zotero.Items.get(itemID);
-							
 							var str = Zotero.File.getSample(file);
 							if (mimeType == 'application/pdf' &&
 									Zotero.MIME.sniffForMIMEType(str) != 'application/pdf') {
@@ -291,9 +293,10 @@ Zotero.Attachments = new function(){
 								return;
 							}
 							
-							attachmentItem.attachmentPath = Zotero.Attachments.getPath(
+							attachmentItem.attachmentPath =
-								file, Zotero.Attachments.LINK_MODE_IMPORTED_URL, itemID
+								Zotero.Attachments.getPath(
-							);
+									file, Zotero.Attachments.LINK_MODE_IMPORTED_URL
+								);
 							attachmentItem.save();
 							
 							Zotero.Notifier.trigger('add', 'item', itemID);
@@ -337,10 +340,9 @@ Zotero.Attachments = new function(){
 					try {
 						// Clean up
 						if (itemID) {
-							var destDir = Zotero.getStorageDirectory();
+							var itemDir = this.getStorageDirectory(itemID);
-							destDir.append(itemID);
+							if (itemDir.exists()) {
-							if (destDir.exists()) {
+								itemDir.remove(true);
-								destDir.remove(true);
 							}
 						}
 					}
@@ -538,8 +540,9 @@ Zotero.Attachments = new function(){
 				wpdDOMSaver.init(file.path, document);
 				wpdDOMSaver.saveHTMLDocument();
 				
-				var path = this.getPath(file, Zotero.Attachments.LINK_MODE_IMPORTED_URL, itemID);
+				attachmentItem.attachmentPath = this.getPath(
-				attachmentItem.attachmentPath = path;
+					file, Zotero.Attachments.LINK_MODE_IMPORTED_URL
+				);
 				attachmentItem.save();
 			}
 			else {
@@ -555,8 +558,10 @@ Zotero.Attachments = new function(){
 				var nsIURL = ioService.newURI(url, null, null);
 				wbp.progressListener = new Zotero.WebProgressFinishListener(function () {
 					try {
-						var path = this.getPath(file, Zotero.Attachments.LINK_MODE_IMPORTED_URL, itemID);
+						attachmentItem.attachmentPath = this.getPath(
-						attachmentItem.attachmentPath = path;
+							file,
+							Zotero.Attachments.LINK_MODE_IMPORTED_URL
+						);
 						attachmentItem.save();
 						
 						Zotero.Notifier.trigger('add', 'item', itemID);
@@ -618,10 +623,9 @@ Zotero.Attachments = new function(){
 			try {
 				// Clean up
 				if (itemID) {
-					var destDir = Zotero.getStorageDirectory();
+					var itemDir = this.getStorageDirectory(itemID);
-					destDir.append(itemID);
+					if (itemDir.exists()) {
-					if (destDir.exists()) {
+						itemDir.remove(true);
-						destDir.remove(true);
 					}
 				}
 			}
@@ -886,6 +890,8 @@ Zotero.Attachments = new function(){
 	
 	/*
 	 * Create directory for attachment files within storage directory
+	 *
+	 * @param	integer		itemID		Item id
 	 */
 	function createDirectoryForItem(itemID) {
 		var dir = this.getStorageDirectory(itemID);
@@ -896,9 +902,35 @@ Zotero.Attachments = new function(){
 	}
 	
 	
+	/*
+	 * Create directory for missing attachment files within storage directory
+	 *
+	 * @param	string	key		Item secondary lookup key
+	 */
+	function createDirectoryForMissingItem(key) {
+		var dir = this.getMissingStorageDirectory(key);
+		if (!dir.exists()) {
+			dir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
+		}
+		return dir;
+	}
+	
+	
 	function getStorageDirectory(itemID) {
+		var item = Zotero.Items.get(itemID);
 		var dir = Zotero.getStorageDirectory();
-		dir.append(itemID);
+		dir.append(item.key);
+		return dir;
+	}
+	
+	
+	function getMissingStorageDirectory(key) {
+		if (typeof key != 'string' || !key.match(/^[A-Z0-9]{8}$/)) {
+			throw ('key must be an 8-character string in '
+				+ 'Zotero.Attachments.getMissingStorageDirectory()')
+		}
+		var dir = Zotero.getStorageDirectory();
+		dir.append(key);
 		return dir;
 	}
 	
@@ -906,41 +938,11 @@ Zotero.Attachments = new function(){
 	/*
 	 * Gets a relative descriptor for imported attachments and a persistent
 	 * descriptor for files outside the storage directory
-	 *
-	 * @param	int		missingItemID		Item id to use if file is missing to
-	 *										generate suitable path
 	 */
-	function getPath(file, linkMode, missingItemID) {
+	function getPath(file, linkMode) {
-		var exists = file.exists();
-		// TODO: can we get the itemID from the path?
-		if (!missingItemID && !exists) {
-			throw ('Zotero.Attachments.getPath() cannot be called on non-existent file without missingItemID');
-		}
-		
-		// If imported file doesn't exist, create one temporarily so we can get
-		// the relative path (which doesn't work on non-existent files)
-		if (!exists && (linkMode == self.LINK_MODE_IMPORTED_URL ||
-				linkMode == self.LINK_MODE_IMPORTED_FILE)) {
-			var missingFile = self.createDirectoryForItem(missingItemID);
-			missingFile.append(file.leafName);
-			missingFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
-			
-			var descriptor = Zotero.Attachments.getPath(missingFile, linkMode);
-			
-			var parentDir = missingFile.parent;
-			missingFile.remove(null);
-			parentDir.remove(null);
-			
-			return descriptor;
-		}
-		
-		file.QueryInterface(Components.interfaces.nsILocalFile);
-		
 		if (linkMode == self.LINK_MODE_IMPORTED_URL ||
 				linkMode == self.LINK_MODE_IMPORTED_FILE) {
-			var storageDir = Zotero.getStorageDirectory();
+			return 'storage:' + file.leafName;
-			storageDir.QueryInterface(Components.interfaces.nsILocalFile);
-			return file.getRelativeDescriptor(storageDir);
 		}
 		
 		return file.persistentDescriptor;
@@ -1009,8 +1011,8 @@ Zotero.Attachments = new function(){
 		
 		// Get path
 		if (file) {
-			var path = Zotero.Attachments.getPath(file, linkMode, attachmentItem.id);
+			attachmentItem.attachmentPath
-			attachmentItem.attachmentPath = path;
+				= Zotero.Attachments.getPath(file, linkMode);
 		}
 		
 		attachmentItem.setSource(sourceItemID);

chrome/content/zotero/xpcom/data/item.js

@@ -2120,49 +2120,48 @@ Zotero.Item.prototype.getFile = function(row, skipExistsCheck) {
 	}
 	
 	if (!row) {
-		var sql = "SELECT linkMode, path FROM itemAttachments WHERE itemID="
+		var sql = "SELECT linkMode, path FROM itemAttachments WHERE itemID=?"
-			+ this.id;
+		var row = Zotero.DB.rowQuery(sql, this.id);
-		var row = Zotero.DB.rowQuery(sql);
 	}
 	
 	if (!row) {
-		throw ('Attachment data not found for item ' + this.id
+		throw ('Attachment data not found for item ' + this.id + ' in getFile()');
-			+ ' in getFile()');
 	}
 	
 	// No associated files for linked URLs
-	if (row['linkMode']==Zotero.Attachments.LINK_MODE_LINKED_URL) {
+	if (row.linkMode == Zotero.Attachments.LINK_MODE_LINKED_URL) {
 		return false;
 	}
 	
-	var file = Components.classes["@mozilla.org/file/local;1"].
+	if (row.linkMode == Zotero.Attachments.LINK_MODE_IMPORTED_URL ||
-		createInstance(Components.interfaces.nsILocalFile);
+			row.linkMode == Zotero.Attachments.LINK_MODE_IMPORTED_FILE) {
-	
-	if (row['linkMode']==Zotero.Attachments.LINK_MODE_IMPORTED_URL ||
-			row['linkMode']==Zotero.Attachments.LINK_MODE_IMPORTED_FILE) {
 		try {
-			var storageDir = Zotero.getStorageDirectory();
+			if (row.path.indexOf("storage:") == -1) {
-			storageDir.QueryInterface(Components.interfaces.nsILocalFile);
+				Zotero.debug("Invalid attachment path '" + row.path + "'");
-			file.setRelativeDescriptor(storageDir, row['path']);
+				throw ('Invalid path');
+			}
+			// Strip "storage:"
+			var path = row.path.substr(8);
+			var file = Zotero.Attachments.getStorageDirectory(this.id);
+			file.append(path);
 			if (!file.exists()) {
-				throw('Invalid relative descriptor');
+				Zotero.debug("Attachment file '" + path + "' not found");
+				throw ('File not found');
 			}
 		}
 		catch (e) {
 			// See if this is a persistent path
 			// (deprecated for imported attachments)
-			Zotero.debug('Invalid relative descriptor -- trying persistent');
+			Zotero.debug('Trying as persistent descriptor');
 			try {
-				file.persistentDescriptor = row['path'];
+				var file = Components.classes["@mozilla.org/file/local;1"].
-				
+					createInstance(Components.interfaces.nsILocalFile);
-				var storageDir = Zotero.getStorageDirectory();
+				file.persistentDescriptor = row.path;
-				storageDir.QueryInterface(Components.interfaces.nsILocalFile);
-				var path = file.getRelativeDescriptor(storageDir);
 				
 				// If valid, convert this to a relative descriptor
 				if (file.exists()) {
 					Zotero.DB.query("UPDATE itemAttachments SET path=? WHERE itemID=?",
-						[path, this.id]);
+						["storage:" + file.leafName, this.id]);
 				}
 			}
 			catch (e) {
@@ -2171,16 +2170,19 @@ Zotero.Item.prototype.getFile = function(row, skipExistsCheck) {
 		}
 	}
 	else {
+		var file = Components.classes["@mozilla.org/file/local;1"].
+			createInstance(Components.interfaces.nsILocalFile);
+		
 		try {
-			file.persistentDescriptor = row['path'];
+			file.persistentDescriptor = row.path;
 		}
 		catch (e) {
 			// See if this is an old relative path (deprecated)
 			Zotero.debug('Invalid persistent descriptor -- trying relative');
 			try {
-				var refDir = (row['linkMode']==this.LINK_MODE_LINKED_FILE)
+				var refDir = (row.linkMode == this.LINK_MODE_LINKED_FILE)
 					? Zotero.getZoteroDirectory() : Zotero.getStorageDirectory();
-				file.setRelativeDescriptor(refDir, row['path']);
+				file.setRelativeDescriptor(refDir, row.path);
 				// If valid, convert this to a persistent descriptor
 				if (file.exists()) {
 					Zotero.DB.query("UPDATE itemAttachments SET path=? WHERE itemID=?",
@@ -2229,7 +2231,7 @@ Zotero.Item.prototype.renameAttachmentFile = function(newName, overwrite) {
 			return -1;
 		}
 		
-		file.moveTo(file.parent, newName);
+		file.moveTo(null, newName);
 		this.relinkAttachmentFile(file);
 		
 		return true;

chrome/content/zotero/xpcom/fulltext.js

@@ -393,7 +393,7 @@ Zotero.Fulltext = new function(){
 		}
 		
 		var item = Zotero.Items.get(itemID);
-		var linkMode = item.getAttachmentLinkMode();
+		var linkMode = item.attachmentLinkMode;
 		// If file is stored outside of Zotero, create a directory for the item
 		// in the storage directory and save the cache file there
 		if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE) {
@@ -644,8 +644,7 @@ Zotero.Fulltext = new function(){
 	 * Gets the number of pages from the PDF info cache file
 	 */
 	function getTotalPagesFromFile(itemID) {
-		var item = Zotero.Items.get(itemID);
+		var file = Zotero.Attachments.getStorageDirectory(itemID);
-		var file = Zotero.Attachments.getStorageDirectory(item.getID());
 		file.append(this.pdfInfoCacheFile);
 		if (!file.exists()) {
 			return false;
@@ -1029,8 +1028,7 @@ Zotero.Fulltext = new function(){
 	
 	
 	function _getItemCacheFile(itemID) {
-		var cacheFile = Zotero.getStorageDirectory();
+		var cacheFile = Zotero.Attachments.getStorageDirectory(itemID);
-		cacheFile.append(itemID);
 		cacheFile.append(self.pdfConverterCacheFile);
 		return cacheFile;
 	}

chrome/content/zotero/xpcom/schema.js

@@ -1531,6 +1531,61 @@ Zotero.Schema = new function(){
 						}
 					}
 					statement.reset();
+					
+					// Migrate attachment folders to secondary keys
+					Zotero.DB.query("UPDATE itemAttachments SET path=REPLACE(path, itemID || '/', 'storage:') WHERE path REGEXP '^[0-9]+/'");
+					
+					if (Zotero.Prefs.get('useDataDir')) {
+						var dataDir = Components.classes["@mozilla.org/file/local;1"].createInstance(Components.interfaces.nsILocalFile);
+						dataDir.persistentDescriptor = Zotero.Prefs.get('dataDir');
+					}
+					else {
+						var dataDir = Zotero.getProfileDirectory();
+						dataDir.append('zotero');
+					}
+					if (!dataDir.exists() || !dataDir.isDirectory()){
+						var e = { name: "NS_ERROR_FILE_NOT_FOUND" };
+						throw (e);
+					}
+					var movedFiles37 = {};
+					var moveReport = '';
+					var orphaned = dataDir.clone();
+					var storage37 = dataDir.clone();
+					var moveReportFile = dataDir.clone();
+					orphaned.append('orphaned-files');
+					storage37.append('storage');
+					moveReportFile.append('zotero.moved-files.' + fromVersion + '.bak');
+					var keys = {};
+					var rows = Zotero.DB.query("SELECT itemID, key FROM items");
+					for each(var row in rows) {
+						keys[row.itemID] = row.key;
+					}
+					var entries = storage37.directoryEntries;
+					while (entries.hasMoreElements()) {
+						var file = entries.getNext();
+						file.QueryInterface(Components.interfaces.nsILocalFile);
+						var id = parseInt(file.leafName); 
+						if (!file.isDirectory() || isNaN(id)) {
+							continue;
+						}
+						if (keys[id]) {
+							file.moveTo(null, keys[id]);
+							moveReport += keys[id] + ' ' + id + "\n";
+						}
+						else {
+							if (!orphaned.exists()) {
+								orphaned.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
+							}
+							file.moveTo(orphaned, null);
+						}
+						movedFiles37[id] = file;
+					}
+					if (moveReport) {
+						moveReport = 'The following directory names in storage were changed:\n'
+									+ '------------------------------------------------------\n'
+									+ moveReport;
+						Zotero.File.putContents(moveReportFile, moveReport);
+					}
 				}
 			}
 			
@@ -1538,7 +1593,15 @@ Zotero.Schema = new function(){
 			
 			Zotero.DB.commitTransaction();
 		}
-		catch(e){
+		catch (e) {
+			if (movedFiles37) {
+				for (var id in movedFiles37) {
+					try {
+						movedFiles37[id].moveTo(storage37, id);
+					}
+					catch (e2) { Zotero.debug(e2); }
+				}
+			}
 			Zotero.DB.rollbackTransaction();
 			throw(e);
 		}