10181f7f56
If a version is returned for an item's full-text content but a 404 is returned for the content itself (because it's missing in Elasticsearch for some reason), don't throw an error. Also remove legacy array comprehensions in fulltext and syncFullTextEngine test files, which apparently weren't being run.
365 lines
10 KiB
JavaScript
365 lines
10 KiB
JavaScript
"use strict";
|
|
|
|
describe("Zotero.Sync.Data.FullTextEngine", function () {
|
|
Components.utils.import("resource://zotero/config.js");
|
|
|
|
var apiKey = Zotero.Utilities.randomString(24);
|
|
var baseURL = "http://local.zotero/";
|
|
var engine, server, client, caller, stub, spy;
|
|
|
|
var responses = {};
|
|
|
|
var setup = Zotero.Promise.coroutine(function* (options = {}) {
|
|
server = sinon.fakeServer.create();
|
|
server.autoRespond = true;
|
|
|
|
Components.utils.import("resource://zotero/concurrentCaller.js");
|
|
var caller = new ConcurrentCaller(1);
|
|
caller.setLogger(msg => Zotero.debug(msg));
|
|
caller.stopOnError = true;
|
|
|
|
var client = new Zotero.Sync.APIClient({
|
|
baseURL,
|
|
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
|
|
apiKey,
|
|
caller,
|
|
background: options.background || true
|
|
});
|
|
|
|
var engine = new Zotero.Sync.Data.FullTextEngine({
|
|
apiClient: client,
|
|
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
|
|
stopOnError: true
|
|
});
|
|
|
|
return { engine, client, caller };
|
|
});
|
|
|
|
function setResponse(response) {
|
|
setHTTPResponse(server, baseURL, response, responses);
|
|
}
|
|
|
|
function generateContent() {
|
|
return new Array(10).fill("").map(x => Zotero.Utilities.randomString()).join(" ");
|
|
}
|
|
|
|
//
|
|
// Tests
|
|
//
|
|
beforeEach(function* () {
|
|
yield resetDB({
|
|
thisArg: this,
|
|
skipBundledFiles: true
|
|
});
|
|
|
|
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
|
|
|
|
yield Zotero.Users.setCurrentUserID(1);
|
|
yield Zotero.Users.setCurrentUsername("testuser");
|
|
})
|
|
|
|
describe("Full-Text Syncing", function () {
|
|
it("should download full-text into a new library and subsequent updates", function* () {
|
|
({ engine, client, caller } = yield setup());
|
|
|
|
var item = yield createDataObject('item');
|
|
var attachment = new Zotero.Item('attachment');
|
|
attachment.parentItemID = item.id;
|
|
attachment.attachmentLinkMode = 'imported_file';
|
|
attachment.attachmentContentType = 'application/pdf';
|
|
attachment.attachmentFilename = 'test.pdf';
|
|
yield attachment.saveTx();
|
|
|
|
var content = generateContent()
|
|
var spy = sinon.spy(Zotero.Fulltext, "startContentProcessor")
|
|
|
|
var itemFullTextVersion = 10;
|
|
var libraryFullTextVersion = 15;
|
|
setResponse({
|
|
method: "GET",
|
|
url: "users/1/fulltext",
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": libraryFullTextVersion
|
|
},
|
|
json: {
|
|
[attachment.key]: itemFullTextVersion
|
|
}
|
|
});
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items/${attachment.key}/fulltext`,
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": itemFullTextVersion
|
|
},
|
|
json: {
|
|
content,
|
|
indexedPages: 1,
|
|
totalPages: 1
|
|
}
|
|
});
|
|
yield engine.start();
|
|
|
|
var dir = Zotero.Attachments.getStorageDirectory(attachment).path;
|
|
var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed');
|
|
assert.isTrue(yield OS.File.exists(unprocessed));
|
|
var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed));
|
|
assert.propertyVal(data, 'text', content);
|
|
assert.propertyVal(data, 'indexedPages', 1);
|
|
assert.propertyVal(data, 'totalPages', 1);
|
|
assert.propertyVal(data, 'version', itemFullTextVersion);
|
|
yield assert.eventually.equal(
|
|
Zotero.FullText.getLibraryVersion(item.libraryID),
|
|
libraryFullTextVersion
|
|
);
|
|
|
|
sinon.assert.calledOnce(spy);
|
|
spy.restore();
|
|
|
|
//
|
|
// Get new content
|
|
//
|
|
({ engine, client, caller } = yield setup());
|
|
|
|
item = yield createDataObject('item');
|
|
attachment = new Zotero.Item('attachment');
|
|
attachment.parentItemID = item.id;
|
|
attachment.attachmentLinkMode = 'imported_file';
|
|
attachment.attachmentContentType = 'application/pdf';
|
|
attachment.attachmentFilename = 'test.pdf';
|
|
yield attachment.saveTx();
|
|
|
|
content = generateContent()
|
|
spy = sinon.spy(Zotero.Fulltext, "startContentProcessor")
|
|
|
|
itemFullTextVersion = 17;
|
|
var lastLibraryFullTextVersion = libraryFullTextVersion;
|
|
libraryFullTextVersion = 20;
|
|
setResponse({
|
|
method: "GET",
|
|
url: "users/1/fulltext?since=" + lastLibraryFullTextVersion,
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": libraryFullTextVersion
|
|
},
|
|
json: {
|
|
[attachment.key]: itemFullTextVersion
|
|
}
|
|
});
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items/${attachment.key}/fulltext`,
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": itemFullTextVersion
|
|
},
|
|
json: {
|
|
content,
|
|
indexedPages: 1,
|
|
totalPages: 1
|
|
}
|
|
});
|
|
yield engine.start();
|
|
|
|
var dir = Zotero.Attachments.getStorageDirectory(attachment).path;
|
|
var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed');
|
|
assert.isTrue(yield OS.File.exists(unprocessed));
|
|
var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed));
|
|
assert.propertyVal(data, 'text', content);
|
|
assert.propertyVal(data, 'indexedPages', 1);
|
|
assert.propertyVal(data, 'totalPages', 1);
|
|
assert.propertyVal(data, 'version', itemFullTextVersion);
|
|
yield assert.eventually.equal(
|
|
Zotero.FullText.getLibraryVersion(item.libraryID),
|
|
libraryFullTextVersion
|
|
);
|
|
|
|
sinon.assert.calledOnce(spy);
|
|
spy.restore();
|
|
})
|
|
|
|
it("should handle remotely missing full-text content", function* () {
|
|
({ engine, client, caller } = yield setup());
|
|
|
|
var item = yield createDataObject('item');
|
|
var attachment = new Zotero.Item('attachment');
|
|
attachment.parentItemID = item.id;
|
|
attachment.attachmentLinkMode = 'imported_file';
|
|
attachment.attachmentContentType = 'application/pdf';
|
|
attachment.attachmentFilename = 'test.pdf';
|
|
yield attachment.saveTx();
|
|
|
|
var itemFullTextVersion = 10;
|
|
var libraryFullTextVersion = 15;
|
|
setResponse({
|
|
method: "GET",
|
|
url: "users/1/fulltext",
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": libraryFullTextVersion
|
|
},
|
|
json: {
|
|
[attachment.key]: itemFullTextVersion
|
|
}
|
|
});
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items/${attachment.key}/fulltext`,
|
|
status: 404,
|
|
headers: {
|
|
"Last-Modified-Version": itemFullTextVersion
|
|
},
|
|
text: ""
|
|
});
|
|
yield engine.start();
|
|
})
|
|
|
|
it("should upload new full-text content and subsequent updates", function* () {
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
var fixSinonBug = ";charset=utf-8";
|
|
|
|
var libraryID = Zotero.Libraries.userLibraryID;
|
|
yield Zotero.Libraries.setVersion(libraryID, 5);
|
|
|
|
({ engine, client, caller } = yield setup());
|
|
|
|
var item = yield createDataObject('item');
|
|
var attachment = new Zotero.Item('attachment');
|
|
attachment.parentItemID = item.id;
|
|
attachment.attachmentLinkMode = 'imported_file';
|
|
attachment.attachmentContentType = 'text/html';
|
|
attachment.attachmentFilename = 'test.html';
|
|
attachment.attachmentCharset = 'utf-8';
|
|
attachment.synced = true;
|
|
yield attachment.saveTx();
|
|
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
|
|
|
var path = attachment.getFilePath();
|
|
var content = generateContent()
|
|
var htmlContent = "<html><body>" + content + "</body></html>";
|
|
yield Zotero.File.putContentsAsync(path, content);
|
|
yield Zotero.Fulltext.indexItems([attachment.id]);
|
|
|
|
var libraryVersion = 15;
|
|
var previousLibraryVersion = libraryVersion;
|
|
|
|
var count = 1;
|
|
setResponse({
|
|
method: "GET",
|
|
url: "users/1/fulltext",
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": libraryVersion
|
|
},
|
|
json: {}
|
|
});
|
|
server.respond(function (req) {
|
|
if (req.method == "PUT") {
|
|
if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) {
|
|
assert.propertyVal(
|
|
req.requestHeaders,
|
|
'Content-Type',
|
|
'application/json' + fixSinonBug
|
|
);
|
|
|
|
let json = JSON.parse(req.requestBody);
|
|
assert.propertyVal(json, 'content', content);
|
|
assert.propertyVal(json, 'indexedChars', content.length);
|
|
assert.propertyVal(json, 'totalChars', content.length);
|
|
assert.propertyVal(json, 'indexedPages', 0);
|
|
assert.propertyVal(json, 'totalPages', 0);
|
|
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": ++libraryVersion
|
|
},
|
|
""
|
|
);
|
|
count--;
|
|
}
|
|
}
|
|
})
|
|
|
|
yield engine.start();
|
|
assert.equal(count, 0);
|
|
yield assert.eventually.equal(
|
|
Zotero.FullText.getItemVersion(attachment.id),
|
|
libraryVersion
|
|
);
|
|
|
|
//
|
|
// Upload new content
|
|
//
|
|
({ engine, client, caller } = yield setup());
|
|
yield Zotero.Libraries.setVersion(libraryID, libraryVersion);
|
|
|
|
item = yield createDataObject('item');
|
|
attachment = new Zotero.Item('attachment');
|
|
attachment.parentItemID = item.id;
|
|
attachment.attachmentLinkMode = 'imported_file';
|
|
attachment.attachmentContentType = 'text/html';
|
|
attachment.attachmentFilename = 'test.html';
|
|
attachment.attachmentCharset = 'utf-8';
|
|
attachment.synced = true;
|
|
yield attachment.saveTx();
|
|
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
|
|
|
path = attachment.getFilePath();
|
|
content = generateContent()
|
|
htmlContent = "<html><body>" + content + "</body></html>";
|
|
yield Zotero.File.putContentsAsync(path, content);
|
|
yield Zotero.Fulltext.indexItems([attachment.id]);
|
|
|
|
count = 1;
|
|
setResponse({
|
|
method: "GET",
|
|
url: "users/1/fulltext?since=" + previousLibraryVersion,
|
|
status: 200,
|
|
headers: {
|
|
"Last-Modified-Version": libraryVersion
|
|
},
|
|
json: {}
|
|
});
|
|
server.respond(function (req) {
|
|
if (req.method == "PUT") {
|
|
if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) {
|
|
assert.propertyVal(req.requestHeaders, 'Zotero-API-Key', apiKey);
|
|
assert.propertyVal(
|
|
req.requestHeaders,
|
|
'Content-Type',
|
|
'application/json' + fixSinonBug
|
|
);
|
|
|
|
let json = JSON.parse(req.requestBody);
|
|
assert.propertyVal(json, 'content', content);
|
|
assert.propertyVal(json, 'indexedChars', content.length);
|
|
assert.propertyVal(json, 'totalChars', content.length);
|
|
assert.propertyVal(json, 'indexedPages', 0);
|
|
assert.propertyVal(json, 'totalPages', 0);
|
|
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": ++libraryVersion
|
|
},
|
|
""
|
|
);
|
|
count--;
|
|
}
|
|
}
|
|
})
|
|
|
|
yield engine.start();
|
|
assert.equal(count, 0);
|
|
yield assert.eventually.equal(
|
|
Zotero.FullText.getItemVersion(attachment.id),
|
|
libraryVersion
|
|
);
|
|
})
|
|
})
|
|
})
|