From de625b60fba3d9a9ff7bbf9126a9f0b89968cb6b Mon Sep 17 00:00:00 2001 From: Matěj Cepl Date: Tue, 6 Sep 2011 10:31:49 +0200 Subject: Cache the configuration JSON. So it is available when the website is down and speeds up startup (I hope). Fixes #94. --- lib/libbugzilla.js | 192 ++++++++++++++++++++++++++++++++++++----------------- lib/main.js | 2 +- 2 files changed, 133 insertions(+), 61 deletions(-) (limited to 'lib') diff --git a/lib/libbugzilla.js b/lib/libbugzilla.js index 928555e..4e9cdd9 100644 --- a/lib/libbugzilla.js +++ b/lib/libbugzilla.js @@ -11,7 +11,9 @@ var passUtils = require("passwords"); var Request = require("request").Request; var selfMod = require("self"); var urlMod = require("url"); +var xhrMod = require("xhr"); var panelMod = require("panel"); +var myStorage = require("simple-storage"); var JSONURLDefault = "https://fedorahosted.org/released"+ "/bugzilla-triage-scripts/Config_data.json"; @@ -357,80 +359,150 @@ var makeJSONRPCCall = exports.makeJSONRPCCall = function makeJSONRPCCall(url, me }).post(); }; -exports.initialize = function initialize(config, callback) { - var prefJSONURLName = BTSPrefNS+"JSONURL"; - var urlStr = preferences.get(prefJSONURLName, JSONURLDefault); - preferences.set(prefJSONURLName, urlStr); +/** + * Preprocess JSON into config data structure. + * + * Should be completely side-effects free pure function. + */ +function processConfigJSON(rawJSON) { + config.gJSONData = rawJSON; + + // Get additional tables + if ("downloadJSON" in config.gJSONData.configData) { + var URLsList = config.gJSONData.configData.downloadJSON; + if (!config.constantData) { + config.constantData = {}; + } + URLsList.forEach(function (arr) { + var title = arr[0]; + var url = arr[1]; + Request({ + url: url, + onComplete: function(response) { + if (response.status == 200) { + config.constantData[title] = response.json; + } else { + console.error("Cannot download " + title + " from URL " + url); + } + } + }).get(); + }); + } + + config.configData = {}; // should we spit out a lot of debugging output var prefDebugName = BTSPrefNS+"debug"; debugOption = preferences.get(prefDebugName, false); preferences.set(prefDebugName, debugOption); + config.configData.debuggingVerbose = debugOption; + // FIXME What are these variables actually good for? Document them. + if ("matches" in config.configData) { + config.configData.skipMatches = config.configData.matches.map(function(x) { + return x.replace("show_bug.cgi.*","((process|post)_bug|attachment)\.cgi$"); + }); + } + + config.constantData = {}; + if ("constantData" in config.gJSONData) { + config.constantData = config.gJSONData.constantData; + config.constantData.queryUpstreamBug = JSON.parse( + selfMod.data.load("queryUpstreamBug.json")); + config.constantData.bugzillaLabelNames = + JSON.parse(selfMod.data.load("bugzillalabelNames.json")); + config.constantData.newUpstreamBug = + JSON.parse(selfMod.data.load("newUpstreamBug.json")); + config.constantData.ProfessionalProducts = + JSON.parse(selfMod.data.load("professionalProducts.json")); + config.constantData.BugzillaAbbreviations = + JSON.parse(selfMod.data.load("bugzillalabelAbbreviations.json")); + } + + if ("CCmaintainer" in config.constantData) { + config.configData.defBugzillaMaintainerArr = config.constantData.CCmaintainer; + } + + copiedAttributes.forEach(function (attrib) { + if (attrib in config.gJSONData.configData) { + config.configData[attrib] = config.gJSONData.configData[attrib]; + } + }); +} + +function fetchConfigurationJSON(url, callback) { + var retValue = null; Request({ - url: urlStr, + url: url, onComplete: function (response) { if (response.status == 200) { - config.gJSONData = response.json; - - // Get additional tables - if ("downloadJSON" in config.gJSONData.configData) { - var URLsList = config.gJSONData.configData.downloadJSON; - var dwnldObj = ""; - URLsList.forEach(function (arr) { - var title = arr[0]; - var url = arr[1]; - Request({ - url: url, - onComplete: function(response) { - if (response.status == 200) { - config.constantData[title] = response.json; - } else { - console.error("Cannot download " + title + " from URL " + url); - } - } - }).get(); - }); - } - - config.configData = {}; - config.configData.debuggingVerbose = debugOption; - config.configData.matches = config.gJSONData.configData.matches; - config.configData.skipMatches = config.configData.matches.map(function(x) { - return x.replace("show_bug.cgi.*","((process|post)_bug|attachment)\.cgi$"); - }); - - config.constantData = {}; - if ("constantData" in config.gJSONData) { - config.constantData = config.gJSONData.constantData; - config.constantData.queryUpstreamBug = JSON.parse( - selfMod.data.load("queryUpstreamBug.json")); - config.constantData.bugzillaLabelNames = - JSON.parse(selfMod.data.load("bugzillalabelNames.json")); - config.constantData.newUpstreamBug = - JSON.parse(selfMod.data.load("newUpstreamBug.json")); - config.constantData.ProfessionalProducts = - JSON.parse(selfMod.data.load("professionalProducts.json")); - config.constantData.BugzillaAbbreviations = - JSON.parse(selfMod.data.load("bugzillalabelAbbreviations.json")); - } + processConfigJSON(response.json); + myStorage.storage.configJSON.meta.eTag = response.headers['Etag']; + myStorage.storage.configJSON.meta.lastModified = + response.headers['Last-Modified']; + myStorage.storage.configJSON.parsedJSON = config; + } + else { + // If we cannot get JSON from the real URL, we are happy + // with getting our fix from anywhere, including (possibly) + // expired cache + if (myStorage.storage.configJSON && + myStorage.storage.configJSON.parsedJSON) { + config = myStorage.storage.configJSON.parsedJSON; + } + } + if ("submitsLogging" in config.gJSONData.configData && + config.gJSONData.configData.submitsLogging) { + logger.initialize(); + } + loginToAllBugzillas(callback); + } + }).get(); +}; - if ("CCmaintainer" in config.constantData) { - config.configData.defBugzillaMaintainerArr = config.constantData.CCmaintainer; - } +exports.initialize = function initialize(callback) { + var prefJSONURLName = BTSPrefNS+"JSONURL"; + var urlStr = preferences.get(prefJSONURLName, JSONURLDefault); + preferences.set(prefJSONURLName, urlStr); - copiedAttributes.forEach(function (attrib) { - if (attrib in config.gJSONData.configData) { - config.configData[attrib] = config.gJSONData.configData[attrib]; - } - }); + if (!myStorage.storage.configJSON) { + myStorage.storage.configJSON = {}; + myStorage.storage.configJSON.meta = { + eTag: "", + lastModified: null + }; + } - if ("submitsLogging" in config.gJSONData.configData && + var req = new xhrMod.XMLHttpRequest(); + req.open("HEAD", urlStr, true); + req.onreadystatechange = function (aEvt) { + if (req.readyState == 4) { + if(req.status == 200) { + var _curETag = req.getResponseHeader("ETag"); + var _curLastModified = new Date(req.getResponseHeader("Last-Modified")); + console.log("_curLastModified = " + _curLastModified); + console.log("cache.lastModified = " + + myStorage.storage.configJSON.meta.lastModified); + if ((_curETag == myStorage.storage.configJSON.meta.eTag) + || (_curLastModified <= + myStorage.storage.configJSON.meta.lastModified)) { + console.log("Loading from cache!"); + // use cached values + config = myStorage.storage.configJSON.parsedJSON; + if ("submitsLogging" in config.gJSONData.configData && config.gJSONData.configData.submitsLogging) { - logger.initialize(); + logger.initialize(); + } + console.log("config.gJSONData = " + config.gJSONData); + console.log("config.config = " + config.configData); + loginToAllBugzillas(callback); + } + else { // cache is not up-to-date + console.log("Too old cache!"); + fetchConfigurationJSON(urlStr, callback); } } - loginToAllBugzillas(callback); } - }).get(); + }; + req.send(); } diff --git a/lib/main.js b/lib/main.js index 0beb947..b7875c3 100644 --- a/lib/main.js +++ b/lib/main.js @@ -117,7 +117,7 @@ var contentScriptLibraries = [ var mainPMRE = new RegExp("http[s]?:\\/\\/bug(zilla|s)\\.[^\\/]*?" + "\\/show_bug.cgi\\?id=.*"); -libbz.initialize(libbz.config, function() { +libbz.initialize(function() { pageMod.PageMod({ include : [ mainPMRE ], contentScriptWhen : 'ready', -- cgit