Cache the configuration JSON.

So it is available when the website is down and speeds up startup (I
hope).

Fixes #94.

4 files changed, 134 insertions, 218 deletions

diff --git a/0001-Make-cached-parameter-for-making-a-resource-explicit.patch b/0001-Make-cached-parameter-for-making-a-resource-explicit.patch
deleted file mode 100644
index 678e0e2..0000000
--- a/0001-Make-cached-parameter-for-making-a-resource-explicit.patch
+++ /dev/null
@@ -1,157 +0,0 @@
-From 5e88bb97d50c1ee32f60aa9ae062a101781de97c Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= <mcepl@redhat.com>
-Date: Sun, 22 May 2011 02:46:39 +0200
-Subject: [PATCH] Make cached parameter for making a resource explicitly
- cached.
-
-Attempt to fix https://bugzilla.mozilla.org/show_bug.cgi?id=655749
-(I wonder how much this cache is different from Firefox own caching).
----
- packages/addon-kit/lib/request.js |   92 +++++++++++++++++++++++++++++++++++--
- 1 files changed, 87 insertions(+), 5 deletions(-)
-
-diff --git a/packages/addon-kit/lib/request.js b/packages/addon-kit/lib/request.js
-index 50da22a..f93716d 100644
---- a/packages/addon-kit/lib/request.js
-+++ b/packages/addon-kit/lib/request.js
-@@ -39,6 +39,8 @@ const xpcom = require("xpcom");
- const xhr = require("xhr");
- const errors = require("errors");
- const apiUtils = require("api-utils");
-+const ss = require("simple-storage");
-+
- 
- // Ugly but will fix with: https://bugzilla.mozilla.org/show_bug.cgi?id=596248
- const EventEmitter = require('events').EventEmitter.compose({
-@@ -62,6 +64,10 @@ const validator = new OptionsValidator({
-   contentType: {
-     map: function (v) v || "application/x-www-form-urlencoded",
-     is:  ["string"]
-+  },
-+  cached: {
-+    map: function (v) v || false,
-+    is:  ["boolean"]
-   }
- });
- 
-@@ -74,9 +80,9 @@ function Request(options) {
-   // request will hold the actual XHR object
-   let request;
-   let response;
--
--  if ('onComplete' in options)
--    self.on('complete', options.onComplete)
-+  if ('onComplete' in options) {
-+    self.on('complete', options.onComplete);
-+  }
-   options = validator.validateOptions(options);
- 
-   // function to prep the request since it's the same between GET and POST
-@@ -86,6 +92,17 @@ function Request(options) {
-       throw new Error(REUSE_ERROR);
-     }
- 
-+    var _recordingCache = (typeof options.recording === "undefined") ? false : options.recording;
-+
-+    if (options.cached && !("httpRequestCache" in ss.storage)) {
-+      ss.storage.httpRequestCache = {};
-+    }
-+
-+    if (options.cached) {
-+      makeCachedRequest(mode);
-+      return;
-+    }
-+
-     request = new xhr.XMLHttpRequest();
- 
-     let url = options.url;
-@@ -111,19 +128,84 @@ function Request(options) {
-     // handle the readystate, create the response, and call the callback
-     request.onreadystatechange = function () {
-       if (request.readyState == 4) {
-+        if (_recordingCache && request.status == 200) {
-+          // index by unmodified URL we don't want to index by data
-+          ss.storage.httpRequestCache[options.url] = {
-+            responseText: request.responseText,
-+            lastModified: new Date(request.getResponseHeader("Last-Modified")),
-+            eTag: request.getResponseHeader("ETag"),
-+            allResponseHeaders: request.getAllResponseHeaders()
-+          };
-+        }
-         response = new Response(request);
-         errors.catchAndLog(function () {
-           self._emit('complete', response);
-         })();
-       }
--    }
-+    };
- 
-     // actually send the request. we only want to send data on POST requests
-     request.send(mode == "POST" ? data : null);
-   }
- 
-+  /**
-+   * return simulated request object or null, if the request is not cached
-+   */
-+  function makeCachedRequest(_mode) {
-+    // request must provide in emitting "complete" at least these properties:
-+    // request.responseText, request.status, request.statusText,
-+    // and request.getAllResponseHeaders()
-+    if (ss.storage.httpRequestCache && (options.url in ss.storage.httpRequestCache)) {
-+      var cache = ss.storage.httpRequestCache[options.url];
-+
-+      // Randomize URL to avoid caching
-+      // TODO see https://fedorahosted.org/bugzilla-triage-scripts/ticket/21
-+      // for more thorough discussion and possible further improvement
-+      options.url += (options.url.match(/\?/) == null ? "?" : "&") + (new Date()).getTime();
-+
-+      // check HEAD request and if (etag != cachedETag) && (last-modified > cachedLastModified)
-+      // recursive call, but with options.cached set to false
-+      var _headRequest = new xhr.XMLHttpRequest();
-+      _headRequest.open("HEAD", options.url, true);
-+      _headRequest.onreadystatechange=function() {
-+        if (_headRequest.readyState==4) {
-+          var _curETag = _headRequest.getResponseHeader("ETag");
-+          var _curLastModified = new Date(_headRequest.getResponseHeader("Last-Modified"));
-+          console.log("_curLastModified = " + _curLastModified);
-+          console.log("cache.lastModified = " + cache.lastModified);
-+          if ((_curETag == cache.eTag) || (_curLastModified <= cache.lastModified)) {
-+            console.log("Loading from cache!");
-+            // create false request with cached values
-+            var fake_response = new Response({
-+              responseText: cache.responseText,
-+              status: 200, // could we have anything else than 200?
-+              statusText: "200 OK",
-+              getAllResponseHeaders: function() {
-+                return cache.allResponseHeaders;
-+              }
-+            });
-+            errors.catchAndLog(function () {
-+              self._emit('complete', fake_response);
-+            })();
-+          } else { // cache is not up-to-date
-+            console.log("Too old cache!");
-+            options.cached = false;
-+            options.recording = true;
-+            makeRequest(_mode);
-+          }
-+        }
-+      }
-+      _headRequest.send(null);
-+    } else { // We don't have this URL in cache at all
-+      console.log("Not cached at all!");
-+      options.cached = false;
-+      options.recording = true;
-+      makeRequest(_mode);
-+    }
-+  }
-+
-   // Map these setters/getters to the options
--  ["url", "headers", "content", "contentType"].forEach(function (k) {
-+  ["url", "headers", "cached", "content", "contentType"].forEach(function (k) {
-     _public.__defineGetter__(k, function () options[k]);
-     _public.__defineSetter__(k, function (v) {
-       // This will automatically rethrow errors from apiUtils.validateOptions.
--- 
-1.7.6.1
-
diff --git a/Makefile b/Makefile
index 08a614d..29fdcc9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,4 @@
+VERNO=$(shell js -e 'print(JSON.parse(read("package.json")).version);')
 WEBDIR=/home/matej/Dokumenty/website/fedorahosted/
 UPSTREAM_XPI_URL=https://fedorahosted.org/released/bugzilla-triage-scripts/bugzilla-triage-$(VERNO).xpi
 UPDATE_URL=https://fedorahosted.org/released/bugzilla-triage-scripts/update.rdf
diff --git a/lib/libbugzilla.js b/lib/libbugzilla.js
index 928555e..4e9cdd9 100644
--- a/lib/libbugzilla.js
+++ b/lib/libbugzilla.js
@@ -11,7 +11,9 @@ var passUtils = require("passwords");
 var Request = require("request").Request;
 var selfMod = require("self");
 var urlMod = require("url");
+var xhrMod = require("xhr");
 var panelMod = require("panel");
+var myStorage = require("simple-storage");
 
 var JSONURLDefault = "https://fedorahosted.org/released"+
 "/bugzilla-triage-scripts/Config_data.json";
@@ -357,80 +359,150 @@ var makeJSONRPCCall = exports.makeJSONRPCCall = function makeJSONRPCCall(url, me
   }).post();
 };
 
-exports.initialize = function initialize(config, callback) {
-  var prefJSONURLName = BTSPrefNS+"JSONURL";
-  var urlStr = preferences.get(prefJSONURLName, JSONURLDefault);
-  preferences.set(prefJSONURLName, urlStr);
+/**
+ * Preprocess JSON into config data structure.
+ *
+ * Should be completely side-effects free pure function.
+ */
+function processConfigJSON(rawJSON) {
+  config.gJSONData = rawJSON;
+
+  // Get additional tables
+  if ("downloadJSON" in config.gJSONData.configData) {
+    var URLsList = config.gJSONData.configData.downloadJSON;
+    if (!config.constantData) {
+      config.constantData = {};
+    }
+    URLsList.forEach(function (arr) {
+      var title = arr[0];
+      var url = arr[1];
+      Request({
+        url: url,
+        onComplete: function(response) {
+          if (response.status == 200) {
+            config.constantData[title] = response.json;
+          } else {
+            console.error("Cannot download " + title + " from URL " + url);
+          }
+        }
+      }).get();
+    });
+  }
+
+  config.configData = {};
 
   // should we spit out a lot of debugging output
   var prefDebugName = BTSPrefNS+"debug";
   debugOption = preferences.get(prefDebugName, false);
   preferences.set(prefDebugName, debugOption);
+  config.configData.debuggingVerbose = debugOption;
 
+  // FIXME What are these variables actually good for? Document them.
+  if ("matches" in config.configData) {
+    config.configData.skipMatches = config.configData.matches.map(function(x) {
+      return x.replace("show_bug.cgi.*","((process|post)_bug|attachment)\.cgi$");
+    });
+  }
+
+  config.constantData = {};
+  if ("constantData" in config.gJSONData) {
+    config.constantData = config.gJSONData.constantData;
+    config.constantData.queryUpstreamBug = JSON.parse(
+        selfMod.data.load("queryUpstreamBug.json"));
+    config.constantData.bugzillaLabelNames =
+      JSON.parse(selfMod.data.load("bugzillalabelNames.json"));
+    config.constantData.newUpstreamBug =
+      JSON.parse(selfMod.data.load("newUpstreamBug.json"));
+    config.constantData.ProfessionalProducts =
+      JSON.parse(selfMod.data.load("professionalProducts.json"));
+    config.constantData.BugzillaAbbreviations =
+     JSON.parse(selfMod.data.load("bugzillalabelAbbreviations.json"));
+  }
+
+  if ("CCmaintainer" in config.constantData) {
+    config.configData.defBugzillaMaintainerArr = config.constantData.CCmaintainer;
+  }
+
+  copiedAttributes.forEach(function (attrib) {
+    if (attrib in config.gJSONData.configData) {
+      config.configData[attrib] = config.gJSONData.configData[attrib];
+    }
+  });
+}
+
+function fetchConfigurationJSON(url, callback) {
+  var retValue = null;
   Request({
-    url: urlStr,
+    url: url,
     onComplete: function (response) {
       if (response.status == 200) {
-        config.gJSONData = response.json;
-
-        // Get additional tables
-        if ("downloadJSON" in config.gJSONData.configData) {
-          var URLsList = config.gJSONData.configData.downloadJSON;
-          var dwnldObj = "";
-          URLsList.forEach(function (arr) {
-            var title = arr[0];
-            var url = arr[1];
-            Request({
-              url: url,
-              onComplete: function(response) {
-                if (response.status == 200) {
-                  config.constantData[title] = response.json;
-                } else {
-                  console.error("Cannot download " + title + " from URL " + url);
-                }
-              }
-            }).get();
-          });
-        }
-
-        config.configData = {};
-        config.configData.debuggingVerbose = debugOption;
-        config.configData.matches = config.gJSONData.configData.matches;
-        config.configData.skipMatches = config.configData.matches.map(function(x) {
-          return x.replace("show_bug.cgi.*","((process|post)_bug|attachment)\.cgi$");
-        });
-
-        config.constantData = {};
-        if ("constantData" in config.gJSONData) {
-          config.constantData = config.gJSONData.constantData;
-          config.constantData.queryUpstreamBug = JSON.parse(
-              selfMod.data.load("queryUpstreamBug.json"));
-          config.constantData.bugzillaLabelNames =
-            JSON.parse(selfMod.data.load("bugzillalabelNames.json"));
-          config.constantData.newUpstreamBug =
-            JSON.parse(selfMod.data.load("newUpstreamBug.json"));
-          config.constantData.ProfessionalProducts =
-            JSON.parse(selfMod.data.load("professionalProducts.json"));
-          config.constantData.BugzillaAbbreviations =
-           JSON.parse(selfMod.data.load("bugzillalabelAbbreviations.json"));
-        }
+        processConfigJSON(response.json);
+        myStorage.storage.configJSON.meta.eTag = response.headers['Etag'];
+        myStorage.storage.configJSON.meta.lastModified =
+          response.headers['Last-Modified'];
+        myStorage.storage.configJSON.parsedJSON = config;
+      }
+      else {
+        // If we cannot get JSON from the real URL, we are happy
+        // with getting our fix from anywhere, including (possibly)
+        // expired cache
+        if (myStorage.storage.configJSON &&
+          myStorage.storage.configJSON.parsedJSON) {
+            config = myStorage.storage.configJSON.parsedJSON;
+          }
+      }
+      if ("submitsLogging" in config.gJSONData.configData &&
+        config.gJSONData.configData.submitsLogging) {
+          logger.initialize();
+      }
+      loginToAllBugzillas(callback);
+    }
+  }).get();
+};
 
-        if ("CCmaintainer" in config.constantData) {
-          config.configData.defBugzillaMaintainerArr = config.constantData.CCmaintainer;
-        }
+exports.initialize = function initialize(callback) {
+  var prefJSONURLName = BTSPrefNS+"JSONURL";
+  var urlStr = preferences.get(prefJSONURLName, JSONURLDefault);
+  preferences.set(prefJSONURLName, urlStr);
 
-        copiedAttributes.forEach(function (attrib) {
-          if (attrib in config.gJSONData.configData) {
-            config.configData[attrib] = config.gJSONData.configData[attrib];
-          }
-        });
+  if (!myStorage.storage.configJSON) {
+    myStorage.storage.configJSON = {};
+    myStorage.storage.configJSON.meta = {
+      eTag: "",
+      lastModified: null
+    };
+  }
 
-        if ("submitsLogging" in config.gJSONData.configData &&
+  var req = new xhrMod.XMLHttpRequest();
+  req.open("HEAD", urlStr, true);
+  req.onreadystatechange = function (aEvt) {
+    if (req.readyState == 4) {
+      if(req.status == 200) {
+        var _curETag = req.getResponseHeader("ETag");
+        var _curLastModified = new Date(req.getResponseHeader("Last-Modified"));
+        console.log("_curLastModified = " + _curLastModified);
+        console.log("cache.lastModified = " +
+          myStorage.storage.configJSON.meta.lastModified);
+        if ((_curETag == myStorage.storage.configJSON.meta.eTag)
+            || (_curLastModified <=
+              myStorage.storage.configJSON.meta.lastModified)) {
+          console.log("Loading from cache!");
+          // use cached values
+          config = myStorage.storage.configJSON.parsedJSON;
+          if ("submitsLogging" in config.gJSONData.configData &&
             config.gJSONData.configData.submitsLogging) {
-          logger.initialize();
+              logger.initialize();
+          }
+          console.log("config.gJSONData = " + config.gJSONData);
+          console.log("config.config = " + config.configData);
+          loginToAllBugzillas(callback);
+        }
+        else { // cache is not up-to-date
+          console.log("Too old cache!");
+          fetchConfigurationJSON(urlStr, callback);
         }
       }
-      loginToAllBugzillas(callback);
     }
-  }).get();
+  };
+  req.send();
 }
diff --git a/lib/main.js b/lib/main.js
index 0beb947..b7875c3 100644
--- a/lib/main.js
+++ b/lib/main.js
@@ -117,7 +117,7 @@ var contentScriptLibraries = [
 var mainPMRE = new RegExp("http[s]?:\\/\\/bug(zilla|s)\\.[^\\/]*?" +
   "\\/show_bug.cgi\\?id=.*");
 
-libbz.initialize(libbz.config, function() {
+libbz.initialize(function() {
   pageMod.PageMod({
     include : [ mainPMRE ],
     contentScriptWhen : 'ready',