fune/toolkit/components/url-classifier/tests/unit/test_partial.js

/**
 * DummyCompleter() lets tests easily specify the results of a partial
 * hash completion request.
 */
function DummyCompleter() {
  this.fragments = {};
  this.queries = [];
  this.tableName = "test-phish-simple";
}

DummyCompleter.prototype =
{
QueryInterface: ChromeUtils.generateQI(["nsIUrlClassifierHashCompleter"]),

complete(partialHash, gethashUrl, tableName, cb) {
  this.queries.push(partialHash);
  var fragments = this.fragments;
  var self = this;
  var doCallback = function() {
      if (self.alwaysFail) {
        cb.completionFinished(Cr.NS_ERROR_FAILURE);
        return;
      }
      if (fragments[partialHash]) {
        for (var i = 0; i < fragments[partialHash].length; i++) {
          var chunkId = fragments[partialHash][i][0];
          var hash = fragments[partialHash][i][1];
          cb.completionV2(hash, self.tableName, chunkId);
        }
      }
    cb.completionFinished(0);
  };
  executeSoon(doCallback);
},

getHash(fragment) {
  var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
  createInstance(Ci.nsIScriptableUnicodeConverter);
  converter.charset = "UTF-8";
  var data = converter.convertToByteArray(fragment);
  var ch = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
  ch.init(ch.SHA256);
  ch.update(data, data.length);
  var hash = ch.finish(false);
  return hash.slice(0, 32);
},

addFragment(chunkId, fragment) {
  this.addHash(chunkId, this.getHash(fragment));
},

// This method allows the caller to generate complete hashes that match the
// prefix of a real fragment, but have different complete hashes.
addConflict(chunkId, fragment) {
  var realHash = this.getHash(fragment);
  var invalidHash = this.getHash("blah blah blah blah blah");
  this.addHash(chunkId, realHash.slice(0, 4) + invalidHash.slice(4, 32));
},

addHash(chunkId, hash) {
  var partial = hash.slice(0, 4);
  if (this.fragments[partial]) {
    this.fragments[partial].push([chunkId, hash]);
  } else {
    this.fragments[partial] = [[chunkId, hash]];
  }
},

compareQueries(fragments) {
  var expectedQueries = [];
  for (let i = 0; i < fragments.length; i++) {
    expectedQueries.push(this.getHash(fragments[i]).slice(0, 4));
  }
  Assert.equal(this.queries.length, expectedQueries.length);
  expectedQueries.sort();
  this.queries.sort();
  for (let i = 0; i < this.queries.length; i++) {
    Assert.equal(this.queries[i], expectedQueries[i]);
  }
}
};

function setupCompleter(table, hits, conflicts) {
  var completer = new DummyCompleter();
  completer.tableName = table;
  for (let i = 0; i < hits.length; i++) {
    let chunkId = hits[i][0];
    let fragments = hits[i][1];
    for (let j = 0; j < fragments.length; j++) {
      completer.addFragment(chunkId, fragments[j]);
    }
  }
  for (let i = 0; i < conflicts.length; i++) {
    let chunkId = conflicts[i][0];
    let fragments = conflicts[i][1];
    for (let j = 0; j < fragments.length; j++) {
      completer.addConflict(chunkId, fragments[j]);
    }
  }

  dbservice.setHashCompleter(table, completer);

  return completer;
}

function installCompleter(table, fragments, conflictFragments) {
  return setupCompleter(table, fragments, conflictFragments);
}

function installFailingCompleter(table) {
  var completer = setupCompleter(table, [], []);
  completer.alwaysFail = true;
  return completer;
}

// Helper assertion for checking dummy completer queries
gAssertions.completerQueried = function(data, cb) {
  var completer = data[0];
  completer.compareQueries(data[1]);
  cb();
};

function doTest(updates, assertions) {
  doUpdateTest(updates, assertions, runNextTest, updateError);
}

// Test an add of two partial urls to a fresh database
function testPartialAdds() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);


  var completer = installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsExist": addUrls,
    "completerQueried": [completer, addUrls]
  };


  doTest([update], assertions);
}

function testPartialAddsWithConflicts() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);

  // Each result will have both a real match and a conflict
  var completer = installCompleter("test-phish-simple",
                                   [[1, addUrls]],
                                   [[1, addUrls]]);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsExist": addUrls,
    "completerQueried": [completer, addUrls]
  };

  doTest([update], assertions);
}

// Test whether the fragmenting code does not cause duplicated completions
function testFragments() {
  var addUrls = [ "foo.com/a/b/c", "foo.net/", "foo.com/c/" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);


  var completer = installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsExist": addUrls,
    "completerQueried": [completer, addUrls]
  };


  doTest([update], assertions);
}

// Test http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
// section 6.2 example 1
function testSpecFragments() {
  var probeUrls = [ "a.b.c/1/2.html?param=1" ];

  var addUrls = [ "a.b.c/1/2.html",
                  "a.b.c/",
                  "a.b.c/1/",
                  "b.c/1/2.html?param=1",
                  "b.c/1/2.html",
                  "b.c/",
                  "b.c/1/",
                  "a.b.c/1/2.html?param=1" ];

  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);


  var completer = installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsExist": probeUrls,
    "completerQueried": [completer, addUrls]
  };

  doTest([update], assertions);

}

// Test http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
// section 6.2 example 2
function testMoreSpecFragments() {
  var probeUrls = [ "a.b.c.d.e.f.g/1.html" ];

  var addUrls = [ "a.b.c.d.e.f.g/1.html",
                  "a.b.c.d.e.f.g/",
                  "c.d.e.f.g/1.html",
                  "c.d.e.f.g/",
                  "d.e.f.g/1.html",
                  "d.e.f.g/",
                  "e.f.g/1.html",
                  "e.f.g/",
                  "f.g/1.html",
                  "f.g/" ];

  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);

  var completer = installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsExist": probeUrls,
    "completerQueried": [completer, addUrls]
  };

  doTest([update], assertions);

}

function testFalsePositives() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);

  // Each result will have no matching complete hashes and a non-matching
  // conflict
  var completer = installCompleter("test-phish-simple", [], [[1, addUrls]]);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsDontExist": addUrls,
    "completerQueried": [completer, addUrls]
  };

  doTest([update], assertions);
}

function testEmptyCompleter() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);

  // Completer will never return full hashes
  var completer = installCompleter("test-phish-simple", [], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsDontExist": addUrls,
    "completerQueried": [completer, addUrls]
  };

  doTest([update], assertions);
}

function testCompleterFailure() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);

  // Completer will never return full hashes
  var completer = installFailingCompleter("test-phish-simple");

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsDontExist": addUrls,
    "completerQueried": [completer, addUrls]
  };

  doTest([update], assertions);
}

function testMixedSizesSameDomain() {
  var add1Urls = [ "foo.com/a" ];
  var add2Urls = [ "foo.com/b" ];

  var update1 = buildPhishingUpdate(
    [
      { "chunkNum": 1,
        "urls": add1Urls }],
    4);
  var update2 = buildPhishingUpdate(
    [
      { "chunkNum": 2,
        "urls": add2Urls }],
    32);

  // We should only need to complete the partial hashes
  var completer = installCompleter("test-phish-simple", [[1, add1Urls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1-2",
    // both urls should match...
    "urlsExist": add1Urls.concat(add2Urls),
    // ... but the completer should only be queried for the partial entry
    "completerQueried": [completer, add1Urls]
  };

  doTest([update1, update2], assertions);
}

function testMixedSizesDifferentDomains() {
  var add1Urls = [ "foo.com/a" ];
  var add2Urls = [ "bar.com/b" ];

  var update1 = buildPhishingUpdate(
    [
      { "chunkNum": 1,
        "urls": add1Urls }],
    4);
  var update2 = buildPhishingUpdate(
    [
      { "chunkNum": 2,
        "urls": add2Urls }],
    32);

  // We should only need to complete the partial hashes
  var completer = installCompleter("test-phish-simple", [[1, add1Urls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1-2",
    // both urls should match...
    "urlsExist": add1Urls.concat(add2Urls),
    // ... but the completer should only be queried for the partial entry
    "completerQueried": [completer, add1Urls]
  };

  doTest([update1, update2], assertions);
}

function testInvalidHashSize() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
        12); // only 4 and 32 are legal hash sizes

  var addUrls2 = [ "zaz.com/a", "xyz.com/b" ];
  var update2 = buildPhishingUpdate(
        [
          { "chunkNum": 2,
            "urls": addUrls2
          }],
        4);

  installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:2",
    "urlsDontExist": addUrls
  };

  // A successful update will trigger an error
  doUpdateTest([update2, update], assertions, updateError, runNextTest);
}

function testWrongTable() {
  var addUrls = [ "foo.com/a" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
        4);
  var completer = installCompleter("test-malware-simple", // wrong table
                                   [[1, addUrls]], []);

  // The above installCompleter installs the completer for test-malware-simple,
  // we want it to be used for test-phish-simple too.
  dbservice.setHashCompleter("test-phish-simple", completer);


  var assertions = {
    "tableData": "test-phish-simple;a:1",
    // The urls were added as phishing urls, but the completer is claiming
    // that they are malware urls, and we trust the completer in this case.
    // The result will be discarded, so we can only check for non-existence.
    "urlsDontExist": addUrls,
    // Make sure the completer was actually queried.
    "completerQueried": [completer, addUrls]
  };

  doUpdateTest([update], assertions,
               function() {
                 // Give the dbservice a chance to (not) cache the result.
                 do_timeout(3000, function() {
                     // The miss earlier will have caused a miss to be cached.
                     // Resetting the completer does not count as an update,
                     // so we will not be probed again.
                     var newCompleter = installCompleter("test-malware-simple", [[1, addUrls]], []); dbservice.setHashCompleter("test-phish-simple",
                                                newCompleter);

                     var assertions1 = {
                       "urlsDontExist": addUrls
                     };
                     checkAssertions(assertions1, runNextTest);
                   });
               }, updateError);
}

function setupCachedResults(addUrls, part2) {
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
        4);

  var completer = installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    // Request the add url.  This should cause the completion to be cached.
    "urlsExist": addUrls,
    // Make sure the completer was actually queried.
    "completerQueried": [completer, addUrls]
  };

  doUpdateTest([update], assertions,
               function() {
                 // Give the dbservice a chance to cache the result.
                 do_timeout(3000, part2);
               }, updateError);
}

function testCachedResults() {
  setupCachedResults(["foo.com/a"], function(add) {
      // This is called after setupCachedResults().  Verify that
      // checking the url again does not cause a completer request.

      // install a new completer, this one should never be queried.
      var newCompleter = installCompleter("test-phish-simple", [[1, []]], []);

      var assertions = {
        "urlsExist": ["foo.com/a"],
        "completerQueried": [newCompleter, []]
      };
      checkAssertions(assertions, runNextTest);
    });
}

function testCachedResultsWithSub() {
  setupCachedResults(["foo.com/a"], function() {
      // install a new completer, this one should never be queried.
      var newCompleter = installCompleter("test-phish-simple", [[1, []]], []);

      var removeUpdate = buildPhishingUpdate(
        [ { "chunkNum": 2,
            "chunkType": "s",
            "urls": ["1:foo.com/a"] }],
        4);

      var assertions = {
        "urlsDontExist": ["foo.com/a"],
        "completerQueried": [newCompleter, []]
      };

      doTest([removeUpdate], assertions);
    });
}

function testCachedResultsWithExpire() {
  setupCachedResults(["foo.com/a"], function() {
      // install a new completer, this one should never be queried.
      var newCompleter = installCompleter("test-phish-simple", [[1, []]], []);

      var expireUpdate =
        "n:1000\n" +
        "i:test-phish-simple\n" +
        "ad:1\n";

      var assertions = {
        "urlsDontExist": ["foo.com/a"],
        "completerQueried": [newCompleter, []]
      };
      doTest([expireUpdate], assertions);
    });
}

function testCachedResultsFailure() {
  var existUrls = ["foo.com/a"];
  setupCachedResults(existUrls, function() {
    // This is called after setupCachedResults().  Verify that
    // checking the url again does not cause a completer request.

    // install a new completer, this one should never be queried.
    var newCompleter = installCompleter("test-phish-simple", [[1, []]], []);

    var assertions = {
      "urlsExist": existUrls,
      "completerQueried": [newCompleter, []]
    };

    checkAssertions(assertions, function() {
      // Apply the update. The cached completes should be gone.
      doErrorUpdate("test-phish-simple,test-malware-simple", function() {
        // Now the completer gets queried again.
        var newCompleter2 = installCompleter("test-phish-simple", [[1, existUrls]], []);
        var assertions2 = {
          "tableData": "test-phish-simple;a:1",
          "urlsExist": existUrls,
          "completerQueried": [newCompleter2, existUrls]
        };
        checkAssertions(assertions2, runNextTest);
      }, updateError);
    });
  });
}

function testErrorList() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": addUrls
          }],
    4);
  // The update failure should will kill the completes, so the above
  // must be a prefix to get any hit at all past the update failure.

  var completer = installCompleter("test-phish-simple", [[1, addUrls]], []);

  var assertions = {
    "tableData": "test-phish-simple;a:1",
    "urlsExist": addUrls,
    // These are complete urls, and will only be completed if the
    // list is stale.
    "completerQueried": [completer, addUrls]
  };

  // Apply the update.
  doStreamUpdate(update, function() {
      // Now the test-phish-simple and test-malware-simple tables are marked
      // as fresh.  Fake an update failure to mark them stale.
      doErrorUpdate("test-phish-simple,test-malware-simple", function() {
          // Now the lists should be marked stale.  Check assertions.
          checkAssertions(assertions, runNextTest);
        }, updateError);
    }, updateError);
}


// Verify that different lists (test-phish-simple,
// test-malware-simple) maintain their freshness separately.
function testErrorListIndependent() {
  var phishUrls = [ "phish.com/a" ];
  var malwareUrls = [ "attack.com/a" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum": 1,
            "urls": phishUrls
          }],
    4);
  // These have to persist past the update failure, so they must be prefixes,
  // not completes.

  update += buildMalwareUpdate(
        [
          { "chunkNum": 2,
            "urls": malwareUrls
          }],
    32);

  var completer = installCompleter("test-phish-simple", [[1, phishUrls]], []);

  var assertions = {
    "tableData": "test-malware-simple;a:2\ntest-phish-simple;a:1",
    "urlsExist": phishUrls,
    "malwareUrlsExist": malwareUrls,
    // Only this phishing urls should be completed, because only the phishing
    // urls will be stale.
    "completerQueried": [completer, phishUrls]
  };

  // Apply the update.
  doStreamUpdate(update, function() {
      // Now the test-phish-simple and test-malware-simple tables are
      // marked as fresh.  Fake an update failure to mark *just*
      // phishing data as stale.
      doErrorUpdate("test-phish-simple", function() {
          // Now the lists should be marked stale.  Check assertions.
          checkAssertions(assertions, runNextTest);
        }, updateError);
    }, updateError);
}

function run_test() {
  runTests([
      testPartialAdds,
      testPartialAddsWithConflicts,
      testFragments,
      testSpecFragments,
      testMoreSpecFragments,
      testFalsePositives,
      testEmptyCompleter,
      testCompleterFailure,
      testMixedSizesSameDomain,
      testMixedSizesDifferentDomains,
      testInvalidHashSize,
      testWrongTable,
      testCachedResults,
      testCachedResultsWithSub,
      testCachedResultsWithExpire,
      testCachedResultsFailure,
      testErrorList,
      testErrorListIndependent
  ]);
}

do_test_pending();