dependency update: wabac.js 2.9.5, replaywebpage 1.5.5, ruffle (2021-10-25), electron 15.3.0

downloader: use sha-1 when downloading as WARC/1.0 for compatibility with older tools
remove duplicate sw.js
bump to 0.7.4

Author: ikreymer

package.json

@@ -1,14 +1,14 @@
 {
   "name": "archiveweb.page",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "^2.9.2",
+    "@webrecorder/wabac": "^2.9.5",
     "browsertrix-behaviors": "^0.2.3",
     "btoa": "^1.2.1",
     "bulma": "^0.9.3",
@@ -20,14 +20,14 @@
     "keyword-mark-element": "^0.1.2",
     "node-fetch": "2.6.2",
     "pretty-bytes": "^5.6.0",
-    "replaywebpage": "^1.5.3",
+    "replaywebpage": "^1.5.5",
     "uuid": "^8.3.2",
     "warcio": "^1.4.7"
   },
   "devDependencies": {
     "copy-webpack-plugin": "^6.4.0",
     "css-loader": "^3.5.3",
-    "electron": "^14.0.0",
+    "electron": "^15.3.0",
     "electron-builder": "^22.11.11",
     "electron-notarize": "^1.0.0",
     "generate-json-webpack-plugin": "1.0.0",

src/downloader.js

@@ -89,6 +89,12 @@ class Downloader
     this.format = format;
     this.warcVersion = (format === "warc1.0") ? "WARC/1.0" : "WARC/1.1";
 
+    if (format === "warc1.0") {
+      this.digestOpts = {algo: "sha-1", prefix: "sha1:", base32: true};
+    } else {
+      this.digestOpts = {algo: "sha-256", prefix: "sha256:"};
+    }
+
     this.filename = filename;
 
     // determine filename from title, if it exists
@@ -577,7 +583,7 @@ class Downloader
     const date = this.createdDate;
 
     const record = await WARCRecord.createWARCInfo({filename, type, date, warcHeaders, warcVersion}, info);
-    const buffer = await WARCSerializer.serialize(record, {gzip: true});
+    const buffer = await WARCSerializer.serialize(record, {gzip: true, digest: this.digestOpts});
     return buffer;
   }
 
@@ -616,6 +622,8 @@ class Downloader
     let type = null;
 
     let refersToUrl, refersToDate;
+    let refersToDigest;
+    let storeDigest = null;
 
     let method = "GET";
     let requestBody;
@@ -647,6 +655,7 @@ class Downloader
 
       refersToUrl = digestOriginal.url;
       refersToDate = digestOriginal.date;
+      refersToDigest = digestOriginal.payloadDigest || resource.digest;
 
     } else if (resource.origURL && resource.origTS) {
       if (!resource.digest) {
@@ -660,6 +669,7 @@ class Downloader
 
       refersToUrl = resource.origURL;
       refersToDate = new Date(resource.origTS).toISOString();
+      refersToDigest = digestOriginal.payloadDigest || resource.digest;
 
     } else {
       type = "response";
@@ -673,7 +683,8 @@ class Downloader
       }
 
       if (method === "GET") {
-        this.digestsVisted[resource.digest] = {url, date};
+        storeDigest = {url, date};
+        this.digestsVisted[resource.digest] = storeDigest;
       }
     }
 
@@ -691,8 +702,8 @@ class Downloader
       warcHeaders["WARC-JSON-Metadata"] = JSON.stringify(resource.extraOpts);
     }
 
-    if (resource.digest) {
-      warcHeaders["WARC-Payload-Digest"] = resource.digest;
+    if (refersToDigest) {
+      warcHeaders["WARC-Payload-Digest"] = refersToDigest;
     }
 
     // remove encoding, set content-length as encoding never preserved in browser-based capture
@@ -702,10 +713,13 @@ class Downloader
       url, date, type, warcVersion, warcHeaders, statusline, httpHeaders,
       refersToUrl, refersToDate}, getPayload(payload));
 
-    const buffer = await WARCSerializer.serialize(record, {gzip: true});
+    const buffer = await WARCSerializer.serialize(record, {gzip: true, digest: this.digestOpts});
     if (!resource.digest) {
       resource.digest = record.warcPayloadDigest;
     }
+    if (storeDigest) {
+      storeDigest.payloadDigest = record.warcPayloadDigest;
+    }
 
     this.lastPageId = pageId;
     this.lastUrl = url;
@@ -730,7 +744,7 @@ class Downloader
         statusline,
       }, getPayload(requestBody));
 
-      records.push(await WARCSerializer.serialize(reqRecord, {gzip: true}));
+      records.push(await WARCSerializer.serialize(reqRecord, {gzip: true, digest: this.digestOpts}));
     }
 
     return records;
@@ -751,7 +765,7 @@ class Downloader
 
     const record = await WARCRecord.create({url, date, warcHeaders, warcVersion, type}, payload);
 
-    const buffer = await WARCSerializer.serialize(record, {gzip: true});
+    const buffer = await WARCSerializer.serialize(record, {gzip: true, digest: this.digestOpts});
     if (!resource.digest) {
       resource.digest = record.warcPayloadDigest;
     }

wr-ext/bg.js

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",