large response stale check: don't commit pending response if a payload is still fetching

behaviors: update to 0.3.0, support for telegram behavior and generic upward scrolling
dependencies: update to wabac.js 2.11.1, replaywebpage 1.5.11, ruffle 2022-04-04
bump to 0.7.9

Author: ikreymer

package.json

@@ -1,16 +1,16 @@
 {
   "name": "@webrecorder/archivewebpage",
   "productName": "ArchiveWeb.page",
-  "version": "0.7.8",
+  "version": "0.7.9",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "^2.10.3",
-    "browsertrix-behaviors": "^0.2.6",
+    "@webrecorder/wabac": "^2.11.1",
+    "browsertrix-behaviors": "^0.3.0",
     "btoa": "^1.2.1",
     "bulma": "^0.9.3",
     "hash-wasm": "^4.9.0",
@@ -22,7 +22,7 @@
     "node-fetch": "2.6.2",
     "pretty-bytes": "^5.6.0",
     "querystring-es3": "^0.2.1",
-    "replaywebpage": "^1.5.10",
+    "replaywebpage": "^1.5.11",
     "stream-browserify": "^3.0.0",
     "url": "^0.11.0",
     "uuid": "^8.3.2",

src/recorder.js

@@ -227,9 +227,12 @@ class Recorder {
         if ((new Date() - reqresp._created) > 20000) {
           if (this.noResponseForStatus(reqresp.status)) {
             console.log("Dropping stale: " + key);
-          } else {
+          } else if (!reqresp.awaitingPayload) {
             console.log(`Committing stale ${reqresp.status} ${reqresp.url}`);
             await this.fullCommit(reqresp, []);
+          } else {
+            console.log(`Waiting for payload for ${reqresp.url}`);
+            continue;
           }
           delete this.pendingRequests[key];
         }
@@ -1316,6 +1319,7 @@ class Recorder {
 
     if (!this.noResponseForStatus(reqresp.status)) {
       try {
+        reqresp.awaitingPayload = true;
         payload = await this.send(method, {requestId: params.requestId}, sessions);
 
         if (payload.base64Encoded) {
@@ -1328,6 +1332,8 @@ class Recorder {
         console.warn("no buffer for: " + reqresp.url + " " + reqresp.status + " " + reqresp.requestId + " " + method);
         console.warn(e);
         return null;
+      } finally {
+        reqresp.awaitingPayload = false;
       }
     } else {
       payload = Buffer.from([]);

wr-ext/bg.js

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.7.8",
+  "version": "0.7.9",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",

wr-ext/manifest.json

@@ -1028,10 +1028,10 @@
   resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-1.6.0.tgz#2c275aa05c895eccebbfc34cfb223c6e8bd591a2"
   integrity sha512-ZkVeqEmRpBV2GHvjjUZqEai2PpUbuq8Bqd//vEYsp63J8WyexI8ppCqVS3Zs0QADf6aWuPdU+0XsPI647PVlQA==
 
-"@webrecorder/wabac@^2.10.3":
-  version "2.10.3"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.10.3.tgz#246cee14ede757648c5c559105f805daabf8a56f"
-  integrity sha512-Awf73AA3KtYkg0wiKRdTQZVwrx9LLF1oCPTxOJ4H2JdUvyKS1XB/Lr2b/0wxGQoeW7CJ7QiVr5HW2drjUbmkNA==
+"@webrecorder/wabac@^2.11.1":
+  version "2.11.1"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.11.1.tgz#292b05f07025f38ad2a00201c1ba86227d4d21d1"
+  integrity sha512-VhQ26umx6m6zP/gXpju15CZJgzh3L0j6W5E5U2L3CmsdGzf41RkCaxyRqMPA1UccTqGKzQj5Gw4PmmhwaXMKmQ==
   dependencies:
     "@webrecorder/wombat" "^3.3.6"
     brotli "github:foliojs/brotli.js"
@@ -1049,7 +1049,7 @@
     path-parser "^6.1.0"
     process "^0.11.10"
     stream-browserify "^3.0.0"
-    warcio "^1.5.0"
+    warcio "^1.5.1"
 
 "@webrecorder/wombat@^3.3.6":
   version "3.3.6"
@@ -1569,10 +1569,10 @@ browserslist@^4.14.5:
     node-releases "^2.0.1"
     picocolors "^1.0.0"
 
-browsertrix-behaviors@^0.2.6:
-  version "0.2.6"
-  resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.2.6.tgz#3d7b4a0444c028495a4443424fa86b32012ba482"
-  integrity sha512-SAT9vbHWzza4SYzutDZl1/zKhWmVLGFjCuBZrw0+f24MMeiZkzeu3urOqOTz60TtMpUZsYVAJmKPnG06FaRtLQ==
+browsertrix-behaviors@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.3.0.tgz#6f424006d9393b760199c144777c529520b24db3"
+  integrity sha512-FI9JuGk15LJ/ofTth3uPWUQHEaeVDCK3l7UfsmGuchH9E1YE8KRocmaCCpMNGgd5ABo3Ymg5BWAFe30eX53u7A==
 
 btoa@^1.2.1:
   version "1.2.1"
@@ -6116,13 +6116,13 @@ registry-url@^5.0.0:
   dependencies:
     rc "^1.2.8"
 
-replaywebpage@^1.5.10:
-  version "1.5.10"
-  resolved "https://registry.yarnpkg.com/replaywebpage/-/replaywebpage-1.5.10.tgz#8b32b7b7ce1e8480d2885483b4c3c650f3b3bc47"
-  integrity sha512-HmdMToSu6kFegZDpoiel8d8FRVtcAemB9II/9w8D2zrTbdZaImEbtP4iiLRWxuferZPyK7EXLDuaIZltfwaP3w==
+replaywebpage@^1.5.11:
+  version "1.5.11"
+  resolved "https://registry.yarnpkg.com/replaywebpage/-/replaywebpage-1.5.11.tgz#fecf164121982fb65086af1024ee57989599e045"
+  integrity sha512-VfE5wbjDxS4paaUO6Z9tkdchgyDEUI+O8ERoN4aD0iUsqU3HZn3dvIRzbQaPcLjai0jLuRdunhxxj4m28w4qNA==
   dependencies:
     "@fortawesome/fontawesome-free" "^5.15.4"
-    "@webrecorder/wabac" "^2.10.3"
+    "@webrecorder/wabac" "^2.11.1"
     bulma "^0.9.3"
     electron-log "^4.4.1"
     electron-updater "^4.3.9"
@@ -7169,6 +7169,19 @@ warcio@^1.5.0:
     uuid-random "^1.3.0"
     yargs "^15.3.1"
 
+warcio@^1.5.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.5.1.tgz#4e0db94ced22919ce2f6daa72a60bcd1cefc5c54"
+  integrity sha512-H1BTLMEufutoR5E3Hrpg3mc2gcWVxl+gwXH0BBglR+ISYoXikgvTA2YuiunoLeSARybCkUQI3WgnyeKEIcHV4Q==
+  dependencies:
+    "@peculiar/webcrypto" "^1.1.1"
+    esm "^3.2.25"
+    hi-base32 "^0.5.0"
+    node-fetch "^2.6.1"
+    pako "^1.0.11"
+    uuid-random "^1.3.0"
+    yargs "^15.3.1"
+
 watchpack@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.3.0.tgz#a41bca3da6afaff31e92a433f4c856a0c25ea0c4"