Skip to content

Commit 4319cbc

Browse files
jcortesjcortes
authored
[ACTION] Google Sheets - Provide polling options as an alternative to current webhook based ones (#19302)
* [ACTION] Google Sheets - Provide polling options as an alternative to current webhook based ones * Fixing OOM error in new-updates source component
1 parent 269e56d commit 4319cbc

File tree

8 files changed

+420
-13
lines changed

8 files changed

+420
-13
lines changed

‎components/google_sheets/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/google_sheets",
3-
"version": "0.12.0",
3+
"version": "0.13.0",
44
"description": "Pipedream Google_sheets Components",
55
"main": "google_sheets.app.mjs",
66
"keywords": [

‎components/google_sheets/sources/common/new-comment.mjs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
export default {
2+
methods: {
3+
_getLastTs() {
4+
return this.db.get("lastTs");
5+
},
6+
_setLastTs(lastTs) {
7+
this.db.set("lastTs", lastTs);
8+
},
9+
generateMeta(comment) {
10+
return {
11+
id: comment.id,
12+
summary: `New Comment: ${comment.content}`,
13+
ts: Date.parse(comment.createdTime),
14+
};
15+
},
16+
getSheetId() {
17+
return this.sheetID.toString();
18+
},
19+
async processSpreadsheet() {
20+
const comments = [];
21+
const lastTs = this._getLastTs();
22+
const results = this.googleSheets.listComments(this.sheetID, lastTs);
23+
for await (const comment of results) {
24+
comments.push(comment);
25+
}
26+
if (!comments.length) {
27+
return;
28+
}
29+
this._setLastTs(comments[0].createdTime);
30+
comments.reverse().forEach((comment) => {
31+
const meta = this.generateMeta(comment);
32+
this.$emit(comment, meta);
33+
});
34+
},
35+
},
36+
};

‎components/google_sheets/sources/common/new-updates.mjs‎

Lines changed: 73 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,21 @@ export default {
2020
label: "Worksheet ID(s)",
2121
description: "Select one or more worksheet(s), or provide an array of worksheet IDs.",
2222
},
23+
maxRows: {
24+
type: "integer",
25+
label: "Max Rows to Monitor",
26+
description: "Maximum number of rows to monitor for changes. Defaults to 10000. Increase with caution as larger values may cause memory issues.",
27+
optional: true,
28+
default: 10000,
29+
},
2330
},
2431
methods: {
32+
getBatchSize() {
33+
return 1000; // Process 1000 rows at a time
34+
},
35+
getMaxRows() {
36+
return this.maxRows || 10000;
37+
},
2538
getMeta(spreadsheet, worksheet) {
2639
const {
2740
sheetId: worksheetId,
@@ -56,6 +69,9 @@ export default {
5669
getWorksheetIds() {
5770
return this.worksheetIDs.map((i) => i.toString());
5871
},
72+
_getBatchKey(baseId, batchIndex) {
73+
return `${baseId}_batch_${batchIndex}`;
74+
},
5975
_getSheetValues(id) {
6076
const stringBuffer = this.db.get(id);
6177

@@ -72,6 +88,51 @@ export default {
7288
const stringBuffer = compressed.toString("base64");
7389
this.db.set(id, stringBuffer);
7490
},
91+
_getBatchedSheetValues(baseId) {
92+
const allValues = [];
93+
let batchIndex = 0;
94+
let hasMore = true;
95+
96+
while (hasMore) {
97+
const batchKey = this._getBatchKey(baseId, batchIndex);
98+
const batchValues = this._getSheetValues(batchKey);
99+
100+
if (!batchValues) {
101+
hasMore = false;
102+
break;
103+
}
104+
105+
allValues.push(...batchValues);
106+
batchIndex++;
107+
}
108+
109+
return allValues.length > 0
110+
? allValues
111+
: null;
112+
},
113+
_setBatchedSheetValues(baseId, sheetValues) {
114+
const batchSize = this.getBatchSize();
115+
const maxRows = this.getMaxRows();
116+
117+
// Limit to maxRows
118+
const limitedValues = sheetValues.slice(0, maxRows);
119+
120+
// Clear old batches first
121+
let batchIndex = 0;
122+
while (this.db.get(this._getBatchKey(baseId, batchIndex))) {
123+
this.db.set(this._getBatchKey(baseId, batchIndex), undefined);
124+
batchIndex++;
125+
}
126+
127+
// Store in batches
128+
batchIndex = 0;
129+
for (let i = 0; i < limitedValues.length; i += batchSize) {
130+
const batch = limitedValues.slice(i, i + batchSize);
131+
const batchKey = this._getBatchKey(baseId, batchIndex);
132+
this._setSheetValues(batchKey, batch);
133+
batchIndex++;
134+
}
135+
},
75136
indexToColumnLabel(index) {
76137
let columnLabel = "";
77138
while (index >= 0) {
@@ -133,10 +194,8 @@ export default {
133194
},
134195
async getContentDiff(spreadsheet, worksheet) {
135196
const sheetId = this.getSheetId();
136-
const oldValues =
137-
this._getSheetValues(
138-
`${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`,
139-
) || null;
197+
const baseId = `${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`;
198+
const oldValues = this._getBatchedSheetValues(baseId) || null;
140199
const currentValues = await this.googleSheets.getSpreadsheetValues(
141200
sheetId,
142201
worksheet.properties.title,
@@ -169,9 +228,11 @@ export default {
169228
continue;
170229
}
171230

172-
const offsetLength = Math.max(values.length - offset, 0);
231+
const maxRows = this.getMaxRows();
232+
const offsetLength = Math.max(Math.min(values.length, maxRows) - offset, 0);
173233
const offsetValues = values.slice(0, offsetLength);
174-
this._setSheetValues(`${sheetId}${worksheetId}`, offsetValues);
234+
const baseId = `${sheetId}${worksheetId}`;
235+
this._setBatchedSheetValues(baseId, offsetValues);
175236
}
176237
},
177238
async processSpreadsheet(spreadsheet) {
@@ -191,8 +252,11 @@ export default {
191252
spreadsheet,
192253
worksheet,
193254
);
194-
const newValues = currentValues.values || [];
255+
const maxRows = this.getMaxRows();
256+
const rawNewValues = currentValues.values || [];
257+
const newValues = rawNewValues.slice(0, maxRows);
195258
let changes = [];
259+
196260
// check if there are differences in the spreadsheet values
197261
if (JSON.stringify(oldValues) !== JSON.stringify(newValues)) {
198262
let rowCount = this.getRowCount(newValues, oldValues);
@@ -214,10 +278,8 @@ export default {
214278
this.getMeta(spreadsheet, worksheet),
215279
);
216280
}
217-
this._setSheetValues(
218-
`${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`,
219-
newValues || [],
220-
);
281+
const baseId = `${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`;
282+
this._setBatchedSheetValues(baseId, newValues || []);
221283
}
222284
},
223285
},

‎components/google_sheets/sources/new-comment-polling/new-comment-polling.mjs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import googleSheets from "../../google_sheets.app.mjs";
2+
import common from "../common/new-comment.mjs";
3+
import base from "../common/http-based/base.mjs";
4+
import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
5+
6+
export default {
7+
...common,
8+
key: "google_sheets-new-comment-polling",
9+
name: "New Comment",
10+
description: "Emit new event each time a comment is added to a spreadsheet.",
11+
version: "0.0.1",
12+
dedupe: "unique",
13+
type: "source",
14+
props: {
15+
googleSheets,
16+
db: "$.service.db",
17+
timer: {
18+
type: "$.interface.timer",
19+
static: {
20+
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
21+
},
22+
},
23+
watchedDrive: {
24+
propDefinition: [
25+
googleSheets,
26+
"watchedDrive",
27+
],
28+
description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
29+
},
30+
sheetID: {
31+
propDefinition: [
32+
googleSheets,
33+
"sheetID",
34+
(c) => ({
35+
driveId: googleSheets.methods.getDriveId(c.watchedDrive),
36+
}),
37+
],
38+
},
39+
...common.props,
40+
},
41+
methods: {
42+
...base.methods,
43+
...common.methods,
44+
},
45+
async run() {
46+
return this.processSpreadsheet();
47+
},
48+
};

‎components/google_sheets/sources/new-comment-polling/test-event.mjs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
export default {
2+
"id": "AAABM3vICvg",
3+
"kind": "drive#comment",
4+
"createdTime": "2024-05-08T21:32:04.823Z",
5+
"modifiedTime": "2024-05-08T21:32:04.823Z",
6+
"anchor": "{\"type\":\"workbook-range\",\"uid\":0,\"range\":\"1600938329\"}",
7+
"replies": [],
8+
"author": {
9+
"displayName": "Test User",
10+
"kind": "drive#user",
11+
"me": true,
12+
"photoLink": "//lh3.googleusercontent.com/a/ACg8ocKv3FxHiUdLT981ghC9w01W50yqe5fi2XWOSA4TgnZf8pCxmg=s50-c-k-no"
13+
},
14+
"deleted": false,
15+
"htmlContent": "comment",
16+
"content": "comment",
17+
"quotedFileContent": {
18+
"mimeType": "text/html",
19+
"value": "1"
20+
}
21+
}

0 commit comments

Comments
 (0)