[ACTION] Google Sheets - Provide polling options as an alternative to current webhook based ones

components/google_sheets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_sheets",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Pipedream Google_sheets Components",
   "main": "google_sheets.app.mjs",
   "keywords": [

components/google_sheets/sources/common/new-comment.mjs

@@ -0,0 +1,36 @@
+export default {
+  methods: {
+    _getLastTs() {
+      return this.db.get("lastTs");
+    },
+    _setLastTs(lastTs) {
+      this.db.set("lastTs", lastTs);
+    },
+    generateMeta(comment) {
+      return {
+        id: comment.id,
+        summary: `New Comment: ${comment.content}`,
+        ts: Date.parse(comment.createdTime),
+      };
+    },
+    getSheetId() {
+      return this.sheetID.toString();
+    },
+    async processSpreadsheet() {
+      const comments = [];
+      const lastTs = this._getLastTs();
+      const results = this.googleSheets.listComments(this.sheetID, lastTs);
+      for await (const comment of results) {
+        comments.push(comment);
+      }
+      if (!comments.length) {
+        return;
+      }
+      this._setLastTs(comments[0].createdTime);
+      comments.reverse().forEach((comment) => {
+        const meta = this.generateMeta(comment);
+        this.$emit(comment, meta);
+      });
+    },
+  },
+};

components/google_sheets/sources/common/new-updates.mjs

@@ -20,8 +20,21 @@ export default {
       label: "Worksheet ID(s)",
       description: "Select one or more worksheet(s), or provide an array of worksheet IDs.",
     },
+    maxRows: {
+      type: "integer",
+      label: "Max Rows to Monitor",
+      description: "Maximum number of rows to monitor for changes. Defaults to 10000. Increase with caution as larger values may cause memory issues.",
+      optional: true,
+      default: 10000,
+    },
   },
   methods: {
+    getBatchSize() {
+      return 1000; // Process 1000 rows at a time
+    },
+    getMaxRows() {
+      return this.maxRows || 10000;
+    },
     getMeta(spreadsheet, worksheet) {
       const {
         sheetId: worksheetId,
@@ -56,6 +69,9 @@ export default {
     getWorksheetIds() {
       return this.worksheetIDs.map((i) => i.toString());
     },
+    _getBatchKey(baseId, batchIndex) {
+      return `${baseId}_batch_${batchIndex}`;
+    },
     _getSheetValues(id) {
       const stringBuffer = this.db.get(id);
 
@@ -72,6 +88,51 @@ export default {
       const stringBuffer = compressed.toString("base64");
       this.db.set(id, stringBuffer);
     },
+    _getBatchedSheetValues(baseId) {
+      const allValues = [];
+      let batchIndex = 0;
+      let hasMore = true;
+
+      while (hasMore) {
+        const batchKey = this._getBatchKey(baseId, batchIndex);
+        const batchValues = this._getSheetValues(batchKey);
+
+        if (!batchValues) {
+          hasMore = false;
+          break;
+        }
+
+        allValues.push(...batchValues);
+        batchIndex++;
+      }
+
+      return allValues.length > 0
+        ? allValues
+        : null;
+    },
+    _setBatchedSheetValues(baseId, sheetValues) {
+      const batchSize = this.getBatchSize();
+      const maxRows = this.getMaxRows();
+
+      // Limit to maxRows
+      const limitedValues = sheetValues.slice(0, maxRows);
+
+      // Clear old batches first
+      let batchIndex = 0;
+      while (this.db.get(this._getBatchKey(baseId, batchIndex))) {
+        this.db.set(this._getBatchKey(baseId, batchIndex), undefined);
+        batchIndex++;
+      }
+
+      // Store in batches
+      batchIndex = 0;
+      for (let i = 0; i < limitedValues.length; i += batchSize) {
+        const batch = limitedValues.slice(i, i + batchSize);
+        const batchKey = this._getBatchKey(baseId, batchIndex);
+        this._setSheetValues(batchKey, batch);
+        batchIndex++;
+      }
+    },
     indexToColumnLabel(index) {
       let columnLabel = "";
       while (index >= 0) {
@@ -133,10 +194,8 @@ export default {
     },
     async getContentDiff(spreadsheet, worksheet) {
       const sheetId = this.getSheetId();
-      const oldValues =
-        this._getSheetValues(
-          `${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`,
-        ) || null;
+      const baseId = `${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`;
+      const oldValues = this._getBatchedSheetValues(baseId) || null;
       const currentValues = await this.googleSheets.getSpreadsheetValues(
         sheetId,
         worksheet.properties.title,
@@ -169,9 +228,11 @@ export default {
           continue;
         }
 
-        const offsetLength = Math.max(values.length - offset, 0);
+        const maxRows = this.getMaxRows();
+        const offsetLength = Math.max(Math.min(values.length, maxRows) - offset, 0);
         const offsetValues = values.slice(0, offsetLength);
-        this._setSheetValues(`${sheetId}${worksheetId}`, offsetValues);
+        const baseId = `${sheetId}${worksheetId}`;
+        this._setBatchedSheetValues(baseId, offsetValues);
       }
     },
     async processSpreadsheet(spreadsheet) {
@@ -191,8 +252,11 @@ export default {
           spreadsheet,
           worksheet,
         );
-        const newValues = currentValues.values || [];
+        const maxRows = this.getMaxRows();
+        const rawNewValues = currentValues.values || [];
+        const newValues = rawNewValues.slice(0, maxRows);
         let changes = [];
+
         // check if there are differences in the spreadsheet values
         if (JSON.stringify(oldValues) !== JSON.stringify(newValues)) {
           let rowCount = this.getRowCount(newValues, oldValues);
@@ -214,10 +278,8 @@ export default {
             this.getMeta(spreadsheet, worksheet),
           );
         }
-        this._setSheetValues(
-          `${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`,
-          newValues || [],
-        );
+        const baseId = `${spreadsheet.spreadsheetId}${worksheet.properties.sheetId}`;
+        this._setBatchedSheetValues(baseId, newValues || []);
       }
     },
   },

components/google_sheets/sources/new-comment-polling/new-comment-polling.mjs

@@ -0,0 +1,48 @@
+import googleSheets from "../../google_sheets.app.mjs";
+import common from "../common/new-comment.mjs";
+import base from "../common/http-based/base.mjs";
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+
+export default {
+  ...common,
+  key: "google_sheets-new-comment-polling",
+  name: "New Comment",
+  description: "Emit new event each time a comment is added to a spreadsheet.",
+  version: "0.0.1",
+  dedupe: "unique",
+  type: "source",
+  props: {
+    googleSheets,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      static: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+    watchedDrive: {
+      propDefinition: [
+        googleSheets,
+        "watchedDrive",
+      ],
+      description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
+    },
+    sheetID: {
+      propDefinition: [
+        googleSheets,
+        "sheetID",
+        (c) => ({
+          driveId: googleSheets.methods.getDriveId(c.watchedDrive),
+        }),
+      ],
+    },
+    ...common.props,
+  },
+  methods: {
+    ...base.methods,
+    ...common.methods,
+  },
+  async run() {
+    return this.processSpreadsheet();
+  },
+};

components/google_sheets/sources/new-comment-polling/test-event.mjs

@@ -0,0 +1,21 @@
+export default {
+  "id": "AAABM3vICvg",
+  "kind": "drive#comment",
+  "createdTime": "2024-05-08T21:32:04.823Z",
+  "modifiedTime": "2024-05-08T21:32:04.823Z",
+  "anchor": "{\"type\":\"workbook-range\",\"uid\":0,\"range\":\"1600938329\"}",
+  "replies": [],
+  "author": {
+    "displayName": "Test User",
+    "kind": "drive#user",
+    "me": true,
+    "photoLink": "//lh3.googleusercontent.com/a/ACg8ocKv3FxHiUdLT981ghC9w01W50yqe5fi2XWOSA4TgnZf8pCxmg=s50-c-k-no"
+  },
+  "deleted": false,
+  "htmlContent": "comment",
+  "content": "comment",
+  "quotedFileContent": {
+    "mimeType": "text/html",
+    "value": "1"
+  }
+}