Initial AI-generated code

Author: GTFalcao

components/spider/actions/scrape-new-page/scrape-new-page.mjs

@@ -0,0 +1,23 @@
+import spider from "../../spider.app.mjs";
+
+export default {
+  key: "spider-scrape-new-page",
+  name: "Scrape New Page",
+  description: "Initiates a new page scrape. [See the documentation](https://spider.cloud/docs/api)",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    spider,
+    url: {
+      propDefinition: [
+        spider,
+        "url",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const content = await this.spider.initiateCrawl();
+    $.export("$summary", `Successfully scraped content from ${this.url}`);
+    return content;
+  },
+};

components/spider/package.json

@@ -12,4 +12,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

components/spider/spider.app.mjs

@@ -1,11 +1,50 @@
+import { axios } from "@pipedream/platform";
+
 export default {
   type: "app",
   app: "spider",
-  propDefinitions: {},
+  propDefinitions: {
+    url: {
+      type: "string",
+      label: "URL to Scrape",
+      description: "The URL of the page to scrape.",
+    },
+  },
   methods: {
     // this.$auth contains connected account data
     authKeys() {
       console.log(Object.keys(this.$auth));
     },
+    _baseUrl() {
+      return "https://api.spider.cloud";
+    },
+    async _makeRequest(opts = {}) {
+      const {
+        $ = this, method = "GET", path = "/", headers, ...otherOpts
+      } = opts;
+      return axios($, {
+        ...otherOpts,
+        method,
+        url: this._baseUrl() + path,
+        headers: {
+          ...headers,
+          "Authorization": `Bearer ${this.$auth.api_key}`,
+          "Content-Type": "application/json",
+        },
+      });
+    },
+    async initiateCrawl() {
+      const response = await this._makeRequest({
+        method: "POST",
+        path: "/crawl",
+        data: {
+          url: this.url,
+        },
+      });
+      if (Array.isArray(response) && response.length > 0) {
+        return response[0].content;
+      }
+      throw new Error("No content returned from crawl");
+    },
   },
-};
+};