From 501a86d1a1ae4d34c0dddd53e7f3c25f7efbeb2a Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Thu, 24 Oct 2024 15:04:45 -0400 Subject: [PATCH 1/3] new components --- .../download-project-report.mjs | 39 ++++++++++++++ .../get-project-status/get-project-status.mjs | 28 ++++++++++ .../actions/submit-project/submit-project.mjs | 40 ++++++++++++++ components/rapid_url_indexer/package.json | 7 ++- .../rapid_url_indexer.app.mjs | 53 +++++++++++++++++-- 5 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 components/rapid_url_indexer/actions/download-project-report/download-project-report.mjs create mode 100644 components/rapid_url_indexer/actions/get-project-status/get-project-status.mjs create mode 100644 components/rapid_url_indexer/actions/submit-project/submit-project.mjs diff --git a/components/rapid_url_indexer/actions/download-project-report/download-project-report.mjs b/components/rapid_url_indexer/actions/download-project-report/download-project-report.mjs new file mode 100644 index 0000000000000..25e91261cc729 --- /dev/null +++ b/components/rapid_url_indexer/actions/download-project-report/download-project-report.mjs @@ -0,0 +1,39 @@ +import rapidUrlIndexer from "../../rapid_url_indexer.app.mjs"; +import fs from "fs"; + +export default { + key: "rapid_url_indexer-download-project-report", + name: "Download Project Report", + description: "Download the report for a specific project. [See the documentation](https://rapidurlindexer.com/indexing-api/).", + type: "action", + version: "0.0.1", + props: { + rapidUrlIndexer, + projectId: { + propDefinition: [ + rapidUrlIndexer, + "projectId", + ], + }, + filename: { + type: "string", + label: "Filename", + description: "A filename to save the report as in the `/tmp` directory. Include the `.csv` extension", + }, + }, + async run({ $ }) { + const response = await this.rapidUrlIndexer.downloadProjectReport({ + $, + projectId: this.projectId, + }); + + const filepath = this.filename.includes("/tmp") + ? this.filename + : `/tmp/${this.filename}`; + + fs.writeFileSync(filepath, response); + + $.export("$summary", `Successfully downloaded report for Project with ID ${this.projectId}`); + return filepath; + }, +}; diff --git a/components/rapid_url_indexer/actions/get-project-status/get-project-status.mjs b/components/rapid_url_indexer/actions/get-project-status/get-project-status.mjs new file mode 100644 index 0000000000000..0dce39b75f554 --- /dev/null +++ b/components/rapid_url_indexer/actions/get-project-status/get-project-status.mjs @@ -0,0 +1,28 @@ +import rapidUrlIndexer from "../../rapid_url_indexer.app.mjs"; + +export default { + key: "rapid_url_indexer-get-project-status", + name: "Get Project Status", + description: "Get the status of a specific project. [See the documentation](https://rapidurlindexer.com/indexing-api/).", + type: "action", + version: "0.0.1", + props: { + rapidUrlIndexer, + projectId: { + propDefinition: [ + rapidUrlIndexer, + "projectId", + ], + }, + }, + async run({ $ }) { + const response = await this.rapidUrlIndexer.getProjectStatus({ + $, + projectId: this.projectId, + }); + if (response.id) { + $.export("$summary", `Successfully retrieved status for Project with ID ${response.id}`); + } + return response; + }, +}; diff --git a/components/rapid_url_indexer/actions/submit-project/submit-project.mjs b/components/rapid_url_indexer/actions/submit-project/submit-project.mjs new file mode 100644 index 0000000000000..338a72ee4463e --- /dev/null +++ b/components/rapid_url_indexer/actions/submit-project/submit-project.mjs @@ -0,0 +1,40 @@ +import rapidUrlIndexer from "../../rapid_url_indexer.app.mjs"; + +export default { + key: "rapid_url_indexer-submit-project", + name: "Submit Poject", + description: "Submit a new project for indexing. [See the documentation](https://rapidurlindexer.com/indexing-api/).", + type: "action", + version: "0.0.1", + props: { + rapidUrlIndexer, + name: { + type: "string", + label: "Project Name", + description: "The name of the project", + }, + urls: { + type: "string[]", + label: "URLs", + description: "An array of URLs to index. URLs must start with either “http://” or “https://”.", + }, + notifyOnStatusChange: { + type: "string", + label: "Notify on Status Change", + description: "If set to `true`, you will receive email notifications when the project status changes", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.rapidUrlIndexer.submitProject({ + $, + data: { + project_name: this.name, + urls: this.urls, + notify_on_status_change: this.notifyOnStatusChange, + }, + }); + $.export("$summary", `${response.message}`); + return response; + }, +}; diff --git a/components/rapid_url_indexer/package.json b/components/rapid_url_indexer/package.json index 2b4427aadaa78..d3d9f342f3855 100644 --- a/components/rapid_url_indexer/package.json +++ b/components/rapid_url_indexer/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/rapid_url_indexer", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Rapid URL Indexer Components", "main": "rapid_url_indexer.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/components/rapid_url_indexer/rapid_url_indexer.app.mjs b/components/rapid_url_indexer/rapid_url_indexer.app.mjs index e09659b2d106c..d5cda1868589d 100644 --- a/components/rapid_url_indexer/rapid_url_indexer.app.mjs +++ b/components/rapid_url_indexer/rapid_url_indexer.app.mjs @@ -1,11 +1,54 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "rapid_url_indexer", - propDefinitions: {}, + propDefinitions: { + projectId: { + type: "string", + label: "Project ID", + description: "The identifier of a project", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://rapidurlindexer.com/wp-json/api/v1"; + }, + _makeRequest({ + $ = this, + path, + ...opts + }) { + return axios($, { + url: `${this._baseUrl()}${path}`, + headers: { + "X-API-Key": `${this.$auth.api_key}`, + }, + ...opts, + }); + }, + getProjectStatus({ + projectId, ...opts + }) { + return this._makeRequest({ + path: `/projects/${projectId}`, + ...opts, + }); + }, + downloadProjectReport({ + projectId, ...opts + }) { + return this._makeRequest({ + path: `/projects/${projectId}/report`, + ...opts, + }); + }, + submitProject(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/projects", + ...opts, + }); }, }, -}; \ No newline at end of file +}; From f9d1cd8baa15eb9d0ba260bd32fedb7608c32ed2 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Thu, 24 Oct 2024 15:06:55 -0400 Subject: [PATCH 2/3] pnpm-lock.yaml --- pnpm-lock.yaml | 107 +++++++++++++++++++++++++------------------------ 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 095e5916ef9b9..11bc142e5d83d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8037,7 +8037,10 @@ importers: specifiers: {} components/rapid_url_indexer: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/rat_genome_database: specifiers: {} @@ -13157,6 +13160,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: + resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sts' + - aws-crt + dev: false + /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13392,7 +13444,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13434,55 +13486,6 @@ packages: - aws-crt dev: false - /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: - resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - aws-crt - dev: false - /@aws-sdk/core/3.556.0: resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -17775,7 +17778,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq + '@aws-sdk/client-sts': 3.600.0 '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6 From a7049a8a135affd895e9d2a7c3544f1198a5ad2c Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 28 Oct 2024 15:41:55 -0400 Subject: [PATCH 3/3] updates --- .../actions/submit-project/submit-project.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/rapid_url_indexer/actions/submit-project/submit-project.mjs b/components/rapid_url_indexer/actions/submit-project/submit-project.mjs index 338a72ee4463e..8ef0cf06d6a4c 100644 --- a/components/rapid_url_indexer/actions/submit-project/submit-project.mjs +++ b/components/rapid_url_indexer/actions/submit-project/submit-project.mjs @@ -2,7 +2,7 @@ import rapidUrlIndexer from "../../rapid_url_indexer.app.mjs"; export default { key: "rapid_url_indexer-submit-project", - name: "Submit Poject", + name: "Submit Project", description: "Submit a new project for indexing. [See the documentation](https://rapidurlindexer.com/indexing-api/).", type: "action", version: "0.0.1", @@ -19,7 +19,7 @@ export default { description: "An array of URLs to index. URLs must start with either “http://” or “https://”.", }, notifyOnStatusChange: { - type: "string", + type: "boolean", label: "Notify on Status Change", description: "If set to `true`, you will receive email notifications when the project status changes", optional: true,