From cf67e232c72d8929ef523847b807bbf5367fa34f Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 16 Dec 2024 15:35:19 -0500 Subject: [PATCH 1/5] new-components --- components/lobste_rs/lobste_rs.app.mjs | 106 +++++++++++++++++- components/lobste_rs/package.json | 9 +- .../new-comment-in-thread.mjs | 43 +++++++ .../new-story-by-user/new-story-by-user.mjs | 55 +++++++++ 4 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs create mode 100644 components/lobste_rs/sources/new-story-by-user/new-story-by-user.mjs diff --git a/components/lobste_rs/lobste_rs.app.mjs b/components/lobste_rs/lobste_rs.app.mjs index f7b6d82bf7ea4..c2ed540a53aaf 100644 --- a/components/lobste_rs/lobste_rs.app.mjs +++ b/components/lobste_rs/lobste_rs.app.mjs @@ -1,11 +1,111 @@ +import { axios } from "@pipedream/platform"; +import FeedParser from "feedparser"; +import hash from "object-hash"; + export default { type: "app", app: "lobste_rs", propDefinitions: {}, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + makeRequest({ + $ = this, ...config + }) { + return axios($, config); + }, + itemTs(item = {}) { + const { + pubdate, pubDate, date_published, + } = item; + const itemPubDate = pubdate ?? pubDate ?? date_published; + if (itemPubDate) { + return +new Date(itemPubDate); + } + return +new Date(); + }, + itemKey(item = {}) { + const { + id, guid, link, title, + } = item; + const itemId = id ?? guid ?? link ?? title; + if (itemId) { + // reduce itemId length for deduping + return itemId.length > 64 + ? itemId.slice(-64) + : itemId; + } + return hash(item); + }, + async fetchFeed(url) { + const res = await axios(this, { + url, + method: "GET", + headers: { + "accept": "text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8, application/json, application/feed+json", + }, + responseType: "stream", // stream is required for feedparser + returnFullResponse: true, + }); + return { + data: res.data, + contentType: res.headers["content-type"], + }; + }, + async parseFeed(stream) { + const feedparser = new FeedParser({ + addmeta: true, + }); + const items = []; + await new Promise((resolve, reject) => { + feedparser.on("error", reject); + feedparser.on("end", resolve); + feedparser.on("readable", function () { + let item = this.read(); + + while (item) { + for (const k in item) { + if (item[`rss:${k}`]) { + delete item[`rss:${k}`]; + continue; + } + const o = item[k]; + if (o == null || (typeof o === "object" && !Object.keys(o).length) || Array.isArray(o) && !o.length) { + delete item[k]; + continue; + } + } + items.push(item); + item = this.read(); + } + }); + stream.pipe(feedparser); + }); + return items; + }, + isJSONFeed(response) { + const acceptedJsonFeedMimes = [ + "application/feed+json", + "application/json", + ]; + return acceptedJsonFeedMimes.includes(response?.contentType?.toLowerCase()); + }, + async parseJSONFeed(stream) { + const buffer = await new Promise((resolve, reject) => { + const _buf = []; + stream.on("data", (chunk) => _buf.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(_buf))); + stream.on("error", (err) => reject(err)); + }); + const contentString = buffer.toString(); + const feed = JSON.parse(contentString); + return feed?.items || []; + }, + async fetchAndParseFeed(url) { + const response = await this.fetchFeed(url); + if (this.isJSONFeed(response)) { + return await this.parseJSONFeed(response.data); + } else { + return await this.parseFeed(response.data); + } }, }, }; diff --git a/components/lobste_rs/package.json b/components/lobste_rs/package.json index 412690b3a7f9a..5372a3a2fb863 100644 --- a/components/lobste_rs/package.json +++ b/components/lobste_rs/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/lobste_rs", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream lobste.rs Components", "main": "lobste_rs.app.mjs", "keywords": [ @@ -11,5 +11,10 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3", + "feedparser": "^2.2.10", + "object-hash": "^3.0.0" } -} \ No newline at end of file +} diff --git a/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs b/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs new file mode 100644 index 0000000000000..81153cf6635aa --- /dev/null +++ b/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs @@ -0,0 +1,43 @@ +import lobsters from "../../lobste_rs.app.mjs"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + key: "lobste_rs-new-comment-in-thread", + name: "New Comment in Thread", + description: "Emit new event when a new coomment is added to a thread.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + lobsters, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + url: { + type: "string", + label: "URL", + description: "The URL of the comment threaad to retrieve. E.g. `https://lobste.rs/s/yqjtvy/cloud_container_iceberg`", + }, + }, + methods: { + generateMeta(comment) { + return { + id: comment.short_id, + summary: comment.comment_plain.substring(0, 50), + ts: Date.parse(comment.created_at), + }; + }, + }, + async run() { + const { comments } = await this.lobsters.makeRequest({ + url: `${this.url}.json`, + }); + for (const comment of comments.reverse()) { + const meta = this.generateMeta(comment); + this.$emit(comment, meta); + } + }, +}; diff --git a/components/lobste_rs/sources/new-story-by-user/new-story-by-user.mjs b/components/lobste_rs/sources/new-story-by-user/new-story-by-user.mjs new file mode 100644 index 0000000000000..2a4b2228c0577 --- /dev/null +++ b/components/lobste_rs/sources/new-story-by-user/new-story-by-user.mjs @@ -0,0 +1,55 @@ +import lobsters from "../../lobste_rs.app.mjs"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + key: "lobste_rs-new-story-by-user", + name: "New Story by User", + description: "Emit new event when a new story is posted by the specified user.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + lobsters, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + username: { + type: "string", + label: "Username", + description: "The user to watch for stories from. E.g. `adamgordonbell`", + }, + publishedAfter: { + type: "string", + label: "Published After", + description: "Emit items published after the specified date in ISO 8601 format .e.g `2022-12-07T12:57:10+07:00`", + default: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + }, + }, + methods: { + generateMeta(item) { + return { + id: this.lobsters.itemKey(item), + summary: item.title, + ts: Date.now(), + }; + }, + }, + async run() { + const url = `https://lobste.rs/~${this.username}/stories.rss`; + + const items = await this.lobsters.fetchAndParseFeed(url); + for (const item of items.reverse()) { + const publishedAfter = +new Date(this.publishedAfter); + const ts = this.lobsters.itemTs(item); + if (Number.isNaN(publishedAfter) || publishedAfter > ts) { + continue; + } + + const meta = this.generateMeta(item); + this.$emit(item, meta); + } + }, +}; From b0f7421adb1d3907383f28d3ee67bd3f2b7ee7a7 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 16 Dec 2024 15:38:58 -0500 Subject: [PATCH 2/5] pnpm-lock.yaml --- pnpm-lock.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0d4136947893..9b6d6aa6baaf9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5872,7 +5872,17 @@ importers: specifier: ^1.0.0 version: 1.3.5 - components/lobste_rs: {} + components/lobste_rs: + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 + feedparser: + specifier: ^2.2.10 + version: 2.2.10 + object-hash: + specifier: ^3.0.0 + version: 3.0.0 components/lodgify: {} From afa90e8373e35a754f7f4a87b535b0974e6e6a46 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 16 Dec 2024 15:42:57 -0500 Subject: [PATCH 3/5] pnpm-lock.yaml --- pnpm-lock.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b6d6aa6baaf9..48275272aa194 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12371,7 +12371,7 @@ importers: version: 3.1.7 ts-jest: specifier: ^29.2.5 - version: 29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) + version: 29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) typescript: specifier: ^5.6 version: 5.7.2 @@ -43156,7 +43156,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-jest@29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): + ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43167,15 +43167,15 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.7.2 + typescript: 5.6.3 yargs-parser: 21.1.1 optionalDependencies: - '@babel/core': 7.26.0 + '@babel/core': 8.0.0-alpha.13 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.26.0) + babel-jest: 29.7.0(@babel/core@8.0.0-alpha.13) - ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): + ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43186,7 +43186,7 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.6.3 + typescript: 5.7.2 yargs-parser: 21.1.1 optionalDependencies: '@babel/core': 8.0.0-alpha.13 From b7343d14910f55503d3306d0173d19bb1b37fdb9 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 16 Dec 2024 15:46:32 -0500 Subject: [PATCH 4/5] pnpm-lock.yaml --- pnpm-lock.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48275272aa194..9b6d6aa6baaf9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12371,7 +12371,7 @@ importers: version: 3.1.7 ts-jest: specifier: ^29.2.5 - version: 29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) + version: 29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) typescript: specifier: ^5.6 version: 5.7.2 @@ -43156,7 +43156,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): + ts-jest@29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43167,15 +43167,15 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.6.3 + typescript: 5.7.2 yargs-parser: 21.1.1 optionalDependencies: - '@babel/core': 8.0.0-alpha.13 + '@babel/core': 7.26.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@8.0.0-alpha.13) + babel-jest: 29.7.0(@babel/core@7.26.0) - ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): + ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43186,7 +43186,7 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.7.2 + typescript: 5.6.3 yargs-parser: 21.1.1 optionalDependencies: '@babel/core': 8.0.0-alpha.13 From ffdc88e89fd3b91cf03fd1376f359d623d4c0122 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 16 Dec 2024 16:00:56 -0500 Subject: [PATCH 5/5] typos --- .../sources/new-comment-in-thread/new-comment-in-thread.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs b/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs index 81153cf6635aa..e1dcec2a8776c 100644 --- a/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs +++ b/components/lobste_rs/sources/new-comment-in-thread/new-comment-in-thread.mjs @@ -4,7 +4,7 @@ import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; export default { key: "lobste_rs-new-comment-in-thread", name: "New Comment in Thread", - description: "Emit new event when a new coomment is added to a thread.", + description: "Emit new event when a new comment is added to a thread.", version: "0.0.1", type: "source", dedupe: "unique", @@ -19,7 +19,7 @@ export default { url: { type: "string", label: "URL", - description: "The URL of the comment threaad to retrieve. E.g. `https://lobste.rs/s/yqjtvy/cloud_container_iceberg`", + description: "The URL of the comment thread to retrieve. E.g. `https://lobste.rs/s/yqjtvy/cloud_container_iceberg`", }, }, methods: {