Initial addition of BitTorrent protocol

.gitignore

@@ -109,4 +109,7 @@ store
 
 *.key
 
+# Don't commit our internal ansibles!
 ansible/prod.yml
+ansible/staging.yml
+ansible/mauve.yml

@types/bt-fetch.d.ts

@@ -0,0 +1,32 @@
+declare module 'bt-fetch' {
+  export type TorrentManagerOptions = Partial<{
+    folder: string
+    timeout: number
+    reloadInterval: number
+  }>
+
+  export interface Torrent {
+    infoHash: Buffer
+    publicKey: Buffer
+  }
+
+  export type TorrentPublishOpts = Partial<{
+    name: string
+    comment: string
+    createdBy: string
+    creationDate: string
+  }>
+
+  export interface KeyPair {
+    publicKey: string
+    secretKey: string
+  }
+
+  export class TorrentManager {
+    constructor (opts: TorrentManagerOptions)
+    stopSeedingPublicKey (publicKey: string): Promise<boolean>
+    republishPublicKey (publicKey: string, secretKey: string, opts: TorrentPublishOpts): Promise<Torrent>
+    createKeypair (petname?: string): KeyPair
+    destroy (): Promise<void>
+  }
+}

ansible/roles/distributed_press/defaults/main.yml

@@ -14,7 +14,7 @@ distributed_press_host: "localhost"
 distributed_press_ipfs_provider: "builtin"
 
 distributed_press_git_repo: "https://github.com/hyphacoop/api.distributed.press.git"
-distributed_press_git_branch: "v1.0.0"
+distributed_press_git_branch: "v1.1.0"
 distributed_press_source: "{{distributed_press_home}}/api.distributed.press"
 
 distributed_press_domain: "example.com"

api/index.ts

@@ -59,6 +59,9 @@ async function apiBuilder (cfg: APIConfig): Promise<FastifyTypebox> {
     hyper: {
       path: path.join(protocolStoragePath, 'hyper')
     },
+    bt: {
+      path: path.join(protocolStoragePath, 'bt')
+    },
     http: {
       path: path.join(protocolStoragePath, 'http')
     }

api/schemas.ts

@@ -22,11 +22,19 @@ export const IPFSProtocolFields = GenericProtocol(Type.Object({
   pubKey: Type.String(), // ipns://{publishKey}
   dnslink: Type.String()
 }))
+export const BitTorrentProtocolFields = GenericProtocol(Type.Object({
+  gateway: Type.String(), // same as gateway in HyperProtocolFields
+  magnet: Type.String(), // Used by most torrent clients. Note: Will not update in regular clients
+  infoHash: Type.String(), // Immutable link, similar to ipfs public key
+  pubKey: Type.String(), // Link to public key for BEP-46, similar to IPNS
+  dnslink: Type.String()
+}))
 
 export const Protocols = Type.Object({
   http: HTTPProtocolFields,
   hyper: HyperProtocolFields,
-  ipfs: IPFSProtocolFields
+  ipfs: IPFSProtocolFields,
+  bt: BitTorrentProtocolFields
 })
 export const ProtocolStatus = Type.Record(Type.KeyOf(Protocols), Type.Boolean())
 export const Site = Type.Object({

dns/index.test.ts

@@ -32,7 +32,8 @@ test('basic dns resolve', async t => {
     protocols: {
       http: false,
       ipfs: true,
-      hyper: true
+      hyper: true,
+      bt: false
     },
     public: true
   })

fixtures/mockProtocols.ts

@@ -1,23 +1,26 @@
 import { Static, TSchema } from '@sinclair/typebox'
-import { HTTPProtocolFields, HyperProtocolFields, IPFSProtocolFields } from '../api/schemas.js'
+import { HTTPProtocolFields, HyperProtocolFields, IPFSProtocolFields, BitTorrentProtocolFields } from '../api/schemas.js'
 import { ProtocolManager } from '../protocols/index.js'
 import Protocol, { Ctx, SyncOptions } from '../protocols/interfaces.js'
 
 export class MockProtocolManager implements ProtocolManager {
   http: MockHTTPProtocol
   ipfs: MockIPFSProtocol
   hyper: MockHyperProtocol
+  bt: MockBitTorrentProtocol
 
   constructor () {
     this.ipfs = new MockIPFSProtocol()
     this.http = new MockHTTPProtocol()
     this.hyper = new MockHyperProtocol()
+    this.bt = new MockBitTorrentProtocol()
   }
 
   async load (): Promise<void> {
     const promises = [
       this.ipfs.load(),
       this.hyper.load(),
+      this.bt.load(),
       this.http.load()
     ]
     await Promise.all(promises)
@@ -27,6 +30,7 @@ export class MockProtocolManager implements ProtocolManager {
     const promises = [
       this.ipfs.unload(),
       this.hyper.unload(),
+      this.bt.unload(),
       this.http.unload()
     ]
     await Promise.all(promises)
@@ -81,3 +85,17 @@ class MockHyperProtocol extends BaseMockProtocol<typeof HyperProtocolFields> {
     }
   }
 }
+
+class MockBitTorrentProtocol extends BaseMockProtocol<typeof BitTorrentProtocolFields> {
+  async sync (_id: string, _folderPath: string, _options?: SyncOptions, _ctx?: Ctx): Promise<Static<typeof BitTorrentProtocolFields>> {
+    return {
+      enabled: true,
+      link: 'bittorrent://example-link',
+      gateway: 'https://example-bittorrent-gateway/example-link',
+      dnslink: '/bt/example-raw',
+      infoHash: 'bittorrent://example-link-infohash',
+      pubKey: 'bittorrent://example-link-publickey',
+      magnet: 'magnet:?xt:urn:btih:example-link&xs=urn:btpk:example-link'
+    }
+  }
+}

fixtures/siteConfig.ts

@@ -13,7 +13,8 @@ export const exampleSiteConfig: Static<typeof NewSite> = {
   protocols: {
     http: true,
     ipfs: false,
-    hyper: false
+    hyper: false,
+    bt: false
   },
   public: true
 }

package.json

@@ -7,13 +7,13 @@
   "scripts": {
     "clean": "rimraf build",
     "build": "npm run clean && tsc",
-    "lint": "ts-standard --fix && tsc --noEmit",
+    "lint": "ts-standard --fix && tsc --skipLibCheck --noEmit",
     "keygen": "ts-node-esm authorization/scripts/keygen.ts",
     "make-admin": "ts-node-esm authorization/scripts/create_admin.ts",
     "dev": "ts-node-esm index.ts | pino-pretty -c -t",
     "watch": "nodemon --watch './**/*.ts' --exec 'node --experimental-specifier-resolution=node --loader ts-node/esm' index.ts | pino-pretty -c -t",
     "start": "node build/index.js",
-    "test": "ava --concurrency 1 --timeout=1m",
+    "test": "ava --verbose --concurrency 1 --timeout=1m",
     "nuke": "ts-node-esm protocols/scripts/nuke.ts"
   },
   "repository": {
@@ -47,6 +47,7 @@
     "@fastify/type-provider-typebox": "^2.4.0",
     "@sinclair/typebox": "^0.25.9",
     "abstract-level": "^1.0.3",
+    "bt-fetch": "^3.3.2",
     "cors": "^2.8.5",
     "dns2": "^2.1.0",
     "env-paths": "^3.0.0",

protocols/bittorrent.ts

@@ -0,0 +1,94 @@
+import { Static } from '@sinclair/typebox'
+import Protocol, { Ctx, SyncOptions } from './interfaces'
+import { BitTorrentProtocolFields } from '../api/schemas'
+
+import path from 'node:path'
+import { cp } from 'node:fs/promises'
+
+import { TorrentManager } from 'bt-fetch'
+
+export interface BitTorrentProtocolOptions {
+  path: string
+}
+
+export class BitTorrentProtocol implements Protocol<Static<typeof BitTorrentProtocolFields>> {
+  options: BitTorrentProtocolOptions
+  manager: TorrentManager | null
+
+  constructor (options: BitTorrentProtocolOptions) {
+    this.options = options
+    this.manager = null
+  }
+
+  async load (): Promise<void> {
+    const folder = this.options.path
+    this.manager = new TorrentManager({ folder })
+  }
+
+  async unload (): Promise<void> {
+    await this.manager?.destroy()
+  }
+
+  async sync (id: string, folderPath: string, options?: SyncOptions, ctx?: Ctx): Promise<Static<typeof BitTorrentProtocolFields>> {
+    ctx?.logger.info('[bittorrent] Sync Start')
+
+    const link = `bittorent://${id}/`
+
+    if (this.manager === null) {
+      throw new Error('[bittorrent] Torrent Manager Not Initialized')
+    }
+
+    const manager: TorrentManager = this.manager
+
+    // Create keypair from the site ID and the seed key
+    const { publicKey, secretKey } = manager.createKeypair(id)
+
+    const storageFolder = path.join(this.options.path, 'data', publicKey, id)
+
+    const torrentInfo = {
+      name: id,
+      comment: `Content for ${link}`,
+      createdBy: 'distributed.press'
+    }
+
+    ctx?.logger.info('[bittorrent] Stop seeding existing')
+
+    await manager.stopSeedingPublicKey(publicKey)
+
+    ctx?.logger.info('[bittorrent] Copy data to storage')
+
+    await cp(folderPath + path.sep, storageFolder + path.sep, { recursive: true })
+
+    ctx?.logger.info('[bittorrent] Generate new torrent and publish to DHT')
+
+    // Pass folder and options
+    const torrent = await manager.republishPublicKey(publicKey, secretKey, torrentInfo)
+
+    const infoHash = torrent.infoHash.toString('hex')
+
+    const subdomain = id.replaceAll('-', '--').replaceAll('.', '-')
+    // TODO: Pass in gateway from config
+    const gateway = `https://${subdomain}.bt.hypha.coop/`
+
+    const magnet = `magnet:?xt:urn:btih:${infoHash}&xs=urn:btpk:${publicKey}`
+
+    const infoHashURL = `bittorrent://${infoHash}/`
+    const publicKeyURL = `bittorrent://${publicKey}/`
+
+    ctx?.logger.info(`[bittorrent] Published: ${link}`)
+
+    const dnslink = `/bt/${publicKey}`
+    return {
+      enabled: true,
+      link,
+      gateway,
+      infoHash: infoHashURL,
+      pubKey: publicKeyURL,
+      magnet,
+      dnslink
+    }
+  }
+
+  async unsync (id: string, _site: Static<typeof BitTorrentProtocolFields>, ctx?: Ctx): Promise<void> {
+  }
+}

protocols/index.test.ts

@@ -8,6 +8,7 @@ import { exampleSiteConfig } from '../fixtures/siteConfig.js'
 import { HyperProtocol } from './hyper.js'
 import Protocol from './interfaces.js'
 import { BUILTIN, IPFSProtocol } from './ipfs.js'
+import { BitTorrentProtocol } from './bittorrent.js'
 
 const paths = envPaths('distributed-press')
 const filename = fileURLToPath(import.meta.url)
@@ -49,3 +50,15 @@ test('hyper: basic e2e sync', async t => {
   t.is(links.enabled, true)
   t.truthy(links.link)
 })
+
+test('bittorrent: basic e2e sync', async t => {
+  const path = await newProtocolTestPath()
+  t.context.protocol = new BitTorrentProtocol({
+    path
+  })
+
+  await t.notThrowsAsync(t.context.protocol.load(), 'initializing bittorrent should work')
+  const links = await t.context.protocol.sync(exampleSiteConfig.domain, fixturePath)
+  t.is(links.enabled, true)
+  t.truthy(links.link)
+})

protocols/index.ts

@@ -1,37 +1,43 @@
 import { IPFSProtocol, IPFSProtocolOptions } from './ipfs.js'
 import { HyperProtocol, HyperProtocolOptions } from './hyper.js'
+import { BitTorrentProtocol, BitTorrentProtocolOptions } from './bittorrent.js'
 import { HTTPProtocol, HTTPProtocolOptions } from './http.js'
 import Protocol from './interfaces.js'
 import { Static } from '@sinclair/typebox'
-import { HTTPProtocolFields, HyperProtocolFields, IPFSProtocolFields } from '../api/schemas.js'
+import { HTTPProtocolFields, HyperProtocolFields, IPFSProtocolFields, BitTorrentProtocolFields } from '../api/schemas.js'
 
 interface ProtocolOptions {
   ipfs: IPFSProtocolOptions
   hyper: HyperProtocolOptions
+  bt: BitTorrentProtocolOptions
   http: HTTPProtocolOptions
 }
 
 export interface ProtocolManager {
   http: Protocol<Static<typeof HTTPProtocolFields>>
   ipfs: Protocol<Static<typeof IPFSProtocolFields>>
   hyper: Protocol<Static<typeof HyperProtocolFields>>
+  bt: Protocol<Static<typeof BitTorrentProtocolFields>>
 }
 
 export class ConcreteProtocolManager implements ProtocolManager {
   http: HTTPProtocol
   ipfs: IPFSProtocol
   hyper: HyperProtocol
+  bt: BitTorrentProtocol
 
   constructor (options: ProtocolOptions) {
-    this.ipfs = new IPFSProtocol(options.ipfs)
     this.http = new HTTPProtocol(options.http)
+    this.ipfs = new IPFSProtocol(options.ipfs)
     this.hyper = new HyperProtocol(options.hyper)
+    this.bt = new BitTorrentProtocol(options.bt)
   }
 
   async load (): Promise<void> {
     const promises = [
       this.ipfs.load(),
       this.hyper.load(),
+      this.bt.load(),
       this.http.load()
     ]
     await Promise.all(promises)
@@ -41,6 +47,7 @@ export class ConcreteProtocolManager implements ProtocolManager {
     const promises = [
       this.ipfs.unload(),
       this.hyper.unload(),
+      this.bt.unload(),
       this.http.unload()
     ]
     await Promise.all(promises)

tsconfig.json

@@ -2,6 +2,9 @@
   "include": [
     "**/*.ts"
   ],
+  "exclude": [
+    "node_modules"
+  ],
   "files": ["@types/fastify.d.ts", "@types/fastify-jwt.d.ts"],
   "compilerOptions": {
     "target": "esnext",