feat: unixfs reifier and path selector

Author: tchardin

package.json

@@ -16,6 +16,7 @@
     "@types/uuid": "^8.3.4",
     "aegir": "^36.2.3",
     "blockstore-core": "^1.0.5",
+    "ipfs-unixfs-importer": "^9.0.6",
     "prettier": "^2.6.1"
   },
   "dependencies": {

src/async-loader.ts

@@ -1,4 +1,10 @@
-import {LinkLoader, Blockstore, blockFromStore} from "./traversal";
+import {
+  LinkLoader,
+  Blockstore,
+  blockFromStore,
+  KnownReifiers,
+  NodeReifier,
+} from "./traversal";
 import type {Block} from "multiformats/block";
 import type {CID} from "multiformats";
 
@@ -22,6 +28,8 @@ export class AsyncLoader implements LinkLoader {
 
   pullQueue: Map<string, Resolvable[]> = new Map();
 
+  reifiers: KnownReifiers = {};
+
   constructor(store: Blockstore, tracker?: BlockNotifyFn) {
     this.store = store;
     this.tracker = tracker;
@@ -79,6 +87,9 @@ export class AsyncLoader implements LinkLoader {
         .then(() => this.pending.delete(blk.cid.toString()));
     }
   }
+  reifier(name: string): NodeReifier {
+    return this.reifiers[name];
+  }
   // cleanup any block in memory
   close() {
     this.pending = new Map();

src/graphsync.ts

@@ -21,8 +21,13 @@ import {
   decodeMessage,
   newRequest,
 } from "./messages";
-import type {SelectorNode, Blockstore} from "./traversal";
-import {Node, walkBlocks, parseContext} from "./traversal";
+import type {
+  SelectorNode,
+  Blockstore,
+  KnownReifiers,
+  NodeReifier,
+} from "./traversal";
+import {Node, walkBlocks, parseContext, unixfsReifier} from "./traversal";
 
 export class GraphSync {
   started = false;
@@ -102,6 +107,10 @@ export class Request extends EventEmitter {
   dialer: ProtocolDialer;
   loader: AsyncLoader;
 
+  reifiers: KnownReifiers = {
+    unixfs: unixfsReifier,
+  };
+
   constructor(
     id: string,
     root: CID,
@@ -136,6 +145,10 @@ export class Request extends EventEmitter {
     );
   }
 
+  reifier(name: string): NodeReifier {
+    return this.reifiers[name];
+  }
+
   // TODO
   close() {}
 

src/index.ts

@@ -1,3 +1,3 @@
 export * from "./traversal";
 export {GraphSync, Request} from "./graphsync";
-export {fetch} from "./resolver";
+export {fetch, unixfsPathSelector} from "./resolver";

src/resolver.ts

@@ -1,10 +1,15 @@
 import {CID} from "multiformats";
 import {UnixFS} from "ipfs-unixfs";
-import * as dagJSON from "multiformats/codecs/json";
-import type {PBLink} from "@ipld/dag-pb";
 import PeerId from "peer-id";
 import type {Multiaddr} from "multiaddr";
-import {allSelector, Node, parseContext, walkBlocks} from "./traversal";
+import {
+  allSelector,
+  Node,
+  Kind,
+  parseContext,
+  walkBlocks,
+  SelectorNode,
+} from "./traversal";
 import mime from "mime/lite";
 import type {GraphSync} from "./graphsync";
 
@@ -19,11 +24,37 @@ export function parsePath(path: string): {root: CID; segments: string[]} {
   const comps = toPathComponents(path);
   const root = CID.parse(comps[0]);
   return {
-    segments: comps,
+    segments: comps.slice(1),
     root,
   };
 }
 
+export function unixfsPathSelector(path: string): {
+  root: CID;
+  selector: SelectorNode;
+} {
+  const {root, segments} = parsePath(path);
+  let selector = allSelector;
+  if (segments.length === 0) {
+    return {root, selector};
+  }
+  for (let i = segments.length - 1; i >= 0; i--) {
+    selector = {
+      "~": {
+        as: "unixfs",
+        ">": {
+          f: {
+            "f>": {
+              [segments[i]]: selector,
+            },
+          },
+        },
+      },
+    };
+  }
+  return {root, selector};
+}
+
 export function getPeerID(addr: Multiaddr): PeerId {
   const addrStr = addr.toString();
   const parts = addrStr.split("/");
@@ -39,17 +70,14 @@ export async function* resolve(
   provider: Multiaddr,
   exchange: GraphSync
 ): AsyncIterable<any> {
-  const {segments, root} = parsePath(path);
-  let cid = root;
-  let segs = segments.slice(1);
-  const sel = allSelector;
+  const {root, selector: sel} = unixfsPathSelector(path);
   const pid = getPeerID(provider);
   exchange.network.peerStore.addressBook.add(pid, [provider]);
-  const request = exchange.request(cid, sel);
+  const request = exchange.request(root, sel);
   const id = Date.now();
   const voucher = {
     ID: id,
-    PayloadCID: cid,
+    PayloadCID: root,
     Params: {
       Selector: sel,
       PieceCID: null,
@@ -63,7 +91,7 @@ export async function* resolve(
     [EXTENSION]: {
       IsRq: true,
       Request: {
-        BCid: cid,
+        BCid: root,
         Type: 0,
         Pull: true,
         Paus: false,
@@ -78,7 +106,7 @@ export async function* resolve(
     },
   });
   for await (const blk of walkBlocks(
-    new Node(cid),
+    new Node(root),
     parseContext().parseSelector(sel),
     request
   )) {
@@ -88,46 +116,22 @@ export async function* resolve(
       case 0x71:
         break;
       default:
-        console.log("raw bytes");
         yield blk.bytes;
         continue;
     }
-    try {
-      const unixfs = UnixFS.unmarshal(blk.value.Data);
-      if (unixfs.isDirectory()) {
-        // if it's a directory and we have a segment to resolve, identify the link
-        if (segs.length > 0) {
-          for (const link of blk.value.Links) {
-            if (link.Name === segs[0]) {
-              cid = link.Hash;
-              segs = segs.slice(1);
-              console.log("found link in directory");
-              continue;
-            }
+    if (blk.value.kind === Kind.Map && blk.value.Data) {
+      try {
+        const unixfs = UnixFS.unmarshal(blk.value.Data);
+        if (unixfs.type === "file") {
+          if (unixfs.data && unixfs.data.length) {
+            yield unixfs.data;
           }
-          throw new Error("key not found: " + segs[0]);
-        } else {
-          // if the block is a directory and we have no key return the entries as JSON
-          yield dagJSON.encode(
-            blk.value.Links.map((l: PBLink) => ({
-              name: l.Name,
-              hash: l.Hash.toString(),
-              size: l.Tsize,
-            }))
-          );
-          break;
+          continue;
         }
-      }
-      if (unixfs.type === "file") {
-        if (unixfs.data && unixfs.data.length) {
-          console.log("unixfs file");
-          yield unixfs.data;
-        }
-        continue;
-      }
-    } catch (e) {}
-    // we're outside of unixfs territory
-    // ignore
+      } catch (e) {}
+      // we're outside of unixfs territory
+      // ignore
+    }
   }
   // tell the loader we're done receiving blocks for this traversal
   request.close();