p_sync: FileSyncer + CLI + Local File Usage

.fernignore

@@ -10,11 +10,15 @@ src/humanloop.client.ts
 src/overload.ts
 src/error.ts
 src/context.ts
+src/cli.ts
+src/cache
+src/sync 
 
 # Tests
 
 # Modified due to issues with OTEL
 tests/unit/fetcher/stream-wrappers/webpack.test.ts
+tests/custom/
 
 # CI Action
 

README.md

@@ -91,6 +91,47 @@ try {
 }
 ```
 
+## Store Humanloop Files in Code
+
+Humanloop allows you to maintain Prompts and Agents in your local filesystem and version control, while still leveraging Humanloop's prompt management capabilities.
+
+### Syncing Files with the CLI
+
+```bash
+# Basic usage
+npx humanloop pull                               # Pull all files to 'humanloop/' directory
+npx humanloop pull --path="examples/chat"        # Pull specific directory
+npx humanloop pull --environment="production"    # Pull from specific environment
+npx humanloop pull --local-files-directory="ai"  # Specify local destination (default: "humanloop")
+
+# View available options
+npx humanloop pull --help
+```
+
+### Using Local Files in the SDK
+
+To use local Files in your code:
+
+```typescript
+// Enable local file support
+const client = new HumanloopClient({
+  apiKey: "YOUR_API_KEY",
+  useLocalFiles: true
+});
+
+// Call a local Prompt file
+const response = await client.prompts.call({
+  path: "examples/chat/basic",  // Looks for humanloop/examples/chat/basic.prompt
+  inputs: { query: "Hello world" }
+});
+
+// The same path-based approach works with prompts.log(), agents.call(), and agents.log()
+```
+
+For detailed instructions, see our [Guide on Storing Files in Code](https://humanloop.com/docs/v5/guides/prompts/store-prompts-in-code).
+
+For information about file formats, see our [File Format Reference](https://humanloop.com/docs/v5/reference/serialized-files).
+
 ## Pagination
 
 List endpoints are paginated. The SDK provides an iterator so that you can simply loop over the items:

jest.config.mjs

@@ -3,6 +3,11 @@ export default {
     preset: "ts-jest",
     testEnvironment: "node",
     moduleNameMapper: {
-        "(.+)\.js$": "$1",
+        // Only map .js files in our src directory, not node_modules
+        "^src/(.+)\\.js$": "<rootDir>/src/$1",
     },
+    // Add transformIgnorePatterns to handle ESM modules in node_modules
+    transformIgnorePatterns: [
+        "node_modules/(?!(@traceloop|js-tiktoken|base64-js)/)",
+    ],
 };

package.json

@@ -27,6 +27,8 @@
         "@traceloop/instrumentation-openai": ">=0.11.3",
         "@traceloop/ai-semantic-conventions": ">=0.11.6",
         "cli-progress": "^3.12.0",
+        "dotenv": "^16.5.0",
+        "commander": "^14.0.0",
         "lodash": "^4.17.21"
     },
     "devDependencies": {
@@ -46,7 +48,6 @@
         "openai": "^4.74.0",
         "@anthropic-ai/sdk": "^0.32.1",
         "cohere-ai": "^7.15.0",
-        "dotenv": "^16.4.6",
         "jsonschema": "^1.4.1",
         "@types/cli-progress": "^3.11.6",
         "@types/lodash": "4.14.74",

src/cache/LRUCache.ts

@@ -0,0 +1,48 @@
+/**
+ * LRU Cache implementation
+ */
+export default class LRUCache<K, V> {
+    private cache: Map<K, V>;
+    private readonly maxSize: number;
+
+    constructor(maxSize: number) {
+        this.cache = new Map<K, V>();
+        this.maxSize = maxSize;
+    }
+
+    get(key: K): V | undefined {
+        if (!this.cache.has(key)) {
+            return undefined;
+        }
+
+        // Get the value
+        const value = this.cache.get(key);
+
+        // Remove key and re-insert to mark as most recently used
+        this.cache.delete(key);
+        this.cache.set(key, value!);
+
+        return value;
+    }
+
+    set(key: K, value: V): void {
+        // If key already exists, refresh its position
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        // If cache is full, remove the least recently used item (first item in the map)
+        else if (this.cache.size >= this.maxSize) {
+            const lruKey = this.cache.keys().next().value;
+            if (lruKey) {
+                this.cache.delete(lruKey);
+            }
+        }
+
+        // Add new key-value pair
+        this.cache.set(key, value);
+    }
+
+    clear(): void {
+        this.cache.clear();
+    }
+}

src/cache/index.ts

@@ -0,0 +1 @@
+export { default as LRUCache } from './LRUCache';

src/cli.ts

@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+import * as dotenv from "dotenv";
+import { Command } from "commander";
+import path from "path";
+
+import { HumanloopClient } from "./humanloop.client";
+import FileSyncer from "./sync/FileSyncer";
+import { SDK_VERSION } from "./version";
+
+dotenv.config();
+
+const LogType = {
+    SUCCESS: "\x1b[92m", // green
+    ERROR: "\x1b[91m", // red
+    INFO: "\x1b[96m", // cyan
+    WARN: "\x1b[93m", // yellow
+    RESET: "\x1b[0m",
+} as const;
+
+function log(message: string, type: keyof typeof LogType): void {
+    console.log(`${LogType[type]}${message}${LogType.RESET}`);
+}
+
+const program = new Command();
+program
+    .name("humanloop")
+    .description("Humanloop CLI for managing sync operations")
+    .version(SDK_VERSION);
+
+interface CommonOptions {
+    apiKey?: string;
+    envFile?: string;
+    baseUrl?: string;
+    localFilesDirectory?: string;
+}
+
+interface PullOptions extends CommonOptions {
+    path?: string;
+    environment?: string;
+    verbose?: boolean;
+    quiet?: boolean;
+}
+
+const addCommonOptions = (command: Command) =>
+    command
+        .option("--api-key <apiKey>", "Humanloop API key")
+        .option("--env-file <envFile>", "Path to .env file")
+        .option("--base-url <baseUrl>", "Base URL for Humanloop API")
+        .option(
+            "--local-dir, --local-files-directory <dir>",
+            "Directory where Humanloop files are stored locally (default: humanloop/)",
+            "humanloop",
+        );
+
+// Instantiate a HumanloopClient for the CLI
+function getClient(options: CommonOptions): HumanloopClient {
+    if (options.envFile) {
+        const result = dotenv.config({ path: options.envFile });
+        if (result.error) {
+            log(
+                `Failed to load environment file: ${options.envFile} (file not found or invalid format)`,
+                "ERROR",
+            );
+            process.exit(1);
+        }
+    }
+
+    const apiKey = options.apiKey || process.env.HUMANLOOP_API_KEY;
+    if (!apiKey) {
+        log(
+            "No API key found. Set HUMANLOOP_API_KEY in .env file or environment, or use --api-key",
+            "ERROR",
+        );
+        process.exit(1);
+    }
+
+    return new HumanloopClient({
+        apiKey,
+        baseUrl: options.baseUrl,
+        localFilesDirectory: options.localFilesDirectory,
+    });
+}
+
+// Helper to handle sync errors
+function handleSyncErrors<T extends CommonOptions>(fn: (options: T) => Promise<void>) {
+    return async (options: T) => {
+        try {
+            await fn(options);
+        } catch (error) {
+            log(`Error: ${error}`, "ERROR");
+            process.exit(1);
+        }
+    };
+}
+
+// Pull command
+addCommonOptions(
+    program
+        .command("pull")
+        .description(
+            "Pull Prompt and Agent files from Humanloop to your local filesystem.\n\n" +
+                "This command will:\n" +
+                "1. Fetch Prompt and Agent files from your Humanloop workspace\n" +
+                "2. Save them to your local filesystem (directory specified by --local-files-directory, default: humanloop/)\n" +
+                "3. Maintain the same directory structure as in Humanloop\n" +
+                "4. Add appropriate file extensions (.prompt or .agent)\n\n" +
+                "For example, with the default --local-files-directory=humanloop, files will be saved as:\n" +
+                "./humanloop/\n" +
+                "├── my_project/\n" +
+                "│   ├── prompts/\n" +
+                "│   │   ├── my_prompt.prompt\n" +
+                "│   │   └── nested/\n" +
+                "│   │       └── another_prompt.prompt\n" +
+                "│   └── agents/\n" +
+                "│       └── my_agent.agent\n" +
+                "└── another_project/\n" +
+                "    └── prompts/\n" +
+                "        └── other_prompt.prompt\n\n" +
+                "If you specify --local-files-directory=data/humanloop, files will be saved in ./data/humanloop/ instead.\n\n" +
+                "If a file exists both locally and in the Humanloop workspace, the local file will be overwritten\n" +
+                "with the version from Humanloop. Files that only exist locally will not be affected.\n\n" +
+                "Currently only supports syncing Prompt and Agent files. Other file types will be skipped.",
+        )
+        .option(
+            "-p, --path <path>",
+            "Path in the Humanloop workspace to pull from (file or directory). " +
+                "You can pull an entire directory (e.g. 'my/directory') or a specific file (e.g. 'my/directory/my_prompt.prompt'). " +
+                "When pulling a directory, all files within that directory and its subdirectories will be included. " +
+                "Paths should not contain leading or trailing slashes. " +
+                "If not specified, pulls from the root of the remote workspace.",
+        )
+        .option(
+            "-e, --environment <env>",
+            "Environment to pull from (e.g. 'production', 'staging')",
+        )
+        .option("-v, --verbose", "Show detailed information about the operation")
+        .option("-q, --quiet", "Suppress output of successful files"),
+).action(
+    handleSyncErrors(async (options: PullOptions) => {
+        const client = getClient(options);
+
+        // Create a separate FileSyncer instance with log level based on verbose flag only
+        const fileSyncer = new FileSyncer(client, {
+            baseDir: options.localFilesDirectory,
+            verbose: options.verbose,
+        });
+
+        log("Pulling files from Humanloop...", "INFO");
+        log(`Path: ${options.path || "(root)"}`, "INFO");
+        log(`Environment: ${options.environment || "(default)"}`, "INFO");
+
+        const startTime = Date.now();
+        const [successfulFiles, failedFiles] = await fileSyncer.pull(
+            options.path,
+            options.environment,
+        );
+        const duration = Date.now() - startTime;
+
+        // Always show operation result
+        const isSuccessful = failedFiles.length === 0;
+        log(`Pull completed in ${duration}ms`, isSuccessful ? "SUCCESS" : "ERROR");
+
+        // Only suppress successful files output if quiet flag is set
+        if (successfulFiles.length > 0 && !options.quiet) {
+            console.log(); // Empty line
+            log(`Successfully pulled ${successfulFiles.length} files:`, "SUCCESS");
+            for (const file of successfulFiles) {
+                log(`  ✓ ${file}`, "SUCCESS");
+            }
+        }
+
+        // Always show failed files
+        if (failedFiles.length > 0) {
+            console.log(); // Empty line
+            log(`Failed to pull ${failedFiles.length} files:`, "ERROR");
+            for (const file of failedFiles) {
+                log(`  ✗ ${file}`, "ERROR");
+            }
+        }
+    }),
+);
+
+program.parse(process.argv);