Experimental Python Support

## Language Support

- Add initial support for Python

## Visualization

- Add support for node clusters. This is used heavily in Python, for context-managers and exception-handling.

## Demo

- Add Python support
- Add sharing - click the "Share" button to get a sharable link to what you currently see

## Testing

- Enable live-testing with the web viewer. Requires that you run both `bun web-tests --watch` and `bun web` at the same time.
- By default, `bun web` only shows failing tests
- `bun web` color-codes tests to note which are failing

## Extension

- No changes

## Known Issues

- Backlinks are no longer thicker than normal links. That said, they were half-broken to begin with and were somewhat arbitrary.

Author: tmr232

CHANGELOG.md

@@ -6,6 +6,33 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how
 
 ## [Unreleased]
 
+### Language Support
+
+- Add initial support for Python
+
+### Visualization
+
+- Add support for node clusters. This is used heavily in Python, for context-managers and exception-handling.
+
+### Demo
+
+- Add Python support
+- Add sharing - click the "Share" button to get a sharable link to what you currently see
+
+### Testing
+
+- Enable live-testing with the web viewer. Requires that you run both `bun web-tests --watch` and `bun web` at the same time.
+- By default, `bun web` only shows failing tests
+- `bun web` color-codes tests to note which are failing
+
+### Extension
+
+- No changes
+
+### Known Issues
+
+- Backlinks are no longer thicker than normal links. That said, they were half-broken to begin with and were somewhat arbitrary.
+
 ## [0.0.4] - 2024-09-10
 
 - Improved comment-test framework to allow writing tests for multiple languages

README.md

@@ -20,6 +20,9 @@ Note that the demo only supports a single function and ignores the cursor locati
 
 - Go
 - C
+- Python (experimental, only in the [interactive demo](https://tmr232.github.io/function-graph-overview/?language=2))
+  - Since this adds _a lot_ of new visualization types, this is marked "experimental"
+    as it is very likely to change.
 
 ## Development
 

bun.lockb

@@ -13,6 +13,7 @@
   "devDependencies": {
     "@codemirror/lang-cpp": "^6.0.2",
     "@codemirror/lang-go": "^6.0.1",
+    "@codemirror/lang-python": "^6.1.6",
     "@eslint/js": "^9.9.1",
     "@rollup/plugin-wasm": "^6.2.2",
     "@sveltejs/vite-plugin-svelte": "^3.1.1",
@@ -22,13 +23,16 @@
     "esbuild": "^0.20.0",
     "esbuild-plugin-copy": "^2.1.1",
     "eslint": "^9.9.1",
+    "graphology-utils": "^2.5.2",
+    "lz-string": "^1.5.0",
     "prettier": "3.3.3",
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^4.2.18",
     "svelte-codemirror-editor": "^1.4.1",
     "tree-sitter-c": "^0.23.0",
     "tree-sitter-cli": "^0.23.0",
     "tree-sitter-go": "^0.23.0",
+    "tree-sitter-python": "^0.23.2",
     "typescript-eslint": "^8.4.0",
     "vite": "^5.4.1"
   },
@@ -44,7 +48,8 @@
     "package": "bun run build && bun run vsce-package",
     "publish": "bun run package && bun run vsce-publish",
     "clean": "rm -r ./dist",
-    "web": "bun run ./scripts/collect-comment-tests.ts && bun run --cwd ./src/frontend/ vite",
+    "web": "bun run --cwd ./src/frontend/ vite",
+    "web-tests": "bun run ./scripts/collect-comment-tests.ts",
     "demo": "bun run --cwd ./src/demo/ vite",
     "build-demo": "bun run --cwd ./src/demo/ vite build --outDir ../../dist/demo --base '/function-graph-overview/'",
     "format": "bun prettier . --write --log-level silent",
@@ -99,4 +104,4 @@
   "engines": {
     "vscode": "^1.86.0"
   }
-}
+}

package.json

@@ -1,7 +1,47 @@
-import { testFunctions as testFuncsForGo } from "../src/test/collect-go";
-import { testFunctions as testFuncsForC } from "../src/test/collect-c";
 import { intoRecords } from "../src/test/commentTestUtils";
+import { watch } from "fs";
+import { parseArgs } from "util";
+import { collectTests } from "../src/test/commentTestCollector";
 
-const records = intoRecords([...testFuncsForC, ...testFuncsForGo]);
+const watchDir = import.meta.dir + "/../src";
 
-Bun.write("./dist/tests/commentTests.json", JSON.stringify(records));
+const { values } = parseArgs({
+  args: Bun.argv,
+  options: {
+    watch: {
+      type: "boolean",
+      default: false,
+    },
+  },
+  strict: true,
+  allowPositionals: true,
+});
+
+async function generateJson() {
+  try {
+    const records = intoRecords(await collectTests());
+    Bun.write("./dist/tests/commentTests.json", JSON.stringify(records));
+  } catch (error) {
+    console.log(error);
+  }
+}
+
+generateJson();
+if (values.watch) {
+  const watcher = watch(
+    watchDir,
+    { recursive: true },
+    async (event, filename) => {
+      console.log(`${event}: ${filename}, regenerating commentTests.json`);
+      await generateJson();
+    },
+  );
+
+  process.on("SIGINT", () => {
+    // close watcher when Ctrl-C is pressed
+    console.log("Closing watcher...");
+    watcher.close();
+
+    process.exit(0);
+  });
+}

parsers/tree-sitter-go.wasm

@@ -3,7 +3,7 @@ import { $ } from "bun";
 const treeSitter = Bun.file("./node_modules/web-tree-sitter/tree-sitter.wasm");
 await Bun.write("./parsers/tree-sitter.wasm", treeSitter);
 
-const parsers = ["tree-sitter-go", "tree-sitter-c"];
+const parsers = ["tree-sitter-go", "tree-sitter-c", "tree-sitter-python"];
 
 for (const name of parsers) {
   await $`bun x --bun tree-sitter build --wasm -o ./parsers/${name}.wasm ./node_modules/${name}/`;

parsers/tree-sitter-python.wasm

@@ -2,6 +2,8 @@ import { MultiDirectedGraph } from "graphology";
 import type Parser from "web-tree-sitter";
 
 export type NodeType =
+  | "YIELD"
+  | "THROW"
   | "MARKER_COMMENT"
   | "LOOP_HEAD"
   | "LOOP_EXIT"
@@ -29,12 +31,29 @@ export type NodeType =
   | "SWITCH_CONDITION"
   | "SWITCH_MERGE"
   | "CASE_CONDITION";
-export type EdgeType = "regular" | "consequence" | "alternative";
+export type EdgeType = "regular" | "consequence" | "alternative" | "exception";
+
+export type ClusterType =
+  | "with"
+  | "try"
+  | "except"
+  | "else"
+  | "finally"
+  | "try-complex";
+export type ClusterId = number;
+export type Cluster = {
+  id: ClusterId;
+  type: ClusterType;
+  parent?: Cluster;
+  depth: number;
+};
+
 export interface GraphNode {
   type: NodeType;
   code: string;
   lines: number;
   markers: string[];
+  cluster?: Cluster;
 }
 
 export interface GraphEdge {
@@ -55,10 +74,13 @@ export interface BasicBlock {
   labels?: Map<string, string>;
   // Target label
   gotos?: Goto[];
+  // Return statements in the block. Needed for exception handling.
+  returns?: string[];
 }
 
+export type CFGGraph = MultiDirectedGraph<GraphNode, GraphEdge>;
 export interface CFG {
-  graph: MultiDirectedGraph<GraphNode, GraphEdge>;
+  graph: CFGGraph;
   entry: string;
 }
 
@@ -67,7 +89,20 @@ export class BlockHandler {
   private continues: string[] = [];
   private labels: Map<string, string> = new Map();
   private gotos: Array<{ label: string; node: string }> = [];
+  /**
+   * All the returns encountered so far.
+   *
+   * This is needed for `finally` clauses in exception handling,
+   * as the return is moved/duplicated to the end of the finally clause.
+   * This means that when processing returns, we expect to get a new set
+   * of returns.
+   */
+  private returns: Array<string> = [];
 
+  /**
+   * Operate on all collected breaks and clear them.
+   * @param callback Handles the breaks, linking them to the relevant nodes.
+   */
   public forEachBreak(callback: (breakNode: string) => void) {
     this.breaks.forEach(callback);
     this.breaks = [];
@@ -78,6 +113,10 @@ export class BlockHandler {
     this.continues = [];
   }
 
+  public forEachReturn(callback: (returnNode: string) => string) {
+    this.returns = this.returns.map(callback);
+  }
+
   public processGotos(callback: (gotoNode: string, labelNode: string) => void) {
     this.gotos.forEach((goto) => {
       const labelNode = this.labels.get(goto.label);
@@ -90,9 +129,10 @@ export class BlockHandler {
   }
 
   public update(block: BasicBlock): BasicBlock {
-    this.breaks.push(...(block.breaks || []));
-    this.continues.push(...(block.continues || []));
-    this.gotos.push(...(block.gotos || []));
+    this.breaks.push(...(block.breaks ?? []));
+    this.continues.push(...(block.continues ?? []));
+    this.gotos.push(...(block.gotos ?? []));
+    this.returns.push(...(block.returns ?? []));
     block.labels?.forEach((value, key) => this.labels.set(key, value));
 
     return {
@@ -102,16 +142,28 @@ export class BlockHandler {
       continues: this.continues,
       gotos: this.gotos,
       labels: this.labels,
+      returns: this.returns,
     };
   }
 }
 
-export function mergeNodeAttrs(from: GraphNode, into: GraphNode): GraphNode {
+export function mergeNodeAttrs(
+  from: GraphNode,
+  into: GraphNode,
+): GraphNode | null {
+  if (from.cluster !== into.cluster) {
+    return null;
+  }
+  const noMergeTypes: NodeType[] = ["YIELD", "THROW"];
+  if (noMergeTypes.includes(from.type) || noMergeTypes.includes(into.type)) {
+    return null;
+  }
   return {
     type: from.type,
     code: `${from.code}\n${into.code}`,
     lines: from.lines + into.lines,
     markers: [...from.markers, ...into.markers],
+    cluster: from.cluster,
   };
 }
 export interface Case {