Implement string table

pablogsal · pablogsal · commit a89c8524415a · 2025-09-14T03:51:11.000+01:00
diff --git a/Lib/profiling/sampling/__init__.py b/Lib/profiling/sampling/__init__.py
@@ -7,5 +7,6 @@
 from .collector import Collector
 from .pstats_collector import PstatsCollector
 from .stack_collector import CollapsedStackCollector
+from .string_table import StringTable
 
-__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector")
+__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "StringTable")
diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js
@@ -1,5 +1,50 @@
 const EMBEDDED_DATA = {{FLAMEGRAPH_DATA}};
 
+// Global string table for resolving string indices
+let stringTable = [];
+
+// Function to resolve string indices to actual strings
+function resolveString(index) {
+    if (typeof index === 'number' && index >= 0 && index < stringTable.length) {
+        return stringTable[index];
+    }
+    // Fallback for non-indexed strings or invalid indices
+    return String(index);
+}
+
+// Function to recursively resolve all string indices in flamegraph data
+function resolveStringIndices(node) {
+    if (!node) return node;
+    
+    // Create a copy to avoid mutating the original
+    const resolved = { ...node };
+    
+    // Resolve string fields
+    if (typeof resolved.name === 'number') {
+        resolved.name = resolveString(resolved.name);
+    }
+    if (typeof resolved.filename === 'number') {
+        resolved.filename = resolveString(resolved.filename);
+    }
+    if (typeof resolved.funcname === 'number') {
+        resolved.funcname = resolveString(resolved.funcname);
+    }
+    
+    // Resolve source lines if present
+    if (Array.isArray(resolved.source)) {
+        resolved.source = resolved.source.map(index => 
+            typeof index === 'number' ? resolveString(index) : index
+        );
+    }
+    
+    // Recursively resolve children
+    if (Array.isArray(resolved.children)) {
+        resolved.children = resolved.children.map(child => resolveStringIndices(child));
+    }
+    
+    return resolved;
+}
+
 // Python color palette - cold to hot
 const pythonColors = [
   "#fff4bf", // Coldest - light yellow (<1%)
@@ -98,16 +143,20 @@ function createPythonTooltip(data) {
         </div>`;
     }
 
+    // Resolve strings for display
+    const funcname = resolveString(d.data.funcname) || resolveString(d.data.name);
+    const filename = resolveString(d.data.filename) || "";
+    
     const tooltipHTML = `
       <div>
         <div style="color: #3776ab; font-weight: 600; font-size: 16px;
                     margin-bottom: 8px; line-height: 1.3;">
-          ${d.data.funcname || d.data.name}
+          ${funcname}
         </div>
         <div style="color: #5a6c7d; font-size: 13px; margin-bottom: 12px;
                     font-family: monospace; background: #f8f9fa;
                     padding: 4px 8px; border-radius: 4px;">
-          ${d.data.filename || ""}${d.data.lineno ? ":" + d.data.lineno : ""}
+          ${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
         </div>
         <div style="display: grid; grid-template-columns: auto 1fr;
                     gap: 8px 16px; font-size: 14px;">
@@ -255,9 +304,9 @@ function updateSearchHighlight(searchTerm, searchInput) {
     let matchCount = 0;
     d3.selectAll("#chart rect").each(function (d) {
       if (d && d.data) {
-        const name = d.data.name || "";
-        const funcname = d.data.funcname || "";
-        const filename = d.data.filename || "";
+        const name = resolveString(d.data.name) || "";
+        const funcname = resolveString(d.data.funcname) || "";
+        const filename = resolveString(d.data.filename) || "";
         const term = searchTerm.toLowerCase();
         const matches =
           name.toLowerCase().includes(term) ||
@@ -315,12 +364,20 @@ function handleResize(chart, data) {
 
 function initFlamegraph() {
   ensureLibraryLoaded();
-  const tooltip = createPythonTooltip(EMBEDDED_DATA);
-  const chart = createFlamegraph(tooltip, EMBEDDED_DATA.value);
-  renderFlamegraph(chart, EMBEDDED_DATA);
+  
+  // Extract string table if present and resolve string indices
+  let processedData = EMBEDDED_DATA;
+  if (EMBEDDED_DATA.strings) {
+    stringTable = EMBEDDED_DATA.strings;
+    processedData = resolveStringIndices(EMBEDDED_DATA);
+  }
+  
+  const tooltip = createPythonTooltip(processedData);
+  const chart = createFlamegraph(tooltip, processedData.value);
+  renderFlamegraph(chart, processedData);
   attachPanelControls();
   initSearchHandlers();
-  handleResize(chart, EMBEDDED_DATA);
+  handleResize(chart, processedData);
 }
 
 if (document.readyState === "loading") {
@@ -336,7 +393,10 @@ function populateStats(data) {
   const functionMap = new Map();
 
   function collectFunctions(node) {
-    if (node.filename && node.funcname) {
+    const filename = resolveString(node.filename);
+    const funcname = resolveString(node.funcname);
+    
+    if (filename && funcname) {
       // Calculate direct samples (this node's value minus children's values)
       let childrenValue = 0;
       if (node.children) {
@@ -345,23 +405,23 @@ function populateStats(data) {
       const directSamples = Math.max(0, node.value - childrenValue);
 
       // Use file:line:funcname as key to ensure uniqueness
-      const funcKey = `${node.filename}:${node.lineno || '?'}:${node.funcname}`;
+      const funcKey = `${filename}:${node.lineno || '?'}:${funcname}`;
 
       if (functionMap.has(funcKey)) {
         const existing = functionMap.get(funcKey);
         existing.directSamples += directSamples;
         existing.directPercent = (existing.directSamples / totalSamples) * 100;
         // Keep the most representative file/line (the one with more samples)
         if (directSamples > existing.maxSingleSamples) {
-          existing.filename = node.filename;
+          existing.filename = filename;
           existing.lineno = node.lineno || '?';
           existing.maxSingleSamples = directSamples;
         }
       } else {
         functionMap.set(funcKey, {
-          filename: node.filename,
+          filename: filename,
           lineno: node.lineno || '?',
-          funcname: node.funcname,
+          funcname: funcname,
           directSamples,
           directPercent: (directSamples / totalSamples) * 100,
           maxSingleSamples: directSamples
diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py
@@ -7,6 +7,7 @@
 import os
 
 from .collector import Collector
+from .string_table import StringTable
 
 
 class StackTraceCollector(Collector):
@@ -50,6 +51,7 @@ def __init__(self):
         self._root = {"samples": 0, "children": {}}
         self._total_samples = 0
         self._func_intern = {}
+        self._string_table = StringTable()
 
     def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None):
         """Set profiling statistics to include in flamegraph data."""
@@ -63,11 +65,13 @@ def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=
     def export(self, filename):
         flamegraph_data = self._convert_to_flamegraph_format()
 
-        # Debug output
+        # Debug output with string table statistics
         num_functions = len(flamegraph_data.get("children", []))
         total_time = flamegraph_data.get("value", 0)
+        string_count = len(self._string_table)
         print(
-            f"Flamegraph data: {num_functions} root functions, total samples: {total_time}"
+            f"Flamegraph data: {num_functions} root functions, total samples: {total_time}, "
+            f"{string_count} unique strings"
         )
 
         if num_functions == 0:
@@ -96,9 +100,14 @@ def _format_function_name(func):
         return f"{funcname} ({filename}:{lineno})"
 
     def _convert_to_flamegraph_format(self):
-        """Convert aggregated trie to d3-flamegraph format."""
+        """Convert aggregated trie to d3-flamegraph format with string table optimization."""
         if self._total_samples == 0:
-            return {"name": "No Data", "value": 0, "children": []}
+            return {
+                "name": self._string_table.intern("No Data"),
+                "value": 0,
+                "children": [],
+                "strings": self._string_table.get_strings()
+            }
 
         def convert_children(children, min_samples):
             out = []
@@ -107,26 +116,33 @@ def convert_children(children, min_samples):
                 if samples < min_samples:
                     continue
 
-                name = self._format_function_name(func)
+                # Intern all string components for maximum efficiency
+                filename_idx = self._string_table.intern(func[0])
+                funcname_idx = self._string_table.intern(func[2])
+                name_idx = self._string_table.intern(self._format_function_name(func))
+
                 child_entry = {
-                    "name": name,
+                    "name": name_idx,
                     "value": samples,
                     "children": [],
-                    "filename": func[0],
+                    "filename": filename_idx,
                     "lineno": func[1],
-                    "funcname": func[2],
+                    "funcname": funcname_idx,
                 }
 
                 source = self._get_source_lines(func)
                 if source:
-                    child_entry["source"] = source
+                    # Intern source lines for memory efficiency
+                    source_indices = [self._string_table.intern(line) for line in source]
+                    child_entry["source"] = source_indices
 
                 # Recurse
                 child_entry["children"] = convert_children(
                     node["children"], min_samples
                 )
                 out.append(child_entry)
 
+            # Sort by value (descending) then by name index for consistent ordering
             out.sort(key=lambda x: (-x["value"], x["name"]))
             return out
 
@@ -136,16 +152,31 @@ def convert_children(children, min_samples):
 
         root_children = convert_children(self._root["children"], min_samples)
         if not root_children:
-            return {"name": "No significant data", "value": 0, "children": []}
+            return {
+                "name": self._string_table.intern("No significant data"),
+                "value": 0,
+                "children": [],
+                "strings": self._string_table.get_strings()
+            }
 
         # If we only have one root child, make it the root to avoid redundant level
         if len(root_children) == 1:
             main_child = root_children[0]
-            main_child["name"] = f"Program Root: {main_child['name']}"
+            # Update the name to indicate it's the program root
+            old_name = self._string_table.get_string(main_child["name"])
+            new_name = f"Program Root: {old_name}"
+            main_child["name"] = self._string_table.intern(new_name)
             main_child["stats"] = self.stats
+            main_child["strings"] = self._string_table.get_strings()
             return main_child
 
-        return {"name": "Program Root", "value": total_samples, "children": root_children, "stats": self.stats}
+        return {
+            "name": self._string_table.intern("Program Root"),
+            "value": total_samples,
+            "children": root_children,
+            "stats": self.stats,
+            "strings": self._string_table.get_strings()
+        }
 
     def process_frames(self, frames):
         # Reverse to root->leaf
diff --git a/Lib/profiling/sampling/string_table.py b/Lib/profiling/sampling/string_table.py
@@ -0,0 +1,53 @@
+"""String table implementation for memory-efficient string storage in profiling data."""
+
+class StringTable:
+    """A string table for interning strings and reducing memory usage."""
+
+    def __init__(self):
+        self._strings = []
+        self._string_to_index = {}
+
+    def intern(self, string):
+        """Intern a string and return its index.
+
+        Args:
+            string: The string to intern
+
+        Returns:
+            int: The index of the string in the table
+        """
+        if not isinstance(string, str):
+            string = str(string)
+
+        if string in self._string_to_index:
+            return self._string_to_index[string]
+
+        index = len(self._strings)
+        self._strings.append(string)
+        self._string_to_index[string] = index
+        return index
+
+    def get_string(self, index):
+        """Get a string by its index.
+
+        Args:
+            index: The index of the string
+
+        Returns:
+            str: The string at the given index, or empty string if invalid
+        """
+        if 0 <= index < len(self._strings):
+            return self._strings[index]
+        return ""
+
+    def get_strings(self):
+        """Get the list of all strings in the table.
+
+        Returns:
+            list: A copy of the strings list
+        """
+        return self._strings.copy()
+
+    def __len__(self):
+        """Return the number of strings in the table."""
+        return len(self._strings)
diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py
@@ -437,7 +437,11 @@ def test_flamegraph_collector_basic(self):
 
         # Empty collector should produce 'No Data'
         data = collector._convert_to_flamegraph_format()
-        self.assertIn(data["name"], ("No Data", "No significant data"))
+        # With string table, name is now an index - resolve it using the strings array
+        strings = data.get("strings", [])
+        name_index = data.get("name", 0)
+        resolved_name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index)
+        self.assertIn(resolved_name, ("No Data", "No significant data"))
 
         # Test collecting sample data
         test_frames = [
@@ -451,14 +455,18 @@ def test_flamegraph_collector_basic(self):
         # Convert and verify structure: func2 -> func1 with counts = 1
         data = collector._convert_to_flamegraph_format()
         # Expect promotion: root is the single child (func2), with func1 as its only child
-        name = data.get("name", "")
+        strings = data.get("strings", [])
+        name_index = data.get("name", 0)
+        name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index)
         self.assertIsInstance(name, str)
         self.assertTrue(name.startswith("Program Root: "))
         self.assertIn("func2 (file.py:20)", name)  # formatted name
         children = data.get("children", [])
         self.assertEqual(len(children), 1)
         child = children[0]
-        self.assertIn("func1 (file.py:10)", child["name"])  # formatted name
+        child_name_index = child.get("name", 0)
+        child_name = strings[child_name_index] if isinstance(child_name_index, int) and 0 <= child_name_index < len(strings) else str(child_name_index)
+        self.assertIn("func1 (file.py:10)", child_name)  # formatted name
         self.assertEqual(child["value"], 1)
 
     def test_flamegraph_collector_export(self):
