Implement command sidebar optimization to fix 15+ second build times

COMMANDS_SIDEBAR_OPTIMIZATION.md

@@ -0,0 +1,31 @@
+# Command Sidebar Optimization
+
+## Problem
+The original command page template had a performance issue where each command page would regenerate the complete sidebar by:
+1. Loading all command section pages (potentially 400+)
+2. For each page, attempting to load 4 different JSON files
+3. Processing all command data to build sidebar entries
+
+This resulted in O(n²) scaling where n = number of commands, leading to build times of 15+ seconds.
+
+## Solution
+Implemented a two-tier optimization approach:
+
+### 1. Pre-generation (Optimal)
+- `build/generate-commands-sidebar.py` processes all command JSON files once during build setup
+- Outputs `_data/commands_sidebar.json` containing pre-computed sidebar data
+- Template loads this single file instead of processing hundreds of JSON files per page
+- Reduces complexity from O(n²) to O(1) per page render
+
+### 2. Fallback (Graceful Degradation)  
+- If pre-generated file doesn't exist, template falls back to original dynamic processing
+- Ensures the site builds correctly even without the optimization script
+- Maintains backward compatibility
+
+## Integration
+The optimization is integrated into `build/init-commands.sh` which runs the pre-generation script after creating command stub files.
+
+## Performance Impact
+- Expected build time reduction: 15+ seconds → <1 second for command processing
+- Eliminates quadratic scaling issue
+- Maintains identical sidebar functionality

_data/commands_sidebar.json

@@ -0,0 +1,3 @@
+{
+  "commands": []
+}

build/generate-commands-sidebar.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""
+Generate commands sidebar data for Valkey documentation site.
+
+This script processes all command JSON files and creates a single
+_data/commands_sidebar.json file containing all command entries.
+This eliminates the need for the template to process hundreds of 
+JSON files on every command page render.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def find_command_json_dirs():
+    """Find all command JSON directories based on symlinks."""
+    base_dir = Path(".")
+    json_dirs = []
+
+    for item in base_dir.iterdir():
+        if item.name.startswith("build-") and item.name.endswith("-command-json") and item.is_symlink():
+            json_dirs.append(item)
+
+    return json_dirs
+
+
+def process_command_json(json_path, slug):
+    """Process a single command JSON file and extract relevant data."""
+    try:
+        with open(json_path, 'r') as f:
+            data = json.load(f)
+
+        # Find the command object name (there should be only one key)
+        command_obj_name = list(data.keys())[0]
+        command_obj = data[command_obj_name]
+
+        # Build command display name
+        command_display = command_obj_name
+        if command_obj.get("container"):
+            command_display = f"{command_obj['container']} {command_display}"
+
+        return {
+            "display": command_display,
+            "permalink": f"/commands/{slug}/",
+            "summary": command_obj.get("summary", ""),
+            "group": command_obj.get("group", "")
+        }
+
+    except (json.JSONDecodeError, KeyError, FileNotFoundError) as e:
+        print(f"Warning: Could not process {json_path}: {e}", file=sys.stderr)
+        return None
+
+
+def generate_commands_sidebar():
+    """Generate the commands sidebar data file."""
+    commands_entries = []
+
+    # Find all command JSON directories
+    json_dirs = find_command_json_dirs()
+
+    if not json_dirs:
+        print("Warning: No command JSON directories found", file=sys.stderr)
+        # Create empty data file
+        output_data = {"commands": []}
+    else:
+        # Process all JSON files in all directories
+        for json_dir in json_dirs:
+            if not json_dir.exists():
+                print(f"Warning: {json_dir} symlink target does not exist", file=sys.stderr)
+                continue
+
+            for json_file in json_dir.glob("*.json"):
+                slug = json_file.stem
+                command_data = process_command_json(json_file, slug)
+
+                if command_data:
+                    commands_entries.append([
+                        command_data["display"],
+                        command_data["permalink"],
+                        command_data["summary"],
+                        command_data["group"]
+                    ])
+
+        output_data = {"commands": commands_entries}
+
+    # Write the output file
+    output_path = Path("_data/commands_sidebar.json")
+    output_path.parent.mkdir(exist_ok=True)
+
+    with open(output_path, 'w') as f:
+        json.dump(output_data, f, indent=2)
+
+    print(f"Generated {output_path} with {len(commands_entries)} commands")
+
+
+if __name__ == "__main__":
+    generate_commands_sidebar()

build/init-commands.sh

@@ -73,6 +73,10 @@ done
 
 echo "Command stub files created."
 
+# Generate optimized commands sidebar data
+echo "Generating commands sidebar data..."
+python3 build/generate-commands-sidebar.py
+
 for datafile in groups.json resp2_replies.json resp3_replies.json modules.json; do
     ln -s "../${1}/../${datafile}" "./_data/${datafile}"
 

templates/command-page.html

@@ -226,33 +226,41 @@ <h3>History</h3>
 {% block related_content %}
 {# Set up variables for sidebar, similar to commands.html #}
 {% set_global group_descriptions = load_data(path= "../_data/groups.json", required= false) %}
-{% set commands_entries = [] %}
-{% set commands_section = get_section(path="commands/_index.md") %}
-{% for page in commands_section.pages %}
-    {% for json_path in [
-        commands::command_json_path(slug=page.slug),
-        commands::command_bloom_json_path(slug=page.slug),
-        commands::command_json_json_path(slug=page.slug),
-        commands::command_search_json_path(slug=page.slug)
-    ] %}
-        {% set command_data = load_data(path= json_path, required= false) %}
-        {% if command_data %}
-            {% set command_obj_name = commands::command_obj_name(command_data= command_data) %}
-            {% set list_command_data_obj = command_data[command_obj_name] %}
-            {% set command_display = command_obj_name %}
-            {% if list_command_data_obj.container %}
-                {% set command_display = list_command_data_obj.container ~ " " ~ command_display %}
+
+{# Load commands data from a pre-generated file if it exists, otherwise compute it #}
+{% set commands_data_file = load_data(path= "../_data/commands_sidebar.json", required= false) %}
+{% if commands_data_file %}
+    {% set commands_entries = commands_data_file.commands %}
+{% else %}
+    {# Fallback to computing commands entries dynamically #}
+    {% set commands_entries = [] %}
+    {% set commands_section = get_section(path="commands/_index.md") %}
+    {% for page in commands_section.pages %}
+        {% for json_path in [
+            commands::command_json_path(slug=page.slug),
+            commands::command_bloom_json_path(slug=page.slug),
+            commands::command_json_json_path(slug=page.slug),
+            commands::command_search_json_path(slug=page.slug)
+        ] %}
+            {% set command_data = load_data(path= json_path, required= false) %}
+            {% if command_data %}
+                {% set command_obj_name = commands::command_obj_name(command_data= command_data) %}
+                {% set list_command_data_obj = command_data[command_obj_name] %}
+                {% set command_display = command_obj_name %}
+                {% if list_command_data_obj.container %}
+                    {% set command_display = list_command_data_obj.container ~ " " ~ command_display %}
+                {% endif %}
+                {% set command_entry = [
+                    command_display,
+                    page.permalink | safe,
+                    list_command_data_obj.summary,
+                    list_command_data_obj.group
+                ] %}
+                {% set commands_entries = commands_entries | concat(with= [ command_entry ]) %}
             {% endif %}
-            {% set command_entry = [
-                command_display,
-                page.permalink | safe,
-                list_command_data_obj.summary,
-                list_command_data_obj.group
-            ] %}
-            {% set_global commands_entries = commands_entries | concat(with= [ command_entry ]) %}
-        {% endif %}
+        {% endfor %}
     {% endfor %}
-{% endfor %}
+{% endif %}
 
 <div class="sb-search-container">
     <input type="text" id="sidebar-search-box" placeholder="Search within documents" onkeyup="searchSidebarCommands()" />