diff --git a/COMMANDS_SIDEBAR_OPTIMIZATION.md b/COMMANDS_SIDEBAR_OPTIMIZATION.md new file mode 100644 index 00000000..1e6d58e9 --- /dev/null +++ b/COMMANDS_SIDEBAR_OPTIMIZATION.md @@ -0,0 +1,31 @@ +# Command Sidebar Optimization + +## Problem +The original command page template had a performance issue where each command page would regenerate the complete sidebar by: +1. Loading all command section pages (potentially 400+) +2. For each page, attempting to load 4 different JSON files +3. Processing all command data to build sidebar entries + +This resulted in O(n²) scaling where n = number of commands, leading to build times of 15+ seconds. + +## Solution +Implemented a two-tier optimization approach: + +### 1. Pre-generation (Optimal) +- `build/generate-commands-sidebar.py` processes all command JSON files once during build setup +- Outputs `_data/commands_sidebar.json` containing pre-computed sidebar data +- Template loads this single file instead of processing hundreds of JSON files per page +- Reduces complexity from O(n²) to O(1) per page render + +### 2. Fallback (Graceful Degradation) +- If pre-generated file doesn't exist, template falls back to original dynamic processing +- Ensures the site builds correctly even without the optimization script +- Maintains backward compatibility + +## Integration +The optimization is integrated into `build/init-commands.sh` which runs the pre-generation script after creating command stub files. + +## Performance Impact +- Expected build time reduction: 15+ seconds → <1 second for command processing +- Eliminates quadratic scaling issue +- Maintains identical sidebar functionality \ No newline at end of file diff --git a/_data/commands_sidebar.json b/_data/commands_sidebar.json new file mode 100644 index 00000000..ac225f03 --- /dev/null +++ b/_data/commands_sidebar.json @@ -0,0 +1,3 @@ +{ + "commands": [] +} \ No newline at end of file diff --git a/build/generate-commands-sidebar.py b/build/generate-commands-sidebar.py new file mode 100755 index 00000000..885f9b4c --- /dev/null +++ b/build/generate-commands-sidebar.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +""" +Generate commands sidebar data for Valkey documentation site. + +This script processes all command JSON files and creates a single +_data/commands_sidebar.json file containing all command entries. +This eliminates the need for the template to process hundreds of +JSON files on every command page render. +""" + +import json +import os +import sys +from pathlib import Path + + +def find_command_json_dirs(): + """Find all command JSON directories based on symlinks.""" + base_dir = Path(".") + json_dirs = [] + + for item in base_dir.iterdir(): + if item.name.startswith("build-") and item.name.endswith("-command-json") and item.is_symlink(): + json_dirs.append(item) + + return json_dirs + + +def process_command_json(json_path, slug): + """Process a single command JSON file and extract relevant data.""" + try: + with open(json_path, 'r') as f: + data = json.load(f) + + # Find the command object name (there should be only one key) + command_obj_name = list(data.keys())[0] + command_obj = data[command_obj_name] + + # Build command display name + command_display = command_obj_name + if command_obj.get("container"): + command_display = f"{command_obj['container']} {command_display}" + + return { + "display": command_display, + "permalink": f"/commands/{slug}/", + "summary": command_obj.get("summary", ""), + "group": command_obj.get("group", "") + } + + except (json.JSONDecodeError, KeyError, FileNotFoundError) as e: + print(f"Warning: Could not process {json_path}: {e}", file=sys.stderr) + return None + + +def generate_commands_sidebar(): + """Generate the commands sidebar data file.""" + commands_entries = [] + + # Find all command JSON directories + json_dirs = find_command_json_dirs() + + if not json_dirs: + print("Warning: No command JSON directories found", file=sys.stderr) + # Create empty data file + output_data = {"commands": []} + else: + # Process all JSON files in all directories + for json_dir in json_dirs: + if not json_dir.exists(): + print(f"Warning: {json_dir} symlink target does not exist", file=sys.stderr) + continue + + for json_file in json_dir.glob("*.json"): + slug = json_file.stem + command_data = process_command_json(json_file, slug) + + if command_data: + commands_entries.append([ + command_data["display"], + command_data["permalink"], + command_data["summary"], + command_data["group"] + ]) + + output_data = {"commands": commands_entries} + + # Write the output file + output_path = Path("_data/commands_sidebar.json") + output_path.parent.mkdir(exist_ok=True) + + with open(output_path, 'w') as f: + json.dump(output_data, f, indent=2) + + print(f"Generated {output_path} with {len(commands_entries)} commands") + + +if __name__ == "__main__": + generate_commands_sidebar() \ No newline at end of file diff --git a/build/init-commands.sh b/build/init-commands.sh index 6e08a4f4..90e6b117 100755 --- a/build/init-commands.sh +++ b/build/init-commands.sh @@ -73,6 +73,10 @@ done echo "Command stub files created." +# Generate optimized commands sidebar data +echo "Generating commands sidebar data..." +python3 build/generate-commands-sidebar.py + for datafile in groups.json resp2_replies.json resp3_replies.json modules.json; do ln -s "../${1}/../${datafile}" "./_data/${datafile}" diff --git a/templates/command-page.html b/templates/command-page.html index 82093fab..de2ca115 100644 --- a/templates/command-page.html +++ b/templates/command-page.html @@ -226,33 +226,41 @@

History

{% block related_content %} {# Set up variables for sidebar, similar to commands.html #} {% set_global group_descriptions = load_data(path= "../_data/groups.json", required= false) %} -{% set commands_entries = [] %} -{% set commands_section = get_section(path="commands/_index.md") %} -{% for page in commands_section.pages %} - {% for json_path in [ - commands::command_json_path(slug=page.slug), - commands::command_bloom_json_path(slug=page.slug), - commands::command_json_json_path(slug=page.slug), - commands::command_search_json_path(slug=page.slug) - ] %} - {% set command_data = load_data(path= json_path, required= false) %} - {% if command_data %} - {% set command_obj_name = commands::command_obj_name(command_data= command_data) %} - {% set list_command_data_obj = command_data[command_obj_name] %} - {% set command_display = command_obj_name %} - {% if list_command_data_obj.container %} - {% set command_display = list_command_data_obj.container ~ " " ~ command_display %} + +{# Load commands data from a pre-generated file if it exists, otherwise compute it #} +{% set commands_data_file = load_data(path= "../_data/commands_sidebar.json", required= false) %} +{% if commands_data_file %} + {% set commands_entries = commands_data_file.commands %} +{% else %} + {# Fallback to computing commands entries dynamically #} + {% set commands_entries = [] %} + {% set commands_section = get_section(path="commands/_index.md") %} + {% for page in commands_section.pages %} + {% for json_path in [ + commands::command_json_path(slug=page.slug), + commands::command_bloom_json_path(slug=page.slug), + commands::command_json_json_path(slug=page.slug), + commands::command_search_json_path(slug=page.slug) + ] %} + {% set command_data = load_data(path= json_path, required= false) %} + {% if command_data %} + {% set command_obj_name = commands::command_obj_name(command_data= command_data) %} + {% set list_command_data_obj = command_data[command_obj_name] %} + {% set command_display = command_obj_name %} + {% if list_command_data_obj.container %} + {% set command_display = list_command_data_obj.container ~ " " ~ command_display %} + {% endif %} + {% set command_entry = [ + command_display, + page.permalink | safe, + list_command_data_obj.summary, + list_command_data_obj.group + ] %} + {% set commands_entries = commands_entries | concat(with= [ command_entry ]) %} {% endif %} - {% set command_entry = [ - command_display, - page.permalink | safe, - list_command_data_obj.summary, - list_command_data_obj.group - ] %} - {% set_global commands_entries = commands_entries | concat(with= [ command_entry ]) %} - {% endif %} + {% endfor %} {% endfor %} -{% endfor %} +{% endif %}