Share more logic for system prompt templating (#727)

Author: aantn

CLAUDE.md

@@ -234,3 +234,12 @@ Check in pyproject.toml and NEVER use a marker/tag that doesn't exist there. Ask
   - `runbooks: {}` - No runbooks available (empty catalog)
   - `runbooks: {catalog: [...]}` - Custom runbook catalog with entries pointing to .md files in the same directory
   - If `runbooks` field is not specified, default system runbooks are used
+
+## Documentation Lookup
+
+When asked about content from the HolmesGPT documentation website (https://robusta-dev.github.io/holmesgpt/), look in the local `docs/` directory:
+- Python SDK examples: `docs/installation/python-installation.md`
+- CLI installation: `docs/installation/cli-installation.md`
+- Kubernetes deployment: `docs/installation/kubernetes-installation.md`
+- Toolset documentation: `docs/data-sources/builtin-toolsets/`
+- API reference: `docs/reference/`

docs/installation/python-installation.md

@@ -18,7 +18,8 @@ pip install "https://github.com/robusta-dev/holmesgpt/archive/refs/heads/master.
 ```python
 import os
 from holmes.config import Config
-from holmes.plugins.prompts import load_and_render_prompt
+from holmes.core.prompt import build_initial_ask_messages
+from rich.console import Console
 
 print("🚀 Initializing HolmesGPT...")
 
@@ -31,22 +32,28 @@ config = Config(
 )
 print(f"✅ Configuration created with model: {config.model}")
 
-# Create AI instance
+# Create AI instance and console
 print("Creating AI instance...")
 ai = config.create_console_toolcalling_llm()
+console = Console()
 print("✅ AI instance ready")
 
 # Ask a question
-print("Loading system prompt...")
-system_prompt = load_and_render_prompt(
-    "builtin://generic_ask.jinja2",
-    {"toolsets": ai.tool_executor.toolsets}
+question = "what pods are failing in production?"
+print(f"\n🔍 Asking: '{question}'")
+
+# Build initial messages with system prompt
+messages = build_initial_ask_messages(
+    console=console,
+    initial_user_prompt=question,
+    file_paths=None,
+    tool_executor=ai.tool_executor,
+    runbooks=config.get_runbook_catalog(),
+    system_prompt_additions=None
 )
-print("✅ System prompt loaded")
 
-print("\n🔍 Asking: 'what pods are failing in production?'")
 print("Holmes is thinking...")
-response = ai.prompt_call(system_prompt, "what pods are failing in production?")
+response = ai.call(messages)
 print(f"Holmes: {response.result}")
 ```
 
@@ -62,7 +69,8 @@ Complete example of using HolmesGPT Python SDK with progress tracking
 
 import os
 from holmes.config import Config
-from holmes.plugins.prompts import load_and_render_prompt
+from holmes.core.prompt import build_initial_ask_messages
+from rich.console import Console
 
 def main():
     print("🚀 Starting HolmesGPT Python SDK Example")
@@ -81,8 +89,9 @@ def main():
     print(f"✅ Configuration created with model: {config.model}")
 
     print("\nStep 2: Creating AI instance...")
-    # Create AI instance
+    # Create AI instance and console
     ai = config.create_console_toolcalling_llm()
+    console = Console()
     print("✅ AI instance created successfully")
 
     print("\nStep 3: Listing available toolsets...")
@@ -99,16 +108,7 @@ def main():
     for tool in sorted(available_tools):
         print(f"   • {tool}")
 
-    print("\nStep 5: Loading system prompt...")
-    # Load system prompt
-    system_prompt = load_and_render_prompt(
-        "builtin://generic_ask.jinja2",
-        {"toolsets": ai.tool_executor.toolsets}
-    )
-    print("✅ System prompt loaded successfully")
-    print(f"Prompt length: {len(system_prompt)} characters")
-
-    print("\nStep 6: Asking questions...")
+    print("\nStep 5: Asking questions...")
     # Ask questions
     questions = [
         "what pods are failing in production?",
@@ -122,7 +122,18 @@ def main():
 
         try:
             print("Holmes is thinking...")
-            response = ai.prompt_call(system_prompt, question)
+
+            # Build initial messages
+            messages = build_initial_ask_messages(
+                console=console,
+                initial_user_prompt=question,
+                file_paths=None,
+                tool_executor=ai.tool_executor,
+                runbooks=config.get_runbook_catalog(),
+                system_prompt_additions=None
+            )
+
+            response = ai.call(messages)
             print(f"Holmes: {response.result}")
 
             # Show tools that were used
@@ -200,20 +211,19 @@ def main():
     ai = config.create_console_toolcalling_llm()
     console = Console()
 
-    # Load system prompt
-    system_prompt = load_and_render_prompt(
-        "builtin://generic_ask.jinja2",
-        {"toolsets": ai.tool_executor.toolsets}
-    )
-
     # First question
     print("\n🔍 First Question:")
     first_question = "what pods are failing in my cluster?"
     print(f"User: {first_question}")
 
-    # Build initial messages (system + first user message)
+    # Build initial messages (includes system prompt + first user message)
     messages = build_initial_ask_messages(
-        console, system_prompt, first_question, None
+        console=console,
+        initial_user_prompt=first_question,
+        file_paths=None,
+        tool_executor=ai.tool_executor,
+        runbooks=config.get_runbook_catalog(),
+        system_prompt_additions=None
     )
 
     # Call AI with initial messages

holmes/core/prompt.py

@@ -1,11 +1,13 @@
 from rich.console import Console
-from typing import Optional, List, Dict
+from typing import Optional, List, Dict, Any, Union
 from pathlib import Path
+from holmes.plugins.prompts import load_and_render_prompt
+from holmes.plugins.runbooks import RunbookCatalog
 
 
 def append_file_to_user_prompt(user_prompt: str, file_path: Path) -> str:
     with file_path.open("r") as f:
-        user_prompt += f"\n\n<attached-file path='{file_path.absolute()}>'\n{f.read()}\n</attached-file>"
+        user_prompt += f"\n\n<attached-file path='{file_path.absolute()}'>\n{f.read()}\n</attached-file>"
 
     return user_prompt
 
@@ -25,11 +27,34 @@ def append_all_files_to_user_prompt(
 
 def build_initial_ask_messages(
     console: Console,
-    system_prompt_rendered: str,
     initial_user_prompt: str,
     file_paths: Optional[List[Path]],
+    tool_executor: Any,  # ToolExecutor type
+    runbooks: Union[RunbookCatalog, Dict, None] = None,
+    system_prompt_additions: Optional[str] = None,
 ) -> List[Dict]:
-    """Build the initial messages for the AI call."""
+    """Build the initial messages for the AI call.
+
+    Args:
+        console: Rich console for output
+        initial_user_prompt: The user's prompt
+        file_paths: Optional list of files to include
+        tool_executor: The tool executor with available toolsets
+        runbooks: Optional runbook catalog
+        system_prompt_additions: Optional additional system prompt content
+    """
+    # Load and render system prompt internally
+    system_prompt_template = "builtin://generic_ask.jinja2"
+    template_context = {
+        "toolsets": tool_executor.toolsets,
+        "runbooks": runbooks or {},
+        "system_prompt_additions": system_prompt_additions or "",
+    }
+    system_prompt_rendered = load_and_render_prompt(
+        system_prompt_template, template_context
+    )
+
+    # Append files to user prompt
     user_prompt_with_files = append_all_files_to_user_prompt(
         console, initial_user_prompt, file_paths
     )

holmes/interactive.py

@@ -761,12 +761,13 @@ def display_recent_tool_outputs(
 def run_interactive_loop(
     ai: ToolCallingLLM,
     console: Console,
-    system_prompt_rendered: str,
     initial_user_input: Optional[str],
     include_files: Optional[List[Path]],
     post_processing_prompt: Optional[str],
     show_tool_output: bool,
     tracer=None,
+    runbooks=None,
+    system_prompt_additions: Optional[str] = None,
 ) -> None:
     # Initialize tracer - use DummyTracer if no tracer provided
     if tracer is None:
@@ -965,7 +966,12 @@ def get_bottom_toolbar():
 
             if messages is None:
                 messages = build_initial_ask_messages(
-                    console, system_prompt_rendered, user_input, include_files
+                    console,
+                    user_input,
+                    include_files,
+                    ai.tool_executor,
+                    runbooks,
+                    system_prompt_additions,
                 )
             else:
                 messages.append({"role": "user", "content": user_input})

holmes/main.py

@@ -180,10 +180,6 @@ def ask(
     destination: Optional[DestinationType] = opt_destination,
     slack_token: Optional[str] = opt_slack_token,
     slack_channel: Optional[str] = opt_slack_channel,
-    # advanced options for this command
-    system_prompt: Optional[str] = typer.Option(
-        "builtin://generic_ask.jinja2", help=system_prompt_help
-    ),
     show_tool_output: bool = typer.Option(
         False,
         "--show-tool-output",
@@ -214,6 +210,11 @@ def ask(
         "--trace",
         help="Enable tracing to the specified provider (e.g., 'braintrust')",
     ),
+    system_prompt_additions: Optional[str] = typer.Option(
+        None,
+        "--system-prompt-additions",
+        help="Additional content to append to the system prompt",
+    ),
 ):
     """
     Ask any question and answer using available tools
@@ -252,12 +253,6 @@ def ask(
         refresh_toolsets=refresh_toolsets,  # flag to refresh the toolset status
         tracer=tracer,
     )
-    template_context = {
-        "toolsets": ai.tool_executor.toolsets,
-        "runbooks": config.get_runbook_catalog(),
-    }
-
-    system_prompt_rendered = load_and_render_prompt(system_prompt, template_context)  # type: ignore
 
     if prompt_file and prompt:
         raise typer.BadParameter(
@@ -292,20 +287,23 @@ def ask(
         run_interactive_loop(
             ai,
             console,
-            system_prompt_rendered,
             prompt,
             include_file,
             post_processing_prompt,
             show_tool_output,
             tracer,
+            config.get_runbook_catalog(),
+            system_prompt_additions,
         )
         return
 
     messages = build_initial_ask_messages(
         console,
-        system_prompt_rendered,
         prompt,  # type: ignore
         include_file,
+        ai.tool_executor,
+        config.get_runbook_catalog(),
+        system_prompt_additions,
     )
 
     with tracer.start_trace(

holmes/plugins/prompts/generic_ask.jinja2

@@ -36,3 +36,7 @@ Relevant logs:
 ```
 
 Validation error led to unhandled Java exception causing a crash.
+
+{% if system_prompt_additions %}
+{{ system_prompt_additions }}
+{% endif %}