Generate spec index pages in mkdocs (#4514)

When clicking on the "specs" links in the readme, it would show a 404.
The simplest solution to this is to generate a basic index page for each
spec. The "tests" link will still 404 but that's alright, no great way
to fix that.

Fixes #4472.

Author: bshastry

.github/workflows/docs.yml

@@ -1,24 +1,36 @@
 
 name: Publish docs
+
+defaults:
+  run:
+    shell: zsh -e {0}
+
 on:
   push:
     branches:
       - master
+
 permissions:
   contents: write
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+
+      - name: Install dependencies to venv
+        run: make pyspec
+
       - name: Build docs
         run: make _copy_docs
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: 3.x
-      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          key: ${{ github.ref }}
-          path: .cache
-      - run: pip install mkdocs==1.4.2 mkdocs-material==9.1.5 mdx-truly-sane-lists==1.3 mkdocs-awesome-pages-plugin==2.8.0
-      - run: mkdocs gh-deploy --force
+
+      - name: Deploy
+        run: venv/bin/mkdocs gh-deploy --force

Makefile

@@ -55,6 +55,7 @@ PYTHON_VENV = $(VENV)/bin/python3
 PIP_VENV = $(VENV)/bin/pip3
 CODESPELL_VENV = $(VENV)/bin/codespell
 MDFORMAT_VENV = $(VENV)/bin/mdformat
+MKDOCS_VENV = $(VENV)/bin/mkdocs
 
 # Make a virtual environment.
 $(VENV):
@@ -164,9 +165,9 @@ _copy_docs:
 	@cp $(CURDIR)/README.md $(DOCS_DIR)/README.md
 
 # Start a local documentation server.
-serve_docs: _copy_docs
-	@mkdocs build
-	@mkdocs serve
+serve_docs: pyspec _copy_docs
+	@$(MKDOCS_VENV) build
+	@$(MKDOCS_VENV) serve
 
 ###############################################################################
 # Checks

mkdocs.yml

@@ -32,6 +32,9 @@ markdown_extensions:
 plugins:
   - search
   - awesome-pages
+  - gen-files:
+      scripts:
+        - scripts/gen_spec_indices.py
 extra_css:
   - stylesheets/extra.css
 extra:

pyproject.toml

@@ -59,6 +59,7 @@ generator = [
 docs = [
   "mdx-truly-sane-lists==1.3",
   "mkdocs-awesome-pages-plugin==2.10.1",
+  "mkdocs-gen-files==0.5.0",
   "mkdocs-material==9.6.17",
   "mkdocs==1.6.1",
 ]

scripts/gen_spec_indices.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""
+Generate index.md files for specification directories to enable directory browsing in mkdocs.
+
+This script is used by the mkdocs-gen-files plugin to create virtual index pages
+that list all files in each specification directory, providing a GitHub-like
+directory browsing experience on the mkdocs documentation site.
+"""
+
+import os
+import mkdocs_gen_files
+
+
+def format_filename_as_title(filename: str) -> str:
+    """Convert a filename to a human-readable title."""
+    name = filename[-3] if filename.endswith(".md") else filename
+
+    replacements = {
+        "api": "API",
+        "bls": "BLS",
+        "das": "DAS",
+        "p2p": "P2P",
+        "ssz": "SSZ",
+    }
+
+    name = name.replace("-", " ").replace("_", " ")
+
+    words = name.split()
+    formatted_words = []
+    for word in words:
+        lower_word = word.lower()
+        if lower_word in replacements:
+            formatted_words.append(replacements[lower_word])
+        else:
+            formatted_words.append(word.title())
+
+    return " ".join(formatted_words)
+
+
+def generate_spec_index(dir_path: str, level: int = 1) -> str:
+    """Generate index content for a specification directory."""
+    files = []
+    subdirs = []
+
+    if os.path.exists(dir_path):
+        for item in sorted(os.listdir(dir_path)):
+            item_path = os.path.join(dir_path, item)
+            if os.path.isdir(item_path):
+                subdirs.append(item)
+            elif item.endswith(".md") and item != "index.md":
+                files.append(item)
+
+    content = ""
+
+    if level == 1:
+        content = "# Index\n\n"
+        if files:
+            content += "## Core\n\n"
+
+    for file in files:
+        name = format_filename_as_title(file)
+        content += f"- [{name}](./{file})\n"
+
+    for subdir in subdirs:
+        formatted_name = format_filename_as_title(subdir)
+        heading_level = "#" * (level + 1)
+        content += f"\n{heading_level} {formatted_name}\n\n"
+        subdir_path = os.path.join(dir_path, subdir)
+        subdir_content = generate_spec_index(subdir_path, level + 1)
+        if subdir_content.strip():
+            content += subdir_content
+        else:
+            content += f"*No files in {subdir}/*\n"
+
+    if not files and not subdirs and level == 1:
+        content += "*No specification files found in this directory.*\n"
+
+    return content
+
+
+print("Generating specification index pages...")
+
+spec_forks = []
+if os.path.exists("specs"):
+    for item in sorted(os.listdir("specs")):
+        item_path = os.path.join("specs", item)
+        if os.path.isdir(item_path) and item not in {"_deprecated", "_features"}:
+            spec_forks.append(item)
+
+for fork in spec_forks:
+    spec_path = f"specs/{fork}"
+    print(f"  - Generating index for {spec_path}")
+    with mkdocs_gen_files.open(f"{spec_path}/index.md", "w") as f:
+        f.write(generate_spec_index(spec_path))

specs/gloas/beacon-chain.md

@@ -79,8 +79,8 @@
 This is the beacon chain specification of the enshrined proposer builder
 separation feature.
 
-*Note*: This specification is built upon [Fulu](../../fulu/beacon-chain.md) and
-is under active development.
+*Note*: This specification is built upon [Fulu](../fulu/beacon-chain.md) and is
+under active development.
 
 This feature adds new staked consensus participants called *Builders* and new
 honest validators duties called *payload timeliness attestations*. The slot is