Fix/whats new page generation

docs/_scripts/fetch_releases.py

@@ -1,135 +1,150 @@
 """
-Dynamically build what's new page based on github releases
+Dynamically build the "What's new?" page from the GitHub releases feed.
+
+The release notes on GitHub are written in Markdown and frequently mix raw
+HTML (``<details><summary>`` blocks wrapping fenced code samples). The
+previous implementation converted the body to RST via ``m2r2``, which left
+the inner Markdown code fences inside ``.. raw:: html`` directives — Sphinx
+then rendered them as literal text. This module instead converts each
+release body to HTML (so fences become ``<pre><code>`` elements) and emits
+a single ``.. raw:: html`` block per release wrapped in a styling hook
+``div.uplt-whats-new-release-body``.
 """
 
+from __future__ import annotations
+
 import re
 from pathlib import Path
+from typing import Iterable
 
+import markdown
 import requests
-from m2r2 import convert
 
 GITHUB_REPO = "ultraplot/ultraplot"
 OUTPUT_RST = Path("whats_new.rst")
+GITHUB_API_URL = f"https://api.github.com/repos/{GITHUB_REPO}/releases"
 
+# Markdown extensions: fenced code (for ```python blocks), tables, attribute
+# lists for class hooks, and md_in_html so block-level HTML such as
+# ``<details>`` correctly contains parsed Markdown children.
+_MD_EXTENSIONS = ("fenced_code", "tables", "attr_list", "md_in_html")
+
+# Strip the trailing "by @user in PR_URL" attribution that GitHub auto-adds
+# to release notes. Keep the PR link in parentheses so credit/traceability
+# remains while removing the contributor handles from rendered output.
+# GitHub author handles can include ``[bot]`` suffixes (``@dependabot[bot]``,
+# ``@pre-commit-ci[bot]``); ``\w`` alone misses the brackets.
+_PR_ATTRIBUTION = re.compile(
+    r" by @[\w.\-]+(?:\[bot\])? in (https://github\.com/[^\s]+)"
+)
+
+# Match an ATX heading line, tolerating up-to-3 leading spaces. Authors
+# occasionally indent whole sections by two spaces in the GitHub release
+# editor (e.g. v2.0.1's "### Layout, Rendering, and Geo Improvements"),
+# which python-markdown then parses as a paragraph rather than a heading.
+# We capture the ``#`` run so we can both strip the indent and downgrade
+# one level — the page already provides the H1 ("What's new?") and each
+# release contributes a per-release RST H2, so body headings start at H2.
+_ATX_HEADING = re.compile(r"^[ ]{0,3}(#{1,5})(?=\s)", flags=re.MULTILINE)
+
+
+def _strip_pr_attribution(text: str) -> str:
+    return _PR_ATTRIBUTION.sub(r" (\1)", text)
 
-GITHUB_API_URL = f"https://api.github.com/repos/{GITHUB_REPO}/releases"
 
+def _downgrade_headings(text: str) -> str:
+    """Demote every Markdown ATX heading by one level (``#`` → ``##``, etc.)."""
+    return _ATX_HEADING.sub(lambda m: "#" + m.group(1), text)
 
-def format_release_body(text):
-    """Formats GitHub release notes for better RST readability."""
-    # Convert Markdown to RST using m2r2
-    formatted_text = convert(text)
 
-    formatted_text = _downgrade_headings(formatted_text)
-    formatted_text = formatted_text.replace("→", "->")
-    formatted_text = re.sub(r"^\\s*`\\s*$", "", formatted_text, flags=re.MULTILINE)
+def _normalize_unicode(text: str) -> str:
+    return text.replace("→", "->")
 
-    # Convert PR references (remove "by @user in ..." but keep the link)
-    formatted_text = re.sub(
-        r" by @\w+ in (https://github.com/[^\s]+)", r" (\1)", formatted_text
-    )
 
-    return formatted_text.strip()
+def _indent_html(html: str, indent: str = "   ") -> str:
+    """Indent every line of ``html`` by ``indent`` for inclusion under ``.. raw:: html``."""
+    return "\n".join(indent + line if line else line for line in html.splitlines())
 
 
-def _downgrade_headings(text):
+def format_release_body(text: str) -> str:
     """
-    Downgrade all heading levels by one to avoid H1/H2 collisions in the TOC.
+    Convert a GitHub release body (Markdown + embedded HTML) into an RST
+    ``.. raw:: html`` block wrapped in ``div.uplt-whats-new-release-body``.
+
+    Parameters
+    ----------
+    text : str
+        Raw Markdown release body as returned by the GitHub releases API.
+
+    Returns
+    -------
+    str
+        Indented RST snippet ready to be appended to ``whats_new.rst``.
     """
-    adornment_map = {
-        "=": "-",
-        "-": "~",
-        "~": "^",
-        "^": '"',
-        '"': "'",
-        "'": "`",
-    }
-    lines = text.splitlines()
-    for idx in range(len(lines) - 1):
-        title = lines[idx]
-        underline = lines[idx + 1]
-        if not title.strip():
-            continue
-        if not underline:
-            continue
-        char = underline[0]
-        if char not in adornment_map:
-            continue
-        if underline.strip(char):
-            continue
-        lines[idx + 1] = adornment_map[char] * len(underline)
-    return "\n".join(lines)
-
-
-def fetch_all_releases():
-    """Fetches all GitHub releases across multiple pages."""
-    releases = []
-    page = 1
+    cleaned = _downgrade_headings(
+        _normalize_unicode(_strip_pr_attribution(text or ""))
+    ).strip()
+    html_body = markdown.markdown(cleaned, extensions=list(_MD_EXTENSIONS))
+    wrapped = f'<div class="uplt-whats-new-release-body">\n{html_body}\n</div>'
+    return ".. raw:: html\n\n" + _indent_html(wrapped) + "\n"
+
 
+def _format_release_title(release: dict) -> str:
+    """
+    Build the per-release section title in ``"<tag>: <name>"`` form,
+    de-duplicating the tag if it is already a prefix of the release name.
+    """
+    tag = release["tag_name"].lower()
+    title = (release.get("name") or "").strip()
+    if title.lower().startswith(tag):
+        title = title[len(tag) :].lstrip(" :-—–")
+    return f"{tag}: {title}" if title else tag
+
+
+def fetch_all_releases() -> list[dict]:
+    """Fetch every GitHub release across paginated responses."""
+    releases: list[dict] = []
+    page = 1
     while True:
         response = requests.get(GITHUB_API_URL, params={"per_page": 30, "page": page})
         if response.status_code != 200:
             print(f"Error fetching releases: {response.status_code}")
             break
-
         page_data = response.json()
-        # If the page is empty, stop fetching
         if not page_data:
             break
-
         releases.extend(page_data)
         page += 1
-
     return releases
 
 
-def fetch_releases():
-    """Fetches the latest releases from GitHub and formats them as RST."""
-    releases = fetch_all_releases()
-    if not releases:
-        print(f"Error fetching releases!")
-        return ""
-
+def _render_releases(releases: Iterable[dict]) -> str:
+    """Render an iterable of release dicts to the full ``whats_new.rst`` body."""
     header = "What's new?"
-    rst_content = f".. _whats_new:\n\n{header}\n{'=' * len(header)}\n\n"  # H1
-
+    out = f".. _whats_new:\n\n{header}\n{'=' * len(header)}\n\n"
     for release in releases:
-        # ensure title is formatted as {tag}: {title}
-        tag = release["tag_name"].lower()
-        title = release["name"]
-        if title.startswith(tag):
-            title = title[len(tag) :]
-            while title:
-                if not title[0].isalpha():
-                    title = title[1:]
-                    title = title.strip()
-                else:
-                    title = title.strip()
-                    break
-
-        if title:
-            title = f"{tag}: {title}"
-        else:
-            title = tag
-
+        title = _format_release_title(release)
         date = release["published_at"][:10]
-        body = format_release_body(release["body"] or "")
-
-        # Version header (H2)
-        rst_content += f"{title} ({date})\n{'-' * (len(title) + len(date) + 3)}\n\n"
+        heading = f"{title} ({date})"
+        out += f"{heading}\n{'-' * len(heading)}\n\n"
+        out += format_release_body(release.get("body") or "") + "\n"
+    return out
 
-        # Process body content
-        rst_content += f"{body}\n\n"
 
-    return rst_content
+def fetch_releases() -> str:
+    """Fetch the latest releases from GitHub and format them as RST."""
+    releases = fetch_all_releases()
+    if not releases:
+        print("Error fetching releases!")
+        return ""
+    return _render_releases(releases)
 
 
-def write_rst():
-    """Writes fetched releases to an RST file."""
+def write_rst() -> None:
+    """Write fetched releases to ``whats_new.rst``."""
     content = fetch_releases()
     if content:
-        with open(OUTPUT_RST, "w", encoding="utf-8") as f:
-            f.write(content)
+        OUTPUT_RST.write_text(content, encoding="utf-8")
         print(f"Updated {OUTPUT_RST}")
     else:
         print("No updates to write.")

docs/_static/custom.css

@@ -1212,3 +1212,41 @@ body.wy-body-for-nav
     border-radius: 0.2rem;
     background: var(--uplt-color-sidebar-bg);
 }
+
+/* What's-new release bodies generated by docs/_scripts/fetch_releases.py.
+   Each release is emitted inside a <div class="uplt-whats-new-release-body">
+   so we have a single styling hook for spacing, code-block padding, and the
+   <details> snippet affordance. */
+.uplt-whats-new-release-body {
+    margin-top: 0.5rem;
+    margin-bottom: 1.5rem;
+}
+
+.uplt-whats-new-release-body h2,
+.uplt-whats-new-release-body h3,
+.uplt-whats-new-release-body h4 {
+    margin-top: 1.25rem;
+    margin-bottom: 0.5rem;
+}
+
+.uplt-whats-new-release-body details {
+    margin: 0.5rem 0 1rem;
+    padding: 0.25rem 0.75rem;
+    border: 1px solid var(--uplt-color-sidebar-bg, #e5e7eb);
+    border-radius: 0.35rem;
+    background: var(--uplt-color-sidebar-bg, #f9fafb);
+}
+
+.uplt-whats-new-release-body details > summary {
+    cursor: pointer;
+    font-weight: 600;
+}
+
+.uplt-whats-new-release-body pre {
+    overflow-x: auto;
+}
+
+.uplt-whats-new-release-body img {
+    max-width: 100%;
+    height: auto;
+}

environment.yml

@@ -20,5 +20,6 @@ dependencies:
   - networkx
   - pyarrow
   - cftime
+  - markdown
   - pip:
       - pycirclize

pyproject.toml

@@ -74,7 +74,7 @@ docs = [
     "jupyter",
     "jupytext",
     "lxml-html-clean",
-    "m2r2",
+    "markdown",
     "mpltern",
     "nbsphinx",
     "sphinx",