diff --git a/docs/_scripts/fetch_releases.py b/docs/_scripts/fetch_releases.py index cb8e78505..d71cff3a6 100644 --- a/docs/_scripts/fetch_releases.py +++ b/docs/_scripts/fetch_releases.py @@ -1,135 +1,150 @@ """ -Dynamically build what's new page based on github releases +Dynamically build the "What's new?" page from the GitHub releases feed. + +The release notes on GitHub are written in Markdown and frequently mix raw +HTML (``
`` blocks wrapping fenced code samples). The +previous implementation converted the body to RST via ``m2r2``, which left +the inner Markdown code fences inside ``.. raw:: html`` directives — Sphinx +then rendered them as literal text. This module instead converts each +release body to HTML (so fences become ``
`` elements) and emits
+a single ``.. raw:: html`` block per release wrapped in a styling hook
+``div.uplt-whats-new-release-body``.
 """
 
+from __future__ import annotations
+
 import re
 from pathlib import Path
+from typing import Iterable
 
+import markdown
 import requests
-from m2r2 import convert
 
 GITHUB_REPO = "ultraplot/ultraplot"
 OUTPUT_RST = Path("whats_new.rst")
+GITHUB_API_URL = f"https://api.github.com/repos/{GITHUB_REPO}/releases"
 
+# Markdown extensions: fenced code (for ```python blocks), tables, attribute
+# lists for class hooks, and md_in_html so block-level HTML such as
+# ``
`` correctly contains parsed Markdown children. +_MD_EXTENSIONS = ("fenced_code", "tables", "attr_list", "md_in_html") + +# Strip the trailing "by @user in PR_URL" attribution that GitHub auto-adds +# to release notes. Keep the PR link in parentheses so credit/traceability +# remains while removing the contributor handles from rendered output. +# GitHub author handles can include ``[bot]`` suffixes (``@dependabot[bot]``, +# ``@pre-commit-ci[bot]``); ``\w`` alone misses the brackets. +_PR_ATTRIBUTION = re.compile( + r" by @[\w.\-]+(?:\[bot\])? in (https://github\.com/[^\s]+)" +) + +# Match an ATX heading line, tolerating up-to-3 leading spaces. Authors +# occasionally indent whole sections by two spaces in the GitHub release +# editor (e.g. v2.0.1's "### Layout, Rendering, and Geo Improvements"), +# which python-markdown then parses as a paragraph rather than a heading. +# We capture the ``#`` run so we can both strip the indent and downgrade +# one level — the page already provides the H1 ("What's new?") and each +# release contributes a per-release RST H2, so body headings start at H2. +_ATX_HEADING = re.compile(r"^[ ]{0,3}(#{1,5})(?=\s)", flags=re.MULTILINE) + + +def _strip_pr_attribution(text: str) -> str: + return _PR_ATTRIBUTION.sub(r" (\1)", text) -GITHUB_API_URL = f"https://api.github.com/repos/{GITHUB_REPO}/releases" +def _downgrade_headings(text: str) -> str: + """Demote every Markdown ATX heading by one level (``#`` → ``##``, etc.).""" + return _ATX_HEADING.sub(lambda m: "#" + m.group(1), text) -def format_release_body(text): - """Formats GitHub release notes for better RST readability.""" - # Convert Markdown to RST using m2r2 - formatted_text = convert(text) - formatted_text = _downgrade_headings(formatted_text) - formatted_text = formatted_text.replace("→", "->") - formatted_text = re.sub(r"^\\s*`\\s*$", "", formatted_text, flags=re.MULTILINE) +def _normalize_unicode(text: str) -> str: + return text.replace("→", "->") - # Convert PR references (remove "by @user in ..." but keep the link) - formatted_text = re.sub( - r" by @\w+ in (https://github.com/[^\s]+)", r" (\1)", formatted_text - ) - return formatted_text.strip() +def _indent_html(html: str, indent: str = " ") -> str: + """Indent every line of ``html`` by ``indent`` for inclusion under ``.. raw:: html``.""" + return "\n".join(indent + line if line else line for line in html.splitlines()) -def _downgrade_headings(text): +def format_release_body(text: str) -> str: """ - Downgrade all heading levels by one to avoid H1/H2 collisions in the TOC. + Convert a GitHub release body (Markdown + embedded HTML) into an RST + ``.. raw:: html`` block wrapped in ``div.uplt-whats-new-release-body``. + + Parameters + ---------- + text : str + Raw Markdown release body as returned by the GitHub releases API. + + Returns + ------- + str + Indented RST snippet ready to be appended to ``whats_new.rst``. """ - adornment_map = { - "=": "-", - "-": "~", - "~": "^", - "^": '"', - '"': "'", - "'": "`", - } - lines = text.splitlines() - for idx in range(len(lines) - 1): - title = lines[idx] - underline = lines[idx + 1] - if not title.strip(): - continue - if not underline: - continue - char = underline[0] - if char not in adornment_map: - continue - if underline.strip(char): - continue - lines[idx + 1] = adornment_map[char] * len(underline) - return "\n".join(lines) - - -def fetch_all_releases(): - """Fetches all GitHub releases across multiple pages.""" - releases = [] - page = 1 + cleaned = _downgrade_headings( + _normalize_unicode(_strip_pr_attribution(text or "")) + ).strip() + html_body = markdown.markdown(cleaned, extensions=list(_MD_EXTENSIONS)) + wrapped = f'
\n{html_body}\n
' + return ".. raw:: html\n\n" + _indent_html(wrapped) + "\n" + +def _format_release_title(release: dict) -> str: + """ + Build the per-release section title in ``": "`` form, + de-duplicating the tag if it is already a prefix of the release name. + """ + tag = release["tag_name"].lower() + title = (release.get("name") or "").strip() + if title.lower().startswith(tag): + title = title[len(tag) :].lstrip(" :-—–") + return f"{tag}: {title}" if title else tag + + +def fetch_all_releases() -> list[dict]: + """Fetch every GitHub release across paginated responses.""" + releases: list[dict] = [] + page = 1 while True: response = requests.get(GITHUB_API_URL, params={"per_page": 30, "page": page}) if response.status_code != 200: print(f"Error fetching releases: {response.status_code}") break - page_data = response.json() - # If the page is empty, stop fetching if not page_data: break - releases.extend(page_data) page += 1 - return releases -def fetch_releases(): - """Fetches the latest releases from GitHub and formats them as RST.""" - releases = fetch_all_releases() - if not releases: - print(f"Error fetching releases!") - return "" - +def _render_releases(releases: Iterable[dict]) -> str: + """Render an iterable of release dicts to the full ``whats_new.rst`` body.""" header = "What's new?" - rst_content = f".. _whats_new:\n\n{header}\n{'=' * len(header)}\n\n" # H1 - + out = f".. _whats_new:\n\n{header}\n{'=' * len(header)}\n\n" for release in releases: - # ensure title is formatted as {tag}: {title} - tag = release["tag_name"].lower() - title = release["name"] - if title.startswith(tag): - title = title[len(tag) :] - while title: - if not title[0].isalpha(): - title = title[1:] - title = title.strip() - else: - title = title.strip() - break - - if title: - title = f"{tag}: {title}" - else: - title = tag - + title = _format_release_title(release) date = release["published_at"][:10] - body = format_release_body(release["body"] or "") - - # Version header (H2) - rst_content += f"{title} ({date})\n{'-' * (len(title) + len(date) + 3)}\n\n" + heading = f"{title} ({date})" + out += f"{heading}\n{'-' * len(heading)}\n\n" + out += format_release_body(release.get("body") or "") + "\n" + return out - # Process body content - rst_content += f"{body}\n\n" - return rst_content +def fetch_releases() -> str: + """Fetch the latest releases from GitHub and format them as RST.""" + releases = fetch_all_releases() + if not releases: + print("Error fetching releases!") + return "" + return _render_releases(releases) -def write_rst(): - """Writes fetched releases to an RST file.""" +def write_rst() -> None: + """Write fetched releases to ``whats_new.rst``.""" content = fetch_releases() if content: - with open(OUTPUT_RST, "w", encoding="utf-8") as f: - f.write(content) + OUTPUT_RST.write_text(content, encoding="utf-8") print(f"Updated {OUTPUT_RST}") else: print("No updates to write.") diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 8cbe3d9e8..4a5390e73 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1212,3 +1212,41 @@ body.wy-body-for-nav border-radius: 0.2rem; background: var(--uplt-color-sidebar-bg); } + +/* What's-new release bodies generated by docs/_scripts/fetch_releases.py. + Each release is emitted inside a
+ so we have a single styling hook for spacing, code-block padding, and the +
snippet affordance. */ +.uplt-whats-new-release-body { + margin-top: 0.5rem; + margin-bottom: 1.5rem; +} + +.uplt-whats-new-release-body h2, +.uplt-whats-new-release-body h3, +.uplt-whats-new-release-body h4 { + margin-top: 1.25rem; + margin-bottom: 0.5rem; +} + +.uplt-whats-new-release-body details { + margin: 0.5rem 0 1rem; + padding: 0.25rem 0.75rem; + border: 1px solid var(--uplt-color-sidebar-bg, #e5e7eb); + border-radius: 0.35rem; + background: var(--uplt-color-sidebar-bg, #f9fafb); +} + +.uplt-whats-new-release-body details > summary { + cursor: pointer; + font-weight: 600; +} + +.uplt-whats-new-release-body pre { + overflow-x: auto; +} + +.uplt-whats-new-release-body img { + max-width: 100%; + height: auto; +} diff --git a/environment.yml b/environment.yml index 0207b556c..a0b16dab9 100644 --- a/environment.yml +++ b/environment.yml @@ -20,5 +20,6 @@ dependencies: - networkx - pyarrow - cftime + - markdown - pip: - pycirclize diff --git a/pyproject.toml b/pyproject.toml index 1f85fd3b6..cfb72d87e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ docs = [ "jupyter", "jupytext", "lxml-html-clean", - "m2r2", + "markdown", "mpltern", "nbsphinx", "sphinx", diff --git a/ultraplot/tests/test_docs_fetch_releases.py b/ultraplot/tests/test_docs_fetch_releases.py new file mode 100644 index 000000000..be4e8b923 --- /dev/null +++ b/ultraplot/tests/test_docs_fetch_releases.py @@ -0,0 +1,172 @@ +from html import unescape +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "docs" / "_scripts" / "fetch_releases.py" + + +def _load_module(): + spec = spec_from_file_location("uplt_fetch_releases", SCRIPT) + module = module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_format_release_body_renders_raw_html(): + module = _load_module() + body = """ +# UltraPlot v9.9.9 + +Highlights +---------- + +* Fix a regression by @cvanelteren in https://github.com/Ultraplot/UltraPlot/pull/123 +* Replace arrows → with ASCII + +```python +print("ok") +``` +""" + rendered = module.format_release_body(body) + + assert rendered.startswith(".. raw:: html") + assert '
' in rendered + assert "

UltraPlot v9.9.9

" in rendered + assert "by @cvanelteren in" not in rendered + assert "->" in unescape(rendered) + assert '' in rendered + + +def test_fetch_releases_formats_titles(monkeypatch): + module = _load_module() + monkeypatch.setattr( + module, + "fetch_all_releases", + lambda: [ + { + "tag_name": "v1.2.3", + "name": "v1.2.3: Release title", + "published_at": "2026-04-02T10:00:00Z", + "body": "Hello world", + } + ], + ) + + rendered = module.fetch_releases() + + assert ".. _whats_new:" in rendered + assert "v1.2.3: Release title (2026-04-02)" in rendered + assert ".. raw:: html" in rendered + + +def test_format_release_body_preserves_code_inside_details(): + """Regression test for the original issue — fenced code blocks nested inside + ``
`` must turn into proper ``
`` HTML, not literal
+    backticks. The previous m2r2 pipeline left them unrendered."""
+    module = _load_module()
+    body = (
+        "# v9.9.9\n\n"
+        "
snippet\n\n" + "```python\n" + "import ultraplot as uplt\n" + "fig, ax = uplt.subplots()\n" + "```\n\n" + "
\n" + ) + + rendered = module.format_release_body(body) + + assert "
snippet" in rendered + assert '
' in rendered
+    assert "import ultraplot as uplt" in rendered
+    # No literal Markdown fences should leak through into the output
+    assert "```python" not in rendered
+
+
+def test_format_release_body_indents_for_raw_html_directive():
+    """Every line of the wrapper must be indented by three spaces so the block
+    is parsed as the body of the ``.. raw:: html`` directive."""
+    module = _load_module()
+    rendered = module.format_release_body("# Heading\n\nBody")
+
+    lines = rendered.splitlines()
+    assert lines[0] == ".. raw:: html"
+    assert lines[1] == ""
+    # All subsequent non-empty lines must start with the 3-space indent
+    for line in lines[2:]:
+        if line:
+            assert line.startswith("   "), line
+
+
+def test_fetch_releases_handles_empty_body(monkeypatch):
+    """A release with an empty body must not crash and must still emit the
+    section heading."""
+    module = _load_module()
+    monkeypatch.setattr(
+        module,
+        "fetch_all_releases",
+        lambda: [
+            {
+                "tag_name": "v0.0.1",
+                "name": "v0.0.1",
+                "published_at": "2026-01-01T00:00:00Z",
+                "body": None,
+            }
+        ],
+    )
+
+    rendered = module.fetch_releases()
+
+    assert "v0.0.1 (2026-01-01)" in rendered
+    assert ".. raw:: html" in rendered
+
+
+def test_fetch_releases_returns_empty_string_when_api_returns_nothing(monkeypatch):
+    module = _load_module()
+    monkeypatch.setattr(module, "fetch_all_releases", lambda: [])
+    assert module.fetch_releases() == ""
+
+
+def test_format_release_body_recognises_indented_atx_headings():
+    """Some GitHub release bodies (e.g. v2.0.1) indent whole sections by two
+    spaces in the source Markdown. python-markdown won't parse ``  ### Foo``
+    as an ATX heading, so without normalisation those headings render as
+    paragraphs (literal ``###`` text). The script must strip up to three
+    leading spaces from heading lines before parsing."""
+    module = _load_module()
+    body = (
+        "  ### Layout, Rendering, and Geo Improvements\n\n"
+        "  - Bullet one\n"
+        "  - Bullet two\n"
+    )
+
+    rendered = module.format_release_body(body)
+
+    assert "

Layout, Rendering, and Geo Improvements

" in rendered + assert "### Layout" not in rendered + assert "

### " not in rendered + + +def test_format_release_body_strips_bot_attribution(): + """``@dependabot[bot]`` and ``@pre-commit-ci[bot]`` style handles must be + stripped along with regular ``@user`` ones; only the PR URL should + remain.""" + module = _load_module() + body = ( + "* Bump deps by @dependabot[bot] in " + "https://github.com/Ultraplot/UltraPlot/pull/671\n" + "* Autoupdate by @pre-commit-ci[bot] in " + "https://github.com/Ultraplot/UltraPlot/pull/674\n" + "* Real fix by @cvanelteren in " + "https://github.com/Ultraplot/UltraPlot/pull/696\n" + ) + + rendered = module.format_release_body(body) + + assert "@dependabot" not in rendered + assert "@pre-commit-ci" not in rendered + assert "@cvanelteren" not in rendered + assert "https://github.com/Ultraplot/UltraPlot/pull/671" in rendered + assert "https://github.com/Ultraplot/UltraPlot/pull/696" in rendered