strip_external_scripts

Make generated API docs obey the site's script-src 'self' CSP (MIP-0044 §5.6).

scaladoc emits four third-party <script src="https://..."> tags on every page — dagre-d3, graphlib-dot, d3 and scastie — and one inline var pathToRoot = "...". A site that advertises no third-party requests cannot serve them. Losing dagre/d3 loses inheritance diagrams and losing scastie loses "run this snippet"; neither is used by this codebase's docs.

pathToRoot is kept, moved to a data-path-to-root attribute on that scaladoc's own scripts read via a tiny same-origin shim, so relative links still resolve without an inline script.

scripts/strip_external_scripts.py out/scala out/python
scripts/strip_external_scripts.py --self-test

View Source

  1#!/usr/bin/env python3
  2"""Make generated API docs obey the site's `script-src 'self'` CSP (MIP-0044 §5.6).
  3
  4scaladoc emits four third-party `<script src="https://...">` tags on every page — dagre-d3,
  5graphlib-dot, d3 and scastie — and one inline `var pathToRoot = "..."`. A site that advertises no
  6third-party requests cannot serve them. Losing dagre/d3 loses inheritance diagrams and losing
  7scastie loses "run this snippet"; neither is used by this codebase's docs.
  8
  9`pathToRoot` is kept, moved to a `data-path-to-root` attribute on <body> that scaladoc's own
 10scripts read via a tiny same-origin shim, so relative links still resolve without an inline script.
 11
 12    scripts/strip_external_scripts.py out/scala out/python
 13    scripts/strip_external_scripts.py --self-test
 14"""
 15
 16from __future__ import annotations
 17
 18import argparse
 19import re
 20import sys
 21from pathlib import Path
 22
 23EXTERNAL_SCRIPT = re.compile(
 24    r"""<script[^>]*\ssrc=["']https?://[^"']*["'][^>]*>\s*</script>""", re.I
 25)
 26PATH_TO_ROOT = re.compile(
 27    r"""<script[^>]*>\s*var\s+pathToRoot\s*=\s*["']([^"']*)["']\s*;?\s*</script>""", re.I
 28)
 29SHIM = '<script src="{root}scripts/path-to-root.js"></script>'
 30
 31
 32def strip(html: str) -> tuple[str, int]:
 33    """Remove third-party scripts and inline pathToRoot; return the new html and how many went."""
 34    removed = 0
 35    root = ""
 36    m = PATH_TO_ROOT.search(html)
 37    if m:
 38        root = m.group(1)
 39        html = PATH_TO_ROOT.sub("", html, count=1)
 40        html = re.sub(r"<body([^>]*)>", rf'<body\1 data-path-to-root="{root}">', html, count=1)
 41        html = html.replace("</head>", SHIM.format(root=root) + "</head>", 1)
 42
 43    def drop(_match: re.Match[str]) -> str:
 44        nonlocal removed
 45        removed += 1
 46        return ""
 47
 48    return EXTERNAL_SCRIPT.sub(drop, html), removed
 49
 50
 51SHIM_JS = """// Restores the `pathToRoot` global scaladoc's own scripts expect, without an inline
 52// <script> — read from <body data-path-to-root>, so the page keeps script-src 'self'. MIP-0044.
 53var pathToRoot = document.body.getAttribute("data-path-to-root") || "";
 54"""
 55
 56
 57def check(dirs: list[Path]) -> int:
 58    """Report any real external <script src> left in the tree, using EXTERNAL_SCRIPT itself.
 59
 60    The workflow used to grep for the looser `src="http`, which matched this file's own
 61    documentation once pdoc rendered it — escaped text about script tags, not a script tag.
 62    """
 63    offenders = [
 64        f
 65        for d in dirs
 66        if d.is_dir()
 67        for f in d.rglob("*.html")
 68        if EXTERNAL_SCRIPT.search(f.read_text(encoding="utf-8", errors="ignore"))
 69    ]
 70    for f in offenders[:5]:
 71        print(f"external script src survives in {f}", file=sys.stderr)
 72    if offenders:
 73        print(f"{len(offenders)} file(s) would violate the site CSP", file=sys.stderr)
 74        return 1
 75    print("no external script src in the generated tree")
 76    return 0
 77
 78
 79def process(dirs: list[Path]) -> int:
 80    total_files = total_removed = 0
 81    for d in dirs:
 82        if not d.is_dir():
 83            print(f"strip_external_scripts: no such directory: {d}", file=sys.stderr)
 84            return 1
 85        (d / "scripts").mkdir(parents=True, exist_ok=True)
 86        (d / "scripts" / "path-to-root.js").write_text(SHIM_JS, encoding="utf-8")
 87        for f in d.rglob("*.html"):
 88            html = f.read_text(encoding="utf-8", errors="ignore")
 89            new, removed = strip(html)
 90            if new != html:
 91                f.write_text(new, encoding="utf-8")
 92            total_files += 1
 93            total_removed += removed
 94    print(f"stripped {total_removed} external <script> tags across {total_files} html files")
 95    return 0
 96
 97
 98def self_test() -> int:
 99    fails = 0
100
101    def ok(got, want, label):
102        nonlocal fails
103        if got == want:
104            print(f"  ok   {label}")
105        else:
106            fails += 1
107            print(f"  FAIL {label} — got {got!r}, want {want!r}")
108
109    page = (
110        '<html><head><script type="text/javascript" src="https://d3js.org/d3.v6.min.js"></script>'
111        '<script src="https://scastie.scala-lang.org/embedded.js"></script>'
112        '<script src="scripts/ux.js"></script></head>'
113        '<body class="theme"><script>var pathToRoot = "../";</script>hi</body></html>'
114    )
115    out, removed = strip(page)
116    ok(removed, 2, "both third-party scripts are removed")
117    ok("d3js.org" in out, False, "no https:// script src survives")
118    ok('src="scripts/ux.js"' in out, True, "the doc's own same-origin scripts are kept")
119    ok("var pathToRoot" in out, False, "the inline pathToRoot script is gone")
120    ok('data-path-to-root="../"' in out, True, "pathToRoot is preserved as a data attribute")
121    ok('class="theme"' in out, True, "existing body attributes survive the rewrite")
122    ok(out.count("path-to-root.js"), 1, "the same-origin shim is linked once")
123    ok(strip("<html><body>plain</body></html>")[1], 0, "a page with no scripts is untouched")
124    # a protocol-relative or http src must go too
125    ok(
126        strip('<script src="http://x.example/a.js"></script>')[1],
127        1,
128        "plain http src is removed as well",
129    )
130    import tempfile
131
132    with tempfile.TemporaryDirectory() as tmp:
133        d = Path(tmp)
134        (d / "clean.html").write_text("<html><body>no scripts</body></html>", encoding="utf-8")
135        ok(check([d]), 0, "--check passes a tree with no external script")
136        # What pdoc emits for this very file: text *about* script tags, with < escaped.
137        (d / "pdoc.html").write_text(
138            '<html><body><code>EXTERNAL_SCRIPT = &lt;script src="https://x/a.js"&gt;</code>'
139            "<p>strips &lt;script src=&quot;https://...&quot;&gt; tags</p></body></html>",
140            encoding="utf-8",
141        )
142        ok(check([d]), 0, "documentation about script tags is not mistaken for one")
143        (d / "real.html").write_text(
144            '<html><head><script src="https://d3js.org/d3.v6.min.js"></script></head></html>',
145            encoding="utf-8",
146        )
147        ok(check([d]), 1, "a real external script tag is still caught")
148
149    if fails:
150        print(f"strip_external_scripts self-test: {fails} failure(s)", file=sys.stderr)
151        return 1
152    print("strip_external_scripts self-test: ok")
153    return 0
154
155
156def main(argv: list[str] | None = None) -> int:
157    ap = argparse.ArgumentParser(
158        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
159    )
160    ap.add_argument("dirs", nargs="*", type=Path)
161    ap.add_argument("--self-test", action="store_true")
162    ap.add_argument("--check", action="store_true", help="report survivors, change nothing")
163    args = ap.parse_args(argv)
164    if args.self_test:
165        return self_test()
166    if not args.dirs:
167        ap.print_help()
168        return 2
169    return check(args.dirs) if args.check else process(args.dirs)
170
171
172if __name__ == "__main__":
173    sys.exit(main())

EXTERNAL_SCRIPT = re.compile('<script[^>]*\\ssrc=["\']https?://[^"\']*["\'][^>]*>\\s*</script>', re.IGNORECASE)

PATH_TO_ROOT = re.compile('<script[^>]*>\\s*var\\s+pathToRoot\\s*=\\s*["\']([^"\']*)["\']\\s*;?\\s*</script>', re.IGNORECASE)

SHIM = '<script src="{root}scripts/path-to-root.js"></script>'

def strip(html: str) -> tuple[str, int]: View Source

33def strip(html: str) -> tuple[str, int]:
34    """Remove third-party scripts and inline pathToRoot; return the new html and how many went."""
35    removed = 0
36    root = ""
37    m = PATH_TO_ROOT.search(html)
38    if m:
39        root = m.group(1)
40        html = PATH_TO_ROOT.sub("", html, count=1)
41        html = re.sub(r"<body([^>]*)>", rf'<body\1 data-path-to-root="{root}">', html, count=1)
42        html = html.replace("</head>", SHIM.format(root=root) + "</head>", 1)
43
44    def drop(_match: re.Match[str]) -> str:
45        nonlocal removed
46        removed += 1
47        return ""
48
49    return EXTERNAL_SCRIPT.sub(drop, html), removed

Remove third-party scripts and inline pathToRoot; return the new html and how many went.

SHIM_JS = '// Restores the `pathToRoot` global scaladoc\'s own scripts expect, without an inline\n// <script> — read from <body data-path-to-root>, so the page keeps script-src \'self\'. MIP-0044.\nvar pathToRoot = document.body.getAttribute("data-path-to-root") || "";\n'

def check(dirs: list[pathlib.Path]) -> int: View Source

58def check(dirs: list[Path]) -> int:
59    """Report any real external <script src> left in the tree, using EXTERNAL_SCRIPT itself.
60
61    The workflow used to grep for the looser `src="http`, which matched this file's own
62    documentation once pdoc rendered it — escaped text about script tags, not a script tag.
63    """
64    offenders = [
65        f
66        for d in dirs
67        if d.is_dir()
68        for f in d.rglob("*.html")
69        if EXTERNAL_SCRIPT.search(f.read_text(encoding="utf-8", errors="ignore"))
70    ]
71    for f in offenders[:5]:
72        print(f"external script src survives in {f}", file=sys.stderr)
73    if offenders:
74        print(f"{len(offenders)} file(s) would violate the site CSP", file=sys.stderr)
75        return 1
76    print("no external script src in the generated tree")
77    return 0

Report any real external