Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>## pylint: disable=R0903,R0912,R0913,R0914,R0917,C0301
"""Install minimal supported requirements for different Sphinx versionsand optionally test the build."""
import argparseimport asyncioimport os.pathimport shutilimport sysimport timeimport subprocess
# Minimal python version supported by the building system.
PYTHON = os.path.basename(sys.executable)
min_python_bin = None
for i in range(9, 13): p = f"python3.{i}" if shutil.which(p): min_python_bin = p break
if not min_python_bin: min_python_bin = PYTHON
# Starting from 8.0, Python 3.9 is not supported anymore.PYTHON_VER_CHANGES = {(8, 0, 0): PYTHON}
DEFAULT_VERSIONS_TO_TEST = [ (3, 4, 3), # Minimal supported version (5, 3, 0), # CentOS Stream 9 / AlmaLinux 9 (6, 1, 1), # Debian 12 (7, 2, 1), # openSUSE Leap 15.6 (7, 2, 6), # Ubuntu 24.04 LTS (7, 4, 7), # Ubuntu 24.10 (7, 3, 0), # openSUSE Tumbleweed (8, 1, 3), # Fedora 42 (8, 2, 3) # Latest version - covers rolling distros]
# Sphinx versions to be installed and their incremental requirementsSPHINX_REQUIREMENTS = { # Oldest versions we support for each package required by Sphinx 3.4.3 (3, 4, 3): { "docutils": "0.16", "alabaster": "0.7.12", "babel": "2.8.0", "certifi": "2020.6.20", "docutils": "0.16", "idna": "2.10", "imagesize": "1.2.0", "Jinja2": "2.11.2", "MarkupSafe": "1.1.1", "packaging": "20.4", "Pygments": "2.6.1", "PyYAML": "5.1", "requests": "2.24.0", "snowballstemmer": "2.0.0", "sphinxcontrib-applehelp": "1.0.2", "sphinxcontrib-devhelp": "1.0.2", "sphinxcontrib-htmlhelp": "1.0.3", "sphinxcontrib-jsmath": "1.0.1", "sphinxcontrib-qthelp": "1.0.3", "sphinxcontrib-serializinghtml": "1.1.4", "urllib3": "1.25.9", },
# Update package dependencies to a more modern base. The goal here # is to avoid to many incremental changes for the next entries (3, 5, 0): { "alabaster": "0.7.13", "babel": "2.17.0", "certifi": "2025.6.15", "idna": "3.10", "imagesize": "1.4.1", "packaging": "25.0", "Pygments": "2.8.1", "requests": "2.32.4", "snowballstemmer": "3.0.1", "sphinxcontrib-applehelp": "1.0.4", "sphinxcontrib-htmlhelp": "2.0.1", "sphinxcontrib-serializinghtml": "1.1.5", "urllib3": "2.0.0", },
# Starting from here, ensure all docutils versions are covered with # supported Sphinx versions. Other packages are upgraded only when # required by pip (4, 0, 0): { "PyYAML": "5.1", }, (4, 1, 0): { "docutils": "0.17", "Pygments": "2.19.1", "Jinja2": "3.0.3", "MarkupSafe": "2.0", }, (4, 3, 0): {}, (4, 4, 0): {}, (4, 5, 0): { "docutils": "0.17.1", }, (5, 0, 0): {}, (5, 1, 0): {}, (5, 2, 0): { "docutils": "0.18", "Jinja2": "3.1.2", "MarkupSafe": "2.0", "PyYAML": "5.3.1", }, (5, 3, 0): { "docutils": "0.18.1", }, (6, 0, 0): {}, (6, 1, 0): {}, (6, 2, 0): { "PyYAML": "5.4.1", }, (7, 0, 0): {}, (7, 1, 0): {}, (7, 2, 0): { "docutils": "0.19", "PyYAML": "6.0.1", "sphinxcontrib-serializinghtml": "1.1.9", }, (7, 2, 6): { "docutils": "0.20", }, (7, 3, 0): { "alabaster": "0.7.14", "PyYAML": "6.0.1", "tomli": "2.0.1", }, (7, 4, 0): { "docutils": "0.20.1", "PyYAML": "6.0.1", }, (8, 0, 0): { "docutils": "0.21", }, (8, 1, 0): { "docutils": "0.21.1", "PyYAML": "6.0.1", "sphinxcontrib-applehelp": "1.0.7", "sphinxcontrib-devhelp": "1.0.6", "sphinxcontrib-htmlhelp": "2.0.6", "sphinxcontrib-qthelp": "1.0.6", }, (8, 2, 0): { "docutils": "0.21.2", "PyYAML": "6.0.1", "sphinxcontrib-serializinghtml": "1.1.9", },}
class AsyncCommands: """Excecute command synchronously"""
def __init__(self, fp=None):
self.stdout = None self.stderr = None self.output = None self.fp = fp
def log(self, out, verbose, is_info=True): out = out.removesuffix('\n')
if verbose: if is_info: print(out) else: print(out, file=sys.stderr)
if self.fp: self.fp.write(out + "\n")
async def _read(self, stream, verbose, is_info): """Ancillary routine to capture while displaying"""
while stream is not None: line = await stream.readline() if line: out = line.decode("utf-8", errors="backslashreplace") self.log(out, verbose, is_info) if is_info: self.stdout += out else: self.stderr += out else: break
async def run(self, cmd, capture_output=False, check=False, env=None, verbose=True):
""" Execute an arbitrary command, handling errors.
Please notice that this class is not thread safe """
self.stdout = "" self.stderr = ""
self.log("$ " + " ".join(cmd), verbose)
proc = await asyncio.create_subprocess_exec(cmd[0], *cmd[1:], env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
# Handle input and output in realtime await asyncio.gather( self._read(proc.stdout, verbose, True), self._read(proc.stderr, verbose, False), )
await proc.wait()
if check and proc.returncode > 0: raise subprocess.CalledProcessError(returncode=proc.returncode, cmd=" ".join(cmd), output=self.stdout, stderr=self.stderr)
if capture_output: if proc.returncode > 0: self.log(f"Error {proc.returncode}", verbose=True, is_info=False) return ""
return self.output
ret = subprocess.CompletedProcess(args=cmd, returncode=proc.returncode, stdout=self.stdout, stderr=self.stderr)
return ret
class SphinxVenv: """ Installs Sphinx on one virtual env per Sphinx version with a minimal set of dependencies, adjusting them to each specific version. """
def __init__(self): """Initialize instance variables"""
self.built_time = {} self.first_run = True
async def _handle_version(self, args, fp, cur_ver, cur_requirements, python_bin): """Handle a single Sphinx version"""
cmd = AsyncCommands(fp)
ver = ".".join(map(str, cur_ver))
if not self.first_run and args.wait_input and args.build: ret = input("Press Enter to continue or 'a' to abort: ").strip().lower() if ret == "a": print("Aborted.") sys.exit() else: self.first_run = False
venv_dir = f"Sphinx_{ver}" req_file = f"requirements_{ver}.txt"
cmd.log(f"\nSphinx {ver} with {python_bin}", verbose=True)
# Create venv await cmd.run([python_bin, "-m", "venv", venv_dir], verbose=args.verbose, check=True) pip = os.path.join(venv_dir, "bin/pip")
# Create install list reqs = [] for pkg, verstr in cur_requirements.items(): reqs.append(f"{pkg}=={verstr}")
reqs.append(f"Sphinx=={ver}")
await cmd.run([pip, "install"] + reqs, check=True, verbose=args.verbose)
# Freeze environment result = await cmd.run([pip, "freeze"], verbose=False, check=True)
# Pip install succeeded. Write requirements file if args.req_file: with open(req_file, "w", encoding="utf-8") as fp: fp.write(result.stdout)
if args.build: start_time = time.time()
# Prepare a venv environment env = os.environ.copy() bin_dir = os.path.join(venv_dir, "bin") env["PATH"] = bin_dir + ":" + env["PATH"] env["VIRTUAL_ENV"] = venv_dir if "PYTHONHOME" in env: del env["PYTHONHOME"]
# Test doc build await cmd.run(["make", "cleandocs"], env=env, check=True) make = ["make"]
if args.output: sphinx_build = os.path.realpath(f"{bin_dir}/sphinx-build") make += [f"O={args.output}", f"SPHINXBUILD={sphinx_build}"]
if args.make_args: make += args.make_args
make += args.targets
if args.verbose: cmd.log(f". {bin_dir}/activate", verbose=True) await cmd.run(make, env=env, check=True, verbose=True) if args.verbose: cmd.log("deactivate", verbose=True)
end_time = time.time() elapsed_time = end_time - start_time hours, minutes = divmod(elapsed_time, 3600) minutes, seconds = divmod(minutes, 60)
hours = int(hours) minutes = int(minutes) seconds = int(seconds)
self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
cmd.log(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}", verbose=True)
async def run(self, args): """ Navigate though multiple Sphinx versions, handling each of them on a loop. """
if args.log: fp = open(args.log, "w", encoding="utf-8") if not args.verbose: args.verbose = False else: fp = None if not args.verbose: args.verbose = True
cur_requirements = {} python_bin = min_python_bin
vers = set(SPHINX_REQUIREMENTS.keys()) | set(args.versions)
for cur_ver in sorted(vers): if cur_ver in SPHINX_REQUIREMENTS: new_reqs = SPHINX_REQUIREMENTS[cur_ver] cur_requirements.update(new_reqs)
if cur_ver in PYTHON_VER_CHANGES: # pylint: disable=R1715 python_bin = PYTHON_VER_CHANGES[cur_ver]
if cur_ver not in args.versions: continue
if args.min_version: if cur_ver < args.min_version: continue
if args.max_version: if cur_ver > args.max_version: break
await self._handle_version(args, fp, cur_ver, cur_requirements, python_bin)
if args.build: cmd = AsyncCommands(fp) cmd.log("\nSummary:", verbose=True) for ver, elapsed_time in sorted(self.built_time.items()): cmd.log(f"\tSphinx {ver} elapsed time: {elapsed_time}", verbose=True)
if fp: fp.close()
def parse_version(ver_str): """Convert a version string into a tuple."""
return tuple(map(int, ver_str.split(".")))
DEFAULT_VERS = " - "DEFAULT_VERS += "\n - ".join(map(lambda v: f"{v[0]}.{v[1]}.{v[2]}", DEFAULT_VERSIONS_TO_TEST))
SCRIPT = os.path.relpath(__file__)
DESCRIPTION = f"""This tool allows creating Python virtual environments for differentSphinx versions that are supported by the Linux Kernel build system.
Besides creating the virtual environment, it can also test buildingthe documentation using "make htmldocs" (and/or other doc targets).
If called without "--versions" argument, it covers the versions shippedon major distros, plus the lowest supported version:
{DEFAULT_VERS}
A typical usage is to run:
{SCRIPT} -m -l sphinx_builds.log
This will create one virtual env for the default version set and run"make htmldocs" for each version, creating a log file with theexcecuted commands on it.
NOTE: The build time can be very long, specially on old versions. Also, thereis a known bug with Sphinx version 6.0.x: each subprocess uses a lot ofmemory. That, together with "-jauto" may cause OOM killer to causefailures at the doc generation. To minimize the risk, you may use the"-a" command line parameter to constrain the built directories and/orreduce the number of threads from "-jauto" to, for instance, "-j4":
{SCRIPT} -m -V 6.0.1 -a "SPHINXDIRS=process" "SPHINXOPTS='-j4'"
"""
MAKE_TARGETS = [ "htmldocs", "texinfodocs", "infodocs", "latexdocs", "pdfdocs", "epubdocs", "xmldocs",]
async def main(): """Main program"""
parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)
ver_group = parser.add_argument_group("Version range options")
ver_group.add_argument('-V', '--versions', nargs="*", default=DEFAULT_VERSIONS_TO_TEST,type=parse_version, help='Sphinx versions to test') ver_group.add_argument('--min-version', "--min", type=parse_version, help='Sphinx minimal version') ver_group.add_argument('--max-version', "--max", type=parse_version, help='Sphinx maximum version') ver_group.add_argument('-f', '--full', action='store_true', help='Add all Sphinx (major,minor) supported versions to the version range')
build_group = parser.add_argument_group("Build options")
build_group.add_argument('-b', '--build', action='store_true', help='Build documentation') build_group.add_argument('-a', '--make-args', nargs="*", help='extra arguments for make, like SPHINXDIRS=netlink/specs', ) build_group.add_argument('-t', '--targets', nargs="+", choices=MAKE_TARGETS, default=[MAKE_TARGETS[0]], help="make build targets. Default: htmldocs.") build_group.add_argument("-o", '--output', help="output directory for the make O=OUTPUT")
other_group = parser.add_argument_group("Other options")
other_group.add_argument('-r', '--req-file', action='store_true', help='write a requirements.txt file') other_group.add_argument('-l', '--log', help='Log command output on a file') other_group.add_argument('-v', '--verbose', action='store_true', help='Verbose all commands') other_group.add_argument('-i', '--wait-input', action='store_true', help='Wait for an enter before going to the next version')
args = parser.parse_args()
if not args.make_args: args.make_args = []
sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys()))
if args.full: args.versions += list(SPHINX_REQUIREMENTS.keys())
venv = SphinxVenv() await venv.run(args)
# Call main methodif __name__ == "__main__": asyncio.run(main())