From cf5730f496af1de006df701ea08fc11838d42699 Mon Sep 17 00:00:00 2001 From: rocketengineer1982 <36516928+rocketengineer1982@users.noreply.github.com> Date: Thu, 12 Sep 2024 21:31:01 -0400 Subject: [PATCH] Fix of pydoctor crash when creating symlink to index.html (#809) When using pydoctor to create documentation for a single root module the symlink to index.html is created before index.html is created. This causes a crash when running pydoctor in Windows under certain restrictions. Now, by default we try to symlink and fallback to hardlink when it fails. Add option --use-hardlinks to enforce copying the files. --------- Co-authored-by: Nathan Kimmel Co-authored-by: tristanlatr --- .github/workflows/unit.yaml | 2 +- README.rst | 2 ++ pydoctor/driver.py | 2 ++ pydoctor/options.py | 4 ++++ pydoctor/templatewriter/__init__.py | 7 ++++++- pydoctor/templatewriter/writer.py | 21 ++++++++++++++------- pydoctor/test/__init__.py | 5 +++++ pydoctor/test/test_commandline.py | 27 +++++++++++++++++++++++++++ pydoctor/test/test_templatewriter.py | 1 + 9 files changed, 62 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 647a52d8e..161a6ba3b 100644 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -20,7 +20,7 @@ jobs: strategy: matrix: # Re-enable 3.13-dev when https://github.com/zopefoundation/zope.interface/issues/292 is fixed - python-version: [pypy-3.7, 3.7, 3.8, 3.9, '3.10', 3.11, '3.12'] + python-version: [pypy-3.8, 3.8, 3.9, '3.10', 3.11, '3.12'] os: [ubuntu-22.04] include: - os: windows-latest diff --git a/README.rst b/README.rst index 27745e6f1..5a920d0d3 100644 --- a/README.rst +++ b/README.rst @@ -77,6 +77,8 @@ in development ^^^^^^^^^^^^^^ * Trigger a warning when several docstrings are detected for the same object. +* Fix WinError caused by the failure of the symlink creation process. + Pydoctor should now run on windows without the need to be administrator. pydoctor 24.3.3 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/driver.py b/pydoctor/driver.py index 1bf8b9cd1..89ac7f418 100644 --- a/pydoctor/driver.py +++ b/pydoctor/driver.py @@ -129,6 +129,8 @@ def make(system: model.System) -> None: if not options.htmlsummarypages: subjects = system.rootobjects writer.writeIndividualFiles(subjects) + if not options.htmlsubjects: + writer.writeLinks(system) if options.makeintersphinx: if not options.makehtml: diff --git a/pydoctor/options.py b/pydoctor/options.py index 60cebc1ae..1c26a9bb0 100644 --- a/pydoctor/options.py +++ b/pydoctor/options.py @@ -244,6 +244,9 @@ def get_parser() -> ArgumentParser: parser.add_argument( '--mod-member-order', dest='mod_member_order', default="alphabetical", choices=["alphabetical", "source"], help=("Presentation order of module/package members. (default: alphabetical)")) + parser.add_argument( + '--use-hardlinks', default=False, action='store_true', dest='use_hardlinks', + help=("Always copy files instead of creating a symlink (hardlinks will be automatically used if the symlink process failed).")) parser.add_argument('-V', '--version', action='version', version=f'%(prog)s {__version__}') @@ -375,6 +378,7 @@ class Options: nosidebar: int = attr.ib() cls_member_order: 'Literal["alphabetical", "source"]' = attr.ib() mod_member_order: 'Literal["alphabetical", "source"]' = attr.ib() + use_hardlinks: bool = attr.ib() def __attrs_post_init__(self) -> None: # do some validations... diff --git a/pydoctor/templatewriter/__init__.py b/pydoctor/templatewriter/__init__.py index c0ba892c1..28896ba58 100644 --- a/pydoctor/templatewriter/__init__.py +++ b/pydoctor/templatewriter/__init__.py @@ -73,7 +73,12 @@ def writeSummaryPages(self, system: System) -> None: def writeIndividualFiles(self, obs: Iterable[Documentable]) -> None: """ - Called last. + Called third. + """ + + def writeLinks(self, system: System) -> None: + """ + Called after writeIndividualFiles when option --html-subject is not used. """ class Template(abc.ABC): diff --git a/pydoctor/templatewriter/writer.py b/pydoctor/templatewriter/writer.py index 06df1d5b4..80802b3cb 100644 --- a/pydoctor/templatewriter/writer.py +++ b/pydoctor/templatewriter/writer.py @@ -3,6 +3,7 @@ import itertools from pathlib import Path +import shutil from typing import IO, Iterable, Type, TYPE_CHECKING from pydoctor import model @@ -97,18 +98,24 @@ def writeSummaryPages(self, system: model.System) -> None: T = time.time() search.write_lunr_index(self.build_directory, system=system) system.msg('html', "took %fs"%(time.time() - T), wantsnl=False) - + + def writeLinks(self, system: model.System) -> None: if len(system.root_names) == 1: # If there is just a single root module it is written to index.html to produce nicer URLs. - # To not break old links we also create a symlink from the full module name to the index.html + # To not break old links we also create a link from the full module name to the index.html # file. This is also good for consistency: every module is accessible by .html root_module_path = (self.build_directory / (list(system.root_names)[0] + '.html')) + root_module_path.unlink(missing_ok=True) # introduced in Python 3.8 + try: - root_module_path.unlink() - # not using missing_ok=True because that was only added in Python 3.8 and we still support Python 3.6 - except FileNotFoundError: - pass - root_module_path.symlink_to('index.html') + if system.options.use_hardlinks: + # The use wants only harlinks, so simulate an OSError + # to jump directly to the hardlink part. + raise OSError() + root_module_path.symlink_to('index.html') + except (OSError, NotImplementedError): # symlink is not implemented for windows on pypy :/ + hardlink_path = (self.build_directory / 'index.html') + shutil.copy(hardlink_path, root_module_path) def _writeDocsFor(self, ob: model.Documentable) -> None: if not ob.isVisible: diff --git a/pydoctor/test/__init__.py b/pydoctor/test/__init__.py index 09a9e65b9..45e8cf406 100644 --- a/pydoctor/test/__init__.py +++ b/pydoctor/test/__init__.py @@ -70,6 +70,11 @@ def writeSummaryPages(self, system: model.System) -> None: Rig the system to not created the inter sphinx inventory. """ system.options.makeintersphinx = False + + def writeLinks(self, system: model.System) -> None: + """ + Does nothing. + """ def _writeDocsFor(self, ob: model.Documentable) -> None: """ diff --git a/pydoctor/test/test_commandline.py b/pydoctor/test/test_commandline.py index 7634b2138..5178af7ea 100644 --- a/pydoctor/test/test_commandline.py +++ b/pydoctor/test/test_commandline.py @@ -272,3 +272,30 @@ def test_make_intersphix(tmp_path: Path) -> None: assert [p.name for p in tmp_path.iterdir()] == ['objects.inv'] assert inventory.is_file() assert b'Project: acme-lib\n# Version: 20.12.0-dev123\n' in inventory.read_bytes() + +def test_index_symlink(tmp_path: Path) -> None: + """ + Test that the default behaviour is to create symlinks, at least on unix. + + For windows users, this has not been a success, so we automatically fallback to copying the file now. + See https://github.com/twisted/pydoctor/issues/808, https://github.com/twisted/pydoctor/issues/720. + """ + import platform + exit_code = driver.main(args=['--html-output', str(tmp_path), 'pydoctor/test/testpackages/basic/']) + assert exit_code == 0 + link = (tmp_path / 'basic.html') + assert link.exists() + if platform.system() == 'Windows': + assert link.is_symlink() or link.is_file() + else: + assert link.is_symlink() + +def test_index_hardlink(tmp_path: Path) -> None: + """ + Test for option --use-hardlink wich enforce the usage of harlinks. + """ + exit_code = driver.main(args=['--use-hardlink', '--html-output', str(tmp_path), 'pydoctor/test/testpackages/basic/']) + assert exit_code == 0 + assert (tmp_path / 'basic.html').exists() + assert not (tmp_path / 'basic.html').is_symlink() + assert (tmp_path / 'basic.html').is_file() diff --git a/pydoctor/test/test_templatewriter.py b/pydoctor/test/test_templatewriter.py index dbc143967..7b64f06c1 100644 --- a/pydoctor/test/test_templatewriter.py +++ b/pydoctor/test/test_templatewriter.py @@ -139,6 +139,7 @@ def test_basic_package(tmp_path: Path) -> None: root, = system.rootobjects w._writeDocsFor(root) w.writeSummaryPages(system) + w.writeLinks(system) for ob in system.allobjects.values(): url = ob.url if '#' in url: