main.py

"""

ReST syntax:
https://docutils.sourceforge.io/docs/ref/rst/directives.html
https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#directives
https://www.sphinx-doc.org/en/master/usage/domains/python.html#directive-py-module
https://www.sphinx-doc.org/en/master/usage/domains/python.html#directive-py-currentmodule

"""

from typing import Any
import ast
import subprocess
import dataclasses
import itertools
import textwrap
import typing
import utils
from typing import Final
import re
import pathlib
import pylsp

rec_directive = utils.rec_directive_with_module
modify_docs = utils.modify_docs


@dataclasses.dataclass(frozen=True)
class DirectiveNode:
    match: re.Match
    level: int
    module: str | None | utils._Missing


def parse_directive_tree(
    txt: str,
    level: int,
    indent: int,
    module_directive: list[str | None | utils._Missing],
    parent_module: str | None | utils._Missing,
):
    ret = []
    for e in rec_directive.finditer(txt):
        head_body = e[0].split("\n\n", maxsplit=1)
        if e.group("directive_type") in ("module", "currentmodule"):
            assert level == 0
            module_raw = typing.cast(str, e.group("first_api"))
            module_directive.append(None if module_raw == "None" else module_raw)
            continue
        head_dedented, head_dedents = utils.dedent(head_body[0])
        module_option = utils.MISSING
        if ":module:" in head_dedented:
            assert "\n   :module:" in head_dedented
            mod_rec = re.compile(r"^   :module:\s*(\S*)\s*$", re.MULTILINE)
            m = mod_rec.search(head_dedented)
            assert m is not None
            module_option = None if m.group(1) == "" else m.group(1)
        module_directive_current = (
            module_directive[-1] if module_directive else "builtins"
        )
        module = (
            module_option
            if module_option is not utils.MISSING
            else parent_module
            if parent_module is not utils.MISSING
            else module_directive_current
        )
        if len(head_body) == 2 and head_body[1].strip() != "":
            body = head_body[1]
            dedented_body, body_indent = utils.dedent(body)
            body_indent_relative = body_indent - indent
            children = parse_directive_tree(
                len(head_body[0]) * " " + "\n\n" + head_body[1],
                level=level + 1,
                indent=body_indent_relative + indent,
                module_directive=module_directive,
                parent_module=module,
            )
            ret.append((DirectiveNode(e, level, module), children))
        else:
            ret.append((DirectiveNode(e, level, module), ()))
    return tuple(ret)


def raw_directive_pattern(api_type_head: str) -> re.Pattern[str]:
    return re.compile(
        f"^{re.escape(api_type_head)}.+(?:\n{' ' * len(api_type_head)}.+)*"
    )


def modify_rst(
    rst_file_stem: str, match: re.Match, rst_code: str, api_type_info: dict
) -> str:
    directive_type = match.group("directive_type")
    if api_type_info == {}:
        return rst_code
    if rst_file_stem == "functions":
        if api_type_info.keys() == {"help"}:
            return rst_code
    api_sig_typeshed_link = list(api_type_info.items())[0][1]

    api_info = get_api_info(rst_file_stem, match)
    if api_sig_typeshed_link is None:
        return rst_code  # FIXME
    hover_txt = api_sig_typeshed_link["hover"]
    # type1 = None
    # if hover_txt.startswith("type["):
    #     type1 = hover_txt
    # elif hover_txt.startswith("TypeAlias["):
    #     type1 = hover_txt.partition(", ")[2].removesuffix("]")
    sig_raw = api_sig_typeshed_link["signatureHelp"]
    sig0 = sig_raw
    api_names_unique = set(api_info.api_names)
    sig1 = sig0 if sig0 is not None and len(api_names_unique) == 1 else None
    if sig1 is not None and len(api_names_unique) == 1:
        api_name = next(iter(api_names_unique))
        sig2 = [
            e
            if e.startswith(api_name) or not e.startswith(("(", "["))
            else api_name + e
            for e in sig1
        ]
        sig2 = [utils.remove_self_parameter(e) for e in sig2]
        if hover_txt.startswith("type["):
            sig2 = [utils.remove_cls_parameter(e) for e in sig2]
    else:
        sig2 = []
    sig2 = [e for e in sig2 if not e.startswith(("type[",))]

    sig2 = [
        list(api_type_info.keys())[0] + e.removeprefix(e.split("(")[0].split("[")[0])
        for e in sig2
    ]
    declaration = api_sig_typeshed_link["declaration"]
    stdlib_path0 = pathlib.Path(declaration["uri"])
    stdlib_path1 = itertools.dropwhile(
        lambda x: not x.startswith("pyrefly_bundled_typeshed_"), stdlib_path0.parts
    )
    next(stdlib_path1)
    stdlib_path = "/".join(stdlib_path1)
    start_line = declaration["range"]["start"]["line"] + 1
    end_line = declaration["range"]["end"]["line"] + 1
    uri_fragment = (
        f"L{start_line}" if start_line == end_line else f"L{start_line}-L{end_line}"
    )
    # link to facebook pyrefly typeshed bundled stdlib for now, waiting for issue https://github.com/facebook/pyrefly/issues/1621
    # to get resolved
    # hyperlink = f"https://github.com/facebook/pyrefly/blob/{utils.pyrefly_revision[:6]}/crates/pyrefly_bundled/third_party/typeshed/stdlib/{stdlib_path}#{uri_fragment}"
    hyperlink = f"https://github.com/python/typeshed/blob/{utils.typeshed_revision[:7]}/stdlib/{stdlib_path}#{uri_fragment}"
    rst_code_hyperlink0, subcount = re.compile("\n\n").subn(
        f"\n\n   (`typeshed <{hyperlink}>`__)\n\n",
        rst_code,
        count=1,
    )
    if subcount == 0:
        # no double new line found, append at end
        rst_code_hyperlink0 = rst_code + f"\n\n   (`typeshed <{hyperlink}>`__)\n"
    rst_code1 = rst_code_hyperlink0
    api_type_head = match.group("api_type_head")
    raw_directive_rec = re.compile(
        f"^{re.escape(api_type_head)}(.+)(?:\n{' ' * len(api_type_head)}(.+))*"
    )

    if sig2 == [] or len(api_names_unique) != 1:
        rst_code2 = rst_code1
    else:
        append_txt = "\n" + textwrap.indent("\n".join(sig2), " " * len(api_type_head))

        if directive_type in (
            "function",
            "exception",
            "class",
            "method",
            "classmethod",
            "staticmethod",
            "decorator",
            "awaitablefunction",
            "awaitablemethod",
        ):
            ## append signatures with types
            def repl(mo: re.Match) -> str:
                return mo.group(0) + append_txt
                ## replace signatures with types
                # return api_type_head + append_txt[len(api_type_head) :]

            if len(sig2) <= 112:  # avoid displaying signatures with too many overloads
                rst_code2 = raw_directive_rec.sub(repl, rst_code1)
            else:
                rst_code2 = rst_code1
            # if (
            #     directive_type in ("class", "exception")
            #     and len(sig2) == 1
            #     and sig2[0].endswith("() -> None")
            # ):
            #     # probably ABC
            #     print(f"-- {rst_file_stem} ----------------------")
            #     print(rst_code1.splitlines()[0])
            #     rst_code2 = rst_code1
            if rst_file_stem == "sys.monitoring":
                # already has signature in the rst doc
                rst_code2 = rst_code1
        elif directive_type in ("attribute", "data"):
            if rst_file_stem == "wsgiref":
                if api_info.api_names == ["WSGIEnvironment"]:
                    sig1 = ["type[dict[str, typing.Any]]"]
                if api_info.api_names == ["WSGIApplication"]:
                    sig1 = [
                        "collections.abc.Callable[[WSGIEnvironment, StartResponse], collections.abc.Iterable[bytes]]"
                    ]
            else:
                assert len(sig1) == 1

            def repl(mo: re.Match) -> str:
                return mo.group(0) + "\n   :type: " + sig1[0]

            if (
                rst_file_stem in ("string.templatelib", "tarfile")
                and "\n   :type:" in rst_code1
            ):
                rst_code2 = rst_code1
            else:
                rst_code2 = raw_directive_rec.sub(repl, rst_code1)
        elif directive_type in ("monitoring-event",):
            """
cpython/Doc/library/sys.monitoring.rst:94: ERROR: Error in "monitoring-event" directive:
unknown option: "type"."""
            rst_code2 = rst_code1
        else:
            raise AssertionError(f"unknown directive_type: {directive_type}")

    # print("-- modified rst code --")
    # print(rst_code2)
    return rst_code2


class APIInfo(typing.NamedTuple):
    api_type: str
    api_names: list[str]


def get_api_info(rst_file_stem: str, match: re.Match) -> APIInfo:
    # https://docutils.sourceforge.io/docs/ref/rst/directives.html
    # https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#directives
    directive_block: Final[str] = textwrap.dedent(match.group())
    api_type_head: Final[str] = match.group("api_type_head")
    api_type: Final[str] = match.group("directive_type")
    raw_directive_rec: Final = raw_directive_pattern(api_type_head)
    raw_directive_match: Final = raw_directive_rec.match(directive_block)
    assert raw_directive_match is not None
    api_raw: Final = textwrap.dedent(
        " " * len(api_type_head) + raw_directive_match.group()[len(api_type_head) :]
    )
    apis = [e for e in re.compile("\\s*\\\\\n\\s*").sub(" ", api_raw).splitlines()]
    api_names = [e.split("(", 1)[0].split("[", 1)[0] for e in apis]

    # sys.monitoring is not a module
    # https://docs.python.org/dev/library/sys.monitoring.html
    # https://raw.githubusercontent.com/python/cpython/refs/heads/main/Doc/tools/extensions/pyspecific.py#:~:text=parse_monitoring_event
    # https://raw.githubusercontent.com/python/cpython/refs/heads/main/Doc/library/sys.monitoring.rst#:~:text=monitoring%2Devent
    if rst_file_stem == "sys.monitoring":
        api_names = [
            "monitoring.events." + api_name
            if api_type == "monitoring-event"
            else "monitoring." + api_name
            for api_name in api_names
        ]

    # repeated module name in api name
    # https://docs.python.org/dev/library/codecs.html#codecs.codecs.escape_encode
    # https://raw.githubusercontent.com/python/cpython/refs/heads/main/Doc/library/codecs.rst#:~:text=codecs%2Eescape%5Fencode
    if rst_file_stem == "codecs":
        if api_names in (["codecs.escape_encode"], ["codecs.escape_decode"]):
            api_names = [e.split(".", maxsplit=1)[-1] for e in api_names]
    # repeated module name in api name
    # https://raw.githubusercontent.com/python/cpython/refs/heads/main/Doc/library/urllib.parse.rst#:~:text=urllib.parse.SplitResult.geturl
    if rst_file_stem == "urllib.parse":
        if api_names == ["urllib.parse.SplitResult.geturl"]:
            api_names = ["SplitResult.geturl"]

    return APIInfo(
        api_type,
        api_names,
    )


def retry_with_new_code_level_0_impl(
    rst_file_stem: str, api: str
) -> str | utils._Missing:
    if rst_file_stem == "__future__":
        if api.startswith("_Feature."):
            # instance of __future__._Feature
            return "from __future__ import annotations\nannotations"
    if rst_file_stem == "asyncio-eventloop":
        if api.startswith("loop."):
            return "import asyncio\nasyncio.new_event_loop()"
    if rst_file_stem == "cmd":
        if api.startswith("Cmd."):
            return "import cmd\ncmd.Cmd()"
    if rst_file_stem == "csv":
        if api.startswith("csvwriter."):
            return "import io,csv\ncsv.writer(io.StringIO())"
        if api.startswith("csvreader."):
            return "import io,csv\ncsv.reader(io.StringIO())"
        if api.startswith("DictReader."):
            return "import io,csv\ncsv.DictReader(io.StringIO('a,b\\n1,2'))"
        if api.startswith("Dialect."):
            return "import io,csv\ncsv.reader(io.StringIO()).dialect"
    if rst_file_stem == "collections":
        if api.startswith("somenamedtuple"):
            # type checker does not work on the code below, so we return code that works
            # namedtuple is special-cased in the type checker
            # https://github.com/python/typeshed/blob/8d2ea19fd937064b6e310eeb30dce2f8b1d9df69/stdlib/collections/__init__.pyi#L35
            # return "import collections\ncollections.namedtuple('somenamedtuple', [])"
            return """import collections
somenamedtuple=collections.namedtuple('somenamedtuple', [])
somenamedtuple()"""
        if api.startswith("ChainMap."):
            return "import collections\ncollections.ChainMap()"
    if rst_file_stem == "ctypes":
        if api == "_CData":
            return "import ctypes\nctypes.c_int"
    if rst_file_stem == "curses.panel":
        if api.startswith("Panel."):
            return """import curses,curses.panel
window = curses.wrapper(lambda x: curses.initscr())
curses.panel.new_panel(window)"""
    if rst_file_stem == "hashlib":
        if api.startswith("hash."):
            return "import hashlib\nhashlib.md5()"
        if api.startswith("shake."):
            return "import hashlib\nhashlib.shake_128()"
    if rst_file_stem == "imaplib":
        if api.startswith("IMAP4."):
            return pathlib.Path("create_imap4_object.py").read_text(encoding="utf-8")
    if rst_file_stem == "netrc":
        if api.startswith("netrc."):
            return "import netrc\nnetrc.netrc(None)"
    if rst_file_stem == "optparse":
        if api.startswith("Option."):
            return (
                "import optparse\noptparse.OptionParser().add_option('-o', '--output')"
            )
    if rst_file_stem == "reprlib":
        if api.startswith("Repr."):
            return "import reprlib\nreprlib.Repr()"
        if api == "Repr.repr_TYPE":
            return utils.MISSING
    if rst_file_stem == "shlex":
        if api.startswith("shlex."):
            return "import shlex\nshlex.shlex('some string')"
    if rst_file_stem == "smtplib":
        if api.startswith("SMTP."):
            return "import smtplib\nsmtplib.SMTP()"
    if rst_file_stem == "socket":
        if api.startswith("socket."):
            return "import socket\nsocket.socket()"
    if rst_file_stem == "ssl":
        if api.startswith("SSLContext."):
            return "import ssl\nssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)"
        if api.startswith("SSLSocket."):
            return """import ssl, socket\nssl.SSLContext(ssl.PROTOCOL_TLS_SERVER).wrap_socket(socket.socket())"""
    if rst_file_stem == "subprocess":
        if api.startswith("Popen."):
            return (
                "import subprocess\nsubprocess.Popen('dir', stdout=subprocess.DEVNULL)"
            )
    if rst_file_stem == "zlib":
        if api.startswith("Compress."):
            return "import zlib\nzlib.compressobj()"
        if api.startswith("Decompress."):
            return "import zlib\nzlib.decompressobj()"
    if rst_file_stem == "xmlrpc.client":
        if api.startswith("ServerProxy."):
            return 'import xmlrpc.client\nxmlrpc.client.ServerProxy("http://localhost")'
    if rst_file_stem == "xml.sax.reader":
        if api.startswith("Attributes."):
            return "import xml.sax.xmlreader\nxml.sax.xmlreader.AttributesImpl({})"
    if rst_file_stem == "xml.sax.reader":
        if api.startswith("AttributesNS."):
            return (
                "import xml.sax.xmlreader\nxml.sax.xmlreader.AttributesNSImpl({}, {})"
            )
    if rst_file_stem == "webbrowser":
        if api.startswith("controller."):
            return "import webbrowser\nwebbrowser.get()"
    if rst_file_stem == "tkinter":
        if api.startswith("Widget."):
            return "import tkinter\ntkinter.Tk()"
    if rst_file_stem == "stdtypes":
        if api.startswith("container."):
            return "import collections.abc\ncollections.abc.Iterable"
        if api.startswith("sequence."):
            if api == "sequence.copy":
                return utils.MISSING
            return "import collections.abc\ncollections.abc.MutableSequence"
        if api.startswith("iterator."):
            return "import collections.abc\ncollections.abc.Iterator"
        if api.startswith("contextmanager."):
            return "import contextlib\ncontextlib.contextmanager(lambda: (yield))()"
    if rst_file_stem == "doctest":
        if api.startswith("DocTestFailure."):
            return """import doctest
dtp = doctest.DocTestParser()
dt = dtp.get_doctest('>>> 1',  {}, 'testname', None, None)
try:
    doctest.DebugRunner().run(dt)
except doctest.DocTestFailure as due:
    exc = due
else:
    raise AssertionError("Expected a DocTestFailure")
exc
"""
        if api.startswith("UnexpectedException."):
            return """import doctest
dtp = doctest.DocTestParser()
dt = dtp.get_doctest('>>> raise Exception',  {}, 'testname', None, None)
try:
    doctest.DebugRunner().run(dt)
except doctest.UnexpectedException as due:
    exc = due
else:
    raise AssertionError("Expected an UnexpectedException")
exc
"""
    if rst_file_stem == "http.client":
        if api.startswith("HTTPConnection."):
            return "import http.client\nhttp.client.HTTPConnection('localhost')"
        if api.startswith("HTTPResponse."):
            return """import http.client, socket
http.client.HTTPResponse(socket.socket())"""
    if rst_file_stem == "http.cookiejar":
        if api.startswith("FileCookieJar."):
            return "import http.cookiejar\nhttp.cookiejar.FileCookieJar('somefile')"
        if api.startswith(("CookiePolicy.", "DefaultCookiePolicy.")):
            return "import http.cookiejar\nhttp.cookiejar.DefaultCookiePolicy()"
        if api.startswith("Cookie."):
            return pathlib.Path("create_http_cookiejar_Cookie.py").read_text(
                encoding="utf-8"
            )
    if rst_file_stem == "select":
        if api.startswith("poll."):
            return "import select\nselect.poll()"
        if api.startswith("devpoll."):
            return "import select\nselect.devpoll()"
        if api.startswith("kqueue."):
            return "import select\nselect.kqueue()"
    if rst_file_stem == "stdtypes":
        if api.startswith("genericalias."):
            return "import typing\ntyping.GenericAlias"
        if api.startswith("definition."):
            if api == "definition.__doc__":
                return "object()"
            return "def f():return\nf"
    if rst_file_stem == "tarfile":
        if api.startswith("TarFile."):
            return "import tarfile,io\ntarfile.TarFile(None, 'w', io.BytesIO())"
        if api.startswith("TarInfo."):
            return "import tarfile\ntarfile.TarInfo()"
    if rst_file_stem == "urllib.request":
        if api.startswith("Request."):
            return "import urllib.request\nurllib.request.Request('http://example.com', method='GET')"
        if api.startswith("BaseHandler."):
            return """import urllib.request
bh = urllib.request.BaseHandler()
bh.add_parent(urllib.request.OpenerDirector())
bh"""
        if api.startswith("HTTPCookieProcessor."):
            return "import urllib.request\nurllib.request.HTTPCookieProcessor()"
    if rst_file_stem == "winreg":
        if api.startswith("PyHKEY."):
            return "import winreg\nwinreg.CreateKey('', None)"
    if rst_file_stem == "pyexpat":
        if api.startswith("ExpatError."):
            return """from xml.parsers.expat import ParserCreate, ExpatError, errors
p = ParserCreate()
try:
    p.Parse('some_invalid_xml', True)
except ExpatError as err:
    exc = err
else:
    raise AssertionError
exc"""
        if api.startswith("xmlparser."):
            return "import xml.parsers.expat\nxml.parsers.expat.ParserCreate()"
    if rst_file_stem == "xml.dom":
        if api.startswith("DOMImplementation."):
            return "import xml.dom\nxml.dom.getDOMImplementation()"
        if api.startswith(("Node.", "Element.")):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
node = doc.createElement("another_tag")
node"""
        if api.startswith("NodeList."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
node = doc.createElement("another_tag")
node.childNodes"""
        if api.startswith("Document."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
doc"""
        if api.startswith("NamedNodeMap."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
doc.createElement("another_tag").attributes"""
        if api.startswith("DocumentType."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc_type: xml.dom.minidom.DocumentType = impl.createDocumentType("some_root", "some_public_id", "some_system_id")
doc_type"""
        if api.startswith("Attr."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
attr = doc.createAttribute("some_attr")
attr"""
        if api.startswith("Comment."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
doc.createComment("some_comment")"""
        if api.startswith("Text."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
doc.createTextNode("some_text")"""
        if api.startswith("ProcessingInstruction."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
doc.createProcessingInstruction("target", "data")"""
    if rst_file_stem == "xml.dom.minidom":
        if api.startswith("Node."):
            return """import xml.dom
import xml.dom.minidom
impl = xml.dom.getDOMImplementation()
doc: xml.dom.minidom.Document = impl.createDocument(None, "some_tag", None)
node = doc.createElement("another_tag")
node"""
    if rst_file_stem == "zipfile":
        if api.startswith("ZipFile."):
            return "import zipfile,io\nzipfile.ZipFile(io.BytesIO(), 'w')"
    return utils.MISSING


def retry_with_new_code_level_0(
    rst_file_stem: str, api_name: str
) -> str | utils._Missing:
    code = retry_with_new_code_level_0_impl(rst_file_stem, api_name)
    if code is utils.MISSING:
        return utils.MISSING
    return code + "." + api_name.split(".", maxsplit=1)[-1]


def retry_with_new_code_level_1(
    rst_file_stem: str, module: str, parent_api: str, api: str
) -> str | utils._Missing:
    if rst_file_stem == "ast":
        if parent_api == "AST":
            # substitude for ast.AST, defined only in ast.expr and ast.stmt
            return "import ast\nast.stmt." + api
    if rst_file_stem == "asyncio-task":
        if parent_api == "timeout" and api == "Timeout":
            # https://docs.python.org/dev/library/asyncio-task.html#asyncio.Timeout
            return "import asyncio\nasyncio.timeout(None)"
    if rst_file_stem == "ctypes":
        if parent_api == "_CData":
            return f"import ctypes\nctypes.c_int.{api}"
    if rst_file_stem == "dbm":
        if parent_api == "open":
            return f"import {module}\n{module}.open('somefile').{api.partition('.')[2]}"
    if rst_file_stem == "functools":
        if parent_api == "lru_cache":
            return f"import functools\nfunctools.lru_cache(lambda x: x).{api}"
        if parent_api == "singledispatch":
            return f"import functools\nfunctools.singledispatch(lambda x: x).{api}"
    if rst_file_stem == "imaplib":
        if parent_api == "IMAP4.idle":
            # https://docs.python.org/dev/library/imaplib.html#imaplib.IMAP4.idle
            return f"import imaplib\nimaplib.IMAP4('localhost').idle().{api.partition('.')[2]}"
    if rst_file_stem == "inspect":
        if parent_api == "Parameter" and api == "kind.description":
            # https://docs.python.org/dev/library/inspect.html#inspect.Parameter.kind.description
            return "import inspect\ninspect.Parameter.POSITIONAL_ONLY.description"
    if rst_file_stem == "multiprocessing":
        return f"import multiprocessing\nmultiprocessing.{parent_api}().{api}"
    if rst_file_stem == "profile":
        if parent_api == "Profile":
            # some apis are only in cProfile(<=3.14)/profiling.tracing(>=3.15)
            # https://docs.python.org/dev/library/profile.html#profile.Profile.enable
            return f"import cProfile\ncProfile.Profile().{api}"
    if rst_file_stem == "os":
        if parent_api == "scandir" and api == "close":
            return "import os\nos.scandir('.').close"
    if rst_file_stem == "profiling.tracing":
        if parent_api == "Profile":
            return f"import cProfile\ncProfile.tracing.Profile().{api}"
    if rst_file_stem == "shutil":
        if parent_api == "rmtree" and api == "avoids_symlink_attacks":
            # https://docs.python.org/dev/library/shutil.html#shutil.rmtree.avoids_symlink_attacks
            return "import shutil\nshutil.rmtree.avoids_symlink_attacks"
    if rst_file_stem == "socketserver":
        if parent_api == "BaseServer" and api in (
            "fileno",
            "address_family",
            "socket",
            "allow_reuse_address",
            "request_queue_size",
            "socket_type",
            "server_bind",
        ):
            # defined in subclasses
            return f"import socketserver\nsocketserver.TCPServer().{api}"
    if rst_file_stem == "typing":
        if parent_api == "TypedDict":
            return f"import typing\nclass TD(typing.TypedDict): pass\nTD.{api}"

    return utils.MISSING


def check_exists_predicate_level_1(
    rst_file_stem: str, parent_api_name: str, api: str
) -> bool:
    if rst_file_stem == "exceptions":
        if parent_api_name == "UnicodeError" and api in (
            "encoding",
            "reason",
            "object",
            "start",
            "end",
        ):
            # https://docs.python.org/dev/library/exceptions.html#UnicodeError
            # in subclasses only
            return False
    if rst_file_stem == "http.cookies":
        if parent_api_name == "Morsel" and api in (
            "expires",
            "path",
            "comment",
            "domain",
            "max-age",
            "secure",
            "version",
            "httponly",
            "samesite",
            "partitioned",
        ):
            # https://docs.python.org/dev/library/http.cookies.html#http.cookies.Morsel
            # dict keys
            return False
    if rst_file_stem == "io":
        if parent_api_name == "BufferedIOBase" and api == "raw":
            # https://docs.python.org/dev/library/io.html#io.BufferedIOBase.raw
            # optional attribute
            return False
        if parent_api_name == "TextIOBase" and api == "buffer":
            # https://docs.python.org/dev/library/io.html#io.TextIOBase.buffer
            # optional attribute
            return False
    if rst_file_stem == "os" and parent_api_name == "stat_result":
        if api == "st_fstype":
            # https://docs.python.org/dev/library/os.html#os.stat_result.st_fstype":
            # On Solaris and derivatives
            return False
        if api in ("st_creator", "st_rsize", "st_type"):
            # https://raw.githubusercontent.com/python/typeshed/main/stdlib/os/__init__.pyi#:~:text=%23%20Attributes%20documented%20as%20sometimes%20appearing%2C%20but%20deliberately%20omitted%20from%20the%20stub
            # https://github.com/python/typeshed/pull/6560#issuecomment-991253327
            return False
    if rst_file_stem == "socketserver":
        if parent_api_name == "ThreadingMixIn" and api == "max_children":
            # https://docs.python.org/dev/library/socketserver.html#socketserver.ThreadingMixIn.max_children
            # attribute for ForkingMixIn only
            return False
        if parent_api_name == "ForkingMixIn" and api == "daemon_threads":
            # https://docs.python.org/dev/library/socketserver.html#socketserver.ThreadingMixIn.daemon_threads
            # attribute for ThreadingMixIn only
            return False
    if rst_file_stem == "test":
        # https://docs.python.org/dev/library/test.html#:~:text=internal%20use%20by%20Python
        # internal use only
        return False
    if rst_file_stem == "xml.etree.elementtree":
        if parent_api_name == "TreeBuilder" and api in (
            "doctype",
            "start_ns",
            "end_ns",
        ):
            # https://docs.python.org/dev/library/xml.etree.elementtree.html#xml.etree.ElementTree.TreeBuilder.doctype
            # optional methods
            return False
    return True


def check_exists_predicate_level_0(rst_file_stem, api_name):
    if rst_file_stem == "constants" and api_name in ("True", "False", "None"):
        return False
    if rst_file_stem == "copy" and api_name in (
        "object.__copy__",
        "object.__deepcopy__",
        "object.__replace__",
    ):
        # https://docs.python.org/dev/library/copy.html#object.__copy__
        # optional methods
        return False
    if rst_file_stem == "ctypes" and api_name == "prototype":
        # prototype is a callable returned from ctypes.CFUNCTYPE, ctypes.PYFUNCTYPE, ctypes.WINFUNCTYPE
        # https://docs.python.org/dev/library/ctypes.html#function-prototypes
        return False
    if rst_file_stem == "dataclasses" and api_name == "__post_init__":
        # https://docs.python.org/dev/library/dataclasses.html#dataclasses.__post_init__
        # optional method
        return False
    if rst_file_stem == "doctest" and api_name == "module.__test__":
        # https://docs.python.org/dev/library/doctest.html#module.__test__
        # __test__ is a module-level variable
        return False
    if rst_file_stem == "pickle" and api_name in (
        "object.__getnewargs_ex__",
        "object.__getnewargs__",
        "object.__getstate__",
        "object.__setstate__",
        "object.__reduce_ex__",
        "object.__reduce__",
    ):
        # https://docs.python.org/dev/library/pickle.html#pickling-class-instances
        # optional methods
        return False
    if rst_file_stem == "reprlib" and api_name == "Repr.repr_TYPE":
        # https://docs.python.org/dev/library/reprlib.html#:~:text=Repr%2Erepr%5FTYPE
        # repr_TYPE is not an actual method
        return False
    if rst_file_stem == "signal" and api_name == "SIG*":
        # https://docs.python.org/dev/library/signal.html#:~:text=SIG*,-All%20the%20signal
        return False
    if (
        rst_file_stem == "socket"
        and api_name.endswith(
            "_*"
        )  # https://docs.python.org/dev/library/socket.html#:~:text=SO_*
        or api_name in ("VMADDR*", "SO_VM*")
        # https://docs.python.org/dev/library/socket.html#:~:text=VMADDR*,SO_VM*
        # list of all constants: https://github.com/torvalds/linux/blob/master/include/uapi/linux/vm_sockets.h
    ):
        return False
    if rst_file_stem == "ssl" and api_name == "ALERT_DESCRIPTION_*":
        # https://docs.python.org/dev/library/ssl.html#:~:text=ALERT_DESCRIPTION_*
        return False
    if rst_file_stem == "stdtypes" and api_name in (
        "sequence.copy",  # optional method
    ):
        return False
    if rst_file_stem == "test":
        # https://docs.python.org/dev/library/test.html#:~:text=internal%20use%20by%20Python
        # internal use only
        return False
    if rst_file_stem == "unittest" and api_name in ("setUpModule", "tearDownModule"):
        # https://docs.python.org/dev/library/unittest.html#setupmodule-and-teardownmodule
        # optional functions, :no-typesetting:
        return False
    if rst_file_stem == "urllib.request":
        if (
            api_name
            in (
                "BaseHandler.default_open",  # optional method, https://docs.python.org/dev/library/urllib.request.html#urllib.request.BaseHandler.default_open
                "BaseHandler.unknown_open",  # optional method, https://docs.python.org/dev/library/urllib.request.html#urllib.request.BaseHandler.unknown_open
                "BaseHandler.http_error_default",  # optional method, https://docs.python.org/dev/library/urllib.request.html#urllib.request.BaseHandler.http_error_default
                "BaseHandler.http_error_<nnn>",  # multiple methods, https://docs.python.org/dev/library/urllib.request.html#http-error-nnn
            )
            or ".<protocol>_" in api_name  # `<protocol>`` can be `http`, `https` etc.
        ):
            return False
    return True


def re_compile_name_sub(names: tuple[str]) -> re.Pattern[str]:
    return re.compile(r"\b(?:" + "|".join(names) + r")\b")


regex_prefix_typing = re_compile_name_sub(("Any", "Literal"))
regex_prefix_collections_abc = re_compile_name_sub(
    (
        "AsyncGenerator",
        "Awaitable",
        "Container",
        "Collection",
        "Coroutine",
        # "Generator", # collections.abc.Generator, email.generator.Generator, typing.Generator
        "Hashable",
        "ValuesView",
        "KeysView",
        "ItemsView",
        "Iterable",
        "Iterator",
        "Mapping",
        "MutableMapping",
        "MutableSequence",
        "OrderedDict",
        "Reversible",
        "Sized",
        "Sequence",
    )
)


def fix_sig_muilt_target(sig: str, rst_file_stem: str) -> str:
    """
    prevent warnings like:
    WARNING: more than one target found for cross-reference
    https://github.com/sphinx-doc/sphinx/blob/master/sphinx/domains/python/__init__.py
    """
    sig = regex_prefix_typing.sub(r"typing.\g<0>", sig)
    sig = regex_prefix_collections_abc.sub(r"collections.abc.\g<0>", sig)
    if rst_file_stem == "annotationlib":
        # ambiguity with `ast`
        sig = re_compile_name_sub(("ParamSpec", "TypeVar", "TypeVarTuple")).sub(
            r"typing.\g<0>", sig
        )
    if rst_file_stem in (
        "asyncio-eventloop",
        "asyncio-extending",
        "asyncio-runner",
        "asyncio-task",
    ):
        # ambiguity with `decimal`
        sig = re_compile_name_sub(("Context",)).sub(r"contextvars.\g<0>", sig)
    if rst_file_stem == "asyncio-subprocess":
        # ambiguity with `codecs`
        sig = re_compile_name_sub(("StreamReader", "StreamWriter")).sub(
            r"asyncio.\g<0>", sig
        )
    if rst_file_stem in ("email.message", "email.compat32-message"):
        sig = re_compile_name_sub(("Generator",)).sub(r"collections.abc.\g<0>", sig)
    if not rst_file_stem.startswith("email."):
        sig = re_compile_name_sub(("Generator",)).sub(r"collections.abc.\g<0>", sig)
    if rst_file_stem == "configparser":
        sig = re_compile_name_sub(("Pattern",)).sub(r"re.\g<0>", sig)
    if rst_file_stem == "dialog":
        # asyncio.Event, threading.Event, multiprocessing.Event
        sig = re_compile_name_sub(("Event",)).sub(r"tkinter.\g<0>", sig)
    if rst_file_stem == "difflib":
        sig = re_compile_name_sub(("Match",)).sub(r"difflib.\g<0>", sig)
    if rst_file_stem.startswith("email.") or rst_file_stem in (
        "http.client",
        "http.server",
        "urllib.error",
        "urllib.request",
        "smtplib",
    ):
        sig = re_compile_name_sub(("Message",)).sub(r"email.message.\g<0>", sig)
    if rst_file_stem == "importlib.resources":
        sig = re_compile_name_sub(("Traversable",)).sub(
            r"importlib.resources.abc.\g<0>", sig
        )
        sig = re_compile_name_sub(("Path",)).sub(r"pathlib.\g<0>", sig)
    if rst_file_stem == "inspect":
        # ambiguity with `typing.ForwardRef`
        sig = re_compile_name_sub(("ForwardRef",)).sub(r"annotationlib.\g<0>", sig)
    if rst_file_stem == "multiprocessing":
        ## Note: the correct type is `multiprocessing.synchronize.Lock` but `multiprocessing.Lock` has documentation
        sig = re_compile_name_sub(("Lock",)).sub(r"multiprocessing.Lock", sig)
        sig = re_compile_name_sub(("Connection",)).sub(
            r"multiprocessing.connection.\g<0>", sig
        )
        sig = re_compile_name_sub(
            (
                "Barrier",
                "BoundedSemaphore",
                "Condition",
                "Event",
                "Queue",
                "RLock",
                "Semaphore",
            )
        ).sub(r"threading.\g<0>", sig)
        sig = sig.replace("-> lock", "-> threading.Lock")
    if rst_file_stem == "sched":
        sig = re_compile_name_sub(("Event",)).sub(r"sched.\g<0>", sig)
    if rst_file_stem == "stdtypes":
        # ambiguity with `ast`
        sig = re_compile_name_sub(("ParamSpec", "TypeVar", "TypeVarTuple")).sub(
            r"typing.\g<0>", sig
        )
    if rst_file_stem == "tkinter.dnd":
        sig = re_compile_name_sub(("Event",)).sub(r"tkinter.\g<0>", sig)
    if rst_file_stem == "unittest":
        sig = re_compile_name_sub(("Pattern",)).sub(r"re.\g<0>", sig)
    if rst_file_stem == "xml.sax.utils":
        # ambiguity with `asyncio`
        sig = re_compile_name_sub(("StreamWriter",)).sub(r"codecs.\g<0>", sig)
    return sig


def replace_collections_abc___(sig: str) -> None:
    # https://github.com/python/typeshed/blob/main/stdlib/_collections_abc.pyi
    # sig = regex_prefix_collections_abc.sub(r"collections.abc.\g<0>", sig)

    import urllib.request
    import itertools, re

    _collections_abc_url = "https://github.com/python/typeshed/raw/refs/heads/main/stdlib/_collections_abc.pyi"
    txt = urllib.request.urlopen(_collections_abc_url).read().decode("utf-8")
    it = itertools.takewhile(
        lambda line: not line.startswith(")"),
        itertools.dropwhile(
            lambda line: not line.startswith("from typing import"), txt.splitlines()
        ),
    )
    next(it)
    it = [re.compile(r"\b(.+?) as (\1)\b").search(e) for e in it]
    [e.group(1) for e in it if e is not None]


in_collections_abc = [
    "AsyncGenerator",
    "AsyncIterable",
    "AsyncIterator",
    "Awaitable",
    "ByteString",
    "Callable",
    "Collection",
    "Container",
    "Coroutine",
    "Generator",
    "Hashable",
    "ItemsView",
    "Iterable",
    "Iterator",
    "KeysView",
    "Mapping",
    "MappingView",
    "MutableMapping",
    "MutableSequence",
    "MutableSet",
    "Reversible",
    "Sequence",
    "Sized",
    "ValuesView",
]

regex_sub_collections_abc = re.compile(rf"\btyping\.({'|'.join(in_collections_abc)})\b")


def replace_collections_abc(sig: str) -> str:
    "https://github.com/python/typeshed/blob/main/stdlib/_collections_abc.pyi"
    sig = sig.replace("ParamSpec(_", "typing.ParamSpec(_")
    sig = sig.replace("_collections_abc.", "collections.abc.")
    sig = sig.replace("typing.AbstractSet", "collections.abc.Set")
    sig = regex_sub_collections_abc.sub(r"collections.abc.\g<1>", sig)
    sig = sig.replace("asyncio.queues.", "asyncio.")
    sig = sig.replace("asyncio.streams.", "asyncio.")
    sig = sig.replace("_contextvars.", "contextvars.")
    return sig


def fix_sig(sig: str, rst_file_stem: str) -> str:
    # sig = fix_sig_muilt_target(sig, rst_file_stem)
    sig = sig.replace("builtins.", "")
    ## in Pyrefly, fully qualified names are not used for `TypeVar`s
    if rst_file_stem == "contextlib":
        sig = re.compile(r"\bAwaitable\b").sub(r"collections.abc.Awaitable", sig)
    if rst_file_stem.startswith("email."):
        sig = re.compile(r"\bMessage\b").sub(r"email.message.Message", sig)
    if rst_file_stem == "getopt":
        sig = re.compile(r"\bSequence\b").sub(r"collections.abc.Sequence", sig)
    if rst_file_stem == "pkgutil":
        sig = re.compile(r"\bIterable\b").sub(r"collections.abc.Iterable", sig)
    if rst_file_stem == "statistics":
        sig = re.compile(r"\bHashable\b").sub(r"collections.abc.Hashable", sig)
    sig = replace_collections_abc(sig)
    # fix Pyrefly output quirks
    # functions without overloads have "def " prefix and ": ..." suffix
    if sig.startswith("def "):
        assert sig.endswith(": ...")
        return sig[len("def ") : -len(": ...")]
    return sig


def get_api_type_info(code: str, rst_file_stem: str) -> dict | None:
    if not modify_docs:
        return None
    # TODO: support Windows and macOS specific APIs
    return get_api_type_info_sys_platform(code, rst_file_stem, "linux")


def get_api_type_info_sys_platform(
    code: str, rst_file_stem: str, sys_platform: pylsp.type_sys_platform
) -> dict | None:
    if code is None:
        return None
    info = {"signatureHelp": [], "hover": None, "declaration": None, "definition": None}

    hover = pylsp.get_LSP_raw(code, sys_platform, "textDocument/hover")["result"]
    type_info_hover = hover["contents"]["value"].splitlines()[1].partition(": ")[2]
    info["hover"] = type_info_hover
    hover_txt = type_info_hover

    signatureHelp = pylsp.get_LSP_raw(
        code + "(_", sys_platform, "textDocument/signatureHelp"
    )["result"]
    sig_raw = [e["label"] for e in signatureHelp["signatures"]]
    if hover_txt.startswith("type[") and hover_txt.endswith(
        "]"
    ):  # check if this is a type
        name = hover_txt[5:-1]
        try:
            ast.parse(name)
        except SyntaxError:
            sig0 = sig_raw
        else:
            sig0 = [
                e[:-4] + name if e.endswith(") -> None") else e for e in sig_raw
            ]  # Pyrefly shows `) -> None` for constructors using __init__
    else:
        sig0 = sig_raw

    info["signatureHelp"] = [fix_sig(e, rst_file_stem) for e in sig0]

    declaration = pylsp.get_LSP_raw(code, sys_platform, "textDocument/declaration")[
        "result"
    ]
    if declaration == []:
        return None
    if "pyrefly_bundled_typeshed_" in declaration["uri"]:
        info["declaration"] = declaration
    else:
        assert False, f"declaration not found for {code=}"
        definition = pylsp.get_LSP_raw(code, sys_platform, "textDocument/definition")[
            "result"
        ]
        assert "pyrefly_bundled_typeshed_" in definition["uri"]
        info["definition"] = definition
    return info


def check_api_level_0(
    rst_file_stem: str,
    api_info: APIInfo,
    module: str | None,
) -> dict:
    api_names = api_info.api_names
    api_type_info = {}
    for api_name, _ in itertools.groupby(api_names):
        if not check_exists_predicate_level_0(rst_file_stem, api_name):
            continue
        if module is None and "." in api_name:
            module, api_name = api_name.rsplit(".", maxsplit=1)
        code = f"""import {module}\n{module}.{api_name}"""
        final_code: str | None = code
        if not pylsp.exists_all_platforms(code):
            code2 = retry_with_new_code_level_0(rst_file_stem, api_name)
            if code2 is utils.MISSING or not pylsp.exists_all_platforms(code2):
                final_code = None
                print(f"missing\t{api_name}\t{module}\t{rst_file_stem}.rst")
            else:
                final_code = code2
        if final_code is not None:
            api_type_info[api_name] = get_api_type_info(final_code, rst_file_stem)
    return api_type_info


def check_api_level_1(
    rst_file_stem: str,
    parent_info: tuple,
    api_info: APIInfo,
    module: str | None,
) -> None:
    api_names = api_info.api_names
    parent_api_names = parent_info[-1].api_names
    parent_api_type = parent_info[-1].api_type
    api_type = api_info.api_type
    api_type_info = {}
    for parent_api_name, api_name_raw in itertools.product(
        [k for k, g in itertools.groupby(parent_api_names)],
        [k for k, g in itertools.groupby(api_names)],
    ):
        if module is None:
            module, parent_api_name = parent_api_name.split(".", maxsplit=1)
        if api_name_raw.startswith(parent_api_name + "."):
            api_name = api_name_raw[len(parent_api_name) + 1 :]
        else:
            api_name = api_name_raw
        if not check_exists_predicate_level_1(rst_file_stem, parent_api_name, api_name):
            continue

        if parent_api_type == "function" and api_type == "data":
            # https://docs.python.org/dev/library/fcntl.html#fcntl.lockf
            # https://docs.python.org/dev/library/locale.html#locale.nl_langinfo
            code = f"""import {module}\n{module}.{api_name}"""
        else:
            code = f"""import {module}\n{module}.{parent_api_name}.{api_name}"""
        final_code: str | None = code
        if not pylsp.exists_all_platforms(code):
            code2 = retry_with_new_code_level_1(
                rst_file_stem, module, parent_api_name, api_name
            )
            if code2 is utils.MISSING or not pylsp.exists_all_platforms(code2):
                final_code = None
                print(
                    f"missing\t{parent_api_name}.{api_name}\t{module}\t{rst_file_stem}.rst"
                )
            else:
                final_code = code2
        if final_code is not None:
            api_type_info[api_name_raw] = get_api_type_info(final_code, rst_file_stem)
    return api_type_info


def check_api_level_2(
    rst_file_stem: str,
    parent_info: tuple,
    api_info: APIInfo,
) -> None:
    code0 = ""
    parent_api_names = tuple(e.api_names for e in parent_info)
    if rst_file_stem == "asyncio-task" and parent_api_names == (
        ["timeout"],
        ["Timeout"],
    ):
        code0 = """
# https://docs.python.org/dev/library/asyncio-task.html#asyncio.Timeout
import asyncio
cm = asyncio.timeout(None)
cm"""
    if (
        rst_file_stem == "ctypes"
        and parent_api_names == (["_CFuncPtr"], ["errcheck"])
        and api_info.api_names == ["callable"]
    ):
        # https://docs.python.org/dev/library/ctypes.html#ctypes._CFuncPtr.errcheck
        # ctypes._CFuncPtr.errcheck.callable is not an attribute
        return
    if rst_file_stem == "importlib.metadata" and parent_api_names == (
        ["DistributionFinder"],
        ["Context"],
    ):
        # https://docs.python.org/dev/library/importlib.metadata.html#importlib.metadata.DistributionFinder.Context
        code0 = """import importlib.metadata
context = importlib.metadata.DistributionFinder.Context()
context"""
    if rst_file_stem == "unittest" and parent_api_names == (
        ["TestCase"],
        ["assertLogs"],
    ):
        # https://docs.python.org/dev/library/unittest.html#unittest.TestCase.assertLogs
        code0 = """
import unittest
context = unittest.TestCase().assertLogs()
cm = context.__enter__()
cm"""

    for api_name in api_info.api_names:
        code = code0 + f".{api_name}"
        if not pylsp.exists_all_platforms(code):
            print(
                f"missing\t{'.'.join(map(str, parent_api_names))}.{api_name}\t{rst_file_stem}.rst"
            )


def check_api_existance(
    rst_file_stem: str, parent_info: tuple, chunk: tuple
) -> tuple[str, Any]:
    directive_node, children = chunk
    (e, level, module0) = dataclasses.astuple(directive_node)
    # sys.monitoring is not a module
    # https://docs.python.org/dev/library/sys.monitoring.html
    # https://raw.githubusercontent.com/python/cpython/refs/heads/main/Doc/tools/extensions/pyspecific.py#:~:text=parse_monitoring_event
    # https://raw.githubusercontent.com/python/cpython/refs/heads/main/Doc/library/sys.monitoring.rst#:~:text=monitoring%2Devent
    module = "sys" if module0 == "sys.monitoring" else module0
    rst_code = e[0]
    # print(rst_code.splitlines()[0])
    if level >= 0:
        api_info = get_api_info(rst_file_stem, e)
        api_type_info = {}
        if level == 0:
            api_type_info = check_api_level_0(rst_file_stem, api_info, module)
        if level == 1:
            api_type_info = check_api_level_1(
                rst_file_stem, parent_info, api_info, module
            )
        if level == 2:
            check_api_level_2(rst_file_stem, parent_info, api_info)
        if level >= 3:
            print("level >=3 api found, no check implemented yet")
            print(rst_file_stem, e, parent_info, api_info, module)
            raise NotImplementedError
    parent_info_for_child = parent_info + ((api_info,) if level >= 0 else ())
    ll = [
        check_api_existance(rst_file_stem, parent_info_for_child, child)
        for child in children
    ]
    for code, match in reversed(ll):
        sp = match.span()
        rst_code = rst_code[: sp[0]] + code + rst_code[sp[1] :]
    if level >= 0:
        if modify_docs:
            dedented_rst, dedent = utils.dedent(rst_code)
            rst_code = modify_rst(
                rst_file_stem,
                rec_directive.fullmatch(dedented_rst),
                dedented_rst,
                api_type_info,
            )
            rst_code = textwrap.indent(rst_code, " " * dedent)
    return rst_code, e


cpython_docs_dir = utils.cpython_docs_dir
sorted_rst_paths = sorted((cpython_docs_dir / "library").glob("**/*.rst"))

subprocess.run(["git", "reset", "--hard"], cwd=cpython_docs_dir, check=True)
subprocess.run(
    ["git", "switch", utils.cpython_branch], cwd=cpython_docs_dir, check=True
)
subprocess.run(["git", "pull"], cwd=cpython_docs_dir, check=True)

conf_py_path = cpython_docs_dir / "conf.py"
# https://www.sphinx-doc.org/en/master/usage/configuration.html
if modify_docs:
    conf_py_path.write_text(
        conf_py_path.read_text(encoding="utf-8")
        + """
python_use_unqualified_type_names = True
python_maximum_signature_line_length = 80
html_last_updated_fmt = '(timestamp removed to reduce diffs)'
""",
        encoding="utf-8",
    )

# started = False
# start_at = "winreg"

for rst_path in sorted_rst_paths:
    # if rst_path.stem not in ("functions", "typing"):
    #     continue

    # if not started:
    #     if rst_path.stem == start_at:
    #         started = True
    #     else:
    #         continue

    txt = rst_path.read_text(encoding="utf-8")
    directive_tree = parse_directive_tree(
        txt, level=0, indent=0, module_directive=[], parent_module=utils.MISSING
    )

    txt2 = check_api_existance(
        rst_path.stem,
        (),
        (
            DirectiveNode(re.compile(".*", re.DOTALL).fullmatch(txt), -1, None),
            directive_tree,
        ),
    )[0]
    if modify_docs:
        rst_path.write_text(txt2, encoding="utf-8")


if modify_docs:
    ## Build the docs
    ## do a `make clean` to resolve issues with audit-event-table directive in library/audit_events.html
    subprocess.run(["make", "clean"], cwd=cpython_docs_dir, check=True)
    subprocess.run(["make", "venv"], cwd=cpython_docs_dir, check=True)
    subprocess.run(["make", "html"], cwd=cpython_docs_dir, check=True)