Refactor component finding logic

src/pulp_docs/cli.py

@@ -5,10 +5,9 @@
 import click
 import git
 from mkdocs.__main__ import cli as mkdocs_cli
-from mkdocs.config import load_config
 
 from pulp_docs.context import ctx_blog, ctx_docstrings, ctx_draft, ctx_dryrun, ctx_path
-from pulp_docs.plugin import load_components
+from pulp_docs.plugin import ComponentLoader, default_lookup_paths
 
 
 def blog_callback(ctx: click.Context, param: click.Parameter, value: bool) -> bool:
@@ -122,16 +121,13 @@ async def clone_repository(repo_url: str) -> None:
 def fetch(dest, config_file, path_exclude):
     """Fetch repositories to destination dir."""
     dest_path = Path(dest)
-    pulpdocs_plugin = load_config(config_file).plugins["PulpDocs"]
-    all_components = pulpdocs_plugin.config.components
-    all_repositories_set = {r.git_url for r in all_components if r.git_url}
-    found_components = load_components(path_exclude, pulpdocs_plugin.config, draft=True)
-    found_repositories_set = {r.git_url for r in found_components}
-    final_repositories_set = all_repositories_set - found_repositories_set
-
+    lookup_paths = default_lookup_paths()
+    component_loader = ComponentLoader(lookup_paths, mkdocs_config=config_file)
+    missing_comps = component_loader.load_all().missing
+    missing_repos = {comp.git_url for comp in missing_comps}
     if not dest_path.exists():
         dest_path.mkdir(parents=True)
-    asyncio.run(clone_repositories(final_repositories_set, dest_path))
+    asyncio.run(clone_repositories(missing_repos, dest_path))
 
 
 main = mkdocs_cli

src/pulp_docs/openapi.py

@@ -10,32 +10,29 @@
 from pathlib import Path
 from typing import Optional
 
-from mkdocs.config import load_config
-
 from pulp_docs.cli import get_default_mkdocs
-from pulp_docs.plugin import ComponentOption
+from pulp_docs.plugin import ComponentLoader, ComponentSpec, default_lookup_paths
 
 BASE_TMPDIR_NAME = "pulpdocs_tmp"
 CURRENT_DIR = Path(__file__).parent.absolute()
 
 
-def main(output_dir: Path, plugins_filter: Optional[list[str]] = None, dry_run: bool = False):
-    """Creates openapi json files for all or selected plugins in output dir."""
+def main(output_dir: Path, filter_list: Optional[list[str]] = None, dry_run: bool = False):
+    """Creates openapi json files for found plugins in the output_dir.
 
-    def filter_plugin(name: str) -> bool:
-        if not plugins_filter:
-            return True
-        return name in plugins_filter or name == "pulpcore"
+    Optionally filter the found plugins with a filter list.
+    """
 
-    def get_plugins() -> list[ComponentOption]:
-        mkdocs_yml = str(get_default_mkdocs())
-        pulpdocs_plugin = load_config(mkdocs_yml).plugins["PulpDocs"]
-        all_components = pulpdocs_plugin.config.components
-        return [c for c in all_components if c.rest_api]
+    def select_component_fn(comp: ComponentSpec) -> bool:
+        name = comp.component_name
+        return (bool(filter_list) and name in filter_list) or name == "pulpcore"
 
-    all_plugins = get_plugins()
-    all_plugins = [p for p in all_plugins if filter_plugin(p.name)]
-    openapi = OpenAPIGenerator(plugins=all_plugins, dry_run=dry_run)
+    mkdocs_config = get_default_mkdocs()
+    lookup_paths = default_lookup_paths()
+    component_loader = ComponentLoader(lookup_paths, mkdocs_config=mkdocs_config)
+    all_specs = component_loader.load_all().all_specs
+    selected = list(filter(select_component_fn, all_specs))
+    openapi = OpenAPIGenerator(plugins=selected, dry_run=dry_run)
     openapi.generate(target_dir=output_dir)
 
 
@@ -49,8 +46,8 @@ class OpenAPIGenerator:
         dry_run: Whether it should execute the commands or just show them.
     """
 
-    def __init__(self, plugins: list[ComponentOption], dry_run=False):
-        self.pulpcore = next(filter(lambda p: p.name == "pulpcore", plugins))
+    def __init__(self, plugins: list[ComponentSpec], dry_run=False):
+        self.pulpcore = next(filter(lambda p: p.component_name == "pulpcore", plugins))
         self.plugins = plugins + [self.pulpcore]
         self.dry_run = dry_run
 
@@ -75,7 +72,7 @@ def generate(self, target_dir: Path):
                 outfile,
             )
 
-    def setup_venv(self, plugin: ComponentOption):
+    def setup_venv(self, plugin: ComponentSpec):
         """
         Creates virtualenv with plugin.
         """
@@ -140,8 +137,8 @@ def parse_args():
     dry_run = args.dry_run
     dest = Path(args.output_dir)
 
-    plugins_filter = []
+    filter_list = []
     if args.plugin_list:
-        plugins_filter = [str(p) for p in args.plugin_list.split(",") if p]
+        filter_list = [str(p) for p in args.plugin_list.split(",") if p]
 
-    main(dest, plugins_filter, dry_run)
+    main(dest, filter_list, dry_run)