diff --git a/.gitignore b/.gitignore index d23644a4..a16abc44 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ tests/test_jupyter/*.txt .ruff_cache .venv docs/jupyter_execute +.DS_Store diff --git a/docs/AGENTS.md b/docs/AGENTS.md index e4fa1bed..db21c5e2 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -1,12 +1,7 @@ # Documentation -- Link to existing docs/API refs instead of re-explaining concepts - reduces duplication - and keeps info in sync - Prevents documentation drift and outdated explanations by - maintaining a single source of truth for each concept -- Link to canonical docs rather than duplicating content - prevents drift and - maintenance burden - Consolidating documentation into existing files with - cross-references keeps information consistent and reduces the effort needed to - update multiple locations when changes occur. +## General + - Document only public APIs and user-facing behavior - exclude internals, framework abstractions, and implementation plumbing - Users need actionable documentation on what they can use, not confusing details about internal mechanics they can't control @@ -17,9 +12,6 @@ comprehensive coverage vs. fragmented mentions - Prevents users from missing features when they approach from different contexts (CLI vs. API) and allows features to be documented holistically rather than buried in subsections. -- Avoid `# ruff: noqa` or `# type: ignore` in doc examples - ensures examples stay - correct and runnable - Skip directives hide bugs and type errors in documentation - code that users will copy, leading to broken examples in the wild - Explicitly mark parameters/features as 'optional' in docs, even when types show it - reduces cognitive load for readers - Users shouldn't need to parse type signatures to understand optionality; explicit labels make documentation scannable and @@ -31,3 +23,21 @@ - Strip boilerplate from docs examples - show only the feature being demonstrated - Reduces cognitive load and helps readers focus on the specific API or pattern being taught without distraction from scaffolding code. + +## Linking + +- Link to existing docs/API refs instead of re-explaining concepts - reduces duplication + and keeps info in sync - Prevents documentation drift and outdated explanations by + maintaining a single source of truth for each concept +- Link to canonical docs rather than duplicating content - prevents drift and + maintenance burden - Consolidating documentation into existing files with + cross-references keeps information consistent and reduces the effort needed to + update multiple locations when changes occur. + +## Code Examples + +- Avoid `# ruff: noqa` or `# type: ignore` in doc examples - ensures examples stay + correct and runnable - Skip directives hide bugs and type errors in documentation + code that users will copy, leading to broken examples in the wild +- Code file examples should have a title that shows the file name. +- Important lines should be highlighted or annotated with a comment. diff --git a/docs/source/_static/md/commands/build-arguments.md b/docs/source/_static/md/commands/build-arguments.md index 1c45a5f0..c7ad3809 100644 --- a/docs/source/_static/md/commands/build-arguments.md +++ b/docs/source/_static/md/commands/build-arguments.md @@ -1,3 +1,3 @@ -| Argument | Description | -| ------------ | ---------------------------------------------------------- | -| `[PATHS]...` | Paths where pytask looks for task files and configuration. | +| Argument | Description | +| ----------------------- | ---------------------------------------------------------- | +| [PATHS]... | Paths where pytask looks for task files and configuration. | diff --git a/docs/source/_static/md/commands/build-options.md b/docs/source/_static/md/commands/build-options.md index d2ef5162..317f7f2a 100644 --- a/docs/source/_static/md/commands/build-options.md +++ b/docs/source/_static/md/commands/build-options.md @@ -1,30 +1,43 @@ -| Option | Default | Description | -| ---------------------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `-c, --config FILE` | - | Path to configuration file. | -| \`--capture \[fd | no | sys | -| `--database-url TEXT` | - | Url to the database. | -| `--debug-pytask` | `false` | Trace all function calls in the plugin framework. | -| `--disable-warnings` | `false` | Disables the summary for warnings. | -| `--dry-run` | `false` | Perform a dry-run. | -| `--editor-url-scheme TEXT` | `file` | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | -| `--explain` | `false` | Explain why tasks need to be executed by showing what changed. | -| `-f, --force` | `false` | Execute a task even if it succeeded successfully before. | -| `--hook-module TEXT` | - | Path to a Python module that contains hook implementations. | -| `--ignore TEXT` | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | -| `-k EXPRESSION` | - | Select tasks via expressions on task ids. | -| `-m MARKER_EXPRESSION` | - | Select tasks via marker expressions. | -| `--max-failures FLOAT RANGE` | `inf` | Stop after some failures. | -| `--n-entries-in-table INTEGER RANGE` | `15` | How many entries to display in the table during the execution. Tasks which are running are always displayed. | -| `--pdb` | `false` | Start the interactive debugger on errors. | -| `--pdbcls module_name:class_name` | - | Start a custom debugger on errors. For example: --pdbcls=IPython.terminal.debugger:TerminalPdb | -| `-s` | `false` | Shortcut for --capture=no. | -| \`--show-capture \[no | stdout | stderr | -| `--show-errors-immediately` | `false` | Show errors with tracebacks as soon as the task fails. | -| `--show-locals` | `false` | Show local variables in tracebacks. | -| `--show-traceback / --show-no-traceback` | `--show-traceback` | Choose whether tracebacks should be displayed or not. | -| `--sort-table / --do-not-sort-table` | `--sort-table` | Sort the table of tasks at the end of the execution. | -| `--strict-markers` | `false` | Raise errors for unknown markers. | -| `--trace` | `false` | Enter debugger in the beginning of each task. | -| `-v, --verbose INTEGER RANGE` | `1` | Make pytask verbose (>= 0) or quiet (= 0). | -| `-x, --stop-after-first-failure` | `false` | Stop after the first failure. | -| `-h, --help` | - | Show this message and exit. | +| Option | Default | Description | +| ---------------------------------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| -c, --config FILE | - | Path to configuration file. | +| --capture [fd\|no\|sys\|tee-sys] | fd | Per task capturing method. | +| --clean-lockfile | false | Rewrite the lockfile with only currently collected tasks. | +| --database-url TEXT | - | Url to the database. | +| --debug-pytask | false | Trace all function calls in the plugin framework. | +| --disable-warnings | false | Disables the summary for warnings. | +| --dry-run | false | Perform a dry-run. | +| --editor-url-scheme TEXT | file | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | +| --explain | false | Explain why tasks need to be executed by showing what changed. | +| -f, --force | false | Execute a task even if it succeeded successfully before. | +| --hook-module TEXT | - | Path to a Python module that contains hook implementations. | +| --ignore TEXT | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | +| -k EXPRESSION | - | Select tasks via expressions on task ids. | +| --log-cli / --no-log-cli | --no-log-cli | Enable live log display during task execution. | +| --log-cli-date-format TEXT | - | Log date format used by the logging module for live logs. | +| --log-cli-format TEXT | - | Log format used by the logging module for live logs. | +| --log-cli-level LEVEL | - | CLI logging level. | +| --log-date-format TEXT | %H:%M:%S | Log date format used by the logging module. | +| --log-file TEXT | - | Path to a file where logging will be written. | +| --log-file-date-format TEXT | - | Log date format used by the logging module for the log file. | +| --log-file-format TEXT | - | Log format used by the logging module for the log file. | +| --log-file-level LEVEL | - | Log file logging level. | +| --log-file-mode [w\|a] | w | Log file open mode. | +| --log-format TEXT | %(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s | Log format used by the logging module. | +| --log-level LEVEL | - | Level of messages to catch/display. Not set by default, so it depends on the logger configuration. | +| -m MARKER_EXPRESSION | - | Select tasks via marker expressions. | +| --max-failures FLOAT RANGE | inf | Stop after some failures. | +| --n-entries-in-table INTEGER RANGE | 15 | How many entries to display in the table during the execution. Tasks which are running are always displayed. | +| --pdb | false | Start the interactive debugger on errors. | +| --pdbcls module_name:class_name | - | Start a custom debugger on errors. For example: --pdbcls=IPython.terminal.debugger:TerminalPdb | +| -s | false | Shortcut for --capture=no. | +| --show-capture [no\|stdout\|stderr\|log\|all] | all | Choose which captured output should be shown for failed tasks. | +| --show-errors-immediately | false | Show errors with tracebacks as soon as the task fails. | +| --show-locals | false | Show local variables in tracebacks. | +| --show-traceback / --show-no-traceback | --show-traceback | Choose whether tracebacks should be displayed or not. | +| --sort-table / --do-not-sort-table | --sort-table | Sort the table of tasks at the end of the execution. | +| --strict-markers | false | Raise errors for unknown markers. | +| --trace | false | Enter debugger in the beginning of each task. | +| -v, --verbose INTEGER RANGE | 1 | Make pytask verbose (>= 0) or quiet (= 0). | +| -x, --stop-after-first-failure | false | Stop after the first failure. | +| `-h, --help` | - | Show this message and exit. | diff --git a/docs/source/_static/md/commands/clean-arguments.md b/docs/source/_static/md/commands/clean-arguments.md index 1c45a5f0..c7ad3809 100644 --- a/docs/source/_static/md/commands/clean-arguments.md +++ b/docs/source/_static/md/commands/clean-arguments.md @@ -1,3 +1,3 @@ -| Argument | Description | -| ------------ | ---------------------------------------------------------- | -| `[PATHS]...` | Paths where pytask looks for task files and configuration. | +| Argument | Description | +| ----------------------- | ---------------------------------------------------------- | +| [PATHS]... | Paths where pytask looks for task files and configuration. | diff --git a/docs/source/_static/md/commands/clean-options.md b/docs/source/_static/md/commands/clean-options.md index f4be2d9b..5c458c80 100644 --- a/docs/source/_static/md/commands/clean-options.md +++ b/docs/source/_static/md/commands/clean-options.md @@ -1,15 +1,15 @@ -| Option | Default | Description | -| -------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `-c, --config FILE` | - | Path to configuration file. | -| `-d, --directories` | `false` | Remove whole directories. | -| `--database-url TEXT` | - | Url to the database. | -| `-e, --exclude PATTERN` | - | A filename pattern to exclude files from the cleaning process. | -| `--editor-url-scheme TEXT` | `file` | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | -| `--hook-module TEXT` | - | Path to a Python module that contains hook implementations. | -| `--ignore TEXT` | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | -| `-k EXPRESSION` | - | Select tasks via expressions on task ids. | -| `-m MARKER_EXPRESSION` | - | Select tasks via marker expressions. | -| \`--mode \[dry-run | force | interactive\]\` | -| `-q, --quiet` | `false` | Do not print the names of the removed paths. | -| `--strict-markers` | `false` | Raise errors for unknown markers. | -| `-h, --help` | - | Show this message and exit. | +| Option | Default | Description | +| ------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| -c, --config FILE | - | Path to configuration file. | +| -d, --directories | false | Remove whole directories. | +| --database-url TEXT | - | Url to the database. | +| -e, --exclude PATTERN | - | A filename pattern to exclude files from the cleaning process. | +| --editor-url-scheme TEXT | file | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | +| --hook-module TEXT | - | Path to a Python module that contains hook implementations. | +| --ignore TEXT | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | +| -k EXPRESSION | - | Select tasks via expressions on task ids. | +| -m MARKER_EXPRESSION | - | Select tasks via marker expressions. | +| --mode [dry-run\|force\|interactive] | dry-run | Choose 'dry-run' to print the paths of files/directories which would be removed, 'interactive' for a confirmation prompt for every path, and 'force' to remove all unknown paths at once. | +| -q, --quiet | false | Do not print the names of the removed paths. | +| --strict-markers | false | Raise errors for unknown markers. | +| `-h, --help` | - | Show this message and exit. | diff --git a/docs/source/_static/md/commands/collect-arguments.md b/docs/source/_static/md/commands/collect-arguments.md index 1c45a5f0..c7ad3809 100644 --- a/docs/source/_static/md/commands/collect-arguments.md +++ b/docs/source/_static/md/commands/collect-arguments.md @@ -1,3 +1,3 @@ -| Argument | Description | -| ------------ | ---------------------------------------------------------- | -| `[PATHS]...` | Paths where pytask looks for task files and configuration. | +| Argument | Description | +| ----------------------- | ---------------------------------------------------------- | +| [PATHS]... | Paths where pytask looks for task files and configuration. | diff --git a/docs/source/_static/md/commands/collect-options.md b/docs/source/_static/md/commands/collect-options.md index 00724b6c..0eda5828 100644 --- a/docs/source/_static/md/commands/collect-options.md +++ b/docs/source/_static/md/commands/collect-options.md @@ -1,12 +1,12 @@ -| Option | Default | Description | -| -------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `-c, --config FILE` | - | Path to configuration file. | -| `--database-url TEXT` | - | Url to the database. | -| `--editor-url-scheme TEXT` | `file` | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | -| `--hook-module TEXT` | - | Path to a Python module that contains hook implementations. | -| `--ignore TEXT` | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | -| `-k EXPRESSION` | - | Select tasks via expressions on task ids. | -| `-m MARKER_EXPRESSION` | - | Select tasks via marker expressions. | -| `--nodes` | `false` | Show a task's dependencies and products. | -| `--strict-markers` | `false` | Raise errors for unknown markers. | -| `-h, --help` | - | Show this message and exit. | +| Option | Default | Description | +| ------------------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | +| -c, --config FILE | - | Path to configuration file. | +| --database-url TEXT | - | Url to the database. | +| --editor-url-scheme TEXT | file | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | +| --hook-module TEXT | - | Path to a Python module that contains hook implementations. | +| --ignore TEXT | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | +| -k EXPRESSION | - | Select tasks via expressions on task ids. | +| -m MARKER_EXPRESSION | - | Select tasks via marker expressions. | +| --nodes | false | Show a task's dependencies and products. | +| --strict-markers | false | Raise errors for unknown markers. | +| `-h, --help` | - | Show this message and exit. | diff --git a/docs/source/_static/md/commands/command-list.md b/docs/source/_static/md/commands/command-list.md index 5b28bdb8..093f3ad2 100644 --- a/docs/source/_static/md/commands/command-list.md +++ b/docs/source/_static/md/commands/command-list.md @@ -1,8 +1,8 @@ -| Command | Description | -| ----------------------------------------- | ------------------------------------------------------------- | -| [`build`](../../../commands/build.md) | Collect tasks, execute them and report the results. | -| [`clean`](../../../commands/clean.md) | Clean the provided paths by removing files unknown to pytask. | -| [`collect`](../../../commands/collect.md) | Collect tasks and report information about them. | -| [`dag`](../../../commands/dag.md) | Create a visualization of the directed acyclic graph. | -| [`markers`](../../../commands/markers.md) | Show all registered markers. | -| [`profile`](../../../commands/profile.md) | Show information about resource consumption. | +| Command | Description | +| ----------------------- | ------------------------------------------------------------- | +| [`build`](build.md) | Collect tasks, execute them and report the results. | +| [`clean`](clean.md) | Clean the provided paths by removing files unknown to pytask. | +| [`collect`](collect.md) | Collect tasks and report information about them. | +| [`dag`](dag.md) | Create a visualization of the directed acyclic graph. | +| [`markers`](markers.md) | Show all registered markers. | +| [`profile`](profile.md) | Show information about resource consumption. | diff --git a/docs/source/_static/md/commands/dag-arguments.md b/docs/source/_static/md/commands/dag-arguments.md index 1c45a5f0..c7ad3809 100644 --- a/docs/source/_static/md/commands/dag-arguments.md +++ b/docs/source/_static/md/commands/dag-arguments.md @@ -1,3 +1,3 @@ -| Argument | Description | -| ------------ | ---------------------------------------------------------- | -| `[PATHS]...` | Paths where pytask looks for task files and configuration. | +| Argument | Description | +| ----------------------- | ---------------------------------------------------------- | +| [PATHS]... | Paths where pytask looks for task files and configuration. | diff --git a/docs/source/_static/md/commands/dag-options.md b/docs/source/_static/md/commands/dag-options.md index 1e642004..8dc55edf 100644 --- a/docs/source/_static/md/commands/dag-options.md +++ b/docs/source/_static/md/commands/dag-options.md @@ -1,9 +1,9 @@ -| Option | Default | Description | -| --------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `-c, --config FILE` | - | Path to configuration file. | -| `--database-url TEXT` | - | Url to the database. | -| `--hook-module TEXT` | - | Path to a Python module that contains hook implementations. | -| `-l, --layout TEXT` | `dot` | The layout determines the structure of the graph. Here you find an overview of all available layouts: https://graphviz.org/docs/layouts. | -| `-o, --output-path FILE` | `dag.pdf` | The output path of the visualization. The format is inferred from the file extension. | -| \`-r, --rank-direction \[TB | LR | BT | -| `-h, --help` | - | Show this message and exit. | +| Option | Default | Description | +| -------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| -c, --config FILE | - | Path to configuration file. | +| --database-url TEXT | - | Url to the database. | +| --hook-module TEXT | - | Path to a Python module that contains hook implementations. | +| -l, --layout TEXT | dot | The layout determines the structure of the graph. Here you find an overview of all available layouts: https://graphviz.org/docs/layouts. | +| -o, --output-path FILE | dag.pdf | The output path of the visualization. The format is inferred from the file extension. | +| -r, --rank-direction [TB\|LR\|BT\|RL] | TB | The direction of the directed graph. It can be ordered from top to bottom, TB, left to right, LR, bottom to top, BT, or right to left, RL. | +| `-h, --help` | - | Show this message and exit. | diff --git a/docs/source/_static/md/commands/markers-arguments.md b/docs/source/_static/md/commands/markers-arguments.md index 1c45a5f0..c7ad3809 100644 --- a/docs/source/_static/md/commands/markers-arguments.md +++ b/docs/source/_static/md/commands/markers-arguments.md @@ -1,3 +1,3 @@ -| Argument | Description | -| ------------ | ---------------------------------------------------------- | -| `[PATHS]...` | Paths where pytask looks for task files and configuration. | +| Argument | Description | +| ----------------------- | ---------------------------------------------------------- | +| [PATHS]... | Paths where pytask looks for task files and configuration. | diff --git a/docs/source/_static/md/commands/markers-options.md b/docs/source/_static/md/commands/markers-options.md index 24518cb8..aaf5aea0 100644 --- a/docs/source/_static/md/commands/markers-options.md +++ b/docs/source/_static/md/commands/markers-options.md @@ -1,5 +1,5 @@ -| Option | Default | Description | -| -------------------- | ------- | ----------------------------------------------------------- | -| `-c, --config FILE` | - | Path to configuration file. | -| `--hook-module TEXT` | - | Path to a Python module that contains hook implementations. | -| `-h, --help` | - | Show this message and exit. | +| Option | Default | Description | +| ------------------------------- | ------- | ----------------------------------------------------------- | +| -c, --config FILE | - | Path to configuration file. | +| --hook-module TEXT | - | Path to a Python module that contains hook implementations. | +| `-h, --help` | - | Show this message and exit. | diff --git a/docs/source/_static/md/commands/profile-arguments.md b/docs/source/_static/md/commands/profile-arguments.md index 1c45a5f0..c7ad3809 100644 --- a/docs/source/_static/md/commands/profile-arguments.md +++ b/docs/source/_static/md/commands/profile-arguments.md @@ -1,3 +1,3 @@ -| Argument | Description | -| ------------ | ---------------------------------------------------------- | -| `[PATHS]...` | Paths where pytask looks for task files and configuration. | +| Argument | Description | +| ----------------------- | ---------------------------------------------------------- | +| [PATHS]... | Paths where pytask looks for task files and configuration. | diff --git a/docs/source/_static/md/commands/profile-options.md b/docs/source/_static/md/commands/profile-options.md index 7023cab5..4b1f5e0c 100644 --- a/docs/source/_static/md/commands/profile-options.md +++ b/docs/source/_static/md/commands/profile-options.md @@ -1,9 +1,9 @@ -| Option | Default | Description | -| -------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `-c, --config FILE` | - | Path to configuration file. | -| `--database-url TEXT` | - | Url to the database. | -| `--editor-url-scheme TEXT` | `file` | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | -| \`--export \[no | json | csv\]\` | -| `--hook-module TEXT` | - | Path to a Python module that contains hook implementations. | -| `--ignore TEXT` | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | -| `-h, --help` | - | Show this message and exit. | +| Option | Default | Description | +| ------------------------------------- | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| -c, --config FILE | - | Path to configuration file. | +| --database-url TEXT | - | Url to the database. | +| --editor-url-scheme TEXT | file | Use file, vscode, pycharm or a custom url scheme to add URLs to task ids to quickly jump to the task definition. Use no_link to disable URLs. | +| --export [no\|json\|csv] | no | Export the profile in the specified format. | +| --hook-module TEXT | - | Path to a Python module that contains hook implementations. | +| --ignore TEXT | - | A pattern to ignore files or directories. Refer to 'pathlib.Path.match' for more info. | +| `-h, --help` | - | Show this message and exit. | diff --git a/docs/source/_static/md/commands/root-options.md b/docs/source/_static/md/commands/root-options.md index 67688857..a13dfe4a 100644 --- a/docs/source/_static/md/commands/root-options.md +++ b/docs/source/_static/md/commands/root-options.md @@ -1,4 +1,4 @@ -| Option | Description | -| ------------ | --------------------------- | -| `--version` | Show the version and exit. | -| `-h, --help` | Show this message and exit. | +| Option | Description | +| ---------------------- | --------------------------- | +| --version | Show the version and exit. | +| `-h, --help` | Show this message and exit. | diff --git a/docs/source/_static/md/logging-live.md b/docs/source/_static/md/logging-live.md new file mode 100644 index 00000000..c42106d4 --- /dev/null +++ b/docs/source/_static/md/logging-live.md @@ -0,0 +1,40 @@ +
+ +```console + +$ pytask --log-cli --log-cli-level=INFO --show-capture=log +───────────────────────── Start pytask session ───────────────────────── +Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 +Root: C:\Users\pytask-dev\git\my_project +Collected 2 tasks. + +10:14:51 INFO build:preparing report.txt +10:14:51 WARNING build:publishing report is about to fail +╭───────────────────────────────────────────┬─────────╮ +│ Task │ Outcome │ +├───────────────────────────────────────────┼─────────┤ +│ task_logging.py::task_prepare_report │ . │ +│ task_logging.py::task_publish_report │ F │ +╰───────────────────────────────────────────┴─────────╯ + +─────────────────────────────── Failures ─────────────────────────────── + +─────────── Task task_logging.py::task_publish_report failed ─────────── + +╭────────────────── Traceback (most recent call last) ──────────────────╮ +│ │ +│ ...\git\my_project\task_logging.py:18 in task_publish_report │ +│ │ +│ 15 │ logger.warning("publishing report is about to fail") │ +│ 16 │ print("stdout from publish") │ +│ 17 │ sys.stderr.write("stderr from publish\n") │ +│ ❱ 18 │ raise RuntimeError("simulated publish failure") │ +│ 19 │ +╰───────────────────────────────────────────────────────────────────────╯ +RuntimeError: simulated publish failure + +─────────────────────── Captured log during call ──────────────────────── +10:14:51 WARNING build:publishing report is about to fail +``` + +
diff --git a/docs/source/how_to_guides/index.md b/docs/source/how_to_guides/index.md index 2fabeeec..ce4084f1 100644 --- a/docs/source/how_to_guides/index.md +++ b/docs/source/how_to_guides/index.md @@ -13,6 +13,7 @@ specific tasks with pytask. - [Remote Files](remote_files.md) - [Functional Interface](functional_interface.md) - [Capture Warnings](capture_warnings.md) +- [Manage Logging](logging.md) - [How To Influence Build Order](how_to_influence_build_order.md) - [Hashing Inputs Of Tasks](hashing_inputs_of_tasks.md) - [Using Task Returns](using_task_returns.md) diff --git a/docs/source/how_to_guides/logging.md b/docs/source/how_to_guides/logging.md new file mode 100644 index 00000000..9072af59 --- /dev/null +++ b/docs/source/how_to_guides/logging.md @@ -0,0 +1,238 @@ +# Manage logging + +pytask can capture log records emitted during task execution, show them for failing +tasks, stream them live to the terminal, and write them to a file. + +If you do not use Python's [`logging`](https://docs.python.org/3/library/logging.html) +module often, think of log records simply as structured status messages such as +"starting download", "loaded 200 rows", or "publishing failed". + +This guide focuses on the most common ways to work with logging in pytask. + +## Quick start + +If you want to... use this: + +- see log messages only when a task fails: run `pytask` +- show only logs in failure reports: run `pytask --show-capture=log` +- see logs immediately while tasks run: run `pytask --log-cli --log-cli-level=INFO` +- save logs to a file: run `pytask --log-file=build.log` +- capture more detailed messages such as `INFO` or `DEBUG`: add `--log-level=INFO` or + `--log-level=DEBUG` + +## A minimal example + +```py title="task_logging.py" +import logging +import sys + + +logger = logging.getLogger(__name__) + + +def task_prepare_report(): + logger.info("preparing report.txt") + + +def task_publish_report(): + logger.warning("publishing report is about to fail") + print("stdout from publish") + sys.stderr.write("stderr from publish\n") + raise RuntimeError("simulated publish failure") +``` + +The most common logging levels are: + +- `DEBUG`: very detailed information for debugging +- `INFO`: normal progress messages +- `WARNING`: something unexpected happened, but execution can continue +- `ERROR`: a more serious problem + +If you are just getting started, `INFO` and `WARNING` are usually the most useful +levels. + +Here is what this looks like with live logging enabled and failure output restricted to +captured logs: + +```console +$ pytask --log-cli --log-cli-level=INFO --show-capture=log +``` + +--8<-- "docs/source/_static/md/logging-live.md" + +## Show captured logs for failing tasks + +Log records emitted with Python's +[`logging`](https://docs.python.org/3/library/logging.html) module are attached to the +report of a failing task in the same way as captured `stdout` and `stderr`. + +```py title="task_logging.py" +import logging + + +logger = logging.getLogger(__name__) + + +def task_example(): + logger.warning("something went wrong") + raise RuntimeError("fail") +``` + +```console +$ pytask +``` + +By default, pytask shows captured log output for failing tasks together with the +traceback and any captured `stdout` or `stderr`. + +This is useful when a task fails and you want to see what happened right before the +error. + +Use `--show-capture` to control which captured output is shown: + +```console +$ pytask --show-capture=log +$ pytask --show-capture=all +$ pytask --show-capture=no +``` + +`--show-capture=log` is useful when you only want log records in the failure report and +want to hide captured `stdout` and `stderr`. + +## Control which log records are captured + +By default, pytask does not change the logging level. Captured output therefore depends +on your normal logging configuration. + +In practice this often means that `WARNING` and `ERROR` messages appear, while `INFO` +and `DEBUG` messages do not, unless you configure logging more explicitly. + +Use `--log-level` to set the threshold for captured log records explicitly: + +```console +$ pytask --log-level=INFO +$ pytask --log-level=DEBUG +``` + +As a rule of thumb: + +- use `INFO` if you want to see normal progress messages, +- use `DEBUG` only when you need very detailed diagnostics. + +This option affects: + +- log records attached to failing task reports, +- live logs shown with `--log-cli`, +- exported logs written with `--log-file`. + +You can customize the formatting of captured log records with: + +```console +$ pytask --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%Y-%m-%d %H:%M:%S" +``` + +## Stream logs live while tasks run + +Use `--log-cli` to print log records directly to the terminal during task execution. + +```console +$ pytask --log-cli --log-cli-level=INFO +``` + +This is helpful when tasks take a while and you want immediate feedback instead of +waiting for the final report. + +You can customize live logs separately from the captured report output: + +```console +$ pytask --log-cli \ + --log-cli-level=INFO \ + --log-cli-format="%(levelname)s:%(message)s" \ + --log-cli-date-format="%H:%M:%S" +``` + +If `--log-cli-format` or `--log-cli-date-format` are not provided, pytask falls back to +`--log-format` and `--log-date-format`. + +## Write logs to a file + +Use `--log-file` to export log records from executed tasks to a file. + +```console +$ pytask --log-file=build.log +``` + +This is useful for CI runs, long builds, or when you want to inspect logs after the run +has finished. + +The file is overwritten by default. Use `--log-file-mode=a` to append instead. + +```console +$ pytask --log-file=build.log --log-file-mode=a +``` + +You can control the file output independently: + +```console +$ pytask --log-file=build.log \ + --log-file-level=INFO \ + --log-file-format="%(asctime)s %(name)s %(levelname)s %(message)s" \ + --log-file-date-format="%Y-%m-%d %H:%M:%S" +``` + +Relative log file paths are resolved relative to the project root detected by pytask. + +## A good beginner setup + +If you want a practical setup without spending much time on logging configuration, this +is a good default: + +```console +$ pytask --log-cli --log-cli-level=INFO --log-file=build.log --show-capture=log +``` + +This gives you: + +- live progress messages in the terminal, +- a log file you can inspect later, +- only log output in failure reports, without extra `stdout` and `stderr` noise. + +## Configure logging defaults in `pyproject.toml` + +All logging options can be configured in `pyproject.toml`. + +```toml title="pyproject.toml" +[tool.pytask.ini_options] +log_level = "INFO" +log_format = "%(asctime)s %(levelname)s %(message)s" +log_date_format = "%Y-%m-%d %H:%M:%S" + +log_cli = true +log_cli_level = "INFO" +log_cli_format = "%(levelname)s:%(message)s" + +log_file = "build.log" +log_file_mode = "w" +log_file_level = "INFO" +log_file_format = "%(asctime)s %(name)s %(levelname)s %(message)s" +log_file_date_format = "%Y-%m-%d %H:%M:%S" +``` + +## Use logging with the programmatic interface + +The same options are available via +[`pytask.build`](../api/functional_interfaces.md#build-workflow). + +```py title="build.py" +from pytask import build + + +session = build( + log_level="INFO", + log_cli=True, + log_cli_level="INFO", + log_file="build.log", + log_file_format="%(levelname)s:%(message)s", +) +``` diff --git a/docs/source/tutorials/capturing_output.md b/docs/source/tutorials/capturing_output.md index 89ebc809..575c79d0 100644 --- a/docs/source/tutorials/capturing_output.md +++ b/docs/source/tutorials/capturing_output.md @@ -10,11 +10,14 @@ default. If the task fails, the output is shown along with the traceback to help you track down the error. -## Default stdout/stderr/stdin capturing behavior +## Default stdout/stderr/logging/stdin capturing behavior Any output sent to `stdout` and `stderr` is captured during task execution. pytask displays it only if the task fails in addition to the traceback. +Log records emitted with Python's `logging` module are also captured during task +execution and shown in their own report section for failing tasks. + In addition, `stdin` is set to a "null" object which will fail on attempts to read from it because it is rarely desired to wait for interactive input when running automated tasks. @@ -23,6 +26,11 @@ By default, capturing is done by intercepting writes to low-level file descripto allows capturing output from simple `print` statements as well as output from a subprocess started by a task. +!!! seealso + + [Manage logging](../how_to_guides/logging.md) for a dedicated guide to captured logs, + live logs, log files, and logging configuration. + ## Setting capturing methods or disabling capturing There are three ways in which `pytask` can perform capturing: diff --git a/justfile b/justfile index 93333683..1e6eba08 100644 --- a/justfile +++ b/justfile @@ -22,14 +22,14 @@ lint: check: lint typing test # Build documentation -docs: +docs *FLAGS: uv run --group plugin-list python scripts/update_plugin_list.py - uv run --group docs zensical build + uv run --group docs zensical build {{FLAGS}} # Serve documentation with auto-reload -docs-serve: +docs-serve *FLAGS: uv run --group plugin-list python scripts/update_plugin_list.py - uv run --group docs zensical serve -a 127.0.0.1:8000 + uv run --group docs zensical serve -a 127.0.0.1:8000 {{FLAGS}} # Run tests with lowest dependency resolution (like CI) test-lowest: diff --git a/mkdocs.yml b/mkdocs.yml index 7e72d685..257741e6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -41,6 +41,7 @@ nav: - Remote Files: how_to_guides/remote_files.md - Functional Interface: how_to_guides/functional_interface.md - Capture Warnings: how_to_guides/capture_warnings.md + - Manage Logging: how_to_guides/logging.md - How To Influence Build Order: how_to_guides/how_to_influence_build_order.md - Hashing Inputs Of Tasks: how_to_guides/hashing_inputs_of_tasks.md - Using Task Returns: how_to_guides/using_task_returns.md diff --git a/scripts/generate_cli_command_docs.py b/scripts/generate_cli_command_docs.py index 311ab8f9..1964c472 100644 --- a/scripts/generate_cli_command_docs.py +++ b/scripts/generate_cli_command_docs.py @@ -3,6 +3,7 @@ from __future__ import annotations import enum +import html import re from pathlib import Path @@ -21,6 +22,17 @@ def _strip_click_suffixes(help_text: str) -> str: return help_text.strip() +def _escape_table_cell(text: str) -> str: + """Escape markdown table separators inside cell content.""" + return text.replace("|", "|").replace("\n", " ") + + +def _format_code(text: str) -> str: + """Format code-like values for markdown tables.""" + escaped = html.escape(text, quote=False).replace("|", "|") + return f"{escaped}" + + def _format_default(option: click.Option) -> str: default = option.default result = "-" @@ -28,21 +40,21 @@ def _format_default(option: click.Option) -> str: if isinstance(default, bool): if option.secondary_opts: active = option.opts[0] if default else option.secondary_opts[0] - result = f"`{active}`" + result = active else: - result = f"`{str(default).lower()}`" + result = str(default).lower() elif default is None or (isinstance(default, tuple | list) and not default): result = "-" elif isinstance(default, enum.Enum): if str(default.value).startswith(" None: continue option_decl, description = help_record + default = _format_default(param) + escaped_description = _escape_table_cell(_strip_click_suffixes(description)) lines.append( "| " - f"`{option_decl}`" + f"{_format_code(option_decl)}" " | " - f"{_format_default(param)}" + f"{_format_code(default) if default != '-' else '-'}" " | " - f"{_strip_click_suffixes(description)}" + f"{escaped_description}" " |" ) @@ -96,7 +110,7 @@ def _write_arguments(command_name: str) -> None: has_arguments = True metavar = param.make_metavar(click.Context(command)).strip() description = "Paths where pytask looks for task files and configuration." - lines.append(f"| `{metavar}` | {description} |") + lines.append(f"| {_format_code(metavar)} | {_escape_table_cell(description)} |") if not has_arguments: lines.append("| - | This command does not take positional arguments. |") @@ -108,7 +122,11 @@ def _write_arguments(command_name: str) -> None: def _write_commands_table() -> None: lines = ["| Command | Description |", "|---|---|"] lines.extend( - f"| [`{name}`]({name}.md) | {cli.commands[name].help} |" for name in COMMANDS + ( + f"| [`{name}`]({name}.md) | " + f"{_escape_table_cell(cli.commands[name].help or '')} |" + ) + for name in COMMANDS ) output = "\n".join(lines) + "\n" @@ -126,7 +144,13 @@ def _write_root_options() -> None: if help_record is None: continue option_decl, description = help_record - lines.append(f"| `{option_decl}` | {_strip_click_suffixes(description)} |") + lines.append( + "| " + f"{_format_code(option_decl)}" + " | " + f"{_escape_table_cell(_strip_click_suffixes(description))}" + " |" + ) lines.append("| `-h, --help` | Show this message and exit. |") diff --git a/src/_pytask/build.py b/src/_pytask/build.py index c18706de..52ce5e17 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -85,8 +85,20 @@ def build( # noqa: PLR0913 paths: Path | Iterable[Path] = (), pdb: bool = False, pdb_cls: str = "", + log_date_format: str = "%H:%M:%S", + log_cli: bool = False, + log_cli_date_format: str | None = None, + log_cli_format: str | None = None, + log_cli_level: int | str | None = None, + log_file: Path | str | None = None, + log_file_date_format: str | None = None, + log_file_format: str | None = None, + log_file_level: int | str | None = None, + log_file_mode: Literal["w", "a"] = "w", + log_format: str = "%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s", + log_level: int | str | None = None, s: bool = False, - show_capture: Literal["no", "stdout", "stderr", "all"] + show_capture: Literal["no", "stdout", "stderr", "log", "all"] | ShowCapture = ShowCapture.ALL, show_errors_immediately: bool = False, show_locals: bool = False, @@ -148,9 +160,35 @@ def build( # noqa: PLR0913 pdb_cls : str, default="" Start a custom debugger on errors. For example: ``--pdbcls=IPython.terminal.debugger:TerminalPdb`` + log_date_format : str, default="%H:%M:%S" + The date format used for captured logs. + log_cli : bool, default=False + Whether log records should be streamed to the terminal while tasks run. + log_cli_date_format : str | None, default=None + The date format used for live logs. Falls back to ``log_date_format``. + log_cli_format : str | None, default=None + The format used for live logs. Falls back to ``log_format``. + log_cli_level : int | str | None, default=None + The level of messages streamed live to the terminal. Falls back to + ``log_level``. + log_file : Path | str | None, default=None + A path to a file where logs from executed tasks should be written. + log_file_date_format : str | None, default=None + The date format used for exported logs. Falls back to ``log_date_format``. + log_file_format : str | None, default=None + The format used for exported logs. Falls back to ``log_format``. + log_file_level : int | str | None, default=None + The level of messages written to the log file. Falls back to ``log_level``. + log_file_mode : Literal["w", "a"], default="w" + The file mode used for the exported log file. + log_format : str, default="%(levelname)-8s %(name)s:%(filename)s:%(lineno)d " + "%(message)s" + The format used for captured logs. + log_level : int | str | None, default=None + The level of messages to capture. If not set, the logger configuration is used. s : bool, default=False Shortcut for ``capture="no"``. - show_capture : Literal["no", "stdout", "stderr", "all"] | ShowCapture + show_capture : Literal["no", "stdout", "stderr", "log", "all"] | ShowCapture Choose which captured output should be shown for failed tasks. show_errors_immediately : bool, default=False Show errors with tracebacks as soon as the task fails. @@ -202,6 +240,18 @@ def build( # noqa: PLR0913 "paths": paths, "pdb": pdb, "pdb_cls": pdb_cls, + "log_date_format": log_date_format, + "log_cli": log_cli, + "log_cli_date_format": log_cli_date_format, + "log_cli_format": log_cli_format, + "log_cli_level": log_cli_level, + "log_file": log_file, + "log_file_date_format": log_file_date_format, + "log_file_format": log_file_format, + "log_file_level": log_file_level, + "log_file_mode": log_file_mode, + "log_format": log_format, + "log_level": log_level, "s": s, "show_capture": show_capture, "show_errors_immediately": show_errors_immediately, diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index ed1d61af..8f0866ad 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -779,6 +779,13 @@ def read(self) -> CaptureResult[str]: assert self._capturing is not None return self._capturing.readouterr() + def write_to_stdout(self, text: str) -> None: + if self._capturing is not None and self._capturing.out is not None: + self._capturing.out.writeorg(text) + return + sys.stdout.write(text) + sys.stdout.flush() + # Helper context managers @contextlib.contextmanager diff --git a/src/_pytask/capture_utils.py b/src/_pytask/capture_utils.py index 47a8376b..a2a926e2 100644 --- a/src/_pytask/capture_utils.py +++ b/src/_pytask/capture_utils.py @@ -9,6 +9,7 @@ class ShowCapture(enum.Enum): NO = "no" STDOUT = "stdout" STDERR = "stderr" + LOG = "log" ALL = "all" diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index c203824d..45b2712a 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -3,8 +3,11 @@ from __future__ import annotations import contextlib +import io +import logging import platform import sys +from pathlib import Path from typing import TYPE_CHECKING from typing import Any from typing import NamedTuple @@ -18,11 +21,17 @@ from _pytask.console import console from _pytask.pluginmanager import hookimpl from _pytask.reports import ExecutionReport +from _pytask.shared import convert_to_enum from _pytask.traceback import Traceback if TYPE_CHECKING: + from collections.abc import Generator + from pluggy._manager import DistFacade + from _pytask.capture import CaptureManager + from _pytask.live import LiveManager + from _pytask.node_protocols import PTask from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.session import Session @@ -32,6 +41,16 @@ pass +if TYPE_CHECKING: + LoggingStreamHandler = logging.StreamHandler[io.StringIO] +else: + LoggingStreamHandler = logging.StreamHandler + + +DEFAULT_LOG_FORMAT = "%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s" +DEFAULT_LOG_DATE_FORMAT = "%H:%M:%S" + + class _TimeUnit(NamedTuple): singular: str plural: str @@ -41,13 +60,104 @@ class _TimeUnit(NamedTuple): @hookimpl def pytask_extend_command_line_interface(cli: click.Group) -> None: - show_locals_option = click.Option( - ["--show-locals"], - is_flag=True, - default=False, - help="Show local variables in tracebacks.", + cli.commands["build"].params.extend( + [ + click.Option( + ["--log-cli/--no-log-cli"], + default=False, + help="Enable live log display during task execution.", + ), + click.Option( + ["--log-cli-level"], + default=None, + metavar="LEVEL", + help="CLI logging level.", + ), + click.Option( + ["--log-cli-format"], + default=None, + help="Log format used by the logging module for live logs.", + ), + click.Option( + ["--log-cli-date-format"], + default=None, + help="Log date format used by the logging module for live logs.", + ), + click.Option( + ["--show-locals"], + is_flag=True, + default=False, + help="Show local variables in tracebacks.", + ), + click.Option( + ["--log-level"], + default=None, + metavar="LEVEL", + help=( + "Level of messages to catch/display. Not set by default, so it " + "depends on the logger configuration." + ), + ), + click.Option( + ["--log-format"], + default=DEFAULT_LOG_FORMAT, + help="Log format used by the logging module.", + ), + click.Option( + ["--log-date-format"], + default=DEFAULT_LOG_DATE_FORMAT, + help="Log date format used by the logging module.", + ), + click.Option( + ["--log-file"], + default=None, + help="Path to a file where logging will be written.", + ), + click.Option( + ["--log-file-mode"], + default="w", + type=click.Choice(["w", "a"]), + help="Log file open mode.", + ), + click.Option( + ["--log-file-level"], + default=None, + metavar="LEVEL", + help="Log file logging level.", + ), + click.Option( + ["--log-file-format"], + default=None, + help="Log format used by the logging module for the log file.", + ), + click.Option( + ["--log-file-date-format"], + default=None, + help="Log date format used by the logging module for the log file.", + ), + ] ) - cli.commands["build"].params.append(show_locals_option) + + +@hookimpl +def pytask_parse_config(config: dict[str, Any]) -> None: + config["show_capture"] = convert_to_enum(config["show_capture"], ShowCapture) + config["log_cli_level"] = _parse_log_level( + config["log_cli_level"], option_name="log_cli_level" + ) + config["log_level"] = _parse_log_level(config["log_level"], option_name="log_level") + config["log_file_level"] = _parse_log_level( + config["log_file_level"], option_name="log_file_level" + ) + + log_file = config["log_file"] + if log_file is None: + return + + log_file = Path(log_file) + if not log_file.is_absolute(): + log_file = config["root"].joinpath(log_file) + config["log_file"] = log_file.resolve() @hookimpl @@ -59,6 +169,8 @@ def pytask_post_parse(config: dict[str, Any]) -> None: ExecutionReport.show_capture = config["show_capture"] ExecutionReport.show_locals = config["show_locals"] + config["pm"].register(LoggingManager.from_config(config), "loggingmanager") + @hookimpl def pytask_log_session_header(session: Session) -> None: @@ -205,3 +317,262 @@ def _humanize_time( # noqa: C901, PLR0912 ) return result + + +class LogCaptureHandler(LoggingStreamHandler): + """Capture logs in a string buffer.""" + + def __init__(self) -> None: + super().__init__(io.StringIO()) + + def reset(self) -> None: + old_stream = self.setStream(io.StringIO()) + if old_stream is not None: + old_stream.close() + + def get_text(self) -> str: + return self.stream.getvalue().strip() + + +class LiveLogHandler(logging.Handler): + """Write log records to the terminal immediately.""" + + def __init__( + self, + *, + plugin_manager: Any, + ) -> None: + super().__init__() + self.plugin_manager = plugin_manager + + def _get_capture_manager(self) -> CaptureManager | None: + capture_manager = self.plugin_manager.get_plugin("capturemanager") + return capture_manager if hasattr(capture_manager, "write_to_stdout") else None + + def _get_live_manager(self) -> LiveManager | None: + live_manager = self.plugin_manager.get_plugin("live_manager") + return live_manager if hasattr(live_manager, "pause") else None + + def emit(self, record: logging.LogRecord) -> None: + message = self.format(record) + capture_manager = self._get_capture_manager() + live_manager = self._get_live_manager() + resume_live = live_manager is not None and live_manager.is_started + + if resume_live and live_manager is not None: + live_manager.pause() + + try: + if capture_manager is not None: + capture_manager.write_to_stdout(message + "\n") + else: + sys.stdout.write(message + "\n") + sys.stdout.flush() + finally: + if resume_live and live_manager is not None: + live_manager.resume() + + +class LoggingManager: + """Capture task logs for reports and optional log files. + + This intentionally follows pytest's handler-on-root design instead of trying to + intercept logging internals. The tradeoff is that task-local logging + reconfiguration is not fully isolated from pytask: + + - ``logging.basicConfig()`` inside a task can become a no-op because pytask has + already attached a handler to the root logger. + - ``logging.basicConfig(force=True)`` or direct mutation of ``root.handlers`` can + remove or close pytask-managed handlers for the remainder of the task/session. + + We accept these limitations for parity with pytest and to keep the integration + simple. A more isolated design would need to avoid attaching handlers to the root + logger in the first place. + """ + + def __init__( # noqa: PLR0913 + self, + *, + formatter: logging.Formatter, + live_log_handler: logging.Handler | None, + log_cli_level: int | None, + log_level: int | None, + log_file_handler: logging.FileHandler | None, + log_file_level: int | None, + ) -> None: + self.live_log_handler = live_log_handler + self.log_cli_level = log_cli_level + self.log_level = log_level + self.log_file_level = log_file_level + self.report_handler = LogCaptureHandler() + self.report_handler.setFormatter(formatter) + self.log_file_handler = log_file_handler + + @classmethod + def from_config(cls, config: dict[str, Any]) -> LoggingManager: + log_cli_level = ( + config["log_cli_level"] + if config["log_cli_level"] is not None + else config["log_level"] + ) + log_file_level = ( + config["log_file_level"] + if config["log_file_level"] is not None + else config["log_level"] + ) + live_log_handler = _create_live_log_handler( + config=config, + log_cli_level=log_cli_level, + ) + log_file_handler = _create_log_file_handler( + log_file=config["log_file"], + log_file_format=config["log_file_format"] or config["log_format"], + log_file_date_format=config["log_file_date_format"] + or config["log_date_format"], + log_file_level=log_file_level, + log_file_mode=config["log_file_mode"], + ) + return cls( + formatter=logging.Formatter( + config["log_format"], datefmt=config["log_date_format"] + ), + live_log_handler=live_log_handler, + log_cli_level=log_cli_level, + log_level=config["log_level"], + log_file_handler=log_file_handler, + log_file_level=log_file_level, + ) + + @contextlib.contextmanager + def _catching_logs(self) -> Generator[None, None, None]: + root_logger = logging.getLogger() + handlers: list[logging.Handler] = [self.report_handler] + if self.live_log_handler is not None: + handlers.append(self.live_log_handler) + if self.log_file_handler is not None: + handlers.append(self.log_file_handler) + + original_level = root_logger.level + configured_levels = [ + level + for level in ( + self.log_cli_level, + self.log_level, + self.log_file_level, + ) + if level not in (None, logging.NOTSET) + ] + + try: + # Attaching handlers to the root logger is the key design choice here. It + # keeps pytask aligned with pytest, but it also means task code that + # reconfigures the root logger can affect pytask's own logging handlers. + for handler in handlers: + handler.setLevel( + self.log_level + if handler is self.report_handler and self.log_level is not None + else handler.level + ) + root_logger.addHandler(handler) + + if configured_levels: + root_logger.setLevel(min(original_level, *configured_levels)) + yield + finally: + for handler in reversed(handlers): + root_logger.removeHandler(handler) + root_logger.setLevel(original_level) + + @contextlib.contextmanager + def _task_logging(self, when: str, task: PTask) -> Generator[None, None, None]: + self.report_handler.reset() + with self._catching_logs(): + try: + yield + finally: + log = self.report_handler.get_text() + if log: + task.report_sections.append((when, "log", log)) + + @hookimpl(wrapper=True) + def pytask_execute_task_setup(self, task: PTask) -> Generator[None, None, None]: + with self._task_logging("setup", task): + return (yield) + + @hookimpl(wrapper=True) + def pytask_execute_task(self, task: PTask) -> Generator[None, None, None]: + with self._task_logging("call", task): + return (yield) + + @hookimpl(wrapper=True) + def pytask_execute_task_teardown(self, task: PTask) -> Generator[None, None, None]: + with self._task_logging("teardown", task): + return (yield) + + @hookimpl + def pytask_unconfigure(self) -> None: + if self.log_file_handler is not None: + self.log_file_handler.close() + + +def _parse_log_level(value: Any, *, option_name: str) -> int | None: + if value is None: + return None + if isinstance(value, int): + return value + if not isinstance(value, str): + msg = f"{option_name!r} must be an int, str, or None, not {type(value)!r}." + raise click.BadParameter(msg) + + normalized_value = value.upper() + level = getattr(logging, normalized_value, None) + if not isinstance(level, int): + msg = f"{value!r} is not recognized as a logging level name." + raise click.BadParameter(msg) + return level + + +def _create_log_file_handler( + *, + log_file: Path | None, + log_file_date_format: str, + log_file_format: str, + log_file_level: int | None, + log_file_mode: str, +) -> logging.FileHandler | None: + if log_file is None: + return None + + log_file.parent.mkdir(parents=True, exist_ok=True) + # This handler is session-scoped and reattached around each task. If user code + # force-reconfigures the root logger, Python may close this handler as part of + # removing existing root handlers. + log_file_handler = logging.FileHandler( + log_file, mode=log_file_mode, encoding="utf-8" + ) + log_file_handler.setFormatter( + logging.Formatter(log_file_format, datefmt=log_file_date_format) + ) + log_file_handler.setLevel( + log_file_level if log_file_level is not None else logging.NOTSET + ) + return log_file_handler + + +def _create_live_log_handler( + *, config: dict[str, Any], log_cli_level: int | None +) -> logging.Handler | None: + if not (config["log_cli"] or config["log_cli_level"] is not None): + return None + + live_log_handler = LiveLogHandler(plugin_manager=config["pm"]) + live_log_handler.setFormatter( + logging.Formatter( + config["log_cli_format"] or config["log_format"], + datefmt=config["log_cli_date_format"] or config["log_date_format"], + ) + ) + live_log_handler.setLevel( + log_cli_level if log_cli_level is not None else logging.NOTSET + ) + return live_log_handler diff --git a/src/_pytask/reports.py b/src/_pytask/reports.py index 0779b967..bf88f33e 100644 --- a/src/_pytask/reports.py +++ b/src/_pytask/reports.py @@ -115,7 +115,7 @@ def __rich_console__( yield "" for when, key, content in self.sections: - if key in ("stdout", "stderr") and self.show_capture in ( + if key in ("stdout", "stderr", "log") and self.show_capture in ( ShowCapture(key), ShowCapture.ALL, ): diff --git a/tests/test_capture.py b/tests/test_capture.py index e0f94073..c899401f 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -29,12 +29,14 @@ from collections.abc import Generator -@pytest.mark.parametrize("show_capture", ["s", "no", "stdout", "stderr", "all"]) +@pytest.mark.parametrize("show_capture", ["s", "no", "stdout", "stderr", "log", "all"]) def test_show_capture(tmp_path, runner, show_capture): source = """ + import logging import sys def task_show_capture(): + logging.getLogger(__name__).warning("yyyy") sys.stdout.write("xxxx") sys.stderr.write("zzzz") raise Exception @@ -46,8 +48,13 @@ def task_show_capture(): assert result.exit_code == ExitCode.FAILED - if show_capture in ("no", "s"): + if show_capture == "no": assert "Captured" not in result.output + elif show_capture == "s": + assert "Captured stdout" not in result.output + assert "Captured stderr" not in result.output + assert "Captured log" in result.output + assert "yyyy" in result.output elif show_capture == "stdout": assert "Captured stdout" in result.output assert "xxxx" in result.output @@ -58,16 +65,24 @@ def task_show_capture(): # assert "xxxx" not in result.output assert "Captured stderr" in result.output assert "zzzz" in result.output + assert "Captured log" not in result.output + elif show_capture == "log": + assert "Captured stdout" not in result.output + assert "Captured stderr" not in result.output + assert "Captured log" in result.output + assert "yyyy" in result.output elif show_capture == "all": assert "Captured stdout" in result.output assert "xxxx" in result.output assert "Captured stderr" in result.output assert "zzzz" in result.output + assert "Captured log" in result.output + assert "yyyy" in result.output else: # pragma: no cover raise NotImplementedError -@pytest.mark.parametrize("show_capture", ["no", "stdout", "stderr", "all"]) +@pytest.mark.parametrize("show_capture", ["no", "stdout", "stderr", "log", "all"]) @pytest.mark.xfail( sys.platform == "win32", reason="Fails on Windows due to encoding.", @@ -75,10 +90,12 @@ def task_show_capture(): ) def test_show_capture_with_build(tmp_path, show_capture): source = f""" + import logging import sys from pytask import build def task_show_capture(): + logging.getLogger(__name__).warning("yyyy") sys.stdout.write("xxxx") sys.stderr.write("zzzz") raise Exception @@ -105,11 +122,19 @@ def task_show_capture(): # assert "xxxx" not in result.stdout assert "Captured stderr" in result.stdout assert "zzzz" in result.stdout + assert "Captured log" not in result.stdout + elif show_capture == "log": + assert "Captured stdout" not in result.stdout + assert "Captured stderr" not in result.stdout + assert "Captured log" in result.stdout + assert "yyyy" in result.stdout elif show_capture == "all": assert "Captured stdout" in result.stdout assert "xxxx" in result.stdout assert "Captured stderr" in result.stdout assert "zzzz" in result.stdout + assert "Captured log" in result.stdout + assert "yyyy" in result.stdout else: # pragma: no cover raise NotImplementedError diff --git a/tests/test_cli.py b/tests/test_cli.py index 4da1cd97..a035852e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -42,3 +42,16 @@ def test_help_texts_are_modified_by_config(runner, tmp_path): ) assert "[default: stdout]" in result.output + + +def test_help_texts_are_modified_by_logging_config(runner, tmp_path): + tmp_path.joinpath("pyproject.toml").write_text( + '[tool.pytask.ini_options]\nlog_level = "INFO"' + ) + + result = runner.invoke( + cli, + ["build", "--help", "--config", tmp_path.joinpath("pyproject.toml").as_posix()], + ) + + assert "[default: INFO]" in result.output diff --git a/tests/test_click.py b/tests/test_click.py index 502e69a2..7bab4cd2 100644 --- a/tests/test_click.py +++ b/tests/test_click.py @@ -11,7 +11,7 @@ def test_choices_are_displayed_in_help_page(runner): result = runner.invoke(cli, ["build", "--help"]) - assert "[no|stdout|stderr|all]" in result.output + assert "[no|stdout|stderr|log|all]" in result.output assert "[fd|no|sys|tee-sys]" in result.output diff --git a/tests/test_logging.py b/tests/test_logging.py index 8ff4bf62..d03c4cad 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import textwrap from contextlib import ExitStack as does_not_raise # noqa: N813 from typing import NamedTuple @@ -12,6 +13,7 @@ from pytask import ExitCode from pytask import TaskOutcome from pytask import cli +from tests.conftest import run_in_subprocess class DummyDist(NamedTuple): @@ -99,6 +101,116 @@ def test_logging_of_outcomes(tmp_path, runner, func, expected_1, expected_2): assert expected_2 in result.output +def test_log_file_exports_logs(tmp_path, runner): + source = """ + import logging + + logger = logging.getLogger(__name__) + + def task_example(): + logger.warning("hello from task") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke( + cli, + [ + tmp_path.as_posix(), + "--log-file=build.log", + "--log-format=%(levelname)s:%(message)s", + ], + ) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("build.log").read_text() == "WARNING:hello from task\n" + + +def test_build_log_file_exports_logs(tmp_path): + source = """ + import logging + import sys + from pytask import build + + logger = logging.getLogger(__name__) + + def task_example(produces="out.txt"): + logger.warning("hello from task") + return "done" + + if __name__ == "__main__": + session = build( + tasks=[task_example], + force=True, + log_file="build.log", + log_format="%(levelname)s:%(message)s", + ) + sys.exit(session.exit_code) + """ + tmp_path.joinpath("workflow.py").write_text(textwrap.dedent(source)) + + result = run_in_subprocess((sys.executable, "workflow.py"), cwd=tmp_path) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("build.log").read_text() == "WARNING:hello from task\n" + + +def test_log_cli_streams_logs(tmp_path): + source = """ + import logging + + logger = logging.getLogger(__name__) + + def task_example(): + logger.info("hello from live log") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = run_in_subprocess( + ( + "pytask", + tmp_path.as_posix(), + "--log-cli", + "--log-cli-level=INFO", + "--log-cli-format=%(levelname)s:%(message)s", + ), + cwd=tmp_path, + ) + + assert result.exit_code == ExitCode.OK + assert "INFO:hello from live log" in result.stdout + + +def test_build_log_cli_streams_logs(tmp_path): + source = """ + import logging + import sys + from pathlib import Path + from pytask import build + + logger = logging.getLogger(__name__) + + def task_example(produces=Path("out.txt")): + logger.info("hello from live log") + produces.write_text("done") + + if __name__ == "__main__": + session = build( + tasks=[task_example], + force=True, + log_cli=True, + log_cli_level="INFO", + log_cli_format="%(levelname)s:%(message)s", + ) + sys.exit(session.exit_code) + """ + tmp_path.joinpath("workflow.py").write_text(textwrap.dedent(source)) + + result = run_in_subprocess((sys.executable, "workflow.py"), cwd=tmp_path) + + assert result.exit_code == ExitCode.OK + assert "INFO:hello from live log" in result.stdout + + @pytest.mark.parametrize( ("amount", "unit", "short_label", "expectation", "expected"), [