Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ select = [
"CPY001", # missing-copyright-notice
"D", # https://docs.astral.sh/ruff/rules/#pydocstyle-d
"DOC", # https://docs.astral.sh/ruff/rules/#pydoclint-doc
"E", # https://docs.astral.sh/ruff/rules/#pycodestyle-e
"F401", # unused-import
"I", # isort
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
Expand Down
4 changes: 2 additions & 2 deletions pyrit/backend/mappers/attack_mappers.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

from __future__ import annotations

"""
Attack mappers – domain ↔ DTO translation for attack-related models.
All functions are pure (no database or service calls) so they are easy to test.
The one exception is `attack_result_to_summary` which receives pre-fetched pieces.
"""

from __future__ import annotations

import mimetypes
import uuid
from datetime import datetime, timezone
Expand Down
4 changes: 3 additions & 1 deletion pyrit/backend/routes/attacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ async def list_attacks(
attack_type: Optional[str] = Query(None, description="Filter by exact attack type name"),
converter_types: Optional[list[str]] = Query(
None,
description="Filter by converter type names (repeatable, AND logic). Pass empty to match no-converter attacks.",
description=(
"Filter by converter type names (repeatable, AND logic). Pass empty to match no-converter attacks."
),
),
outcome: Optional[Literal["undetermined", "success", "failure"]] = Query(None, description="Filter by outcome"),
label: Optional[list[str]] = Query(None, description="Filter by labels (format: key:value, repeatable)"),
Expand Down
10 changes: 5 additions & 5 deletions pyrit/cli/pyrit_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@
from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
from typing import Optional

# Ensure emoji and other Unicode characters don't crash on Windows consoles
# that use legacy encodings like cp1252.
sys.stdout.reconfigure(errors="replace") # type: ignore[union-attr]
sys.stderr.reconfigure(errors="replace") # type: ignore[union-attr]

from pyrit.cli import frontend_core


Expand Down Expand Up @@ -196,6 +191,11 @@ def main(*, args: Optional[list[str]] = None) -> int:
Returns:
int: Exit code (0 for success, 1 for error).
"""
# Ensure emoji and other Unicode characters don't crash on Windows consoles
# that use legacy encodings like cp1252.
sys.stdout.reconfigure(errors="replace") # type: ignore[union-attr]
sys.stderr.reconfigure(errors="replace") # type: ignore[union-attr]

try:
parsed_args = parse_args(args=args)
except SystemExit as e:
Expand Down
38 changes: 27 additions & 11 deletions pyrit/cli/pyrit_shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,19 @@ def do_run(self, line: str) -> None:
--log-level <level> Override default log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
Examples:
run garak.encoding --initializers openai_objective_target load_default_datasets
run garak.encoding --initializers custom_target load_default_datasets --strategies base64 rot13
run foundry --initializers openai_objective_target load_default_datasets --max-concurrency 10 --max-retries 3
run garak.encoding --initializers custom_target load_default_datasets --memory-labels '{"run_id":"test123","env":"dev"}'
run foundry --initializers openai_objective_target load_default_datasets -s jailbreak crescendo
run garak.encoding --initializers openai_objective_target load_default_datasets --database InMemory --log-level DEBUG
run garak.encoding --initializers openai_objective_target \
load_default_datasets
run garak.encoding --initializers custom_target \
load_default_datasets --strategies base64 rot13
run foundry --initializers openai_objective_target \
load_default_datasets --max-concurrency 10 --max-retries 3
run garak.encoding --initializers custom_target \
load_default_datasets \
--memory-labels '{"run_id":"test123","env":"dev"}'
run foundry --initializers openai_objective_target \
load_default_datasets -s jailbreak crescendo
run garak.encoding --initializers openai_objective_target \
load_default_datasets --database InMemory --log-level DEBUG
run foundry --initialization-scripts ./my_custom_init.py -s all
Note:
Expand All @@ -182,17 +189,19 @@ def do_run(self, line: str) -> None:
print("\nOptions:")
print(f" --initializers <name> ... {frontend_core.ARG_HELP['initializers']} (REQUIRED)")
print(
f" --initialization-scripts <...> {frontend_core.ARG_HELP['initialization_scripts']} (alternative to --initializers)"
f" --initialization-scripts <...> {frontend_core.ARG_HELP['initialization_scripts']}"
" (alternative to --initializers)"
)
print(f" --strategies, -s <s1> <s2> ... {frontend_core.ARG_HELP['scenario_strategies']}")
print(f" --max-concurrency <N> {frontend_core.ARG_HELP['max_concurrency']}")
print(f" --max-retries <N> {frontend_core.ARG_HELP['max_retries']}")
print(f" --memory-labels <JSON> {frontend_core.ARG_HELP['memory_labels']}")
print(
f" --database <type> Override default database ({frontend_core.IN_MEMORY}, {frontend_core.SQLITE}, {frontend_core.AZURE_SQL})"
f" --database <type> Override default database"
f" ({frontend_core.IN_MEMORY}, {frontend_core.SQLITE}, {frontend_core.AZURE_SQL})"
)
print(
f" --log-level <level> Override default log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)"
" --log-level <level> Override default log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)"
)
print("\nExample:")
print(" run foundry --initializers openai_objective_target load_default_datasets")
Expand Down Expand Up @@ -464,15 +473,22 @@ def main() -> int:
"--database",
choices=[frontend_core.IN_MEMORY, frontend_core.SQLITE, frontend_core.AZURE_SQL],
default=frontend_core.SQLITE,
help=f"Default database type to use ({frontend_core.IN_MEMORY}, {frontend_core.SQLITE}, {frontend_core.AZURE_SQL}) (default: {frontend_core.SQLITE}, can be overridden per-run)",
help=(
f"Default database type to use"
f" ({frontend_core.IN_MEMORY}, {frontend_core.SQLITE}, {frontend_core.AZURE_SQL})"
f" (default: {frontend_core.SQLITE}, can be overridden per-run)"
),
)

parser.add_argument(
"--log-level",
type=str,
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
default="WARNING",
help="Default logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) (default: WARNING, can be overridden per-run)",
help=(
"Default logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)"
" (default: WARNING, can be overridden per-run)"
),
)

parser.add_argument(
Expand Down
3 changes: 2 additions & 1 deletion pyrit/datasets/jailbreak/text_jailbreak.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ def get_jailbreak_templates(cls, num_templates: Optional[int] = None) -> list[st
if num_templates:
if num_templates > len(jailbreak_template_names):
raise ValueError(
f"Attempted to pull {num_templates} jailbreaks from a dataset with only {len(jailbreak_template_names)} jailbreaks!"
f"Attempted to pull {num_templates} jailbreaks from a dataset"
f" with only {len(jailbreak_template_names)} jailbreaks!"
)
jailbreak_template_names = random.choices(jailbreak_template_names, k=num_templates)
return jailbreak_template_names
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,11 @@ async def _fetch_and_save_image_async(self, image_url: str, group_id: str) -> st

# Add browser-like headers for better success rate
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
" AppleWebKit/537.36 (KHTML, like Gecko)"
" Chrome/120.0.0.0 Safari/537.36"
),
"Accept": "image/webp,image/apng,image/*,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
Expand Down
3 changes: 2 additions & 1 deletion pyrit/executor/attack/multi_turn/chunked_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ def __init__(
chunk_size (int): Size of each chunk to request (default: 50).
total_length (int): Estimated total length of the target value (default: 200).
chunk_type (str): Type of chunk to request (e.g., "characters", "bytes", "words").
request_template (str): Template for generating chunk requests (default: "Give me {chunk_type} {start}-{end} of '{objective}'").
request_template (str): Template for generating chunk requests
(default: "Give me {chunk_type} {start}-{end} of '{objective}'").
attack_converter_config (Optional[AttackConverterConfig]): Configuration for prompt converters.
attack_scoring_config (Optional[AttackScoringConfig]): Configuration for scoring components.
prompt_normalizer (Optional[PromptNormalizer]): Normalizer for handling prompts.
Expand Down
4 changes: 3 additions & 1 deletion pyrit/prompt_target/openai/openai_image_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ def __init__(
max_requests_per_minute (int, Optional): Number of requests the target can handle per
minute before hitting a rate limit. The number of requests sent to the target
will be capped at the value provided.
image_size (Literal["256x256", "512x512", "1024x1024", "1536x1024", "1024x1536", "1792x1024", "1024x1792"], Optional): The size of the generated image.
image_size (Literal, Optional): The size of the generated image.
Accepts "256x256", "512x512", "1024x1024", "1536x1024",
"1024x1536", "1792x1024", or "1024x1792".
Different models support different image sizes.
GPT image models support "1024x1024", "1536x1024" and "1024x1536".
DALL-E-3 supports "1024x1024", "1792x1024" and "1024x1792".
Expand Down
3 changes: 2 additions & 1 deletion pyrit/prompt_target/openai/openai_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ def _ensure_async_token_provider(

# Wrap synchronous token provider in async function
logger.info(
"Detected synchronous token provider. Automatically wrapping in async function for compatibility with AsyncOpenAI."
"Detected synchronous token provider."
" Automatically wrapping in async function for compatibility with AsyncOpenAI."
)

async def async_token_provider() -> str:
Expand Down
4 changes: 3 additions & 1 deletion pyrit/registry/discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,9 @@ def discover_in_package(
Tuples of (registry_name, class) for each discovered subclass.
"""
if name_builder is None:
name_builder = lambda prefix, name: name if not prefix else f"{prefix}.{name}"

def name_builder(prefix: str, name: str) -> str:
return name if not prefix else f"{prefix}.{name}"

for _, module_name, is_pkg in pkgutil.iter_modules([str(package_path)]):
if module_name.startswith("_"):
Expand Down
4 changes: 2 additions & 2 deletions pyrit/scenario/core/scenario_strategy.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

from __future__ import annotations

"""
Base class for scenario attack strategies with group-based aggregation.
Expand All @@ -13,6 +11,8 @@
It also provides ScenarioCompositeStrategy for representing composed attack strategies.
"""

from __future__ import annotations

from enum import Enum
from typing import TYPE_CHECKING, TypeVar

Expand Down
3 changes: 2 additions & 1 deletion pyrit/scenario/scenarios/airt/jailbreak.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ def __init__(
jailbreak_names = []
if jailbreak_names and num_templates:
raise ValueError(
"Please provide only one of `num_templates` (random selection) or `jailbreak_names` (specific selection)."
"Please provide only one of `num_templates` (random selection)"
" or `jailbreak_names` (specific selection)."
)

if not objective_scorer:
Expand Down
3 changes: 2 additions & 1 deletion pyrit/score/audio_transcript_scorer.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,8 @@ def extract_audio_from_video(video_path: str) -> Optional[str]:
# Ensure 16-bit audio
if audio.sample_width != AudioTranscriptHelper._DEFAULT_SAMPLE_WIDTH:
logger.info(
f"Converting sample width from {audio.sample_width * 8}-bit to {AudioTranscriptHelper._DEFAULT_SAMPLE_WIDTH * 8}-bit"
f"Converting sample width from {audio.sample_width * 8}-bit"
f" to {AudioTranscriptHelper._DEFAULT_SAMPLE_WIDTH * 8}-bit"
)
audio = audio.set_sample_width(AudioTranscriptHelper._DEFAULT_SAMPLE_WIDTH)

Expand Down
3 changes: 2 additions & 1 deletion pyrit/score/scorer_evaluation/scorer_metrics_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,8 @@ def replace_evaluation_results(
replaced = len(existing_entries) != len(filtered_entries)
action = "Replaced" if replaced else "Added"
logger.info(
f"{action} metrics for {scorer_identifier.class_name} (eval_hash={eval_hash[:8]}...) in {file_path.name}"
f"{action} metrics for {scorer_identifier.class_name}"
f" (eval_hash={eval_hash[:8]}...) in {file_path.name}"
)

except Exception as e:
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/executor/attack/multi_turn/test_red_teaming.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def test_init_with_seed_prompt_variations(
)

assert attack._adversarial_chat_seed_prompt.value == expected_value
if expected_type == str:
if expected_type is str:
assert attack._adversarial_chat_seed_prompt.data_type == "text"

def test_init_with_invalid_system_prompt_path_raises_error(
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/score/test_scorer_prompt_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ def test_max_text_length_default_none_allows_all(self):
assert validator.is_message_piece_supported(very_long_piece) is True

def test_validate_raises_when_all_pieces_filtered_by_length(self):
"""Test that validate raises error when all pieces are filtered due to length and raise_on_no_valid_pieces=True."""
"""Test validate raises error when all pieces filtered by length and raise_on_no_valid_pieces=True."""
validator = ScorerPromptValidator(
supported_data_types=["text"], max_text_length=100, raise_on_no_valid_pieces=True
)
Expand Down