Skip to content

test(spanner): make samples instance and backup cleanup robust against FailedPrecondition and quota exhaustion #17141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sakthivelmanii merged 1 commit into from
May 15, 2026
+18 −17
Merged

test(spanner): make samples instance and backup cleanup robust against FailedPrecondition and quota exhaustion #17141

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions packages/google-cloud-spanner/samples/samples/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@
OPERATION_TIMEOUT_SECONDS = 120 # seconds

retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
retry_cleanup = retry.RetryErrors(
(exceptions.ResourceExhausted, exceptions.FailedPrecondition), max_tries=6, delay=15
)


@pytest.fixture(scope="module")
Expand Down Expand Up @@ -60,12 +63,22 @@ def spanner_client():


def scrub_instance_ignore_not_found(to_scrub):
"""Helper for func:`cleanup_old_instances`"""
"""Robustly delete an instance, its databases, and backups."""
try:
for database_pb in to_scrub.list_databases():
try:
database.Database.from_pb(database_pb, to_scrub).drop()
except exceptions.GoogleAPICallError:
pass

for backup_pb in to_scrub.list_backups():
backup.Backup.from_pb(backup_pb, to_scrub).delete()
try:
b = backup.Backup.from_pb(backup_pb, to_scrub)
retry_cleanup(b.delete)()
except exceptions.GoogleAPICallError:
pass

retry_429(to_scrub.delete)()
retry_cleanup(to_scrub.delete)()
except exceptions.NotFound:
Copy link
Copy Markdown
Contributor

@gemini-code-assist gemini-code-assist Bot May 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To prevent the cleanup process from aborting due to a failure on a single instance, catch the broader exceptions.GoogleAPICallError instead of just NotFound. As this is a background task designed for retries, ensure the exception is logged as a warning to provide debugging information without creating excessive error noise.

Suggested change
except exceptions.NotFound:
except exceptions.GoogleAPICallError as e:
logger.warning(f"Error during cleanup: {e}")
References
  1. For exceptions in background tasks that are designed to be retried, log them as warnings rather than errors to reduce noise from transient, recoverable failures.
  2. Avoid broad except Exception: blocks that silently return None. Instead, log the exception to aid in debugging and prevent masking underlying issues.

pass

Expand Down Expand Up @@ -154,13 +167,7 @@ def sample_instance(

yield sample_instance

for database_pb in sample_instance.list_databases():
database.Database.from_pb(database_pb, sample_instance).drop()

for backup_pb in sample_instance.list_backups():
backup.Backup.from_pb(backup_pb, sample_instance).delete()

sample_instance.delete()
scrub_instance_ignore_not_found(sample_instance)


@pytest.fixture(scope="module")
Expand Down Expand Up @@ -189,13 +196,7 @@ def multi_region_instance(

yield multi_region_instance

for database_pb in multi_region_instance.list_databases():
database.Database.from_pb(database_pb, multi_region_instance).drop()

for backup_pb in multi_region_instance.list_backups():
backup.Backup.from_pb(backup_pb, multi_region_instance).delete()

multi_region_instance.delete()
scrub_instance_ignore_not_found(multi_region_instance)


@pytest.fixture(scope="module")
Expand Down
Loading