From 24f06c3469e21b6037a60242d5e6d5b63b1409c9 Mon Sep 17 00:00:00 2001 From: Carlos Fernandez Date: Sat, 8 Aug 2026 21:33:58 -0700 Subject: [PATCH] fix(ci): treat ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS as retryable GCP reports zone exhaustion under two codes: ZONE_RESOURCE_POOL_EXHAUSTED and ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS, the latter naming the resource that ran out. Only the bare code was listed in GCP_RETRYABLE_ERRORS and GCP_ERROR_MESSAGES, and is_retryable_gcp_error() matches by exact set membership, so the _WITH_DETAILS variant fell through to mark_test_failed(). In practice _WITH_DETAILS is the variant Compute Engine returns for instance inserts, so the retry path we already have almost never ran. A PR that hit an exhausted zone was failed permanently instead of staying pending for the next cron run, and the contributor got the unknown-code fallback message ("Please contact the administrator") rather than the one that explains it will be retried. Both codes now map to the same message and are both retryable. Nothing else changes: the test is left pending, has no GcpInstance row and no terminal TestProgress, so gcp_instance() picks it up on the next run exactly as the bare code already did. Seen on ccextractor PR #2309, where CI - linux and CI - windows both failed with "VM creation failed (ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS). Please contact the administrator." and reopening the PR just reproduced it. --- mod_ci/controllers.py | 19 +++++++++++---- tests/test_ci/test_controllers.py | 39 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index 1e0498f4..d66c1e62 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -126,12 +126,20 @@ def safe_db_commit(db, operation_description: str = "database operation") -> boo return False -# User-friendly messages for known GCP error codes +_ZONE_EXHAUSTED_MESSAGE = ( + "GCP resources temporarily unavailable in the configured zone. " + "The test will be retried automatically when resources become available." +) + +# User-friendly messages for known GCP error codes. +# +# GCP reports zone exhaustion under two codes: the bare ZONE_RESOURCE_POOL_EXHAUSTED and +# ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS, which carries the specific resource that ran +# out. Both mean the same thing to us, and in practice the _WITH_DETAILS variant is the one +# Compute Engine returns for instance inserts. GCP_ERROR_MESSAGES = { - 'ZONE_RESOURCE_POOL_EXHAUSTED': ( - "GCP resources temporarily unavailable in the configured zone. " - "The test will be retried automatically when resources become available." - ), + 'ZONE_RESOURCE_POOL_EXHAUSTED': _ZONE_EXHAUSTED_MESSAGE, + 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS': _ZONE_EXHAUSTED_MESSAGE, 'QUOTA_EXCEEDED': ( "GCP quota limit reached. " "The test will be retried automatically when resources become available." @@ -145,6 +153,7 @@ def safe_db_commit(db, operation_description: str = "database operation") -> boo # Tests encountering these errors will remain pending and be picked up on the next cron run. GCP_RETRYABLE_ERRORS = { 'ZONE_RESOURCE_POOL_EXHAUSTED', + 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS', 'QUOTA_EXCEEDED', } diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index 3b55ef89..b8472f34 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -3294,6 +3294,45 @@ def test_parse_gcp_error_zone_resource_exhausted(self): # Should NOT contain raw technical details self.assertNotIn("us-central1-a", error_msg) + def test_parse_gcp_error_zone_resource_exhausted_with_details(self): + """Test that the _WITH_DETAILS variant gets the same user-friendly message.""" + from mod_ci.controllers import parse_gcp_error + + mock_log = MagicMock() + result = { + 'status': 'DONE', + 'error': { + 'errors': [{ + 'code': 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS', + 'message': "The zone 'projects/test/zones/us-central1-a' does not have enough " + "resources available to fulfill the request. '(resource type:compute)'." + }] + } + } + + error_msg = parse_gcp_error(result, log=mock_log) + self.assertIn("GCP resources temporarily unavailable", error_msg) + self.assertIn("retried automatically", error_msg) + # Must not fall through to the generic "contact the administrator" message + self.assertNotIn("contact the administrator", error_msg) + self.assertNotIn("us-central1-a", error_msg) + + def test_zone_resource_exhausted_variants_are_retryable(self): + """Both zone-exhaustion codes must be retryable so the test stays pending.""" + from mod_ci.controllers import is_retryable_gcp_error + + for code in ('ZONE_RESOURCE_POOL_EXHAUSTED', 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS'): + with self.subTest(code=code): + result = {'error': {'errors': [{'code': code, 'message': 'no capacity'}]}} + self.assertTrue(is_retryable_gcp_error(result)) + + def test_non_transient_gcp_error_is_not_retryable(self): + """A genuine failure must still be marked failed rather than retried forever.""" + from mod_ci.controllers import is_retryable_gcp_error + + result = {'error': {'errors': [{'code': 'RESOURCE_NOT_FOUND', 'message': 'nope'}]}} + self.assertFalse(is_retryable_gcp_error(result)) + def test_parse_gcp_error_quota_exceeded(self): """Test that QUOTA_EXCEEDED returns user-friendly message.""" from mod_ci.controllers import parse_gcp_error