diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index 1e0498f4..d66c1e62 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -126,12 +126,20 @@ def safe_db_commit(db, operation_description: str = "database operation") -> boo return False -# User-friendly messages for known GCP error codes +_ZONE_EXHAUSTED_MESSAGE = ( + "GCP resources temporarily unavailable in the configured zone. " + "The test will be retried automatically when resources become available." +) + +# User-friendly messages for known GCP error codes. +# +# GCP reports zone exhaustion under two codes: the bare ZONE_RESOURCE_POOL_EXHAUSTED and +# ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS, which carries the specific resource that ran +# out. Both mean the same thing to us, and in practice the _WITH_DETAILS variant is the one +# Compute Engine returns for instance inserts. GCP_ERROR_MESSAGES = { - 'ZONE_RESOURCE_POOL_EXHAUSTED': ( - "GCP resources temporarily unavailable in the configured zone. " - "The test will be retried automatically when resources become available." - ), + 'ZONE_RESOURCE_POOL_EXHAUSTED': _ZONE_EXHAUSTED_MESSAGE, + 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS': _ZONE_EXHAUSTED_MESSAGE, 'QUOTA_EXCEEDED': ( "GCP quota limit reached. " "The test will be retried automatically when resources become available." @@ -145,6 +153,7 @@ def safe_db_commit(db, operation_description: str = "database operation") -> boo # Tests encountering these errors will remain pending and be picked up on the next cron run. GCP_RETRYABLE_ERRORS = { 'ZONE_RESOURCE_POOL_EXHAUSTED', + 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS', 'QUOTA_EXCEEDED', } diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index 3b55ef89..b8472f34 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -3294,6 +3294,45 @@ def test_parse_gcp_error_zone_resource_exhausted(self): # Should NOT contain raw technical details self.assertNotIn("us-central1-a", error_msg) + def test_parse_gcp_error_zone_resource_exhausted_with_details(self): + """Test that the _WITH_DETAILS variant gets the same user-friendly message.""" + from mod_ci.controllers import parse_gcp_error + + mock_log = MagicMock() + result = { + 'status': 'DONE', + 'error': { + 'errors': [{ + 'code': 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS', + 'message': "The zone 'projects/test/zones/us-central1-a' does not have enough " + "resources available to fulfill the request. '(resource type:compute)'." + }] + } + } + + error_msg = parse_gcp_error(result, log=mock_log) + self.assertIn("GCP resources temporarily unavailable", error_msg) + self.assertIn("retried automatically", error_msg) + # Must not fall through to the generic "contact the administrator" message + self.assertNotIn("contact the administrator", error_msg) + self.assertNotIn("us-central1-a", error_msg) + + def test_zone_resource_exhausted_variants_are_retryable(self): + """Both zone-exhaustion codes must be retryable so the test stays pending.""" + from mod_ci.controllers import is_retryable_gcp_error + + for code in ('ZONE_RESOURCE_POOL_EXHAUSTED', 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS'): + with self.subTest(code=code): + result = {'error': {'errors': [{'code': code, 'message': 'no capacity'}]}} + self.assertTrue(is_retryable_gcp_error(result)) + + def test_non_transient_gcp_error_is_not_retryable(self): + """A genuine failure must still be marked failed rather than retried forever.""" + from mod_ci.controllers import is_retryable_gcp_error + + result = {'error': {'errors': [{'code': 'RESOURCE_NOT_FOUND', 'message': 'nope'}]}} + self.assertFalse(is_retryable_gcp_error(result)) + def test_parse_gcp_error_quota_exceeded(self): """Test that QUOTA_EXCEEDED returns user-friendly message.""" from mod_ci.controllers import parse_gcp_error