diff --git a/pytest.ini b/pytest.ini index 2529b9e..1d483fb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,22 @@ [pytest] markers = smoke: Build verification test + +filterwarnings = + ignore: WARNING the new order is not taken into account:UserWarning + ignore::urllib3.exceptions.InsecureRequestWarning + ignore::cryptography.utils.CryptographyDeprecationWarning + ignore: Use ProtectionLevel enum instead:DeprecationWarning + ignore: Use protection_level parameter instead:DeprecationWarning + ignore: pkg_resources is deprecated as an API:DeprecationWarning + +log_format=%(asctime)s %(levelname)s:%(name)s:%(message)s +log_date_format=%H:%M:%S %z +log_level=INFO +junit_logging=all +junit_family=xunit2 +junit_log_passing_tests=0 +asyncio_mode=auto +addopts = --last-failed-no-failures=none + +#addopts = --pdbcls=IPython.terminal.debugger:Pdb diff --git a/threescale_api_crd/defaults.py b/threescale_api_crd/defaults.py index fcaa6ba..4584c5d 100644 --- a/threescale_api_crd/defaults.py +++ b/threescale_api_crd/defaults.py @@ -1,5 +1,6 @@ """ Module with default objects """ +import backoff import logging import copy import random @@ -63,6 +64,14 @@ def disable_crd_implemented(self): """Set False to crd is implemented attribute""" self.__class__.CRD_IMPLEMENTED = False + def after_create(self, params, instance): + """Called after create with the created instance. + + Override this method in subclasses to perform post-creation actions + like creating related CRDs (e.g., ApplicationAuth for OIDC services). + """ + pass + def fetch_crd_entity(self, name: str): """Fetches the entity based on crd name Args: @@ -200,7 +209,10 @@ def create(self, params: dict = None, **kwargs) -> "DefaultResourceCRD": spec["metadata"]["namespace"] = self.threescale_client.ocp_namespace spec["metadata"]["name"] = name spec = self._set_provider_ref_new_crd(spec) - self.before_create(params, spec) + # before_create may return dependent objects that need to be waited on + dependencies = self.before_create(params, spec) or [] + if not isinstance(dependencies, list): + dependencies = [dependencies] spec["spec"].update(self.translate_to_crd(params)) DefaultClientCRD.cleanup_spec(spec, self.KEYS, params) @@ -238,13 +250,47 @@ def create(self, params: dict = None, **kwargs) -> "DefaultResourceCRD": assert created_objects assert success + # Wait for dependent objects created in before_create to be fully ready with IDs + for dep in dependencies: + self._wait_for_dependency_ready(dep) + instance = (self._create_instance(response=created_objects)[:1] or [None])[ 0 ] + + # Call after_create hook if defined + self.after_create(params, instance) + return instance return threescale_api.defaults.DefaultClient.create(self, params, **kwargs) + def _wait_for_dependency_ready(self, dep): + """Wait for a dependent resource to be fully ready with its ID. + + Args: + dep: tuple of (resource_instance, id_name) where resource_instance + is the created dependent resource and id_name is the status + field name for its ID (e.g., 'developerUserID') + """ + if dep is None: + return + resource, id_name = dep + if resource is None or not hasattr(resource, 'crd'): + return + + @backoff.on_predicate(backoff.fibo, lambda x: not x, max_tries=12, jitter=None) + def _wait(): + resource.crd = resource.crd.refresh() + return self._is_ready_with_id(resource.crd, id_name) + + _wait() + # Update the resource's entity with the ID so it doesn't need to fetch again + status = resource.crd.as_dict().get("status", {}) + new_id = status.get(id_name) + if new_id: + resource._entity["id"] = new_id + def _set_provider_ref_new_crd(self, spec): """set provider reference to new crd""" if self.threescale_client.ocp_provider_ref is None: @@ -259,21 +305,44 @@ def _set_provider_ref_new_crd(self, spec): return spec def _is_ready(self, obj): - """Is object ready?""" + """Is object ready? + + Ready states: + - Synced=True or Ready=True (with valid ID) + - Orphan=True (waiting for parent resource) with no Failed/Invalid + """ if not ("status" in obj.model and "conditions" in obj.model.status): return False status = obj.as_dict()["status"] new_id = status.get(self.ID_NAME, 0) + state = {"Failed": True, "Invalid": True, "Synced": False, "Ready": False, "Orphan": False} + for sta in status["conditions"]: + state[sta["type"]] = sta["status"] == "True" + + if state["Failed"] or state["Invalid"]: + return False + # Orphan is valid (waiting for parent), or Synced/Ready with valid ID + return state["Orphan"] or ((state["Synced"] or state["Ready"]) and (new_id != 0)) + + def _is_ready_with_id(self, obj, id_name): + """Is object ready with ID populated? + + Unlike _is_ready, this does NOT consider Orphan state as ready. + Requires Synced=True or Ready=True with a valid ID. + Used for waiting on dependent objects created in before_create. + """ + if not ("status" in obj.model and "conditions" in obj.model.status): + return False + status = obj.as_dict()["status"] + new_id = status.get(id_name, 0) state = {"Failed": True, "Invalid": True, "Synced": False, "Ready": False} for sta in status["conditions"]: state[sta["type"]] = sta["status"] == "True" - return ( - not state["Failed"] - and not state["Invalid"] - and (state["Synced"] or state["Ready"]) - and (new_id != 0) - ) + if state["Failed"] or state["Invalid"]: + return False + # Require Synced or Ready with valid ID (not Orphan) + return (state["Synced"] or state["Ready"]) and new_id != 0 def _create_instance(self, response, klass=None, collection: bool = False): klass = klass or self._instance_klass @@ -283,7 +352,8 @@ def _create_instance(self, response, klass=None, collection: bool = False): else: extracted = self._extract_resource(response, collection) instance = self._instantiate(extracted=extracted, klass=klass) - LOG.info("[INSTANCE] CRD Created instance: %s", str(instance)) + # Avoid str(instance) as it may trigger __repr__ which accesses entity_id + LOG.info("[INSTANCE] CRD Created instance: %s", type(instance).__name__) return instance def _extract_resource_crd(self, response, collection, klass): @@ -375,6 +445,16 @@ def update( if result.status(): LOG.error("[INSTANCE] Update CRD failed: %s", str(result)) raise Exception(str(result)) + + # Wait for the CRD to be synced after update + @backoff.on_predicate(backoff.fibo, lambda x: not x, max_tries=12, jitter=None) + def _wait_for_sync(): + resource.crd = resource.crd.refresh() + return self._is_ready(resource.crd) + + if not _wait_for_sync(): + LOG.warning("[UPDATE] CRD update did not reach ready state") + # return self.read(resource.entity_id) return resource @@ -579,25 +659,51 @@ def crd(self, value): @property def entity_id(self) -> int: - return self._entity_id or self._entity.get("id") or self.get_id_from_crd() + """Returns entity ID, fetching from CRD if needed. + + If the CRD is in Orphan state (waiting for parent), returns None + instead of blocking. Use get_id_from_crd() to explicitly wait. + """ + if self._entity_id: + return self._entity_id + if self._entity.get("id"): + return self._entity.get("id") + # Don't block if CRD is in Orphan state - return None + if self._crd and self._is_orphan(): + return None + return self.get_id_from_crd() @entity_id.setter def entity_id(self, value): self._entity_id = value + def _is_orphan(self): + """Check if the CRD is in Orphan state (waiting for parent resource).""" + if not self._crd: + return False + crd_dict = self._crd.as_dict() + status = crd_dict.get("status") + if not status: + return False + conditions = status.get("conditions", []) + for cond in conditions: + if cond.get("type") == "Orphan" and cond.get("status") == "True": + return True + return False + def get_id_from_crd(self): - """Returns object id extracted from CRD.""" - counter = 5 - while counter > 0: + """Returns object id extracted from CRD. + + This will wait with backoff until the ID is available. + """ + # 12 tries with fibonacci backoff: 1+1+2+3+5+8+13+21+34+55+89+144 ≈ 376 seconds (~6 min) + @backoff.on_predicate(backoff.fibo, lambda x: x is None, max_tries=12, jitter=None) + def _get_id(): self.crd = self.crd.refresh() status = self.crd.as_dict()["status"] - ret_id = status.get(self.client.ID_NAME, None) - if ret_id: - return ret_id - time.sleep(20) - counter -= 1 + return status.get(self.client.ID_NAME, None) - return None + return _get_id() def get_path(self): """ diff --git a/threescale_api_crd/resources.py b/threescale_api_crd/resources.py index ae5288b..68e960e 100644 --- a/threescale_api_crd/resources.py +++ b/threescale_api_crd/resources.py @@ -1,5 +1,6 @@ """ Module with resources for CRD for Threescale client """ +import backoff import logging import copy import json @@ -25,6 +26,54 @@ from threescale_api_crd import constants LOG = logging.getLogger(__name__) +BACKOFF_MAX_TRIES = 12 + + +class CRDNotReadyError(Exception): + """Raised when CRD is not yet ready or synced.""" + pass + + +def _is_crd_ready(crd): + """Check if CRD has status.conditions indicating it's ready to use. + + Ready states: + - Synced=True or Ready=True + - Orphan=True (waiting for parent resource) with no Failed/Invalid + """ + crd_dict = crd.as_dict() + status = crd_dict.get("status") + if not status: + return False + conditions = status.get("conditions", []) + state = {"Failed": False, "Invalid": False, "Synced": False, "Ready": False, "Orphan": False} + for cond in conditions: + cond_type = cond.get("type") + if cond_type in state: + state[cond_type] = cond.get("status") == "True" + # Fail if Failed or Invalid + if state["Failed"] or state["Invalid"]: + return False + # Ready if Synced, Ready, or Orphan (waiting for parent) + return state["Synced"] or state["Ready"] or state["Orphan"] + + +def _crd_status_info(crd, crd_type: str) -> str: + """Generate descriptive error message for CRD not ready state.""" + crd_dict = crd.as_dict() + name = crd_dict.get("metadata", {}).get("name", "unknown") + namespace = crd_dict.get("metadata", {}).get("namespace", "unknown") + status = crd_dict.get("status") + if not status: + return f"{crd_type} '{name}' in namespace '{namespace}' has no status" + conditions = status.get("conditions", []) + if not conditions: + return f"{crd_type} '{name}' in namespace '{namespace}' has no conditions" + cond_summary = ", ".join( + f"{c.get('type')}={c.get('status')}" + (f" ({c.get('message')})" if c.get("message") else "") + for c in conditions + ) + return f"{crd_type} '{name}' in namespace '{namespace}' not ready: {cond_summary}" class Services(DefaultClientCRD, threescale_api.resources.Services): @@ -1004,14 +1053,22 @@ def __init__( ) def before_create(self, params, spec): - """Called before create.""" + """Called before create. + + Returns: + List of (resource, id_name) tuples for dependent resources that + need to be waited on after the main object is created. + """ if "username" in params: pars = params.copy() pars["account_name"] = pars["name"] pars["name"] = secrets.token_urlsafe(8) # first user should be admin pars["role"] = "admin" - self.parent.threescale_client.account_users.create(params=pars) + user = self.parent.threescale_client.account_users.create(params=pars) + # Return the user so that create() waits for it to have its ID + return [(user, AccountUsers.ID_NAME)] + return [] def before_update(self, new_params, resource): """Called before update.""" @@ -1979,6 +2036,13 @@ def __init__(self, entity_name="system_name", **kwargs): for cey, walue in constants.KEYS_SERVICE.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "Service")) + + _wait() entity["id"] = crd.as_dict().get("status").get(Services.ID_NAME) # add ids to cache if entity["id"] and entity[entity_name]: @@ -2040,6 +2104,13 @@ def __init__(self, **kwargs): crd = kwargs.pop("crd") self.spec_path = [] entity = {} + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "Proxy")) + + _wait() # there is no attribute which can simulate Proxy id, service id should be used entity["id"] = crd.as_dict().get("status").get(Services.ID_NAME) # apicastHosted or ApicastSelfManaged @@ -2335,6 +2406,13 @@ def __init__(self, entity_name="system_name", **kwargs): for cey, walue in constants.KEYS_ACTIVE_DOC.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "ActiveDoc")) + + _wait() entity["id"] = crd.as_dict().get("status").get(ActiveDocs.ID_NAME) if "service_id" in entity: ide = Service.system_name_to_id.get(entity["service_id"], None) @@ -2374,6 +2452,13 @@ def __init__(self, entity_name="name", **kwargs): for cey, walue in constants.KEYS_POLICY_REG.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "PolicyRegistry")) + + _wait() entity["id"] = crd.as_dict().get("status").get(PoliciesRegistry.ID_NAME) super().__init__(crd=crd, entity=entity, entity_name=entity_name, **kwargs) else: @@ -2399,6 +2484,13 @@ def __init__(self, entity_name="system_name", **kwargs): for cey, walue in constants.KEYS_BACKEND.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "Backend")) + + _wait() entity["id"] = crd.as_dict().get("status").get(Backends.ID_NAME) super().__init__(crd=crd, entity=entity, entity_name=entity_name, **kwargs) @@ -2526,6 +2618,13 @@ def __init__(self, entity_name="system_name", **kwargs): for cey, walue in constants.KEYS_BACKEND_USAGE.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "BackendUsage")) + + _wait() entity["service_id"] = int( crd.as_dict().get("status", {}).get(Services.ID_NAME, 0) ) @@ -2644,6 +2743,13 @@ def __init__(self, entity_name="org_name", **kwargs): for cey, walue in constants.KEYS_ACCOUNT.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "Account")) + + _wait() status = crd.as_dict().get("status", None) if status: entity["id"] = status.get(Accounts.ID_NAME) @@ -2705,6 +2811,13 @@ def __init__(self, entity_name="username", **kwargs): for cey, walue in constants.KEYS_ACCOUNT_USER.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "AccountUser")) + + _wait() status = crd.as_dict().get("status", None) if status: entity["id"] = status.get(AccountUsers.ID_NAME) @@ -2768,6 +2881,13 @@ def __init__(self, entity_name="name", **kwargs): for cey, walue in constants.KEYS_POLICY.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "Policy")) + + _wait() entity["service_id"] = int( crd.as_dict().get("status", {}).get(Services.ID_NAME, 0) ) @@ -2814,6 +2934,13 @@ def __init__(self, entity_name="name", **kwargs): for cey, walue in constants.KEYS_OPEN_API.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "OpenApi")) + + _wait() status = crd.as_dict().get("status") entity["id"] = status.get(OpenApis.ID_NAME) entity["productResourceName"] = status.get("productResourceName", {}).get( @@ -3115,6 +3242,13 @@ def __init__(self, entity_name="name", **kwargs): for cey, walue in constants.KEYS_APPLICATION.items(): if key == walue: entity[cey] = value + + @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None) + def _wait(): + if not _is_crd_ready(crd): + raise CRDNotReadyError(_crd_status_info(crd, "Application")) + + _wait() status = crd.as_dict().get("status") entity["id"] = status.get(Applications.ID_NAME) entity["state"] = status.get("state")