csp provider tests requests.HTTPError exceptions

atst/domain/csp/cloud/azure_cloud_provider.py

@@ -136,14 +136,15 @@ class AzureCloudProvider(CloudProviderInterface):
         )
         try:
             return secret_client.set_secret(secret_key, secret_value)
-        except self.azure_exceptions.HttpResponseError:
+        except self.sdk.azure_exceptions.HttpResponseError:
             app.logger.error(
                 f"Could not SET secret in Azure keyvault for key {secret_key}.",
                 exc_info=1,
             )
+            creds = self._source_creds()
             raise SecretException(
+                creds.tenant_id,
                 f"Could not SET secret in Azure keyvault for key {secret_key}.",
-                exc.message,
             )
 
     def get_secret(self, secret_key):
@@ -153,14 +154,15 @@ class AzureCloudProvider(CloudProviderInterface):
         )
         try:
             return secret_client.get_secret(secret_key).value
-        except self.azure_exceptions.HttpResponseError:
+        except self.sdk.azure_exceptions.HttpResponseError:
             app.logger.error(
                 f"Could not GET secret in Azure keyvault for key {secret_key}.",
                 exc_info=1,
             )
+            creds = self._source_creds()
             raise SecretException(
+                creds.tenant_id,
                 f"Could not GET secret in Azure keyvault for key {secret_key}.",
-                exc.message,
             )
 
     def create_environment(self, payload: EnvironmentCSPPayload):
@@ -339,8 +341,16 @@ class AzureCloudProvider(CloudProviderInterface):
                 f"Could not create tenant. Request timed out.", exc_info=1,
             )
             raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
+        except self.sdk.requests.exceptions.HTTPError as exc:
-            raise UnknownServerException("azure application error creating tenant")
+            app.logger.error(
+                result.status_code,
+                "azure application error creating tenant",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating tenant. {str(exc)}",
+            )
 
         result_dict = result.json()
         tenant_id = result_dict.get("tenantId")
@@ -392,16 +402,24 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create billing profile. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating billing profile")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create billing profile. Request timed out.", exc_info=1,
             )
             raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
+        except self.sdk.requests.exceptions.HTTPError as exc:
-            raise UnknownServerException("azure application error creating tenant")
+            app.logger.error(
+                result.status_code,
+                "azure application error creating billing profile",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating billing profile. {str(exc)}",
+            )
 
     def create_billing_profile_verification(
         self, payload: BillingProfileVerificationCSPPayload
@@ -429,16 +447,30 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not verify billing profile creation. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error creating billing profile verification"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not verify billing profile creation. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "timout error during billing profile verification"
+            )
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error during billing profile verification",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error during billing profile verification. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_billing_profile_tenant_access(
         self, payload: BillingProfileTenantAccessCSPPayload
@@ -461,21 +493,36 @@ class AzureCloudProvider(CloudProviderInterface):
             result = self.sdk.requests.post(
                 url, headers=headers, json=request_body, timeout=30,
             )
+            result.raise_for_status()
             if result.status_code == 201:
                 return BillingProfileTenantAccessCSPResult(**result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create billing profile tenant access. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error creating billing profile tenant access"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create billing profile tenant access. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "timout error creating billing profile tenant access"
+            )
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating billing profile tenant access",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating billing profile tenant access. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_task_order_billing_creation(
         self, payload: TaskOrderBillingCreationCSPPayload
@@ -509,16 +556,24 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create task order billing. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating task order billing")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create task order billing. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error creating task order billing")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating task order billing",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating task order billing. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_task_order_billing_verification(
         self, payload: TaskOrderBillingVerificationCSPPayload
@@ -547,16 +602,27 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not verify task order billing. Connection Error", exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error during task order billing verification"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create verify task order billing. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error in task order billing verification")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error in task order billing verification",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error in task order billing verification. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_billing_instruction(self, payload: BillingInstructionCSPPayload):
         sp_token = self._get_root_provisioning_token()
@@ -588,16 +654,25 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create billing instructions. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating billing instructions")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create billing instructions. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error creating billing instructions")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error in creating billing instructions",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error in creating billing instructions. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_subscription(self, payload: SubscriptionCreationCSPPayload):
         sp_token = self._get_tenant_principal_token(payload.tenant_id)
@@ -622,22 +697,31 @@ class AzureCloudProvider(CloudProviderInterface):
             result = self.sdk.requests.put(
                 url, headers=auth_header, json=request_body, timeout=30
             )
+            result.raise_for_status()
             if result.status_code in [200, 202]:
                 # 202 has location/retry after headers
                 return SubscriptionCreationCSPResult(**result.headers, **result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create subscription. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating subscription")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create subscription. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error creating subscription")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating subscription",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating subscription. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_subscription_creation(self, payload: SubscriptionCreationCSPPayload):
         return self.create_subscription(payload)
@@ -666,16 +750,26 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not verify subscription. Connection Error", exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error during subscription verification"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not verify subscription. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error during subscription verification")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error during subscription verification",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error during subscription verification. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_product_purchase(self, payload: ProductPurchaseCSPPayload):
         sp_token = self._get_root_provisioning_token()
@@ -713,16 +807,24 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not purchase product. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error during product purchase")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not purchase product. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error during product purchase")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error during product purchase",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error during product purchase. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_product_purchase_verification(
         self, payload: ProductPurchaseVerificationCSPPayload
@@ -753,16 +855,28 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not verify product purchase. Connection Error", exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error during product purchase verification"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not verify product purchase. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException(
+                "timout error during product purchase verification"
+            )
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error during product purchase verification",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error during product purchase verification. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_tenant_admin_ownership(self, payload: TenantAdminOwnershipCSPPayload):
         mgmt_token = self._get_elevated_management_token(payload.tenant_id)
@@ -785,25 +899,37 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.sdk.cloud.endpoints.resource_manager}/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleAssignments/{assignment_guid}?api-version=2015-07-01"
 
         try:
-            response = self.sdk.requests.put(
+            result = self.sdk.requests.put(
                 url, headers=auth_header, json=request_body, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
 
-            return TenantAdminOwnershipCSPResult(**response.json())
+            return TenantAdminOwnershipCSPResult(**result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create tenant admin ownership. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error creating tenant admin ownership"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create tenant admin ownership. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error creating tenant admin ownership")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating tenant admin ownership",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating tenant admin ownership. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_tenant_principal_ownership(
         self, payload: TenantPrincipalOwnershipCSPPayload
@@ -829,24 +955,38 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.sdk.cloud.endpoints.resource_manager}/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleAssignments/{assignment_guid}?api-version=2015-07-01"
 
         try:
-            response = self.sdk.requests.put(
+            result = self.sdk.requests.put(
                 url, headers=auth_header, json=request_body, timeout=30,
             )
-            response.raise_for_status()
+            result.raise_for_status()
-            return TenantPrincipalOwnershipCSPResult(**response.json())
+            return TenantPrincipalOwnershipCSPResult(**result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create tenant principal ownership. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error creating tenant principal ownership"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create tenant principal ownership. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "timout error creating tenant prinicpal ownership"
+            )
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating tenant principal ownership",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating tenant principal ownership. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_tenant_principal_app(self, payload: TenantPrincipalAppCSPPayload):
         graph_token = self._get_tenant_admin_token(
@@ -866,24 +1006,33 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}/v1.0/applications"
 
         try:
-            response = self.sdk.requests.post(
+            result = self.sdk.requests.post(
                 url, json=request_body, headers=auth_header, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
-            return TenantPrincipalAppCSPResult(**response.json())
+            return TenantPrincipalAppCSPResult(**result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create tenant principal app. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating tenant principal app")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create tenant principal app. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error creating tenant principal app")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating tenant principal app",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating tenant principal app. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_tenant_principal(self, payload: TenantPrincipalCSPPayload):
         graph_token = self._get_tenant_admin_token(
@@ -903,24 +1052,32 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}/beta/servicePrincipals"
 
         try:
-            response = self.sdk.requests.post(
+            result = self.sdk.requests.post(
                 url, json=request_body, headers=auth_header, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
-            return TenantPrincipalCSPResult(**response.json())
+            return TenantPrincipalCSPResult(**result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create tenant principal. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating tenant principal")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create tenant principal. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error creating tenant principal")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating tenant principal",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating tenant principal. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_tenant_principal_credential(
         self, payload: TenantPrincipalCredentialCSPPayload
@@ -944,16 +1101,16 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}/v1.0/applications/{payload.principal_app_object_id}/addPassword"
 
         try:
-            response = self.sdk.requests.post(
+            result = self.sdk.requests.post(
                 url, json=request_body, headers=auth_header, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
-            result = response.json()
+            result_json = result.json()
             self.update_tenant_creds(
                 payload.tenant_id,
                 KeyVaultCredentials(
                     tenant_id=payload.tenant_id,
-                    tenant_sp_key=result.get("secretText"),
+                    tenant_sp_key=result_json.get("secretText"),
                     tenant_sp_client_id=payload.principal_app_id,
                 ),
             )
@@ -964,16 +1121,30 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create tenant principal credential. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error creating tenant principal credential"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create tenant principal credential. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "timout error creating tenant principal credential"
+            )
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating tenant principal credential",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating tenant principal credential. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_admin_role_definition(self, payload: AdminRoleDefinitionCSPPayload):
         graph_token = self._get_tenant_admin_token(
@@ -1010,16 +1181,25 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create admin role definition. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating admin role definition")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create admin role definition. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error creating admin role definition")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                response.status_code,
+                "azure application error creating admin role definition",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                response.status_code,
+                f"azure application error creating admin role definition. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_principal_admin_role(self, payload: PrincipalAdminRoleCSPPayload):
         graph_token = self._get_tenant_admin_token(
@@ -1043,24 +1223,33 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}/beta/roleManagement/directory/roleAssignments"
 
         try:
-            response = self.sdk.requests.post(
+            result = self.sdk.requests.post(
                 url, headers=auth_header, json=request_body, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
-            return PrincipalAdminRoleCSPResult(**response.json())
+            return PrincipalAdminRoleCSPResult(**result.json())
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create principal admin role. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating principal admin role")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create principal admin role. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error creating principal admin role")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating principal admin role",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating principal admin role. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_billing_owner(self, payload: BillingOwnerCSPPayload):
         graph_token = self._get_tenant_principal_token(
@@ -1098,24 +1287,30 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}/beta/roleManagement/directory/roleAssignments"
 
         try:
-            response = self.sdk.requests.post(
+            result = self.sdk.requests.post(url, headers=auth_header, json=request_body)
-                url, headers=auth_header, json=request_body
+            result.raise_for_status()
-            )
-            response.raise_for_status()
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not assign billing owner role. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error assigning billing owner role")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not assign billing owner role. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error assigning billing owner role")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error assigning billing owner role",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error assigning billing owner role. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
-        if response.ok:
+        if result.ok:
             return True
         else:
             raise UserProvisioningException("Could not assign billing admin role")
@@ -1127,23 +1322,31 @@ class AzureCloudProvider(CloudProviderInterface):
 
         url = f"{self.graph_resource}/v1.0/directoryRoles"
         try:
-            response = self.sdk.requests.get(url, headers=auth_header)
+            result = self.sdk.requests.get(url, headers=auth_header)
-            response.raise_for_status()
+            result.raise_for_status()
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not get billing owner role. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error getting billing owner role")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not get billing owner role. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error getting billing owner role")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error getting billing owner role",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error getting billing owner role. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
-        if response.ok:
+        if result.ok:
-            result = response.json()
+            result = result.json()
             for role in result["value"]:
                 if role["displayName"] == "Billing Administrator":
                     return role["id"]
@@ -1152,12 +1355,6 @@ class AzureCloudProvider(CloudProviderInterface):
                 "Could not find Billing Administrator role ID; role may not be enabled."
             )
 
-    def force_tenant_admin_pw_update(self, creds, tenant_owner_id):
-        # use creds to update to force password recovery?
-        # not sure what the endpoint/method for this is, yet
-
-        return {}
-
     def _get_management_service_principal(self):
         # we really should be using graph.microsoft.com, but i'm getting
         # "expired token" errors for that
@@ -1251,27 +1448,34 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}v1.0/users"
 
         try:
-            response = self.sdk.requests.post(
+            result = self.sdk.requests.post(
                 url, headers=auth_header, json=request_body, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
 
-            return UserCSPResult(**response.json())
+            return UserCSPResult(**result.json())
-
-        # raise UserProvisioningException(f"Failed to create user: {response.json()}")
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not create active directory user. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error creating active directory user")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not create active directory user. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error creating active directory user")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error creating active directory user",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error creating active directory user. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def _update_active_directory_user_email(self, graph_token, user_id, payload):
         request_body = {"otherMails": [payload.email]}
@@ -1283,12 +1487,12 @@ class AzureCloudProvider(CloudProviderInterface):
         url = f"{self.graph_resource}v1.0/users/{user_id}"
 
         try:
-            response = self.sdk.requests.patch(
+            result = self.sdk.requests.patch(
                 url, headers=auth_header, json=request_body, timeout=30
             )
-            response.raise_for_status()
+            result.raise_for_status()
 
-            if response.ok:
+            if result.ok:
                 return True
             else:
                 raise UserProvisioningException(
@@ -1297,16 +1501,30 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not update active directory user email. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error updating active directory user email"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not update active directory user email. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "timout error updating active directory user email"
+            )
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error updating active directory user email",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error updating active directory user email. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def create_user_role(self, payload: UserRoleCSPPayload):
         graph_token = self._get_tenant_principal_token(payload.tenant_id)
@@ -1446,16 +1664,28 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not get elevated management token. Connection Error",
+                exc_info=1,
+            )
+            raise ConnectionException(
+                "connection error getting elevated management token"
             )
-            raise ConnectionException("connection error creating tenant")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not get elevated management token. Request timed out.",
+                exc_info=1,
+            )
+            raise ConnectionException("timout error getting elevated management token")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error getting elevated management token",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error getting elevated management token. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")
 
     def _source_creds(self, tenant_id=None) -> KeyVaultCredentials:
         if tenant_id:
@@ -1523,13 +1753,21 @@ class AzureCloudProvider(CloudProviderInterface):
 
         except self.sdk.requests.exceptions.ConnectionError:
             app.logger.error(
-                f"Could not create tenant. Connection Error", exc_info=1,
+                f"Could not get reporting data. Connection Error", exc_info=1,
             )
-            raise ConnectionException("connection error creating tenant")
+            raise ConnectionException("connection error getting reporting data")
         except self.sdk.requests.exceptions.Timeout:
             app.logger.error(
-                f"Could not create tenant. Request timed out.", exc_info=1,
+                f"Could not get reporting data. Request timed out.", exc_info=1,
+            )
+            raise ConnectionException("timout error getting reporting data")
+        except self.sdk.requests.exceptions.HTTPError as exc:
+            app.logger.error(
+                result.status_code,
+                "azure application error getting reporting data",
+                exc_info=1,
+            )
+            raise UnknownServerException(
+                result.status_code,
+                f"azure application error getting reporting data. {str(exc)}",
             )
-            raise ConnectionException("timout error creating tenant")
-        except self.sdk.requests.exceptions.HTTPError:
-            raise UnknownServerException("azure application error creating tenant")

atst/domain/csp/cloud/exceptions.py

@@ -61,12 +61,13 @@ class UnknownServerException(GeneralCSPException):
     """An error occured on the CSP side (5xx) and we don't know why
     """
 
-    def __init__(self, server_error):
+    def __init__(self, status_code, server_error):
+        self.status_code = status_code
         self.server_error = server_error
 
     @property
     def message(self):
-        return "A server error occured: {}".format(self.server_error)
+        return f"A server error with status code [{self.status_code}] occured: {self.server_error}"
 
 
 class EnvironmentCreationException(GeneralCSPException):

atst/domain/csp/cloud/mock_cloud_provider.py

@@ -72,7 +72,7 @@ class MockCloudProvider(CloudProviderInterface):
     AUTHENTICATION_EXCEPTION = AuthenticationException("Authentication failure.")
     AUTHORIZATION_EXCEPTION = AuthorizationException("Not authorized.")
     NETWORK_EXCEPTION = ConnectionException("Network failure.")
-    SERVER_EXCEPTION = UnknownServerException("Not our fault.")
+    SERVER_EXCEPTION = UnknownServerException(500, "Not our fault.")
 
     SERVER_FAILURE_PCT = 1
     NETWORK_FAILURE_PCT = 7

atst/models/portfolio_state_machine.py

@@ -145,7 +145,7 @@ class PortfolioStateMachine(
                     self.trigger("fail")
 
             elif self.current_state == FSMStates.FAILED:
-                # get the first trigger that starts with 'create_'
+                # get the first trigger that starts with 'resume_progress_'
                 resume_progress_trigger = next(
                     filter(
                         lambda trigger: trigger.startswith("resume_progress_"),

tests/domain/cloud/test_azure_csp.py

@@ -73,6 +73,23 @@ from atst.domain.csp.cloud.exceptions import UserProvisioningException
 BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31"
 
 
+def mock_requests_response(
+    status=200, content="CONTENT", json_data=None, raise_for_status=None
+):
+    mock_resp = Mock()
+    # mock raise_for_status call w/optional error
+    mock_resp.raise_for_status = Mock()
+    if raise_for_status:
+        mock_resp.raise_for_status.side_effect = raise_for_status
+    # set status code and content
+    mock_resp.status_code = status
+    mock_resp.content = content
+    # add json data if provided
+    if json_data:
+        mock_resp.json = mock.Mock(return_value=json_data)
+    return mock_resp
+
+
 def mock_management_group_create(mock_azure, spec_dict):
     mock_azure.sdk.managementgroups.ManagementGroupsAPI.return_value.management_groups.create_or_update.return_value.result.return_value = (
         spec_dict
@@ -190,7 +207,6 @@ def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider):
 
 
 def test_create_tenant(mock_azure: AzureCloudProvider):
-
     mock_result = Mock()
     mock_result.json.return_value = {
         "objectId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d",
@@ -198,14 +214,18 @@ def test_create_tenant(mock_azure: AzureCloudProvider):
         "userId": "1153801116406515559",
     }
     mock_result.status_code = 200
-
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
-
     payload = TenantCSPPayload(
         **dict(
             user_id="admin",
@@ -223,7 +243,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider):
         mock_azure.create_tenant(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_tenant(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_tenant(payload)
 
     result: TenantCSPResult = mock_azure.create_tenant(payload)
@@ -242,10 +262,16 @@ def test_create_billing_profile_creation(mock_azure: AzureCloudProvider):
     }
     mock_result.status_code = 202
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -268,7 +294,7 @@ def test_create_billing_profile_creation(mock_azure: AzureCloudProvider):
         mock_azure.create_billing_profile_creation(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_billing_profile_creation(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_billing_profile_creation(payload)
 
     body: BillingProfileCreationCSPResult = mock_azure.create_billing_profile_creation(
@@ -313,10 +339,16 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider):
         },
         "type": "Microsoft.Billing/billingAccounts/billingProfiles",
     }
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.get.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -330,7 +362,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider):
         mock_azure.create_billing_profile_verification(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_billing_profile_verification(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_billing_profile_verification(payload)
 
     body: BillingProfileVerificationCSPResult = mock_azure.create_billing_profile_verification(
@@ -364,10 +396,16 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider):
         "type": "Microsoft.Billing/billingRoleAssignments",
     }
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -383,7 +421,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider):
         mock_azure.create_billing_profile_tenant_access(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_billing_profile_tenant_access(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_billing_profile_tenant_access(payload)
 
     body: BillingProfileTenantAccessCSPResult = mock_azure.create_billing_profile_tenant_access(
@@ -407,10 +445,16 @@ def test_create_task_order_billing_creation(mock_azure: AzureCloudProvider):
         "Retry-After": "10",
     }
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.patch.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -425,7 +469,7 @@ def test_create_task_order_billing_creation(mock_azure: AzureCloudProvider):
         mock_azure.create_task_order_billing_creation(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_task_order_billing_creation(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_task_order_billing_creation(payload)
 
     body: TaskOrderBillingCreationCSPResult = mock_azure.create_task_order_billing_creation(
@@ -480,10 +524,16 @@ def test_create_task_order_billing_verification(mock_azure):
         },
         "type": "Microsoft.Billing/billingAccounts/billingProfiles",
     }
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.get.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -497,7 +547,7 @@ def test_create_task_order_billing_verification(mock_azure):
         mock_azure.create_task_order_billing_verification(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_task_order_billing_verification(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_task_order_billing_verification(payload)
 
     body: TaskOrderBillingVerificationCSPResult = mock_azure.create_task_order_billing_verification(
@@ -527,10 +577,16 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider):
         "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions",
     }
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.put.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -550,7 +606,7 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider):
         mock_azure.create_billing_instruction(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_billing_instruction(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_billing_instruction(payload)
     body: BillingInstructionCSPResult = mock_azure.create_billing_instruction(payload)
     assert body.reported_clin_name == "TO1:CLIN001"
@@ -568,10 +624,16 @@ def test_create_product_purchase(mock_azure: AzureCloudProvider):
         "Retry-After": "10",
     }
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -586,7 +648,7 @@ def test_create_product_purchase(mock_azure: AzureCloudProvider):
         mock_azure.create_product_purchase(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_product_purchase(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_product_purchase(payload)
 
     body: ProductPurchaseCSPResult = mock_azure.create_product_purchase(payload)
@@ -624,10 +686,16 @@ def test_create_product_purchase_verification(mock_azure):
         "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections/products",
     }
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.get.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -641,7 +709,7 @@ def test_create_product_purchase_verification(mock_azure):
         mock_azure.create_product_purchase_verification(payload)
     with pytest.raises(ConnectionException):
         mock_azure.create_product_purchase_verification(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.create_product_purchase_verification(payload)
 
     body: ProductPurchaseVerificationCSPResult = mock_azure.create_product_purchase_verification(
@@ -662,10 +730,16 @@ def test_create_tenant_principal_app(mock_azure: AzureCloudProvider):
         mock_result.ok = True
         mock_result.json.return_value = {"appId": "appId", "id": "id"}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.post.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -678,7 +752,7 @@ def test_create_tenant_principal_app(mock_azure: AzureCloudProvider):
             mock_azure.create_tenant_principal_app(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_tenant_principal_app(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_tenant_principal_app(payload)
 
         result: TenantPrincipalAppCSPResult = mock_azure.create_tenant_principal_app(
@@ -700,10 +774,16 @@ def test_create_tenant_principal(mock_azure: AzureCloudProvider):
         mock_result.ok = True
         mock_result.json.return_value = {"id": "principal_id"}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.post.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -719,7 +799,7 @@ def test_create_tenant_principal(mock_azure: AzureCloudProvider):
             mock_azure.create_tenant_principal(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_tenant_principal(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_tenant_principal(payload)
 
         result: TenantPrincipalCSPResult = mock_azure.create_tenant_principal(payload)
@@ -739,10 +819,16 @@ def test_create_tenant_principal_credential(mock_azure: AzureCloudProvider):
         mock_result.ok = True
         mock_result.json.return_value = {"secretText": "new secret key"}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.post.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -759,7 +845,7 @@ def test_create_tenant_principal_credential(mock_azure: AzureCloudProvider):
             mock_azure.create_tenant_principal_credential(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_tenant_principal_credential(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_tenant_principal_credential(payload)
 
         result: TenantPrincipalCredentialCSPResult = mock_azure.create_tenant_principal_credential(
@@ -786,10 +872,16 @@ def test_create_admin_role_definition(mock_azure: AzureCloudProvider):
             ]
         }
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.get.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
         mock_azure = mock_get_secret(mock_azure)
@@ -801,7 +893,7 @@ def test_create_admin_role_definition(mock_azure: AzureCloudProvider):
             mock_azure.create_admin_role_definition(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_admin_role_definition(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_admin_role_definition(payload)
 
         result: AdminRoleDefinitionCSPResult = mock_azure.create_admin_role_definition(
@@ -823,10 +915,16 @@ def test_create_tenant_admin_ownership(mock_azure: AzureCloudProvider):
         mock_result.ok = True
         mock_result.json.return_value = {"id": "id"}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.put.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -840,7 +938,7 @@ def test_create_tenant_admin_ownership(mock_azure: AzureCloudProvider):
             mock_azure.create_tenant_admin_ownership(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_tenant_admin_ownership(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_tenant_admin_ownership(payload)
 
         result: TenantAdminOwnershipCSPResult = mock_azure.create_tenant_admin_ownership(
@@ -862,10 +960,16 @@ def test_create_tenant_principal_ownership(mock_azure: AzureCloudProvider):
         mock_result.ok = True
         mock_result.json.return_value = {"id": "id"}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.put.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -879,7 +983,7 @@ def test_create_tenant_principal_ownership(mock_azure: AzureCloudProvider):
             mock_azure.create_tenant_principal_ownership(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_tenant_principal_ownership(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_tenant_principal_ownership(payload)
 
         result: TenantPrincipalOwnershipCSPResult = mock_azure.create_tenant_principal_ownership(
@@ -901,10 +1005,16 @@ def test_create_principal_admin_role(mock_azure: AzureCloudProvider):
         mock_result.ok = True
         mock_result.json.return_value = {"id": "id"}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.post.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -919,7 +1029,7 @@ def test_create_principal_admin_role(mock_azure: AzureCloudProvider):
             mock_azure.create_principal_admin_role(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_principal_admin_role(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_principal_admin_role(payload)
 
         result: PrincipalAdminRoleCSPResult = mock_azure.create_principal_admin_role(
@@ -945,10 +1055,16 @@ def test_create_subscription_creation(mock_azure: AzureCloudProvider):
         }
         mock_result.json.return_value = {}
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.put.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -967,7 +1083,7 @@ def test_create_subscription_creation(mock_azure: AzureCloudProvider):
             mock_azure.create_subscription_creation(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_subscription_creation(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_subscription_creation(payload)
 
         result: SubscriptionCreationCSPResult = mock_azure.create_subscription_creation(
@@ -991,10 +1107,16 @@ def test_create_subscription_verification(mock_azure: AzureCloudProvider):
             "subscriptionLink": "/subscriptions/60fbbb72-0516-4253-ab18-c92432ba3230"
         }
 
+        mock_http_error_resp = mock_requests_response(
+            status=500,
+            raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+                "500 Server Error"
+            ),
+        )
         mock_azure.sdk.requests.get.side_effect = [
             mock_azure.sdk.requests.exceptions.ConnectionError,
             mock_azure.sdk.requests.exceptions.Timeout,
-            mock_azure.sdk.requests.exceptions.HTTPError,
+            mock_http_error_resp,
             mock_result,
         ]
 
@@ -1008,7 +1130,7 @@ def test_create_subscription_verification(mock_azure: AzureCloudProvider):
             mock_azure.create_subscription_verification(payload)
         with pytest.raises(ConnectionException):
             mock_azure.create_subscription_verification(payload)
-        with pytest.raises(UnknownServerException):
+        with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
             mock_azure.create_subscription_verification(payload)
 
         result: SuscriptionVerificationCSPResult = mock_azure.create_subscription_verification(
@@ -1039,10 +1161,16 @@ def test_get_reporting_data(mock_azure: AzureCloudProvider):
     }
     mock_result.ok = True
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -1068,7 +1196,7 @@ def test_get_reporting_data(mock_azure: AzureCloudProvider):
         mock_azure.get_reporting_data(payload)
     with pytest.raises(ConnectionException):
         mock_azure.get_reporting_data(payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure.get_reporting_data(payload)
 
     data: CostManagementQueryCSPResult = mock_azure.get_reporting_data(payload)
@@ -1082,10 +1210,16 @@ def test_get_reporting_data_malformed_payload(mock_azure: AzureCloudProvider):
     mock_result = Mock()
     mock_result.ok = True
 
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -1125,6 +1259,26 @@ def test_get_secret(mock_azure: AzureCloudProvider):
         assert mock_azure.get_secret("secret key") == "my secret"
 
 
+def test_get_secret_secret_exception(mock_azure: AzureCloudProvider):
+    with patch.object(
+        AzureCloudProvider,
+        "_get_client_secret_credential_obj",
+        wraps=mock_azure._get_client_secret_credential_obj,
+    ) as _get_client_secret_credential_obj:
+        _get_client_secret_credential_obj.return_value = {}
+
+        mock_azure.sdk.secrets.SecretClient.return_value.get_secret.return_value = (
+            "my secret"
+        )
+
+        mock_azure.sdk.secrets.SecretClient.return_value.get_secret.side_effect = [
+            mock_azure.sdk.azure_exceptions.HttpResponseError,
+        ]
+
+        with pytest.raises(SecretException):
+            mock_azure.get_secret("secret key") == "my secret"
+
+
 def test_set_secret(mock_azure: AzureCloudProvider):
     with patch.object(
         AzureCloudProvider,
@@ -1136,18 +1290,43 @@ def test_set_secret(mock_azure: AzureCloudProvider):
         mock_azure.sdk.secrets.SecretClient.return_value.set_secret.return_value = (
             "my secret"
         )
-
         assert mock_azure.set_secret("secret key", "secret_value") == "my secret"
 
 
+def test_set_secret_secret_exception(mock_azure: AzureCloudProvider):
+    with patch.object(
+        AzureCloudProvider,
+        "_get_client_secret_credential_obj",
+        wraps=mock_azure._get_client_secret_credential_obj,
+    ) as _get_client_secret_credential_obj:
+        _get_client_secret_credential_obj.return_value = {}
+
+        mock_azure.sdk.secrets.SecretClient.return_value.set_secret.return_value = (
+            "my secret"
+        )
+
+        mock_azure.sdk.secrets.SecretClient.return_value.set_secret.side_effect = [
+            mock_azure.sdk.azure_exceptions.HttpResponseError,
+        ]
+
+        with pytest.raises(SecretException):
+            mock_azure.set_secret("secret key", "secret_value")
+
+
 def test_create_active_directory_user(mock_azure: AzureCloudProvider):
     mock_result = Mock()
     mock_result.ok = True
     mock_result.json.return_value = {"id": "id"}
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.post.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
 
@@ -1162,7 +1341,7 @@ def test_create_active_directory_user(mock_azure: AzureCloudProvider):
         mock_azure._create_active_directory_user("token", payload)
     with pytest.raises(ConnectionException):
         mock_azure._create_active_directory_user("token", payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure._create_active_directory_user("token", payload)
 
     result = mock_azure._create_active_directory_user("token", payload)
@@ -1173,10 +1352,16 @@ def test_create_active_directory_user(mock_azure: AzureCloudProvider):
 def test_update_active_directory_user_email(mock_azure: AzureCloudProvider):
     mock_result = Mock()
     mock_result.ok = True
+    mock_http_error_resp = mock_requests_response(
+        status=500,
+        raise_for_status=mock_azure.sdk.requests.exceptions.HTTPError(
+            "500 Server Error"
+        ),
+    )
     mock_azure.sdk.requests.patch.side_effect = [
         mock_azure.sdk.requests.exceptions.ConnectionError,
         mock_azure.sdk.requests.exceptions.Timeout,
-        mock_azure.sdk.requests.exceptions.HTTPError,
+        mock_http_error_resp,
         mock_result,
     ]
     payload = UserCSPPayload(
@@ -1190,7 +1375,7 @@ def test_update_active_directory_user_email(mock_azure: AzureCloudProvider):
         mock_azure._update_active_directory_user_email("token", uuid4().hex, payload)
     with pytest.raises(ConnectionException):
         mock_azure._update_active_directory_user_email("token", uuid4().hex, payload)
-    with pytest.raises(UnknownServerException):
+    with pytest.raises(UnknownServerException, match=r".*500 Server Error.*"):
         mock_azure._update_active_directory_user_email("token", uuid4().hex, payload)
 
     result = mock_azure._update_active_directory_user_email(

tests/domain/test_portfolio_state_machine.py

@@ -143,11 +143,9 @@ def test_fail_stage(portfolio):
     sm.state = FSMStates.TENANT_IN_PROGRESS
     sm.fail_stage("tenant")
     assert sm.state == FSMStates.TENANT_FAILED
-    # import ipdb;ipdb.set_trace()
 
 
 def test_stage_state_to_stage_name():
-    # sm = PortfolioStateMachineFactory.create(portfolio=portfolio)
     stage = _stage_state_to_stage_name(
         FSMStates.TENANT_IN_PROGRESS, StageStates.IN_PROGRESS
     )

tests/mock_azure.py

@@ -69,7 +69,7 @@ def mock_policy():
 def mock_azure_exceptions():
     from azure.core import exceptions
 
-    return Mock(spec=exceptions)
+    return exceptions
 
 
 def mock_adal():