Re-enable Native Auth SSPR tests with Mail.tm, Fixes AB#3729032

azure-pipelines/pull-request-validation/pr-msal.yml

@@ -75,6 +75,59 @@ stages:
           tasks: msal:jacocoTestReport  -PcodeCoverageEnabled=true -ProbolectricSdkVersion=${{variables.robolectricSdkVersion}} -PmockApiUrl=$(MOCK_API_URL) -PnativeAuthConfigString=$(NATIVE_AUTH_CONFIG_STRING)
           javaHomeSelection: $(BuildParameters.javaHomeSelection)
           jdkVersion: 1.17
+      - bash: |
+          set -euo pipefail
+          SKIP_LABEL="skip-native-auth-e2e-tests"
+          # AGENT_TEMPDIRECTORY is a Windows path on the windows-2022 pool; normalise the
+          # separators so curl/grep never see a backslash path under Git Bash/MSYS.
+          TEMP_DIR="${AGENT_TEMPDIRECTORY:-.}"
+          LABELS_FILE="${TEMP_DIR//\\//}/native-auth-pr-labels.json"
+
+          if [ -z "${PR_NUMBER:-}" ]; then
+            echo "No PR number available (not a PR build). Skipping Native Auth E2E tests."
+            echo "##vso[task.setvariable variable=SKIP_NATIVE_AUTH_E2E]true"
+            exit 0
+          fi
+
+          LABELS_URL="https://api.github.com/repos/${REPO}/issues/${PR_NUMBER}/labels"
+          echo "Fetching PR labels from ${LABELS_URL}"
+          # Capture the body and the HTTP status separately, so that a failed API call is
+          # distinguishable from a successful call that simply returned no matching label.
+          HTTP_STATUS=$(curl -sS -o "$LABELS_FILE" -w "%{http_code}" "$LABELS_URL" || echo "000")
+
+          if [ "$HTTP_STATUS" != "200" ]; then
+            echo "GitHub labels API returned HTTP ${HTTP_STATUS}. Skipping Native Auth E2E tests as a preventive measure (fail-safe to skipping, never to blocking)."
+            echo "##vso[task.setvariable variable=SKIP_NATIVE_AUTH_E2E]true"
+            exit 0
+          fi
+
+          # jq is not reliably available on the windows-2022 pool, so match on the raw JSON.
+          if grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${SKIP_LABEL}\"" "$LABELS_FILE"; then
+            echo "Label '${SKIP_LABEL}' is present on PR #${PR_NUMBER}. Skipping Native Auth E2E tests."
+            echo "##vso[task.setvariable variable=SKIP_NATIVE_AUTH_E2E]true"
+          else
+            echo "Label '${SKIP_LABEL}' is not present on PR #${PR_NUMBER}. Running Native Auth E2E tests."
+            echo "##vso[task.setvariable variable=SKIP_NATIVE_AUTH_E2E]false"
+          fi
+        displayName: Check for skip-native-auth-e2e-tests label
+        env:
+          PR_NUMBER: $(System.PullRequest.PullRequestNumber)
+          REPO: $(Build.Repository.Name)
+      - task: Gradle@2
+        displayName: Run Native Auth E2E Tests
+        # Mirrors iOS (microsoft-authentication-library-for-objc) pr-validation.yml, which guards its
+        # native auth E2E jobs with the same opt-out label and a 30 minute timeout.
+        condition: and(succeeded(), ne(variables['SKIP_NATIVE_AUTH_E2E'], 'true'))
+        timeoutInMinutes: 30
+        inputs:
+          tasks: msal:testLocalDebugUnitTest -Plabtest -ProbolectricSdkVersion=${{variables.robolectricSdkVersion}} -PnativeAuthConfigString=$(NATIVE_AUTH_CONFIG_STRING) --tests com.microsoft.identity.client.e2e.tests.network.nativeauth.*
+          javaHomeSelection: $(BuildParameters.javaHomeSelection)
+          jdkVersion: 1.17
+          publishJUnitResults: true
+          testResultsFiles: '**/build/test-results/testLocalDebugUnitTest/TEST-*.xml'
+          testRunTitle: Native Auth E2E Tests (PR Branch)
+        env:
+          EMAIL_PROVIDER_PASSWORD: $(EMAIL_PROVIDER_PASSWORD)
       - publish: $(Build.SourcesDirectory)/msal/build/reports/jacoco/jacocoTestReport/jacocoTestReport.xml
         artifact: jacocoReport
         displayName: 'Publish JaCoCo Report Artifact (PR Branch)'

msal/src/test/java/com/microsoft/identity/client/e2e/tests/network/nativeauth/NativeAuthPublicClientApplicationAbstractTest.kt

@@ -30,11 +30,13 @@ import com.google.gson.reflect.TypeToken
 import com.microsoft.identity.client.PublicClientApplication
 import com.microsoft.identity.client.e2e.shadows.ShadowAndroidSdkStorageEncryptionManager
 import com.microsoft.identity.client.e2e.tests.IPublicClientApplicationTest
+import com.microsoft.identity.client.e2e.utils.NativeAuthEmailOTPErrorClassifier
 import com.microsoft.identity.client.exception.MsalException
 import com.microsoft.identity.common.internal.controllers.CommandDispatcherHelper
 import com.microsoft.identity.common.java.nativeauth.BuildValues
 import com.microsoft.identity.internal.testutils.TestUtils
 import com.microsoft.identity.internal.testutils.nativeauth.ConfigType
+import com.microsoft.identity.internal.testutils.nativeauth.api.TemporaryEmailService
 import com.microsoft.identity.internal.testutils.nativeauth.api.models.NativeAuthTestConfig
 import com.microsoft.identity.labapi.utilities.BuildConfig
 import com.microsoft.identity.labapi.utilities.authentication.LabApiAuthenticationClient
@@ -69,13 +71,31 @@ abstract class NativeAuthPublicClientApplicationAbstractTest : IPublicClientAppl
         const val INVALID_PASSWORD = "password"
         const val INCORRECT_CODE = "00000000"
 
+        /**
+         * Retry budget for OTP-throttled (AADSTS701014) auth flows.
+         *
+         * Each retry re-runs the whole flow (new temporary inbox + signup + OTP request), so
+         * attempts are expensive in wall-clock time on top of the backoff itself. With 3 retries
+         * and the delay capped at [MAX_RETRY_DELAY_MILLIS], a persistently throttled test sleeps at
+         * most 5s + 10s + 20s = 35s across 4 attempts.
+         *
+         * iOS keeps 5 attempts only because XCTest's plan-level `maximumTestRepetitions` /
+         * `retryOnFailure` costs no extra wall-clock on success, and because its throttle path
+         * raises `XCTSkip` (via `skipIfEmailOTPThrottled`) instead of retrying into the throttle.
+         * This hand-rolled loop has neither property, hence the tighter budget.
+         */
+        const val MAX_THROTTLE_RETRIES = 3
+        private const val RETRY_BASE_DELAY_MILLIS = 5_000L
+        private const val MAX_RETRY_DELAY_MILLIS = 20_000L
+
         private val labApiAuthenticationClient: LabApiAuthenticationClient =
             LabApiAuthenticationClient(BuildConfig.LAB_CLIENT_SECRET)
         val labClient: LabClient = LabClient(labApiAuthenticationClient)
     }
 
     private lateinit var context: Context
     private lateinit var activity: Activity
+    protected val tempEmailApi = TemporaryEmailService()
 
     // Remove default Coroutine test timeout of 10 seconds.
     private val testDispatcher = StandardTestDispatcher()
@@ -150,29 +170,38 @@ abstract class NativeAuthPublicClientApplicationAbstractTest : IPublicClientAppl
     }
 
     fun <T> retryOperation(
-        maxRetries: Int = 5,
+        maxRetries: Int = MAX_THROTTLE_RETRIES,
         authFlow: () -> T
     ) {
         var retryCount = 0
-        var shouldRetry = true
 
-        while (shouldRetry) {
+        while (true) {
             try {
                 authFlow()
-                shouldRetry = false // authFlow() has succeeded, so we don't need to retry.
-            } catch (e: Exception) {
-                //1secmail occasionally has a delay for emails to arrive / return from the API, or throws an internal server error, which causes tests to fail
-                //In this case, retry the test
-                if (retryCount >= maxRetries) {
-                    Assert.fail(e.message)
-                    shouldRetry = false
-                } else {
-                    retryCount++
+                return
+            } catch (e: AssertionError) {
+                // Classification (typed errorCodes first, assertion message as unconditional
+                // fallback) lives in the classifier so it is unit-testable without standing up
+                // this Robolectric-backed abstract test.
+                if (!NativeAuthEmailOTPErrorClassifier.isThrottleError(e)) {
+                    throw e
                 }
+                retryOrFail(e, retryCount++, maxRetries)
             }
         }
     }
 
+    private fun retryOrFail(error: Throwable, retryCount: Int, maxRetries: Int) {
+        if (retryCount >= maxRetries) {
+            throw AssertionError(error.message).apply { initCause(error) }
+        }
+
+        // Avoid repeatedly requesting OTPs while the Native Auth test tenant is throttling them.
+        Thread.sleep(
+            minOf(RETRY_BASE_DELAY_MILLIS * (1L shl retryCount), MAX_RETRY_DELAY_MILLIS)
+        )
+    }
+
     private fun readConfigFile(filePath: String): String {
         val sb = StringBuilder()
         try {

msal/src/test/java/com/microsoft/identity/client/e2e/tests/network/nativeauth/NativeAuthPublicClientApplicationAbstractTestTest.kt

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation.
+// All rights reserved.
+//
+// This code is licensed under the MIT License.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files(the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+package com.microsoft.identity.client.e2e.tests.network.nativeauth
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertSame
+import org.junit.Assert.assertThrows
+import org.junit.Test
+
+class NativeAuthPublicClientApplicationAbstractTestTest :
+    NativeAuthPublicClientApplicationAbstractTest() {
+
+    @Test
+    fun retryOperationDoesNotRetryNonThrottleExceptions() {
+        val expected = IllegalStateException("Non-throttle failure")
+        var attempts = 0
+
+        val actual = assertThrows(IllegalStateException::class.java) {
+            retryOperation(maxRetries = 1) {
+                attempts++
+                throw expected
+            }
+        }
+
+        assertSame(expected, actual)
+        assertEquals(1, attempts)
+    }
+}

msal/src/test/java/com/microsoft/identity/client/e2e/tests/network/nativeauth/SSPRTest.kt

@@ -25,7 +25,6 @@ package com.microsoft.identity.client.e2e.tests.network.nativeauth
 
 import com.microsoft.identity.client.e2e.utils.assertResult
 import com.microsoft.identity.internal.testutils.nativeauth.ConfigType
-import com.microsoft.identity.internal.testutils.nativeauth.api.TemporaryEmailService
 import com.microsoft.identity.internal.testutils.nativeauth.api.models.NativeAuthTestConfig
 import com.microsoft.identity.nativeauth.INativeAuthPublicClientApplication
 import com.microsoft.identity.nativeauth.parameters.NativeAuthResetPasswordParameters
@@ -42,20 +41,18 @@ import org.junit.Test
 
 class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
 
-    private val tempEmailApi = TemporaryEmailService()
-
     lateinit var application: INativeAuthPublicClientApplication
     lateinit var config: NativeAuthTestConfig.Config
 
     private val defaultConfigType = ConfigType.SSPR
     private val defaultChallengeTypes = listOf("password", "oob")
     private val defaultCapabilities = listOf("mfa_required", "registration_required")
+    private val invalidPasswordTooShort = "a"
 
     /**
      * Verify email with email OTP first and then reset password.
      * (hero scenario 8 & 17, use case 3.1.1)
      */
-    @Ignore("Retrieving OTP code failure")
     @Test
     fun testSSPRSuccess() {
         config = getConfig(defaultConfigType)
@@ -67,6 +64,7 @@ class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
             runBlocking {
                 val user = config.email
                 val param = NativeAuthResetPasswordParameters(username = user)
+                tempEmailApi.markCheckpoint(user)
                 result = application.resetPassword(param)
                 assertResult<ResetPasswordStartResult.CodeRequired>(result)
 
@@ -85,7 +83,6 @@ class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
      * New password being set doesn’t meet password complexity requirements set on portal
      * (use case 3.1.3)
      */
-    @Ignore("Retrieving OTP code failure.")
     @Test
     fun testErrorInvalidPasswordFormat() {
         config = getConfig(defaultConfigType)
@@ -97,17 +94,22 @@ class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
             runBlocking {
                 val user = config.email
                 val param = NativeAuthResetPasswordParameters(username = user)
+                tempEmailApi.markCheckpoint(user)
                 result = application.resetPassword(param)
                 assertResult<ResetPasswordStartResult.CodeRequired>(result)
 
                 val otp = tempEmailApi.retrieveCodeFromInbox(user)
                 val submitCodeResult = (result as ResetPasswordStartResult.CodeRequired).nextState.submitCode(otp)
                 assertResult<ResetPasswordSubmitCodeResult.PasswordRequired>(submitCodeResult)
 
-                val password = INVALID_PASSWORD
+                val password = invalidPasswordTooShort
                 val submitPasswordResult = (submitCodeResult as ResetPasswordSubmitCodeResult.PasswordRequired).nextState.submitPassword(password.toCharArray())
                 Assert.assertTrue(submitPasswordResult is ResetPasswordSubmitPasswordError)
-                Assert.assertTrue((submitPasswordResult as ResetPasswordSubmitPasswordError).isInvalidPassword())
+                val error = submitPasswordResult as ResetPasswordSubmitPasswordError
+                Assert.assertTrue(
+                    "Expected invalid password error, but received errorType=${error.errorType}, error=${error.error}, subError=${error.subError}",
+                    error.isInvalidPassword()
+                )
             }
         }
     }
@@ -116,7 +118,6 @@ class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
      * Resend Code.
      * (use case 3.1.4)
      */
-    @Ignore("Retrieving OTP code failure.")
     @Test
     fun testResendCode() {
         config = getConfig(defaultConfigType)
@@ -128,11 +129,13 @@ class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
             runBlocking {
                 val user = config.email
                 val param = NativeAuthResetPasswordParameters(username = user)
+                tempEmailApi.markCheckpoint(user)
                 result = application.resetPassword(param)
                 assertResult<ResetPasswordStartResult.CodeRequired>(result)
 
                 val otp1 = tempEmailApi.retrieveCodeFromInbox(user)
                 val codeRequiredState = (result as ResetPasswordStartResult.CodeRequired).nextState
+                tempEmailApi.markCheckpoint(user)
                 val resendCodeResult = codeRequiredState.resendCode()
                 assertResult<ResetPasswordResendCodeResult.Success>(resendCodeResult)
 
@@ -159,7 +162,7 @@ class SSPRTest : NativeAuthPublicClientApplicationAbstractTest() {
         application = setupPCA(config, defaultChallengeTypes, defaultCapabilities)
 
         runBlocking {
-            val username = tempEmailApi.generateRandomEmailAddressLocally()
+            val username = tempEmailApi.generateRandomUnregisteredEmailAddress()
             val param = NativeAuthResetPasswordParameters(username = username)
             val result = application.resetPassword(param)
             Assert.assertTrue(result is ResetPasswordError)

msal/src/test/java/com/microsoft/identity/client/e2e/tests/network/nativeauth/SignInEmailOTPTest.kt

@@ -25,7 +25,6 @@ package com.microsoft.identity.client.e2e.tests.network.nativeauth
 
 import com.microsoft.identity.client.e2e.utils.assertResult
 import com.microsoft.identity.internal.testutils.nativeauth.ConfigType
-import com.microsoft.identity.internal.testutils.nativeauth.api.TemporaryEmailService
 import com.microsoft.identity.internal.testutils.nativeauth.api.models.NativeAuthTestConfig
 import com.microsoft.identity.nativeauth.INativeAuthPublicClientApplication
 import com.microsoft.identity.nativeauth.parameters.NativeAuthSignInParameters
@@ -40,8 +39,6 @@ import org.junit.Test
 
 class SignInEmailOTPTest : NativeAuthPublicClientApplicationAbstractTest() {
 
-    private val tempEmailApi = TemporaryEmailService()
-
     lateinit var application: INativeAuthPublicClientApplication
     lateinit var config: NativeAuthTestConfig.Config
 
@@ -63,6 +60,7 @@ class SignInEmailOTPTest : NativeAuthPublicClientApplicationAbstractTest() {
             runBlocking {
                 val user = config.email
                 val param = NativeAuthSignInParameters(username = user)
+                tempEmailApi.markCheckpoint(user)
                 val signInResult = application.signIn(param)
                 assertResult<SignInResult.CodeRequired>(signInResult)
                 val otp = tempEmailApi.retrieveCodeFromInbox(user)
@@ -83,7 +81,7 @@ class SignInEmailOTPTest : NativeAuthPublicClientApplicationAbstractTest() {
 
         retryOperation {
             runBlocking {
-                val username = tempEmailApi.generateRandomEmailAddressLocally()
+                val username = tempEmailApi.generateRandomUnregisteredEmailAddress()
                 val param = NativeAuthSignInParameters(username = username)
                 val signInResult = application.signIn(param)
                 Assert.assertTrue(signInResult is SignInError)
@@ -146,11 +144,13 @@ class SignInEmailOTPTest : NativeAuthPublicClientApplicationAbstractTest() {
             runBlocking {
                 val user = config.email
                 val param = NativeAuthSignInParameters(username = user)
+                tempEmailApi.markCheckpoint(user)
                 val signInResult = application.signIn(param)
                 assertResult<SignInResult.CodeRequired>(signInResult)
 
                 val otp1 = tempEmailApi.retrieveCodeFromInbox(user)
                 val codeRequiredState = (signInResult as SignInResult.CodeRequired).nextState
+                tempEmailApi.markCheckpoint(user)
                 val resendCodeResult = codeRequiredState.resendCode()
                 assertResult<SignInResendCodeResult.Success>(resendCodeResult)