diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 8600c927c3f5b3..5d5059d800c0dd 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -171,6 +171,9 @@ else()
   set(CACHE_TAR_DIR "${FA_BUILD_DIR}/flashattn_libs_${FLASHATTN_TAG}")
 
   set(SKIP_BUILD_FA OFF)
+  if(WITH_ARM)
+    set(WITH_FA_BUILD_WITH_CACHE OFF)
+  endif()
   if(WITH_FA_BUILD_WITH_CACHE)
 
     message(STATUS "Downloading ${TAR_FILE_URL} to ${CACHE_TAR_PATH}")
diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
index 290d1fd198e3ac..5663a4dcd56ecb 100755
--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -349,7 +349,6 @@ include(external/glog) # download, build, install glog
 
 ########################### include third_party according to flags ###############################
 if(WITH_GPU
-   AND NOT WITH_ARM
    AND NOT WIN32
    AND NOT APPLE)
   if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.0)
@@ -601,13 +600,12 @@ if(WITH_ROCM)
 endif()
 
 if(WITH_GPU
-   AND NOT WITH_ARM
    AND NOT WIN32
    AND NOT APPLE)
   if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 12.3
      AND ${CMAKE_CUDA_COMPILER_VERSION} LESS_EQUAL 13.0)
     foreach(arch ${NVCC_ARCH_BIN})
-      if(${arch} GREATER_EQUAL 90)
+      if(${arch} EQUAL 90)
         set(WITH_FLASHATTN_V3 ON)
         break()
       endif()