diff --git a/cmake/external/flashattn.cmake b/cmake/external/flashattn.cmake
index 6aec251be0215..7dfcec2c166fc 100644
--- a/cmake/external/flashattn.cmake
+++ b/cmake/external/flashattn.cmake
@@ -184,6 +184,9 @@ else()
   set(CACHE_TAR_DIR "${FA_BUILD_DIR}/flashattn_libs_${FLASHATTN_TAG}")
 
   set(SKIP_BUILD_FA OFF)
+  if(WITH_ARM)
+    set(WITH_FA_BUILD_WITH_CACHE OFF)
+  endif()
   if(WITH_FA_BUILD_WITH_CACHE)
 
     message(STATUS "Downloading ${TAR_FILE_URL} to ${CACHE_TAR_PATH}")
diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
index 290d1fd198e3a..70ff374b35ee8 100755
--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -349,7 +349,6 @@ include(external/glog) # download, build, install glog
 
 ########################### include third_party according to flags ###############################
 if(WITH_GPU
-   AND NOT WITH_ARM
    AND NOT WIN32
    AND NOT APPLE)
   if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.0)
@@ -601,7 +600,6 @@ if(WITH_ROCM)
 endif()
 
 if(WITH_GPU
-   AND NOT WITH_ARM
    AND NOT WIN32
    AND NOT APPLE)
   if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 12.3