diff --git a/crates/witness/src/lib.rs b/crates/witness/src/lib.rs
index 73f8a96..b5f82b4 100644
--- a/crates/witness/src/lib.rs
+++ b/crates/witness/src/lib.rs
@@ -310,10 +310,13 @@ impl<T: Sized + Sync + Clone + Send + Copy + Default + FieldAlgebra> RowMajorMat
     }
 
     pub fn padding_by_strategy(&mut self) {
-        self.invalidate_device_backing();
         let num_rotation = Self::num_rotation(self.log2_num_rotation);
         let start_index = self.num_instances() * num_rotation * self.n_col();
 
+        if matches!(self.padding_strategy, InstancePaddingStrategy::Custom(_)) {
+            self.invalidate_device_backing();
+        }
+
         match &self.padding_strategy {
             InstancePaddingStrategy::Default => (),
             InstancePaddingStrategy::Custom(fun) => {