diff --git a/requirements.txt b/requirements.txt
index d85cfbb..36818e4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ hydra-submitit-launcher==1.2.0
 torch-scatter==2.0.8
 pytorch-lightning==1.6.4
 fairscale==0.4.6
-transformers==3.4.0
+transformers==5.0.0rc3
 sentence_splitter>=1.4
 pyyaml
 tqdm