diff --git a/pyproject.toml b/pyproject.toml
index 5f63b6a..fd6c463 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,6 +15,7 @@ dependencies = [
     "torch==2.0.1",
     "transformers==4.38.2",
     "timm>=0.9.16",
+    "xformers>=0.0.21",
     "accelerate",
     "sentencepiece",
     "attrdict",
diff --git a/requirements.txt b/requirements.txt
index c7acd26..b89b147 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
 torch==2.0.1
 transformers==4.38.2
+xformers>=0.0.21
 timm>=0.9.16
 accelerate
 sentencepiece