torch.cuda.is_available() is redundant as apex handles that internally (#9350)

examples/language-modeling/README.md

@@ -30,15 +30,6 @@ need extra processing on your datasets.
 The following examples, will run on a datasets hosted on our [hub](https://huggingface.co/datasets) or with your own
 text files for training and validation. We give examples of both below.
 
-### Datasets
-
-The main dataset used in the examples below is:
-
-```bash
-wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip
-unzip wikitext-2-raw-v1.zip
-```
-
 ### GPT-2/GPT and causal language modeling
 
 The following example fine-tunes GPT-2 on WikiText-2. We're using the raw WikiText-2 (no tokens were replaced before

src/transformers/models/bart/modeling_bart.py

@@ -110,13 +110,12 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int]
 
 
 def BartLayerNorm(normalized_shape: torch.Size, eps: float = 1e-5, elementwise_affine: bool = True):
-    if torch.cuda.is_available():
-        try:
-            from apex.normalization import FusedLayerNorm
+    try:
+        from apex.normalization import FusedLayerNorm
 
-            return FusedLayerNorm(normalized_shape, eps, elementwise_affine)
-        except ImportError:
-            pass
+        return FusedLayerNorm(normalized_shape, eps, elementwise_affine)
+    except ImportError:
+        pass
     return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine)
 
 

src/transformers/models/fsmt/modeling_fsmt.py

@@ -265,14 +265,12 @@ FSMT_INPUTS_DOCSTRING = r"""
 
 
 have_fused_layer_norm = False
-if torch.cuda.is_available():
-    try:
-        from apex.normalization import FusedLayerNorm
-
-        have_fused_layer_norm = True
-    except ImportError:
-        pass
+try:
+    from apex.normalization import FusedLayerNorm
 
+    have_fused_layer_norm = True
+except ImportError:
+    pass
 LayerNorm = FusedLayerNorm if have_fused_layer_norm else torch.nn.LayerNorm
 
 

src/transformers/models/prophetnet/modeling_prophetnet.py

@@ -511,13 +511,12 @@ class ProphetNetDecoderLMOutput(ModelOutput):
 
 
 def ProphetNetLayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True):
-    if torch.cuda.is_available():
-        try:
-            from apex.normalization import FusedLayerNorm
+    try:
+        from apex.normalization import FusedLayerNorm
 
-            return FusedLayerNorm(normalized_shape, eps, elementwise_affine)
-        except ImportError:
-            pass
+        return FusedLayerNorm(normalized_shape, eps, elementwise_affine)
+    except ImportError:
+        pass
     return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine)