diff --git a/TTS/utils/synthesizer.py b/TTS/utils/synthesizer.py
index d1d978d8..27628225 100644
--- a/TTS/utils/synthesizer.py
+++ b/TTS/utils/synthesizer.py
@@ -168,14 +168,14 @@ class Synthesizer(object):
         if self._is_use_speaker_embedding():
             if self.tts_speakers_file:
                 speaker_manager = SpeakerManager(speaker_id_file_path=self.tts_speakers_file)
-            if speakers_file:
+            elif speakers_file:
                 speaker_manager = SpeakerManager(speaker_id_file_path=speakers_file)
 
         if self._is_use_d_vector_file():
             d_vector_file = get_from_config_or_model_args_with_default(self.tts_config, "d_vector_file", None)
             if self.tts_speakers_file:
                 speaker_manager = SpeakerManager(d_vectors_file_path=self.tts_speakers_file)
-            if d_vector_file:
+            elif d_vector_file:
                 speaker_manager = SpeakerManager(d_vectors_file_path=d_vector_file)
         return speaker_manager