removed unnecessary files and updated app_py

README1.md

@@ -1,15 +0,0 @@
-# FastPitchMulti
-Experimental multi-lingual FastPitch
-
-What's done:
-- [x] Conditioning on language and speaker labels
-- [x] Dataset and preprocessing of Sámi data
-- [x] Combined character set for the Sámi languages
-- [x] Train a model on Sámi languages
-- [x] Selecting Estonian data
-- [x] Processing Estonian data
-- [ ] Train a model on Sámi x 3, Finnish, Estonian
-
-Ideas:
-- Move the language embedding to the very beginning of the encoder
-  

app.py

@@ -11,7 +11,7 @@ speakers={"aj0": 0,
           "aj1": 1,
           "am": 2,
           "bi": 3,
-          "kd": 4,
+          #"kd": 4,
           "ln": 5,
           "lo": 6,
           "ms": 7,

common/text/symbols_2.py

@@ -1,64 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='english_basic'):
-    if symbol_set == 'english_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        _letters = 'abcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćž'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        # _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    elif symbol_set == 'sme_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëńñöøćčžđšŧ' #also north sámi letters...
-        # _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='english_basic'):
-    if symbol_set in {'english_basic', 'english_basic_lowercase', 'smj_expanded', 'sme_expanded'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/text/symbols_BACKUP_MARCH_2024.py

@@ -1,64 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='english_basic'):
-    if symbol_set == 'english_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        _letters = 'abcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćž'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        # _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTŦUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz' ########################## Ŧ ########################
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    elif symbol_set == 'sme_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëńñöøćčžđšŧ' #also north sámi letters...
-        # _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCČDĐEFGHIJKLMNŊOØÖPQRSŠTŦUVWXYZŽaáæåäbcčdđefghijklmnŋoøöpqrsštŧuvwxyzž'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='english_basic'):
-    if symbol_set in {'english_basic', 'english_basic_lowercase', 'smj_expanded', 'sme_expanded'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/text/symbols_ORIGINAL.py

@@ -1,65 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='english_basic'):
-    if symbol_set == 'english_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        _letters = 'abcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćž'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        # _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTŦUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    elif symbol_set == 'sme_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëńñöøćčžđšŧ' #also north sámi letters...
-        # _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCČDĐEFGHIJKLMNŊOØÖPQRSŠTŦUVWXYZŽaáæåäbcčdđefghijklmnŋoøöpqrsštŧuvwxyzž'
-        # _letters = 'AÁÆÅÄBCDĐEFGHIJKLMNŊOØÖPQRSŠTŦUVWXYZŽaáæåäbcčdđefghijklmnŋoøöpqrsštŧuvwxyzž'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='english_basic'):
-    if symbol_set in {'english_basic', 'english_basic_lowercase', 'smj_expanded', 'sme_expanded'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/text/symbols_backup.py

@@ -1,54 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='english_basic'):
-    if symbol_set == 'english_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        _letters = 'abcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćž'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        # _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='english_basic'):
-    if symbol_set in {'english_basic', 'english_basic_lowercase', 'smj_expanded'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/text/symbols_sme.py

@@ -1,64 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='english_basic'):
-    if symbol_set == 'english_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        _letters = 'abcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćž'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        # _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    elif symbol_set == 'sme_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëńñöøćčžđšŧ' #also north sámi letters...
-        # _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='english_basic'):
-    if symbol_set in {'english_basic', 'english_basic_lowercase', 'smj_expanded', 'sme_expanded'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/text/symbols_sme_1.py

@@ -1,64 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='english_basic'):
-    if symbol_set == 'english_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        _letters = 'abcdefghijklmnopqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'english_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćž'
-        _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        # _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    elif symbol_set == 'sme_expanded':
-        _punctuation = '!\'",.:;?- '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëńñöøćčžđšŧ' #also north sámi letters...
-        # _accented = 'áçéêëñöø' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDĐEFGHIJKLMNŊOØÖPQRSŠTUVWXYZŽaáæåäbcdđefghijklmnŋoøöpqrsštŧuvwxyzž'
-        # symbols = list(_punctuation + _math + _special + _accented + _letters) #+ _arpabet
-        symbols = list(_punctuation + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='english_basic'):
-    if symbol_set in {'english_basic', 'english_basic_lowercase', 'smj_expanded', 'sme_expanded'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/text/symbols_smj.py

@@ -1,48 +0,0 @@
-""" from https://github.com/keithito/tacotron """
-
-'''
-Defines the set of symbols used in text input to the model.
-
-The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
-from .cmudict import valid_symbols
-
-
-# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
-_arpabet = ['@' + s for s in valid_symbols]
-
-
-def get_symbols(symbol_set='smj_basic'):
-    if symbol_set == 'smj_basic':
-        _pad = '_'
-        _punctuation = '!\'(),.:;? '
-        _special = '-'
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        # LETTERS = 'AaÁáBbCcDdEeFfGgHhIiJjKkLlMmNnŊŋOoPpRrSsTtUuVvZzŃńÑñÆæØøÅåÄäÖö'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'smj_basic_lowercase':
-        _pad = '_'
-        _punctuation = '!\'"(),.:;? '
-        _special = '-'
-        # _letters = 'abcdefghijklmnopqrstuvwxyz'
-        _letters = 'aáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        symbols = list(_pad + _special + _punctuation + _letters) + _arpabet
-    elif symbol_set == 'smj_expanded':
-        _punctuation = '!\'",.:;? '
-        _math = '#%&*+-/[]()'
-        _special = '_@©°½—₩€$'
-        _accented = 'áçéêëñöøćžđšŧ' #also north sámi letters...
-        # _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-        _letters = 'AÁÆÅÄBCDEFGHIJKLMNŊŃÑOØÖPQRSTUVWXYZaáæåäbcdefghijklmnŋńñoøöpqrstuvwxyz'
-        symbols = list(_punctuation + _math + _special + _accented + _letters) + _arpabet
-    else:
-        raise Exception("{} symbol set does not exist".format(symbol_set))
-
-    return symbols
-
-
-def get_pad_idx(symbol_set='smj_basic'):
-    if symbol_set in {'smj_basic', 'smj_basic_lowercase'}:
-        return 0
-    else:
-        raise Exception("{} symbol set not used yet".format(symbol_set))

common/utils_hfg.py

@@ -1,14 +0,0 @@
-
-##############################################################################
-# Foreing utils.py from HiFi-GAN
-##############################################################################
-
-
-def init_weights(m, mean=0.0, std=0.01):
-    classname = m.__class__.__name__
-    if classname.find("Conv") != -1:
-        m.weight.data.normal_(mean, std)
-
-
-def get_padding(kernel_size, dilation=1):
-    return int((kernel_size*dilation - dilation)/2)

common/utils_ok.py

@@ -1,291 +0,0 @@
-# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#           http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#  MIT License
-#
-#  Copyright (c) 2020 Jungil Kong
-#
-#  Permission is hereby granted, free of charge, to any person obtaining a copy
-#  of this software and associated documentation files (the "Software"), to deal
-#  in the Software without restriction, including without limitation the rights
-#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-#  copies of the Software, and to permit persons to whom the Software is
-#  furnished to do so, subject to the following conditions:
-#
-#  The above copyright notice and this permission notice shall be included in all
-#  copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-#  SOFTWARE.
-
-# The following functions/classes were based on code from https://github.com/jik876/hifi-gan:
-# init_weights, get_padding, AttrDict
-
-import ctypes
-import glob
-import os
-import re
-import shutil
-import warnings
-from collections import defaultdict, OrderedDict
-from pathlib import Path
-from typing import Optional
-
-import librosa
-import numpy as np
-
-import torch
-import torch.distributed as dist
-from scipy.io.wavfile import read
-
-
-def mask_from_lens(lens, max_len: Optional[int] = None):
-    if max_len is None:
-        max_len = lens.max()
-    ids = torch.arange(0, max_len, device=lens.device, dtype=lens.dtype)
-    mask = torch.lt(ids, lens.unsqueeze(1))
-    return mask
-
-
-def load_wav(full_path, torch_tensor=False):
-    import soundfile  # flac
-    data, sampling_rate = soundfile.read(full_path, dtype='int16')
-    if torch_tensor:
-        return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-    else:
-        return data, sampling_rate
-
-
-def load_wav_to_torch(full_path, force_sampling_rate=None):
-    if force_sampling_rate is not None:
-        data, sampling_rate = librosa.load(full_path, sr=force_sampling_rate)
-    else:
-        sampling_rate, data = read(full_path)
-
-    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-
-
-def load_filepaths_and_text(dataset_path, fnames, has_speakers=False, split="|"):
-    def split_line(root, line):
-        parts = line.strip().split(split)
-        if has_speakers:
-            paths, non_paths = parts[:-2], parts[-2:]
-        else:
-            paths, non_paths = parts[:-1], parts[-1:]
-        return tuple(str(Path(root, p)) for p in paths) + tuple(non_paths)
-
-    fpaths_and_text = []
-    for fname in fnames:
-        with open(fname, encoding='utf-8') as f:
-            fpaths_and_text += [split_line(dataset_path, line) for line in f]
-    return fpaths_and_text
-
-
-def to_gpu(x):
-    x = x.contiguous()
-    return x.cuda(non_blocking=True) if torch.cuda.is_available() else x
-
-
-def l2_promote():
-    _libcudart = ctypes.CDLL('libcudart.so')
-    # Set device limit on the current device
-    # cudaLimitMaxL2FetchGranularity = 0x05
-    pValue = ctypes.cast((ctypes.c_int*1)(), ctypes.POINTER(ctypes.c_int))
-    _libcudart.cudaDeviceSetLimit(ctypes.c_int(0x05), ctypes.c_int(128))
-    _libcudart.cudaDeviceGetLimit(pValue, ctypes.c_int(0x05))
-    assert pValue.contents.value == 128
-
-
-def prepare_tmp(path):
-    if path is None:
-        return
-    p = Path(path)
-    if p.is_dir():
-        warnings.warn(f'{p} exists. Removing...')
-        shutil.rmtree(p, ignore_errors=True)
-    p.mkdir(parents=False, exist_ok=False)
-
-
-def print_once(*msg):
-    if not dist.is_initialized() or dist.get_rank() == 0:
-        print(*msg)
-
-
-def init_weights(m, mean=0.0, std=0.01):
-    classname = m.__class__.__name__
-    if classname.find("Conv") != -1:
-        m.weight.data.normal_(mean, std)
-
-
-def get_padding(kernel_size, dilation=1):
-    return int((kernel_size*dilation - dilation)/2)
-
-
-class AttrDict(dict):
-    def __init__(self, *args, **kwargs):
-        super(AttrDict, self).__init__(*args, **kwargs)
-        self.__dict__ = self
-
-
-class DefaultAttrDict(defaultdict):
-    def __init__(self, *args, **kwargs):
-        super(DefaultAttrDict, self).__init__(*args, **kwargs)
-        self.__dict__ = self
-
-    def __getattr__(self, item):
-        return self[item]
-
-
-class BenchmarkStats:
-    """ Tracks statistics used for benchmarking. """
-    def __init__(self):
-        self.num_frames = []
-        self.losses = []
-        self.mel_losses = []
-        self.took = []
-
-    def update(self, num_frames, losses, mel_losses, took):
-        self.num_frames.append(num_frames)
-        self.losses.append(losses)
-        self.mel_losses.append(mel_losses)
-        self.took.append(took)
-
-    def get(self, n_epochs):
-        frames_s = sum(self.num_frames[-n_epochs:]) / sum(self.took[-n_epochs:])
-        return {'frames/s': frames_s,
-                'loss': np.mean(self.losses[-n_epochs:]),
-                'mel_loss': np.mean(self.mel_losses[-n_epochs:]),
-                'took': np.mean(self.took[-n_epochs:]),
-                'benchmark_epochs_num': n_epochs}
-
-    def __len__(self):
-        return len(self.losses)
-
-
-class Checkpointer:
-
-    def __init__(self, save_dir, keep_milestones=[]):
-        self.save_dir = save_dir
-        self.keep_milestones = keep_milestones
-
-        find = lambda name: [
-            (int(re.search("_(\d+).pt", fn).group(1)), fn)
-            for fn in glob.glob(f"{save_dir}/{name}_checkpoint_*.pt")]
-
-        tracked = sorted(find("FastPitch"), key=lambda t: t[0])
-        self.tracked = OrderedDict(tracked)
-
-    def last_checkpoint(self, output):
-
-        def corrupted(fpath):
-            try:
-                torch.load(fpath, map_location="cpu")
-                return False
-            except:
-                warnings.warn(f"Cannot load {fpath}")
-                return True
-
-        saved = sorted(
-            glob.glob(f"{output}/FastPitch_checkpoint_*.pt"),
-            key=lambda f: int(re.search("_(\d+).pt", f).group(1)))
-
-        if len(saved) >= 1 and not corrupted(saved[-1]):
-            return saved[-1]
-        elif len(saved) >= 2:
-            return saved[-2]
-        else:
-            return None
-
-    def maybe_load(self, model, optimizer, scaler, train_state, args,
-                   ema_model=None):
-
-        assert args.checkpoint_path is None or args.resume is False, (
-            "Specify a single checkpoint source")
-
-        fpath = None
-        if args.checkpoint_path is not None:
-            fpath = args.checkpoint_path
-            self.tracked = OrderedDict()  # Do not track/delete prev ckpts
-        elif args.resume:
-            fpath = self.last_checkpoint(args.output)
-
-        if fpath is None:
-            return
-
-        print_once(f"Loading model and optimizer state from {fpath}")
-        ckpt = torch.load(fpath, map_location="cpu")
-        train_state["epoch"] = ckpt["epoch"] + 1
-        train_state["total_iter"] = ckpt["iteration"]
-
-        no_pref = lambda sd: {re.sub("^module.", "", k): v for k, v in sd.items()}
-        unwrap = lambda m: getattr(m, "module", m)
-
-        unwrap(model).load_state_dict(no_pref(ckpt["state_dict"]))
-
-        if ema_model is not None:
-            unwrap(ema_model).load_state_dict(no_pref(ckpt["ema_state_dict"]))
-
-        optimizer.load_state_dict(ckpt["optimizer"])
-
-        if "scaler" in ckpt:
-            scaler.load_state_dict(ckpt["scaler"])
-        else:
-            warnings.warn("AMP scaler state missing from the checkpoint.")
-
-    def maybe_save(self, args, model, ema_model, optimizer, scaler, epoch,
-                   total_iter, config):
-
-        intermediate = (args.epochs_per_checkpoint > 0
-                        and epoch % args.epochs_per_checkpoint == 0)
-        final = epoch == args.epochs
-
-        if not intermediate and not final and epoch not in self.keep_milestones:
-            return
-
-        rank = 0
-        if dist.is_initialized():
-            dist.barrier()
-            rank = dist.get_rank()
-
-        if rank != 0:
-            return
-
-        unwrap = lambda m: getattr(m, "module", m)
-        ckpt = {"epoch": epoch,
-                "iteration": total_iter,
-                "config": config,
-                "train_setup": args.__dict__,
-                "state_dict": unwrap(model).state_dict(),
-                "optimizer": optimizer.state_dict(),
-                "scaler": scaler.state_dict()}
-        if ema_model is not None:
-            ckpt["ema_state_dict"] = unwrap(ema_model).state_dict()
-
-        fpath = Path(args.output, f"FastPitch_checkpoint_{epoch}.pt")
-        print(f"Saving model and optimizer state at epoch {epoch} to {fpath}")
-        torch.save(ckpt, fpath)
-
-        # Remove old checkpoints; keep milestones and the last two
-        self.tracked[epoch] = fpath
-        for epoch in set(list(self.tracked)[:-2]) - set(self.keep_milestones):
-            try:
-                os.remove(self.tracked[epoch])
-            except:
-                pass
-            del self.tracked[epoch]

fastpitch/data_function (copy).py.txt

@@ -1,425 +0,0 @@
-# *****************************************************************************
-#  Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions are met:
-#      * Redistributions of source code must retain the above copyright
-#        notice, this list of conditions and the following disclaimer.
-#      * Redistributions in binary form must reproduce the above copyright
-#        notice, this list of conditions and the following disclaimer in the
-#        documentation and/or other materials provided with the distribution.
-#      * Neither the name of the NVIDIA CORPORATION nor the
-#        names of its contributors may be used to endorse or promote products
-#        derived from this software without specific prior written permission.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-#  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-#  DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
-#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-#  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-#  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# *****************************************************************************
-
-import functools
-import json
-import re
-from pathlib import Path
-
-import librosa
-import numpy as np
-import torch
-import torch.nn.functional as F
-from scipy import ndimage
-from scipy.stats import betabinom
-
-import common.layers as layers
-from common.text.text_processing import TextProcessing
-from common.utils import load_wav_to_torch, load_filepaths_and_text, to_gpu
-
-
-class BetaBinomialInterpolator:
-    """Interpolates alignment prior matrices to save computation.
-
-    Calculating beta-binomial priors is costly. Instead cache popular sizes
-    and use img interpolation to get priors faster.
-    """
-    def __init__(self, round_mel_len_to=100, round_text_len_to=20):
-        self.round_mel_len_to = round_mel_len_to
-        self.round_text_len_to = round_text_len_to
-        self.bank = functools.lru_cache(beta_binomial_prior_distribution)
-
-    def round(self, val, to):
-        return max(1, int(np.round((val + 1) / to))) * to
-
-    def __call__(self, w, h):
-        bw = self.round(w, to=self.round_mel_len_to)
-        bh = self.round(h, to=self.round_text_len_to)
-        ret = ndimage.zoom(self.bank(bw, bh).T, zoom=(w / bw, h / bh), order=1)
-        assert ret.shape[0] == w, ret.shape
-        assert ret.shape[1] == h, ret.shape
-        return ret
-
-
-def beta_binomial_prior_distribution(phoneme_count, mel_count, scaling=1.0):
-    P = phoneme_count
-    M = mel_count
-    x = np.arange(0, P)
-    mel_text_probs = []
-    for i in range(1, M+1):
-        a, b = scaling * i, scaling * (M + 1 - i)
-        rv = betabinom(P, a, b)
-        mel_i_prob = rv.pmf(x)
-        mel_text_probs.append(mel_i_prob)
-    return torch.tensor(np.array(mel_text_probs))
-
-
-def estimate_pitch(wav, mel_len, method='pyin', normalize_mean=None,
-                   normalize_std=None, n_formants=1):
-
-    if type(normalize_mean) is float or type(normalize_mean) is list:
-        normalize_mean = torch.tensor(normalize_mean)
-
-    if type(normalize_std) is float or type(normalize_std) is list:
-        normalize_std = torch.tensor(normalize_std)
-
-    if method == 'pyin':
-
-        snd, sr = librosa.load(wav)
-        pitch_mel, voiced_flag, voiced_probs = librosa.pyin(
-            snd, fmin=librosa.note_to_hz('C2'),
-            # fmax=librosa.note_to_hz('C7'), frame_length=1024)
-            fmax=400, frame_length=1024)
-        assert np.abs(mel_len - pitch_mel.shape[0]) <= 1.0
-
-        pitch_mel = np.where(np.isnan(pitch_mel), 0.0, pitch_mel)
-        pitch_mel = torch.from_numpy(pitch_mel).unsqueeze(0)
-        pitch_mel = F.pad(pitch_mel, (0, mel_len - pitch_mel.size(1)))
-
-        if n_formants > 1:
-            raise NotImplementedError
-
-    else:
-        raise ValueError
-
-    pitch_mel = pitch_mel.float()
-
-    if normalize_mean is not None:
-        assert normalize_std is not None
-        pitch_mel = normalize_pitch(pitch_mel, normalize_mean, normalize_std)
-
-    return pitch_mel
-
-
-def normalize_pitch(pitch, mean, std):
-    zeros = (pitch == 0.0)
-    pitch -= mean[:, None]
-    pitch /= std[:, None]
-    pitch[zeros] = 0.0
-    return pitch
-
-
-class TTSDataset(torch.utils.data.Dataset):
-    """
-        1) loads audio,text pairs
-        2) normalizes text and converts them to sequences of one-hot vectors
-        3) computes mel-spectrograms from audio files.
-    """
-    def __init__(self,
-                 dataset_path,
-                 audiopaths_and_text,
-                 text_cleaners,
-                 n_mel_channels,
-                 symbol_set='english_basic',
-                 p_arpabet=1.0,
-                 n_speakers=1,
-                 load_mel_from_disk=True,
-                 load_pitch_from_disk=True,
-                 pitch_mean=214.72203,  # LJSpeech defaults
-                 pitch_std=65.72038,
-                 max_wav_value=None,
-                 sampling_rate=None,
-                 filter_length=None,
-                 hop_length=None,
-                 win_length=None,
-                 mel_fmin=None,
-                 mel_fmax=None,
-                 prepend_space_to_text=False,
-                 append_space_to_text=False,
-                 pitch_online_dir=None,
-                 betabinomial_online_dir=None,
-                 use_betabinomial_interpolator=True,
-                 pitch_online_method='pyin',
-                 **ignored):
-
-        # Expect a list of filenames
-        if type(audiopaths_and_text) is str:
-            audiopaths_and_text = [audiopaths_and_text]
-
-        self.dataset_path = dataset_path
-        self.audiopaths_and_text = load_filepaths_and_text(
-            dataset_path, audiopaths_and_text,
-            has_speakers=(n_speakers > 1))
-        self.load_mel_from_disk = load_mel_from_disk
-        if not load_mel_from_disk:
-            self.max_wav_value = max_wav_value
-            self.sampling_rate = sampling_rate
-            self.stft = layers.TacotronSTFT(
-                filter_length, hop_length, win_length,
-                n_mel_channels, sampling_rate, mel_fmin, mel_fmax)
-        self.load_pitch_from_disk = load_pitch_from_disk
-
-        self.prepend_space_to_text = prepend_space_to_text
-        self.append_space_to_text = append_space_to_text
-
-        assert p_arpabet == 0.0 or p_arpabet == 1.0, (
-            'Only 0.0 and 1.0 p_arpabet is currently supported. '
-            'Variable probability breaks caching of betabinomial matrices.')
-
-        self.tp = TextProcessing(symbol_set, text_cleaners, p_arpabet=p_arpabet)
-        self.n_speakers = n_speakers
-        self.pitch_tmp_dir = pitch_online_dir
-        self.f0_method = pitch_online_method
-        self.betabinomial_tmp_dir = betabinomial_online_dir
-        self.use_betabinomial_interpolator = use_betabinomial_interpolator
-
-        if use_betabinomial_interpolator:
-            self.betabinomial_interpolator = BetaBinomialInterpolator()
-
-        expected_columns = (2 + int(load_pitch_from_disk) + (n_speakers > 1))
-
-        assert not (load_pitch_from_disk and self.pitch_tmp_dir is not None)
-
-        if len(self.audiopaths_and_text[0]) < expected_columns:
-            raise ValueError(f'Expected {expected_columns} columns in audiopaths file. '
-                             'The format is <mel_or_wav>|[<pitch>|]<text>[|<speaker_id>]')
-
-        if len(self.audiopaths_and_text[0]) > expected_columns:
-            print('WARNING: Audiopaths file has more columns than expected')
-
-        to_tensor = lambda x: torch.Tensor([x]) if type(x) is float else x
-        self.pitch_mean = to_tensor(pitch_mean)
-        self.pitch_std = to_tensor(pitch_std)
-
-    def __getitem__(self, index):
-        # Separate filename and text
-        if self.n_speakers > 1:
-            audiopath, *extra, text, speaker = self.audiopaths_and_text[index]
-            speaker = int(speaker)
-        else:
-            audiopath, *extra, text = self.audiopaths_and_text[index]
-            speaker = None
-
-        mel = self.get_mel(audiopath)
-        text = self.get_text(text)
-        # print(text)
-        pitch = self.get_pitch(index, mel.size(-1))
-        energy = torch.norm(mel.float(), dim=0, p=2)
-        attn_prior = self.get_prior(index, mel.shape[1], text.shape[0])
-
-        assert pitch.size(-1) == mel.size(-1)
-
-        # No higher formants?
-        if len(pitch.size()) == 1:
-            pitch = pitch[None, :]
-
-	
-        return (text, mel, len(text), pitch, energy, speaker, attn_prior,
-                audiopath)
-
-    def __len__(self):
-        return len(self.audiopaths_and_text)
-
-    def get_mel(self, filename):
-        if not self.load_mel_from_disk:
-            audio, sampling_rate = load_wav_to_torch(filename)
-            if sampling_rate != self.stft.sampling_rate:
-                raise ValueError("{} SR doesn't match target {} SR".format(
-                    sampling_rate, self.stft.sampling_rate))
-            audio_norm = audio / self.max_wav_value
-            audio_norm = audio_norm.unsqueeze(0)
-            audio_norm = torch.autograd.Variable(audio_norm,
-                                                 requires_grad=False)
-            melspec = self.stft.mel_spectrogram(audio_norm)
-            melspec = torch.squeeze(melspec, 0)
-        else:
-            melspec = torch.load(filename)
-            assert melspec.size(0) == self.stft.n_mel_channels, (
-                'Mel dimension mismatch: given {}, expected {}'.format(
-                 melspec.size(0), self.stft.n_mel_channels))
-                 
-################ Plotting mels ########################################
-        import matplotlib.pyplot as plt
-        # plt.imshow(melspec.detach().cpu().T,aspect="auto")
-        fig, ax1 = plt.subplots(ncols=1)
-        pos = ax1.imshow(melspec.cpu().numpy().T,aspect="auto")
-        fig.colorbar(pos, ax=ax1)
-        plt.show()
-#######################################################################
-
-        return melspec
-
-    def get_text(self, text):
-        text = self.tp.encode_text(text)
-        space = [self.tp.encode_text("A A")[1]]
-
-        if self.prepend_space_to_text:
-            text = space + text
-
-        if self.append_space_to_text:
-            text = text + space
-
-        return torch.LongTensor(text)
-
-    def get_prior(self, index, mel_len, text_len):
-
-        if self.use_betabinomial_interpolator:
-            return torch.from_numpy(self.betabinomial_interpolator(mel_len,
-                                                                   text_len))
-
-        if self.betabinomial_tmp_dir is not None:
-            audiopath, *_ = self.audiopaths_and_text[index]
-            fname = Path(audiopath).relative_to(self.dataset_path)
-            fname = fname.with_suffix('.pt')
-            cached_fpath = Path(self.betabinomial_tmp_dir, fname)
-
-            if cached_fpath.is_file():
-                return torch.load(cached_fpath)
-
-        attn_prior = beta_binomial_prior_distribution(text_len, mel_len)
-
-        if self.betabinomial_tmp_dir is not None:
-            cached_fpath.parent.mkdir(parents=True, exist_ok=True)
-            torch.save(attn_prior, cached_fpath)
-
-        return attn_prior
-
-    def get_pitch(self, index, mel_len=None):
-        audiopath, *fields = self.audiopaths_and_text[index]
-
-        if self.n_speakers > 1:
-            spk = int(fields[-1])
-        else:
-            spk = 0
-
-        if self.load_pitch_from_disk:
-            pitchpath = fields[0]
-            pitch = torch.load(pitchpath)
-            if self.pitch_mean is not None:
-                assert self.pitch_std is not None
-                pitch = normalize_pitch(pitch, self.pitch_mean, self.pitch_std)
-            return pitch
-
-        if self.pitch_tmp_dir is not None:
-            fname = Path(audiopath).relative_to(self.dataset_path)
-            fname_method = fname.with_suffix('.pt')
-            cached_fpath = Path(self.pitch_tmp_dir, fname_method)
-            if cached_fpath.is_file():
-                return torch.load(cached_fpath)
-
-        # No luck so far - calculate
-        wav = audiopath
-        if not wav.endswith('.wav'):
-            wav = re.sub('/mels/', '/wavs/', wav)
-            wav = re.sub('.pt$', '.wav', wav)
-
-        pitch_mel = estimate_pitch(wav, mel_len, self.f0_method,
-                                   self.pitch_mean, self.pitch_std)
-
-        if self.pitch_tmp_dir is not None and not cached_fpath.is_file():
-            cached_fpath.parent.mkdir(parents=True, exist_ok=True)
-            torch.save(pitch_mel, cached_fpath)
-
-        return pitch_mel
-
-
-class TTSCollate:
-    """Zero-pads model inputs and targets based on number of frames per step"""
-
-    def __call__(self, batch):
-        """Collate training batch from normalized text and mel-spec"""
-        # Right zero-pad all one-hot text sequences to max input length
-        input_lengths, ids_sorted_decreasing = torch.sort(
-            torch.LongTensor([len(x[0]) for x in batch]),
-            dim=0, descending=True)
-        max_input_len = input_lengths[0]
-
-        text_padded = torch.LongTensor(len(batch), max_input_len)
-        text_padded.zero_()
-        for i in range(len(ids_sorted_decreasing)):
-            text = batch[ids_sorted_decreasing[i]][0]
-            text_padded[i, :text.size(0)] = text
-
-        # Right zero-pad mel-spec
-        num_mels = batch[0][1].size(0)
-        max_target_len = max([x[1].size(1) for x in batch])
-
-        # Include mel padded and gate padded
-        mel_padded = torch.FloatTensor(len(batch), num_mels, max_target_len)
-        mel_padded.zero_()
-        output_lengths = torch.LongTensor(len(batch))
-        for i in range(len(ids_sorted_decreasing)):
-            mel = batch[ids_sorted_decreasing[i]][1]
-            mel_padded[i, :, :mel.size(1)] = mel
-            output_lengths[i] = mel.size(1)
-
-        n_formants = batch[0][3].shape[0]
-        pitch_padded = torch.zeros(mel_padded.size(0), n_formants,
-                                   mel_padded.size(2), dtype=batch[0][3].dtype)
-        energy_padded = torch.zeros_like(pitch_padded[:, 0, :])
-
-        for i in range(len(ids_sorted_decreasing)):
-            pitch = batch[ids_sorted_decreasing[i]][3]
-            energy = batch[ids_sorted_decreasing[i]][4]
-            pitch_padded[i, :, :pitch.shape[1]] = pitch
-            energy_padded[i, :energy.shape[0]] = energy
-
-        if batch[0][5] is not None:
-            speaker = torch.zeros_like(input_lengths)
-            for i in range(len(ids_sorted_decreasing)):
-                speaker[i] = batch[ids_sorted_decreasing[i]][5]
-        else:
-            speaker = None
-
-        attn_prior_padded = torch.zeros(len(batch), max_target_len,
-                                        max_input_len)
-        attn_prior_padded.zero_()
-        for i in range(len(ids_sorted_decreasing)):
-            prior = batch[ids_sorted_decreasing[i]][6]
-            attn_prior_padded[i, :prior.size(0), :prior.size(1)] = prior
-
-        # Count number of items - characters in text
-        len_x = [x[2] for x in batch]
-        len_x = torch.Tensor(len_x)
-
-        audiopaths = [batch[i][7] for i in ids_sorted_decreasing]
-
-        return (text_padded, input_lengths, mel_padded, output_lengths, len_x,
-                pitch_padded, energy_padded, speaker, attn_prior_padded,
-                audiopaths)
-
-
-def batch_to_gpu(batch):
-    (text_padded, input_lengths, mel_padded, output_lengths, len_x,
-     pitch_padded, energy_padded, speaker, attn_prior, audiopaths) = batch
-
-    text_padded = to_gpu(text_padded).long()
-    input_lengths = to_gpu(input_lengths).long()
-    mel_padded = to_gpu(mel_padded).float()
-    output_lengths = to_gpu(output_lengths).long()
-    pitch_padded = to_gpu(pitch_padded).float()
-    energy_padded = to_gpu(energy_padded).float()
-    attn_prior = to_gpu(attn_prior).float()
-    if speaker is not None:
-        speaker = to_gpu(speaker).long()
-
-    # Alignments act as both inputs and targets - pass shallow copies
-    x = [text_padded, input_lengths, mel_padded, output_lengths,
-         pitch_padded, energy_padded, speaker, attn_prior, audiopaths]
-    y = [mel_padded, input_lengths, output_lengths]
-    len_x = torch.sum(output_lengths)
-    return (x, y, len_x)

fastpitch/data_function_model_py.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7977160d12775529ba3426181093c8bca7927e52024f6d4faa91e1a0e53ef008
-size 9564

fastpitch/utils_trainplot_transformers.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5d3b72d713798a0552c0939fded67cb89655e12cc406a6fe793fcc7c9f63456a
-size 16365

fastpitch/utils_trainplot_transformers/train_1_with_plot.py

@@ -1,591 +0,0 @@
-# *****************************************************************************
-#  Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions are met:
-#      * Redistributions of source code must retain the above copyright
-#        notice, this list of conditions and the following disclaimer.
-#      * Redistributions in binary form must reproduce the above copyright
-#        notice, this list of conditions and the following disclaimer in the
-#        documentation and/or other materials provided with the distribution.
-#      * Neither the name of the NVIDIA CORPORATION nor the
-#        names of its contributors may be used to endorse or promote products
-#        derived from this software without specific prior written permission.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-#  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-#  DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
-#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-#  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-#  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# *****************************************************************************
-
-import argparse
-import copy
-import os
-import time
-from collections import defaultdict, OrderedDict
-from itertools import cycle
-
-import numpy as np
-import torch
-import torch.distributed as dist
-import amp_C
-from apex.optimizers import FusedAdam, FusedLAMB
-from torch.nn.parallel import DistributedDataParallel
-from torch.utils.data import DataLoader
-from torch.utils.data.distributed import DistributedSampler
-
-import common.tb_dllogger as logger
-import models
-from common.tb_dllogger import log
-from common.repeated_dataloader import (RepeatedDataLoader,
-                                        RepeatedDistributedSampler)
-from common.text import cmudict
-from common.utils import BenchmarkStats, Checkpointer, prepare_tmp
-from fastpitch.attn_loss_function import AttentionBinarizationLoss
-from fastpitch.data_function import batch_to_gpu, TTSCollate, TTSDataset
-from fastpitch.loss_function import FastPitchLoss
-
-import matplotlib.pyplot as plt
-
-def parse_args(parser):
-    parser.add_argument('-o', '--output', type=str, required=True,
-                        help='Directory to save checkpoints')
-    parser.add_argument('-d', '--dataset-path', type=str, default='./',
-                        help='Path to dataset')
-    parser.add_argument('--log-file', type=str, default=None,
-                        help='Path to a DLLogger log file')
-
-    train = parser.add_argument_group('training setup')
-    train.add_argument('--epochs', type=int, required=True,
-                       help='Number of total epochs to run')
-    train.add_argument('--epochs-per-checkpoint', type=int, default=50,
-                       help='Number of epochs per checkpoint')
-    train.add_argument('--checkpoint-path', type=str, default=None,
-                       help='Checkpoint path to resume training')
-    train.add_argument('--keep-milestones', default=list(range(100, 1000, 100)),
-                       type=int, nargs='+',
-                       help='Milestone checkpoints to keep from removing')
-    train.add_argument('--resume', action='store_true',
-                       help='Resume training from the last checkpoint')
-    train.add_argument('--seed', type=int, default=1234,
-                       help='Seed for PyTorch random number generators')
-    train.add_argument('--amp', action='store_true',
-                       help='Enable AMP')
-    train.add_argument('--cuda', action='store_true',
-                       help='Run on GPU using CUDA')
-    train.add_argument('--cudnn-benchmark', action='store_true',
-                       help='Enable cudnn benchmark mode')
-    train.add_argument('--ema-decay', type=float, default=0,
-                       help='Discounting factor for training weights EMA')
-    train.add_argument('--grad-accumulation', type=int, default=1,
-                       help='Training steps to accumulate gradients for')
-    train.add_argument('--kl-loss-start-epoch', type=int, default=250,
-                       help='Start adding the hard attention loss term')
-    train.add_argument('--kl-loss-warmup-epochs', type=int, default=100,
-                       help='Gradually increase the hard attention loss term')
-    train.add_argument('--kl-loss-weight', type=float, default=1.0,
-                       help='Gradually increase the hard attention loss term')
-    train.add_argument('--benchmark-epochs-num', type=int, default=20,
-                        help='Number of epochs for calculating final stats')
-    train.add_argument('--validation-freq', type=int, default=1,
-                       help='Validate every N epochs to use less compute')
-
-    opt = parser.add_argument_group('optimization setup')
-    opt.add_argument('--optimizer', type=str, default='lamb',
-                     help='Optimization algorithm')
-    opt.add_argument('-lr', '--learning-rate', type=float, required=True,
-                     help='Learing rate')
-    opt.add_argument('--weight-decay', default=1e-6, type=float,
-                     help='Weight decay')
-    opt.add_argument('--grad-clip-thresh', default=1000.0, type=float,
-                     help='Clip threshold for gradients')
-    opt.add_argument('-bs', '--batch-size', type=int, required=True,
-                     help='Batch size per GPU')
-    opt.add_argument('--warmup-steps', type=int, default=1000,
-                     help='Number of steps for lr warmup')
-    opt.add_argument('--dur-predictor-loss-scale', type=float,
-                     default=1.0, help='Rescale duration predictor loss')
-    opt.add_argument('--pitch-predictor-loss-scale', type=float,
-                     default=1.0, help='Rescale pitch predictor loss')
-    opt.add_argument('--attn-loss-scale', type=float,
-                     default=1.0, help='Rescale alignment loss')
-
-    data = parser.add_argument_group('dataset parameters')
-    data.add_argument('--training-files', type=str, nargs='*', required=True,
-                      help='Paths to training filelists.')
-    data.add_argument('--validation-files', type=str, nargs='*',
-                      required=True, help='Paths to validation filelists')
-    data.add_argument('--text-cleaners', nargs='*',
-                      default=['english_cleaners'], type=str,
-                      help='Type of text cleaners for input text')
-    data.add_argument('--symbol-set', type=str, default='english_basic',
-                      help='Define symbol set for input text')
-    data.add_argument('--p-arpabet', type=float, default=0.0,
-                      help='Probability of using arpabets instead of graphemes '
-                           'for each word; set 0 for pure grapheme training')
-    data.add_argument('--heteronyms-path', type=str, default='cmudict/heteronyms',
-                      help='Path to the list of heteronyms')
-    data.add_argument('--cmudict-path', type=str, default='cmudict/cmudict-0.7b',
-                      help='Path to the pronouncing dictionary')
-    data.add_argument('--prepend-space-to-text', action='store_true',
-                      help='Capture leading silence with a space token')
-    data.add_argument('--append-space-to-text', action='store_true',
-                      help='Capture trailing silence with a space token')
-    data.add_argument('--num-workers', type=int, default=2, # 6
-                      help='Subprocesses for train and val DataLoaders')
-    data.add_argument('--trainloader-repeats', type=int, default=100,
-                      help='Repeats the dataset to prolong epochs')
-
-    cond = parser.add_argument_group('data for conditioning')
-    cond.add_argument('--n-speakers', type=int, default=1,
-                      help='Number of speakers in the dataset. '
-                           'n_speakers > 1 enables speaker embeddings')
-    cond.add_argument('--load-pitch-from-disk', action='store_true',
-                      help='Use pitch cached on disk with prepare_dataset.py')
-    cond.add_argument('--pitch-online-method', default='pyin',
-                      choices=['pyin'],
-                      help='Calculate pitch on the fly during trainig')
-    cond.add_argument('--pitch-online-dir', type=str, default=None,
-                      help='A directory for storing pitch calculated on-line')
-    cond.add_argument('--pitch-mean', type=float, default=125.626816, #default=214.72203,
-                      help='Normalization value for pitch')
-    cond.add_argument('--pitch-std', type=float, default=37.52, #default=65.72038,
-                      help='Normalization value for pitch')
-    cond.add_argument('--load-mel-from-disk', action='store_true',
-                      help='Use mel-spectrograms cache on the disk')  # XXX
-
-    audio = parser.add_argument_group('audio parameters')
-    audio.add_argument('--max-wav-value', default=32768.0, type=float,
-                       help='Maximum audiowave value')
-    audio.add_argument('--sampling-rate', default=22050, type=int,
-                       help='Sampling rate')
-    audio.add_argument('--filter-length', default=1024, type=int,
-                       help='Filter length')
-    audio.add_argument('--hop-length', default=256, type=int,
-                       help='Hop (stride) length')
-    audio.add_argument('--win-length', default=1024, type=int,
-                       help='Window length')
-    audio.add_argument('--mel-fmin', default=0.0, type=float,
-                       help='Minimum mel frequency')
-    audio.add_argument('--mel-fmax', default=8000.0, type=float,
-                       help='Maximum mel frequency')
-
-    dist = parser.add_argument_group('distributed setup')
-    dist.add_argument('--local_rank', type=int, default=os.getenv('LOCAL_RANK', 0),
-                      help='Rank of the process for multiproc; do not set manually')
-    dist.add_argument('--world_size', type=int, default=os.getenv('WORLD_SIZE', 1),
-                      help='Number of processes for multiproc; do not set manually')
-    return parser
-
-
-def reduce_tensor(tensor, num_gpus):
-    rt = tensor.clone()
-    dist.all_reduce(rt, op=dist.ReduceOp.SUM)
-    return rt.true_divide(num_gpus)
-
-
-def init_distributed(args, world_size, rank):
-    assert torch.cuda.is_available(), "Distributed mode requires CUDA."
-    print("Initializing distributed training")
-
-    # Set cuda device so everything is done on the right GPU.
-    torch.cuda.set_device(rank % torch.cuda.device_count())
-
-    # Initialize distributed communication
-    dist.init_process_group(backend=('nccl' if args.cuda else 'gloo'),
-                            init_method='env://')
-    print("Done initializing distributed training")
-
-
-def validate(model, epoch, total_iter, criterion, val_loader, distributed_run,
-             batch_to_gpu, local_rank, ema=False):
-    was_training = model.training
-    model.eval()
-
-    tik = time.perf_counter()
-    with torch.no_grad():
-        val_meta = defaultdict(float)
-        val_num_frames = 0
-        for i, batch in enumerate(val_loader):
-            x, y, num_frames = batch_to_gpu(batch)
-            y_pred = model(x)
-            loss, meta = criterion(y_pred, y, is_training=False, meta_agg='sum')
-
-            if distributed_run:
-                for k, v in meta.items():
-                    val_meta[k] += reduce_tensor(v, 1)
-                val_num_frames += reduce_tensor(num_frames.data, 1).item()
-            else:
-                for k, v in meta.items():
-                    val_meta[k] += v
-                val_num_frames += num_frames.item()
-
-            # NOTE: ugly patch to visualize the first utterance of the validation corpus.
-            #       The goal is to determine if the training is progressing properly
-            if (i == 0) and (local_rank == 0) and (not ema):
-                # Plot some debug information
-                fig, axs = plt.subplots(2, 2, figsize=(21,14))
-
-                #  - Mel-spectrogram
-                pred_mel = y_pred[0][0, :, :].cpu().detach().numpy().astype(np.float32).T
-                orig_mel = y[0][0, :, :].cpu().detach().numpy().astype(np.float32)
-                axs[0,0].imshow(orig_mel, aspect='auto', origin='lower', interpolation='nearest')
-                axs[1,0].imshow(pred_mel, aspect='auto', origin='lower', interpolation='nearest')
-
-                # Prosody
-                f0_pred = y_pred[4][0, :].cpu().detach().numpy().astype(np.float32)
-                f0_ori = y_pred[5][0, :].cpu().detach().numpy().astype(np.float32)
-                axs[1,1].plot(f0_ori)
-                axs[1,1].plot(f0_pred)
-
-                # # Duration
-                # att_pred = y_pred[2][0, :].cpu().detach().numpy().astype(np.float32)
-                # att_ori = x[7][0,:].cpu().detach().numpy().astype(np.float32)
-                # axs[0,1].imshow(att_ori, aspect='auto', origin='lower', interpolation='nearest')
-
-                if not os.path.exists("debug_epoch/"):
-                    os.makedirs("debug_epoch_laila/")
-
-                fig.savefig(f'debug_epoch/{epoch:06d}.png', bbox_inches='tight')
-
-        val_meta = {k: v / len(val_loader.dataset) for k, v in val_meta.items()}
-
-    val_meta['took'] = time.perf_counter() - tik
-
-    log((epoch,) if epoch is not None else (), tb_total_steps=total_iter,
-        subset='val_ema' if ema else 'val',
-        data=OrderedDict([
-            ('loss', val_meta['loss'].item()),
-            ('mel_loss', val_meta['mel_loss'].item()),
-            ('frames/s', val_num_frames / val_meta['took']),
-            ('took', val_meta['took'])]),
-        )
-
-    if was_training:
-        model.train()
-    return val_meta
-
-
-def adjust_learning_rate(total_iter, opt, learning_rate, warmup_iters=None):
-    if warmup_iters == 0:
-        scale = 1.0
-    elif total_iter > warmup_iters:
-        scale = 1. / (total_iter ** 0.5)
-    else:
-        scale = total_iter / (warmup_iters ** 1.5)
-
-    for param_group in opt.param_groups:
-        param_group['lr'] = learning_rate * scale
-
-
-def apply_ema_decay(model, ema_model, decay):
-    if not decay:
-        return
-    st = model.state_dict()
-    add_module = hasattr(model, 'module') and not hasattr(ema_model, 'module')
-    for k, v in ema_model.state_dict().items():
-        if add_module and not k.startswith('module.'):
-            k = 'module.' + k
-        v.copy_(decay * v + (1 - decay) * st[k])
-
-
-def init_multi_tensor_ema(model, ema_model):
-    model_weights = list(model.state_dict().values())
-    ema_model_weights = list(ema_model.state_dict().values())
-    ema_overflow_buf = torch.cuda.IntTensor([0])
-    return model_weights, ema_model_weights, ema_overflow_buf
-
-
-def apply_multi_tensor_ema(decay, model_weights, ema_weights, overflow_buf):
-    amp_C.multi_tensor_axpby(
-        65536, overflow_buf, [ema_weights, model_weights, ema_weights],
-        decay, 1-decay, -1)
-
-
-def main():
-    parser = argparse.ArgumentParser(description='PyTorch FastPitch Training',
-                                     allow_abbrev=False)
-    parser = parse_args(parser)
-    args, _ = parser.parse_known_args()
-
-    if args.p_arpabet > 0.0:
-        cmudict.initialize(args.cmudict_path, args.heteronyms_path)
-
-    distributed_run = args.world_size > 1
-
-    torch.manual_seed(args.seed + args.local_rank)
-    np.random.seed(args.seed + args.local_rank)
-
-    if args.local_rank == 0:
-        if not os.path.exists(args.output):
-            os.makedirs(args.output)
-
-    log_fpath = args.log_file or os.path.join(args.output, 'nvlog.json')
-    tb_subsets = ['train', 'val']
-    if args.ema_decay > 0.0:
-        tb_subsets.append('val_ema')
-
-    logger.init(log_fpath, args.output, enabled=(args.local_rank == 0),
-                tb_subsets=tb_subsets)
-    logger.parameters(vars(args), tb_subset='train')
-
-    parser = models.parse_model_args('FastPitch', parser)
-    args, unk_args = parser.parse_known_args()
-    if len(unk_args) > 0:
-        raise ValueError(f'Invalid options {unk_args}')
-
-    torch.backends.cudnn.benchmark = args.cudnn_benchmark
-
-    if distributed_run:
-        init_distributed(args, args.world_size, args.local_rank)
-    else:
-        if args.trainloader_repeats > 1:
-            print('WARNING: Disabled --trainloader-repeats, supported only for'
-                  ' multi-GPU data loading.')
-            args.trainloader_repeats = 1
-
-    device = torch.device('cuda' if args.cuda else 'cpu')
-    model_config = models.get_model_config('FastPitch', args)
-    model = models.get_model('FastPitch', model_config, device)
-
-    attention_kl_loss = AttentionBinarizationLoss()
-
-    # Store pitch mean/std as params to translate from Hz during inference
-    model.pitch_mean[0] = args.pitch_mean
-    model.pitch_std[0] = args.pitch_std
-
-    kw = dict(lr=args.learning_rate, betas=(0.9, 0.98), eps=1e-9,
-              weight_decay=args.weight_decay)
-    if args.optimizer == 'adam':
-        optimizer = FusedAdam(model.parameters(), **kw)
-        # optimizer = torch.optim.Adam(model.parameters(), **kw)
-    elif args.optimizer == 'lamb':
-
-        optimizer = FusedLAMB(model.parameters(), **kw)
-        # optimizer = torch.optim.Adam(model.parameters(), **kw)
-    else:
-        raise ValueError
-
-    scaler = torch.cuda.amp.GradScaler(enabled=args.amp)
-
-    if args.ema_decay > 0:
-        ema_model = copy.deepcopy(model)
-    else:
-        ema_model = None
-
-    if distributed_run:
-        model = DistributedDataParallel(
-            model, device_ids=[args.local_rank], output_device=args.local_rank,
-            find_unused_parameters=True)
-
-    train_state = {'epoch': 1, 'total_iter': 1}
-    checkpointer = Checkpointer(args.output, args.keep_milestones)
-
-    checkpointer.maybe_load(model, optimizer, scaler, train_state, args,
-                            ema_model)
-
-    start_epoch = train_state['epoch']
-    total_iter = train_state['total_iter']
-
-    criterion = FastPitchLoss(
-        dur_predictor_loss_scale=args.dur_predictor_loss_scale,
-        pitch_predictor_loss_scale=args.pitch_predictor_loss_scale,
-        attn_loss_scale=args.attn_loss_scale)
-
-    collate_fn = TTSCollate()
-
-    if args.local_rank == 0:
-        prepare_tmp(args.pitch_online_dir)
-
-    trainset = TTSDataset(audiopaths_and_text=args.training_files, **vars(args))
-    valset = TTSDataset(audiopaths_and_text=args.validation_files, **vars(args))
-
-    if distributed_run:
-        train_sampler = RepeatedDistributedSampler(args.trainloader_repeats,
-                                                   trainset, drop_last=True)
-        val_sampler = DistributedSampler(valset)
-        shuffle = False
-    else:
-        train_sampler, val_sampler, shuffle = None, None, False ########### was True
-
-    # 4 workers are optimal on DGX-1 (from epoch 2 onwards)
-    kw = {'num_workers': args.num_workers, 'batch_size': args.batch_size,
-          'collate_fn': collate_fn}
-    train_loader = RepeatedDataLoader(args.trainloader_repeats, trainset,
-                                      shuffle=shuffle, drop_last=True,
-                                      sampler=train_sampler, pin_memory=True,
-                                      persistent_workers=True, **kw)
-    val_loader = DataLoader(valset, shuffle=False, sampler=val_sampler,
-                            pin_memory=False, **kw)
-    if args.ema_decay:
-        mt_ema_params = init_multi_tensor_ema(model, ema_model)
-
-    model.train()
-    bmark_stats = BenchmarkStats()
-
-    torch.cuda.synchronize()
-    for epoch in range(start_epoch, args.epochs + 1):
-        epoch_start_time = time.perf_counter()
-
-        epoch_loss = 0.0
-        epoch_mel_loss = 0.0
-        epoch_num_frames = 0
-        epoch_frames_per_sec = 0.0
-
-        if distributed_run:
-            train_loader.sampler.set_epoch(epoch)
-
-        iter_loss = 0
-        iter_num_frames = 0
-        iter_meta = {}
-        iter_start_time = time.perf_counter()
-
-        epoch_iter = 1
-        for batch, accum_step in zip(train_loader,
-                                     cycle(range(1, args.grad_accumulation + 1))):
-            if accum_step == 1:
-                adjust_learning_rate(total_iter, optimizer, args.learning_rate,
-                                     args.warmup_steps)
-
-                model.zero_grad(set_to_none=True)
-
-            x, y, num_frames = batch_to_gpu(batch)
-
-            with torch.cuda.amp.autocast(enabled=args.amp):
-                y_pred = model(x)
-                loss, meta = criterion(y_pred, y)
-
-                if (args.kl_loss_start_epoch is not None
-                        and epoch >= args.kl_loss_start_epoch):
-
-                    if args.kl_loss_start_epoch == epoch and epoch_iter == 1:
-                        print('Begin hard_attn loss')
-
-                    _, _, _, _, _, _, _, _, attn_soft, attn_hard, _, _ = y_pred
-                    binarization_loss = attention_kl_loss(attn_hard, attn_soft)
-                    kl_weight = min((epoch - args.kl_loss_start_epoch) / args.kl_loss_warmup_epochs, 1.0) * args.kl_loss_weight
-                    meta['kl_loss'] = binarization_loss.clone().detach() * kl_weight
-                    loss += kl_weight * binarization_loss
-
-                else:
-                    meta['kl_loss'] = torch.zeros_like(loss)
-                    kl_weight = 0
-                    binarization_loss = 0
-
-                loss /= args.grad_accumulation
-
-            meta = {k: v / args.grad_accumulation
-                    for k, v in meta.items()}
-
-            if args.amp:
-                scaler.scale(loss).backward()
-            else:
-                loss.backward()
-
-            if distributed_run:
-                reduced_loss = reduce_tensor(loss.data, args.world_size).item()
-                reduced_num_frames = reduce_tensor(num_frames.data, 1).item()
-                meta = {k: reduce_tensor(v, args.world_size) for k, v in meta.items()}
-            else:
-                reduced_loss = loss.item()
-                reduced_num_frames = num_frames.item()
-            if np.isnan(reduced_loss):
-                raise Exception("loss is NaN")
-
-            iter_loss += reduced_loss
-            iter_num_frames += reduced_num_frames
-            iter_meta = {k: iter_meta.get(k, 0) + meta.get(k, 0) for k in meta}
-
-            if accum_step % args.grad_accumulation == 0:
-
-                logger.log_grads_tb(total_iter, model)
-                if args.amp:
-                    scaler.unscale_(optimizer)
-                    torch.nn.utils.clip_grad_norm_(
-                        model.parameters(), args.grad_clip_thresh)
-                    scaler.step(optimizer)
-                    scaler.update()
-                else:
-                    torch.nn.utils.clip_grad_norm_(
-                        model.parameters(), args.grad_clip_thresh)
-                    optimizer.step()
-
-                if args.ema_decay > 0.0:
-                    apply_multi_tensor_ema(args.ema_decay, *mt_ema_params)
-
-                iter_mel_loss = iter_meta['mel_loss'].item()
-                iter_kl_loss = iter_meta['kl_loss'].item()
-                iter_time = time.perf_counter() - iter_start_time
-                epoch_frames_per_sec += iter_num_frames / iter_time
-                epoch_loss += iter_loss
-                epoch_num_frames += iter_num_frames
-                epoch_mel_loss += iter_mel_loss
-
-                num_iters = len(train_loader) // args.grad_accumulation
-                log((epoch, epoch_iter, num_iters), tb_total_steps=total_iter,
-                    subset='train', data=OrderedDict([
-                        ('loss', iter_loss),
-                        ('mel_loss', iter_mel_loss),
-                        ('kl_loss', iter_kl_loss),
-                        ('kl_weight', kl_weight),
-                        ('frames/s', iter_num_frames / iter_time),
-                        ('took', iter_time),
-                        ('lrate', optimizer.param_groups[0]['lr'])]),
-                )
-
-                iter_loss = 0
-                iter_num_frames = 0
-                iter_meta = {}
-                iter_start_time = time.perf_counter()
-
-                if epoch_iter == num_iters:
-                    break
-                epoch_iter += 1
-                total_iter += 1
-
-        # Finished epoch
-        epoch_loss /= epoch_iter
-        epoch_mel_loss /= epoch_iter
-        epoch_time = time.perf_counter() - epoch_start_time
-
-        log((epoch,), tb_total_steps=None, subset='train_avg',
-            data=OrderedDict([
-                ('loss', epoch_loss),
-                ('mel_loss', epoch_mel_loss),
-                ('frames/s', epoch_num_frames / epoch_time),
-                ('took', epoch_time)]),
-        )
-        bmark_stats.update(epoch_num_frames, epoch_loss, epoch_mel_loss,
-                           epoch_time)
-
-        if epoch % args.validation_freq == 0:
-            validate(model, epoch, total_iter, criterion, val_loader,
-                     distributed_run, batch_to_gpu, ema=False, local_rank=args.local_rank)
-
-            if args.ema_decay > 0:
-                validate(ema_model, epoch, total_iter, criterion, val_loader,
-                         distributed_run, batch_to_gpu, args.local_rank, ema=True)
-
-        # save before making sched.step() for proper loading of LR
-        checkpointer.maybe_save(args, model, ema_model, optimizer, scaler,
-                                epoch, total_iter, model_config)
-        logger.flush()
-
-    # Finished training
-    if len(bmark_stats) > 0:
-        log((), tb_total_steps=None, subset='train_avg',
-            data=bmark_stats.get(args.benchmark_epochs_num))
-
-    validate(model, None, total_iter, criterion, val_loader, distributed_run,
-             batch_to_gpu)
-
-
-if __name__ == '__main__':
-    main()

fastpitch/utils_trainplot_transformers/transformer.py

@@ -1,213 +0,0 @@
-# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from common.utils import mask_from_lens
-
-
-class PositionalEmbedding(nn.Module):
-    def __init__(self, demb):
-        super(PositionalEmbedding, self).__init__()
-        self.demb = demb
-        inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb))
-        self.register_buffer('inv_freq', inv_freq)
-
-    def forward(self, pos_seq, bsz=None):
-        sinusoid_inp = torch.matmul(torch.unsqueeze(pos_seq, -1),
-                                    torch.unsqueeze(self.inv_freq, 0))
-        pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=1)
-        if bsz is not None:
-            return pos_emb[None, :, :].expand(bsz, -1, -1)
-        else:
-            return pos_emb[None, :, :]
-
-
-class PositionwiseConvFF(nn.Module):
-    def __init__(self, d_model, d_inner, kernel_size, dropout, pre_lnorm=False):
-        super(PositionwiseConvFF, self).__init__()
-
-        self.d_model = d_model
-        self.d_inner = d_inner
-        self.dropout = dropout
-
-        self.CoreNet = nn.Sequential(
-            nn.Conv1d(d_model, d_inner, kernel_size, 1, (kernel_size // 2)),
-            nn.ReLU(),
-            # nn.Dropout(dropout),  # worse convergence
-            nn.Conv1d(d_inner, d_model, kernel_size, 1, (kernel_size // 2)),
-            nn.Dropout(dropout),
-        )
-        self.layer_norm = nn.LayerNorm(d_model)
-        self.pre_lnorm = pre_lnorm
-
-    def forward(self, inp):
-        return self._forward(inp)
-
-    def _forward(self, inp):
-        if self.pre_lnorm:
-            # layer normalization + positionwise feed-forward
-            core_out = inp.transpose(1, 2)
-            core_out = self.CoreNet(self.layer_norm(core_out).to(inp.dtype))
-            core_out = core_out.transpose(1, 2)
-
-            # residual connection
-            output = core_out + inp
-        else:
-            # positionwise feed-forward
-            core_out = inp.transpose(1, 2)
-            core_out = self.CoreNet(core_out)
-            core_out = core_out.transpose(1, 2)
-
-            # residual connection + layer normalization
-            output = self.layer_norm(inp + core_out).to(inp.dtype)
-
-        return output
-
-
-class MultiHeadAttn(nn.Module):
-    def __init__(self, n_head, d_model, d_head, dropout, dropatt=0.1,
-                 pre_lnorm=False):
-        super(MultiHeadAttn, self).__init__()
-
-        self.n_head = n_head
-        self.d_model = d_model
-        self.d_head = d_head
-        self.scale = 1 / (d_head ** 0.5)
-        self.pre_lnorm = pre_lnorm
-
-        self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head)
-        self.drop = nn.Dropout(dropout)
-        self.dropatt = nn.Dropout(dropatt)
-        self.o_net = nn.Linear(n_head * d_head, d_model, bias=False)
-        self.layer_norm = nn.LayerNorm(d_model)
-
-    def forward(self, inp, attn_mask=None):
-        return self._forward(inp, attn_mask)
-
-    def _forward(self, inp, attn_mask=None):
-        residual = inp
-
-        if self.pre_lnorm:
-            # layer normalization
-            inp = self.layer_norm(inp)
-
-        n_head, d_head = self.n_head, self.d_head
-
-        head_q, head_k, head_v = torch.chunk(self.qkv_net(inp), 3, dim=2)
-        head_q = head_q.view(inp.size(0), inp.size(1), n_head, d_head)
-        head_k = head_k.view(inp.size(0), inp.size(1), n_head, d_head)
-        head_v = head_v.view(inp.size(0), inp.size(1), n_head, d_head)
-
-        q = head_q.permute(2, 0, 1, 3).reshape(-1, inp.size(1), d_head)
-        k = head_k.permute(2, 0, 1, 3).reshape(-1, inp.size(1), d_head)
-        v = head_v.permute(2, 0, 1, 3).reshape(-1, inp.size(1), d_head)
-
-        attn_score = torch.bmm(q, k.transpose(1, 2))
-        attn_score.mul_(self.scale)
-
-        if attn_mask is not None:
-            attn_mask = attn_mask.unsqueeze(1).to(attn_score.dtype)
-            attn_mask = attn_mask.repeat(n_head, attn_mask.size(2), 1)
-            attn_score.masked_fill_(attn_mask.to(torch.bool), -float('inf'))
-
-        attn_prob = F.softmax(attn_score, dim=2)
-        attn_prob = self.dropatt(attn_prob)
-        attn_vec = torch.bmm(attn_prob, v)
-
-        attn_vec = attn_vec.view(n_head, inp.size(0), inp.size(1), d_head)
-        attn_vec = attn_vec.permute(1, 2, 0, 3).contiguous().view(
-            inp.size(0), inp.size(1), n_head * d_head)
-
-        # linear projection
-        attn_out = self.o_net(attn_vec)
-        attn_out = self.drop(attn_out)
-
-        if self.pre_lnorm:
-            # residual connection
-            output = residual + attn_out
-        else:
-            # residual connection + layer normalization
-            output = self.layer_norm(residual + attn_out)
-
-        output = output.to(attn_out.dtype)
-
-        return output
-
-
-class TransformerLayer(nn.Module):
-    def __init__(self, n_head, d_model, d_head, d_inner, kernel_size, dropout,
-                 **kwargs):
-        super(TransformerLayer, self).__init__()
-
-        self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs)
-        self.pos_ff = PositionwiseConvFF(d_model, d_inner, kernel_size, dropout,
-                                         pre_lnorm=kwargs.get('pre_lnorm'))
-
-    def forward(self, dec_inp, mask=None):
-        output = self.dec_attn(dec_inp, attn_mask=~mask.squeeze(2))
-        output *= mask
-        output = self.pos_ff(output)
-        output *= mask
-        return output
-
-
-class FFTransformer(nn.Module):
-    def __init__(self, n_layer, n_head, d_model, d_head, d_inner, kernel_size,
-                 dropout, dropatt, dropemb=0.0, embed_input=True,
-                 n_embed=None, d_embed=None, padding_idx=0, pre_lnorm=False):
-        super(FFTransformer, self).__init__()
-        self.d_model = d_model
-        self.n_head = n_head
-        self.d_head = d_head
-        self.padding_idx = padding_idx
-
-        if embed_input:
-            self.word_emb = nn.Embedding(n_embed, d_embed or d_model,
-                                         padding_idx=self.padding_idx)
-        else:
-            self.word_emb = None
-
-        self.pos_emb = PositionalEmbedding(self.d_model)
-        self.drop = nn.Dropout(dropemb)
-        self.layers = nn.ModuleList()
-
-        for _ in range(n_layer):
-            self.layers.append(
-                TransformerLayer(
-                    n_head, d_model, d_head, d_inner, kernel_size, dropout,
-                    dropatt=dropatt, pre_lnorm=pre_lnorm)
-            )
-
-    def forward(self, dec_inp, seq_lens=None, conditioning=0):
-        if self.word_emb is None:
-            inp = dec_inp
-            mask = mask_from_lens(seq_lens).unsqueeze(2)
-        else:
-            inp = self.word_emb(dec_inp)
-            # [bsz x L x 1]
-            mask = (dec_inp != self.padding_idx).unsqueeze(2)
-
-        pos_seq = torch.arange(inp.size(1), device=inp.device).to(inp.dtype)
-        pos_emb = self.pos_emb(pos_seq) * mask
-
-        out = self.drop(inp + pos_emb + conditioning)
-
-        for layer in self.layers:
-            out = layer(out, mask=mask)
-
-        # out = self.drop(out)
-        return out, mask

fastpitch/utils_trainplot_transformers/transformer_jit.py

@@ -1,255 +0,0 @@
-# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import List, Optional
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from common.utils import mask_from_lens
-
-
-class PositionalEmbedding(nn.Module):
-    def __init__(self, demb):
-        super(PositionalEmbedding, self).__init__()
-        self.demb = demb
-        inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb))
-        self.register_buffer('inv_freq', inv_freq)
-
-    def forward(self, pos_seq, bsz: Optional[int] = None):
-        sinusoid_inp = torch.ger(pos_seq, self.inv_freq)
-        pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=1)
-        if bsz is not None:
-            return pos_emb[None, :, :].expand(bsz, -1, -1)
-        else:
-            return pos_emb[None, :, :]
-
-
-class PositionwiseFF(nn.Module):
-    def __init__(self, d_model, d_inner, dropout, pre_lnorm=False):
-        super(PositionwiseFF, self).__init__()
-
-        self.d_model = d_model
-        self.d_inner = d_inner
-        self.dropout = dropout
-
-        self.CoreNet = nn.Sequential(
-            nn.Linear(d_model, d_inner), nn.ReLU(),
-            nn.Dropout(dropout),
-            nn.Linear(d_inner, d_model),
-            nn.Dropout(dropout),
-        )
-
-        self.layer_norm = nn.LayerNorm(d_model)
-        self.pre_lnorm = pre_lnorm
-
-    def forward(self, inp):
-        if self.pre_lnorm:
-            # layer normalization + positionwise feed-forward
-            core_out = self.CoreNet(self.layer_norm(inp))
-
-            # residual connection
-            output = core_out + inp
-        else:
-            # positionwise feed-forward
-            core_out = self.CoreNet(inp)
-
-            # residual connection + layer normalization
-            output = self.layer_norm(inp + core_out)
-
-        return output
-
-
-class PositionwiseConvFF(nn.Module):
-    def __init__(self, d_model, d_inner, kernel_size, dropout, pre_lnorm=False):
-        super(PositionwiseConvFF, self).__init__()
-
-        self.d_model = d_model
-        self.d_inner = d_inner
-        self.dropout = dropout
-
-        self.CoreNet = nn.Sequential(
-            nn.Conv1d(d_model, d_inner, kernel_size, 1, (kernel_size // 2)),
-            nn.ReLU(),
-            # nn.Dropout(dropout),  # worse convergence
-            nn.Conv1d(d_inner, d_model, kernel_size, 1, (kernel_size // 2)),
-            nn.Dropout(dropout),
-        )
-        self.layer_norm = nn.LayerNorm(d_model)
-        self.pre_lnorm = pre_lnorm
-
-    def forward(self, inp):
-        if self.pre_lnorm:
-            # layer normalization + positionwise feed-forward
-            core_out = inp.transpose(1, 2)
-            core_out = self.CoreNet(self.layer_norm(core_out))
-            core_out = core_out.transpose(1, 2)
-
-            # residual connection
-            output = core_out + inp
-        else:
-            # positionwise feed-forward
-            core_out = inp.transpose(1, 2)
-            core_out = self.CoreNet(core_out)
-            core_out = core_out.transpose(1, 2)
-
-            # residual connection + layer normalization
-            output = self.layer_norm(inp + core_out)
-
-        return output
-
-
-class MultiHeadAttn(nn.Module):
-    def __init__(self, n_head, d_model, d_head, dropout, dropatt=0.1,
-                 pre_lnorm=False):
-        super(MultiHeadAttn, self).__init__()
-
-        self.n_head = n_head
-        self.d_model = d_model
-        self.d_head = d_head
-        self.scale = 1 / (d_head ** 0.5)
-        self.dropout = dropout
-        self.pre_lnorm = pre_lnorm
-
-        self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head)
-        self.drop = nn.Dropout(dropout)
-        self.dropatt = nn.Dropout(dropatt)
-        self.o_net = nn.Linear(n_head * d_head, d_model, bias=False)
-        self.layer_norm = nn.LayerNorm(d_model)
-
-
-    def forward(self, inp, attn_mask: Optional[torch.Tensor] = None):
-        residual = inp
-
-        if self.pre_lnorm:
-            # layer normalization
-            inp = self.layer_norm(inp)
-
-        n_head, d_head = self.n_head, self.d_head
-
-        head_q, head_k, head_v = torch.chunk(self.qkv_net(inp), 3, dim=-1)
-        head_q = head_q.view(inp.size(0), inp.size(1), n_head, d_head)
-        head_k = head_k.view(inp.size(0), inp.size(1), n_head, d_head)
-        head_v = head_v.view(inp.size(0), inp.size(1), n_head, d_head)
-
-        q = head_q.permute(0, 2, 1, 3).reshape(-1, inp.size(1), d_head)
-        k = head_k.permute(0, 2, 1, 3).reshape(-1, inp.size(1), d_head)
-        v = head_v.permute(0, 2, 1, 3).reshape(-1, inp.size(1), d_head)
-
-        attn_score = torch.bmm(q, k.transpose(1, 2))
-        attn_score.mul_(self.scale)
-
-        if attn_mask is not None:
-            attn_mask = attn_mask.unsqueeze(1)
-            attn_mask = attn_mask.repeat(n_head, attn_mask.size(2), 1)
-            attn_score.masked_fill_(attn_mask, -float('inf'))
-
-        attn_prob = F.softmax(attn_score, dim=2)
-        attn_prob = self.dropatt(attn_prob)
-        attn_vec = torch.bmm(attn_prob, v)
-
-        attn_vec = attn_vec.view(n_head, inp.size(0), inp.size(1), d_head)
-        attn_vec = attn_vec.permute(1, 2, 0, 3).contiguous().view(
-            inp.size(0), inp.size(1), n_head * d_head)
-
-        # linear projection
-        attn_out = self.o_net(attn_vec)
-        attn_out = self.drop(attn_out)
-
-        if self.pre_lnorm:
-            # residual connection
-            output = residual + attn_out
-        else:
-            # residual connection + layer normalization
-
-            # XXX Running TorchScript on 20.02 and 20.03 containers crashes here
-            # XXX Works well with 20.01-py3 container.
-            # XXX dirty fix is:
-            # XXX     output = self.layer_norm(residual + attn_out).half()
-            output = self.layer_norm(residual + attn_out)
-
-        return output
-
-
-class TransformerLayer(nn.Module):
-    def __init__(self, n_head, d_model, d_head, d_inner, kernel_size, dropout,
-                 **kwargs):
-        super(TransformerLayer, self).__init__()
-
-        self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs)
-        self.pos_ff = PositionwiseConvFF(d_model, d_inner, kernel_size, dropout,
-                                         pre_lnorm=kwargs.get('pre_lnorm'))
-
-    def forward(self, dec_inp, mask):
-        output = self.dec_attn(dec_inp, attn_mask=~mask.squeeze(2))
-        output *= mask
-        output = self.pos_ff(output)
-        output *= mask
-        return output
-
-
-class FFTransformer(nn.Module):
-    def __init__(self, n_layer, n_head, d_model, d_head, d_inner, kernel_size,
-                 dropout, dropatt, dropemb=0.0, embed_input=True,
-                 n_embed=None, d_embed=None, padding_idx=0, pre_lnorm=False):
-        super(FFTransformer, self).__init__()
-        self.d_model = d_model
-        self.n_head = n_head
-        self.d_head = d_head
-        self.padding_idx = padding_idx
-        self.n_embed = n_embed
-
-        self.embed_input = embed_input
-        if embed_input:
-            print(padding_idx) #########################################        	
-            self.word_emb = nn.Embedding(n_embed, d_embed or d_model,
-                                         padding_idx=self.padding_idx)
-        else:
-            self.word_emb = nn.Identity()
-
-        self.pos_emb = PositionalEmbedding(self.d_model)
-        self.drop = nn.Dropout(dropemb)
-        self.layers = nn.ModuleList()
-
-        for _ in range(n_layer):
-            self.layers.append(
-                TransformerLayer(
-                    n_head, d_model, d_head, d_inner, kernel_size, dropout,
-                    dropatt=dropatt, pre_lnorm=pre_lnorm)
-            )
-
-    def forward(self, dec_inp, seq_lens: Optional[torch.Tensor] = None,
-                conditioning: Optional[torch.Tensor] = None):
-        if not self.embed_input:
-            inp = dec_inp
-            assert seq_lens is not None
-            mask = mask_from_lens(seq_lens).unsqueeze(2)
-        else:
-            inp = self.word_emb(dec_inp)
-            # [bsz x L x 1]
-            mask = (dec_inp != self.padding_idx).unsqueeze(2)
-
-        pos_seq = torch.arange(inp.size(1), device=inp.device, dtype=inp.dtype)
-        pos_emb = self.pos_emb(pos_seq) * mask
-        if conditioning is not None:
-            out = self.drop(inp + pos_emb + conditioning)
-        else:
-            out = self.drop(inp + pos_emb)
-
-        for layer in self.layers:
-            out = layer(out, mask=mask)
-
-        # out = self.drop(out)
-        return out, mask

fastpitch/utils_trainplot_transformers/utils.py

@@ -1,291 +0,0 @@
-# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#           http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#  MIT License
-#
-#  Copyright (c) 2020 Jungil Kong
-#
-#  Permission is hereby granted, free of charge, to any person obtaining a copy
-#  of this software and associated documentation files (the "Software"), to deal
-#  in the Software without restriction, including without limitation the rights
-#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-#  copies of the Software, and to permit persons to whom the Software is
-#  furnished to do so, subject to the following conditions:
-#
-#  The above copyright notice and this permission notice shall be included in all
-#  copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-#  SOFTWARE.
-
-# The following functions/classes were based on code from https://github.com/jik876/hifi-gan:
-# init_weights, get_padding, AttrDict
-
-import ctypes
-import glob
-import os
-import re
-import shutil
-import warnings
-from collections import defaultdict, OrderedDict
-from pathlib import Path
-from typing import Optional
-
-import librosa
-import numpy as np
-
-import torch
-import torch.distributed as dist
-from scipy.io.wavfile import read
-
-
-def mask_from_lens(lens, max_len: Optional[int] = None):
-    if max_len is None:
-        max_len = lens.max()
-    ids = torch.arange(0, max_len, device=lens.device, dtype=lens.dtype)
-    mask = torch.lt(ids, lens.unsqueeze(1))
-    return mask
-
-
-def load_wav(full_path, torch_tensor=False):
-    import soundfile  # flac
-    data, sampling_rate = soundfile.read(full_path, dtype='int16')
-    if torch_tensor:
-        return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-    else:
-        return data, sampling_rate
-
-
-def load_wav_to_torch(full_path, force_sampling_rate=None):
-    if force_sampling_rate is not None:
-        data, sampling_rate = librosa.load(full_path, sr=force_sampling_rate)
-    else:
-        sampling_rate, data = read(full_path)
-
-    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
-
-
-def load_filepaths_and_text(dataset_path, fnames, has_speakers=False, split="|"):
-    def split_line(root, line):
-        parts = line.strip().split(split)
-        if has_speakers:
-            paths, non_paths = parts[:-2], parts[-2:]
-        else:
-            paths, non_paths = parts[:-1], parts[-1:]
-        return tuple(str(Path(root, p)) for p in paths) + tuple(non_paths)
-
-    fpaths_and_text = []
-    for fname in fnames:
-        with open(fname, encoding='utf-8') as f:
-            fpaths_and_text += [split_line(dataset_path, line) for line in f]
-    return fpaths_and_text
-
-
-def to_gpu(x):
-    x = x.contiguous()
-    return x.cuda(non_blocking=True) if torch.cuda.is_available() else x
-
-
-def l2_promote():
-    _libcudart = ctypes.CDLL('libcudart.so')
-    # Set device limit on the current device
-    # cudaLimitMaxL2FetchGranularity = 0x05
-    pValue = ctypes.cast((ctypes.c_int*1)(), ctypes.POINTER(ctypes.c_int))
-    _libcudart.cudaDeviceSetLimit(ctypes.c_int(0x05), ctypes.c_int(128))
-    _libcudart.cudaDeviceGetLimit(pValue, ctypes.c_int(0x05))
-    assert pValue.contents.value == 128
-
-
-def prepare_tmp(path):
-    if path is None:
-        return
-    p = Path(path)
-    if p.is_dir():
-        warnings.warn(f'{p} exists. Removing...')
-        shutil.rmtree(p, ignore_errors=True)
-    p.mkdir(parents=False, exist_ok=False)
-
-
-def print_once(*msg):
-    if not dist.is_initialized() or dist.get_rank() == 0:
-        print(*msg)
-
-
-def init_weights(m, mean=0.0, std=0.01):
-    classname = m.__class__.__name__
-    if classname.find("Conv") != -1:
-        m.weight.data.normal_(mean, std)
-
-
-def get_padding(kernel_size, dilation=1):
-    return int((kernel_size*dilation - dilation)/2)
-
-
-class AttrDict(dict):
-    def __init__(self, *args, **kwargs):
-        super(AttrDict, self).__init__(*args, **kwargs)
-        self.__dict__ = self
-
-
-class DefaultAttrDict(defaultdict):
-    def __init__(self, *args, **kwargs):
-        super(DefaultAttrDict, self).__init__(*args, **kwargs)
-        self.__dict__ = self
-
-    def __getattr__(self, item):
-        return self[item]
-
-
-class BenchmarkStats:
-    """ Tracks statistics used for benchmarking. """
-    def __init__(self):
-        self.num_frames = []
-        self.losses = []
-        self.mel_losses = []
-        self.took = []
-
-    def update(self, num_frames, losses, mel_losses, took):
-        self.num_frames.append(num_frames)
-        self.losses.append(losses)
-        self.mel_losses.append(mel_losses)
-        self.took.append(took)
-
-    def get(self, n_epochs):
-        frames_s = sum(self.num_frames[-n_epochs:]) / sum(self.took[-n_epochs:])
-        return {'frames/s': frames_s,
-                'loss': np.mean(self.losses[-n_epochs:]),
-                'mel_loss': np.mean(self.mel_losses[-n_epochs:]),
-                'took': np.mean(self.took[-n_epochs:]),
-                'benchmark_epochs_num': n_epochs}
-
-    def __len__(self):
-        return len(self.losses)
-
-
-class Checkpointer:
-
-    def __init__(self, save_dir, keep_milestones=[]):
-        self.save_dir = save_dir
-        self.keep_milestones = keep_milestones
-
-        find = lambda name: [
-            (int(re.search("_(\d+).pt", fn).group(1)), fn)
-            for fn in glob.glob(f"{save_dir}/{name}_checkpoint_*.pt")]
-
-        tracked = sorted(find("FastPitch"), key=lambda t: t[0])
-        self.tracked = OrderedDict(tracked)
-
-    def last_checkpoint(self, output):
-
-        def corrupted(fpath):
-            try:
-                torch.load(fpath, map_location="cpu")
-                return False
-            except:
-                warnings.warn(f"Cannot load {fpath}")
-                return True
-
-        saved = sorted(
-            glob.glob(f"{output}/FastPitch_checkpoint_*.pt"),
-            key=lambda f: int(re.search("_(\d+).pt", f).group(1)))
-
-        if len(saved) >= 1 and not corrupted(saved[-1]):
-            return saved[-1]
-        elif len(saved) >= 2:
-            return saved[-2]
-        else:
-            return None
-
-    def maybe_load(self, model, optimizer, scaler, train_state, args,
-                   ema_model=None):
-
-        assert args.checkpoint_path is None or args.resume is False, (
-            "Specify a single checkpoint source")
-
-        fpath = None
-        if args.checkpoint_path is not None:
-            fpath = args.checkpoint_path
-            self.tracked = OrderedDict()  # Do not track/delete prev ckpts
-        elif args.resume:
-            fpath = self.last_checkpoint(args.output)
-
-        if fpath is None:
-            return
-
-        print_once(f"Loading model and optimizer state from {fpath}")
-        ckpt = torch.load(fpath, map_location="cpu")
-        train_state["epoch"] = ckpt["epoch"] + 1
-        train_state["total_iter"] = ckpt["iteration"]
-
-        no_pref = lambda sd: {re.sub("^module.", "", k): v for k, v in sd.items()}
-        unwrap = lambda m: getattr(m, "module", m)
-
-        unwrap(model).load_state_dict(no_pref(ckpt["state_dict"]))
-
-        if ema_model is not None:
-            unwrap(ema_model).load_state_dict(no_pref(ckpt["ema_state_dict"]))
-
-        optimizer.load_state_dict(ckpt["optimizer"])
-
-        if "scaler" in ckpt:
-            scaler.load_state_dict(ckpt["scaler"])
-        else:
-            warnings.warn("AMP scaler state missing from the checkpoint.")
-
-    def maybe_save(self, args, model, ema_model, optimizer, scaler, epoch,
-                   total_iter, config):
-
-        intermediate = (args.epochs_per_checkpoint > 0
-                        and epoch % args.epochs_per_checkpoint == 0)
-        final = epoch == args.epochs
-
-        if not intermediate and not final and epoch not in self.keep_milestones:
-            return
-
-        rank = 0
-        if dist.is_initialized():
-            dist.barrier()
-            rank = dist.get_rank()
-
-        if rank != 0:
-            return
-
-        unwrap = lambda m: getattr(m, "module", m)
-        ckpt = {"epoch": epoch,
-                "iteration": total_iter,
-                "config": config,
-                "train_setup": args.__dict__,
-                "state_dict": unwrap(model).state_dict(),
-                "optimizer": optimizer.state_dict(),
-                "scaler": scaler.state_dict()}
-        if ema_model is not None:
-            ckpt["ema_state_dict"] = unwrap(ema_model).state_dict()
-
-        fpath = Path(args.output, f"FastPitch_checkpoint_{epoch}.pt")
-        print(f"Saving model and optimizer state at epoch {epoch} to {fpath}")
-        torch.save(ckpt, fpath)
-
-        # Remove old checkpoints; keep milestones and the last two
-        self.tracked[epoch] = fpath
-        for epoch in set(list(self.tracked)[:-2]) - set(self.keep_milestones):
-            try:
-                os.remove(self.tracked[epoch])
-            except:
-                pass
-            del self.tracked[epoch]

gradio_gui.py

@@ -1,74 +0,0 @@
-import gradio as gr
-import syn_hifigan as syn
-#import syn_k_univnet_multi as syn
-import os, tempfile
-        
-languages = {"South Sámi":0,
-          "North Sámi":1,
-          "Lule Sámi":2}
-
-speakers={"aj0": 0,
-          "aj1": 1,
-          "am": 2,
-          "bi": 3,
-          "kd": 4,
-          "ln": 5,
-          "lo": 6,
-          "ms": 7,
-          "mu": 8,
-          "sa": 9
-}
-public=False
-
-tempdir = tempfile.gettempdir()
-
-tts = syn.Synthesizer()
-
-
-
-def speak(text, language,speaker,l_weight, s_weight, pace, postfilter): #pitch_shift,pitch_std):
-
-
-
-    # text frontend not implemented...
-    text = text.replace("...", "…")
-    print(speakers[speaker])
-    audio = tts.speak(text, output_file=f'{tempdir}/tmp', lang=languages[language],
-                      spkr=speakers[speaker], l_weight=l_weight, s_weight=s_weight,
-                      pace=pace, clarity=postfilter)
-
-    if not public:
-        try:
-            os.system("play "+tempdir+"/tmp.wav &")
-        except:
-            pass
-
-    return (22050, audio)
-
-
-
-controls = []
-controls.append(gr.Textbox(label="text", value="Suohtas duinna deaivvadit."))
-controls.append(gr.Dropdown(list(languages.keys()), label="language", value="North Sámi"))
-controls.append(gr.Dropdown(list(speakers.keys()), label="speaker", value="ms"))
-
-controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1, label="language weight"))
-controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1, label="speaker weight"))
-
-controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1.0, label="speech rate"))
-controls.append(gr.Slider(minimum=0., maximum=2, step=0.05, value=1.0, label="post-processing"))
-
-
-
-
-tts_gui = gr.Interface(
-    fn=speak,
-    inputs=controls,
-    outputs= gr.Audio(label="output"),
-    live=False
-
-)
-
-
-if __name__ == "__main__":
-    tts_gui.launch(share=public)

gradio_gui_katri.py

@@ -1,73 +0,0 @@
-import gradio as gr
-#import syn_hifigan as syn
-import syn_k_univnet_multi as syn
-import os, tempfile
-        
-languages = {"South Sámi":0,
-          "North Sámi":1,
-          "Lule Sámi":2}
-
-speakers={"aj0": 0,
-          "aj1": 1,
-          "am": 2,
-          "bi": 3,
-          "kd": 4,
-          "ln": 5,
-          "lo": 6,
-          "ms": 7,
-          "mu": 8,
-          "sa": 9
-}
-public=True
-
-tempdir = tempfile.gettempdir()
-
-tts = syn.Synthesizer()
-
-
-
-def speak(text, language,speaker,l_weight, s_weight, pace): #pitch_shift,pitch_std):
-
-
-
-    # text frontend not implemented...
-    text = text.replace("...", "…")
-    print(speakers[speaker])
-    audio = tts.speak(text, output_file=f'{tempdir}/tmp', lang=languages[language],
-                      spkr=speakers[speaker], l_weight=l_weight, s_weight=s_weight,
-                      pace=pace)
-
-    if not public:
-        try:
-            os.system("play "+tempdir+"/tmp.wav &")
-        except:
-            pass
-
-    return (22050, audio)
-
-
-
-controls = []
-controls.append(gr.Textbox(label="text", value="Suohtas duinna deaivvadit."))
-controls.append(gr.Dropdown(list(languages.keys()), label="language", value="North Sámi"))
-controls.append(gr.Dropdown(list(speakers.keys()), label="speaker", value="ms"))
-
-controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1, label="Language weight"))
-controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1, label="Speaker weight"))
-#controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1.0, label="Pitch variance"))
-controls.append(gr.Slider(minimum=0.5, maximum=1.5, step=0.05, value=1.0, label="speech rate"))
-
-
-
-
-tts_gui = gr.Interface(
-    fn=speak,
-    inputs=controls,
-    outputs= gr.Audio(label="output"),
-    live=False
-
-)
-
-
-if __name__ == "__main__":
-    tts_gui.launch(share=public)

prepare_dataset.py

@@ -1,180 +0,0 @@
-# *****************************************************************************
-#  Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions are met:
-#      * Redistributions of source code must retain the above copyright
-#        notice, this list of conditions and the following disclaimer.
-#      * Redistributions in binary form must reproduce the above copyright
-#        notice, this list of conditions and the following disclaimer in the
-#        documentation and/or other materials provided with the distribution.
-#      * Neither the name of the NVIDIA CORPORATION nor the
-#        names of its contributors may be used to endorse or promote products
-#        derived from this software without specific prior written permission.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-#  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-#  DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
-#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-#  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-#  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# *****************************************************************************
-
-import argparse
-import time
-from pathlib import Path
-
-import torch
-import tqdm
-import dllogger as DLLogger
-from dllogger import StdOutBackend, JSONStreamBackend, Verbosity
-from torch.utils.data import DataLoader
-
-from fastpitch.data_function import TTSCollate, TTSDataset
-
-
-def parse_args(parser):
-    """
-    Parse commandline arguments.
-    """
-    parser.add_argument('-d', '--dataset-path', type=str,
-                        default='./', help='Path to dataset')
-    parser.add_argument('--wav-text-filelists', required=True, nargs='+',
-                        type=str, help='Files with audio paths and text')
-    parser.add_argument('--extract-mels', action='store_true',
-                        help='Calculate spectrograms from .wav files')
-    parser.add_argument('--extract-pitch', action='store_true',
-                        help='Extract pitch')
-    parser.add_argument('--save-alignment-priors', action='store_true',
-                        help='Pre-calculate diagonal matrices of alignment of text to audio')
-    parser.add_argument('--log-file', type=str, default='preproc_log.json',
-                         help='Filename for logging')
-    parser.add_argument('--n-speakers', type=int, default=1)
-    parser.add_argument('--n-languages', type=int, default=1)
-    # Mel extraction
-    parser.add_argument('--max-wav-value', default=32768.0, type=float,
-                        help='Maximum audiowave value')
-    parser.add_argument('--sampling-rate', default=22050, type=int,
-                        help='Sampling rate')
-    parser.add_argument('--filter-length', default=1024, type=int,
-                        help='Filter length')
-    parser.add_argument('--hop-length', default=256, type=int,
-                        help='Hop (stride) length')
-    parser.add_argument('--win-length', default=1024, type=int,
-                        help='Window length')
-    parser.add_argument('--mel-fmin', default=0.0, type=float,
-                        help='Minimum mel frequency')
-    parser.add_argument('--mel-fmax', default=8000.0, type=float,
-                        help='Maximum mel frequency')
-    parser.add_argument('--n-mel-channels', type=int, default=80)
-    # Pitch extraction
-    parser.add_argument('--f0-method', default='pyin', type=str,
-                        choices=['pyin'], help='F0 estimation method')
-    parser.add_argument('--pitch-mean', default='214', type=float, ###
-                        help='F0 estimation method')
-    parser.add_argument('--pitch-std', default='65', type=float, ####
-                        help='F0 estimation method')
-    # Performance
-    parser.add_argument('-b', '--batch-size', default=1, type=int)
-    parser.add_argument('--n-workers', type=int, default=16)
-    return parser
-
-
-def main():
-    parser = argparse.ArgumentParser(description='FastPitch Data Pre-processing')
-    parser = parse_args(parser)
-    args, unk_args = parser.parse_known_args()
-    if len(unk_args) > 0:
-        raise ValueError(f'Invalid options {unk_args}')
-
-    DLLogger.init(backends=[JSONStreamBackend(Verbosity.DEFAULT, Path(args.dataset_path, args.log_file)),
-                            StdOutBackend(Verbosity.VERBOSE)])
-    for k, v in vars(args).items():
-        DLLogger.log(step="PARAMETER", data={k: v})
-    DLLogger.flush()
-
-    if args.extract_mels:
-        Path(args.dataset_path, 'mels').mkdir(parents=False, exist_ok=True)
-
-    if args.extract_pitch:
-        Path(args.dataset_path, 'pitch').mkdir(parents=False, exist_ok=True)
-
-    if args.save_alignment_priors:
-        Path(args.dataset_path, 'alignment_priors').mkdir(parents=False, exist_ok=True)
-
-    for filelist in args.wav_text_filelists:
-
-        print(f'Processing {filelist}...')
-
-        dataset = TTSDataset(
-            args.dataset_path,
-            filelist,
-            text_cleaners=['basic_cleaners'],
-            n_mel_channels=args.n_mel_channels,
-            p_arpabet=0.0,
-            n_speakers=args.n_speakers,
-            n_languages=args.n_languages,
-            load_mel_from_disk=False,
-            load_pitch_from_disk=False,
-            pitch_mean=args.pitch_mean,
-            pitch_std=args.pitch_std,
-            max_wav_value=args.max_wav_value,
-            sampling_rate=args.sampling_rate,
-            filter_length=args.filter_length,
-            hop_length=args.hop_length,
-            win_length=args.win_length,
-            mel_fmin=args.mel_fmin,
-            mel_fmax=args.mel_fmax,
-            betabinomial_online_dir=None,
-            pitch_online_dir=None,
-            pitch_online_method=args.f0_method)
-
-        data_loader = DataLoader(
-            dataset,
-            batch_size=args.batch_size,
-            shuffle=False,
-            sampler=None,
-            num_workers=args.n_workers,
-            collate_fn=TTSCollate(),
-            pin_memory=False,
-            drop_last=False)
-
-        all_filenames = set()
-        for i, batch in enumerate(tqdm.tqdm(data_loader)):
-            tik = time.time()
-
-            _, input_lens, mels, mel_lens, _, pitch, _, _, _, attn_prior, fpaths = batch
-
-            # Ensure filenames are unique
-            for p in fpaths:
-                fname = Path(p).name
-                if fname in all_filenames:
-                    raise ValueError(f'Filename is not unique: {fname}')
-                all_filenames.add(fname)
-
-            if args.extract_mels:
-                for j, mel in enumerate(mels):
-                    fname = Path(fpaths[j]).with_suffix('.pt').name
-                    fpath = Path(args.dataset_path, 'mels', fname)
-                    torch.save(mel[:, :mel_lens[j]], fpath)
-
-            if args.extract_pitch:
-                for j, p in enumerate(pitch):
-                    fname = Path(fpaths[j]).with_suffix('.pt').name
-                    fpath = Path(args.dataset_path, 'pitch', fname)
-                    torch.save(p[:mel_lens[j]], fpath)
-
-            if args.save_alignment_priors:
-                for j, prior in enumerate(attn_prior):
-                    fname = Path(fpaths[j]).with_suffix('.pt').name
-                    fpath = Path(args.dataset_path, 'alignment_priors', fname)
-                    torch.save(prior[:mel_lens[j], :input_lens[j]], fpath)
-
-
-if __name__ == '__main__':
-    main()

run_training_cluster_s.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=train_fastpitch
-#SBATCH --account=nn9866k
-#SBATCH --time=11:50:00
-#SBATCH --mem=16G
-#SBATCH --partition=accel
-#SBATCH --gres=gpu:1
-
-# == Logging
-
-#SBATCH --error="log_err" # Save the error messages
-#SBATCH --output="log_out" # Save the stdout
-
-## Set up job environment:
-# set -o errexit  # Exit the script on any error
-# set -o nounset  # Treat any unset variables as an error
-
-## Activate environment
-# source ~/.bashrc
-
-eval "$(conda shell.bash hook)"
-conda activate fastpitch
-
-# Setup monitoring
-nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory \
-        --format=csv --loop=1 > "gpu_util-$SLURM_JOB_ID.csv" &
-NVIDIA_MONITOR_PID=$!  # Capture PID of monitoring process
-
-# Run our computation
-bash scripts/train_2.sh
-
-# After computation stop monitoring
-kill -SIGINT "$NVIDIA_MONITOR_PID"

scripts/docker/build.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-docker build . -t fastpitch:latest

scripts/docker/interactive.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-PORT=${PORT:-8888}
-
-docker run --gpus=all -it --rm -e CUDA_VISIBLE_DEVICES --ipc=host -p $PORT:$PORT -v $PWD:/workspace/fastpitch/ fastpitch:latest bash 

scripts/download_cmudict.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-: ${CMUDICT_DIR:="cmudict"}
-
-if [ ! -f $CMUDICT_DIR/cmudict-0.7b ]; then
-  echo "Downloading cmudict-0.7b ..."
-  wget https://github.com/Alexir/CMUdict/raw/master/cmudict-0.7b -qO $CMUDICT_DIR/cmudict-0.7b
-fi

scripts/download_dataset.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-scripts/download_cmudict.sh
-
-DATA_DIR="LJSpeech-1.1"
-LJS_ARCH="LJSpeech-1.1.tar.bz2"
-LJS_URL="http://data.keithito.com/data/speech/${LJS_ARCH}"
-
-if [ ! -d ${DATA_DIR} ]; then
-  echo "Downloading ${LJS_ARCH} ..."
-  wget -q ${LJS_URL}
-  echo "Extracting ${LJS_ARCH} ..."
-  tar jxvf ${LJS_ARCH}
-  rm -f ${LJS_ARCH}
-fi

scripts/download_models.sh

@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-MODEL_NAMES="$@"
-[ -z "$MODEL_NAMES" ] && { echo "Usage: $0 [fastpitch|waveglow|hifigan|hifigan-finetuned-fastpitch]"; exit 1; }
-
-function download_ngc_model() {
-  mkdir -p "$MODEL_DIR"
-
-  if [ ! -f "${MODEL_DIR}/${MODEL_ZIP}" ]; then
-    echo "Downloading ${MODEL_ZIP} ..."
-    wget --content-disposition -O ${MODEL_DIR}/${MODEL_ZIP} ${MODEL_URL} \
-         || { echo "ERROR: Failed to download ${MODEL_ZIP} from NGC"; exit 1; }
-  fi
-
-  if [ ! -f "${MODEL_DIR}/${MODEL}" ]; then
-    echo "Extracting ${MODEL} ..."
-    unzip -qo ${MODEL_DIR}/${MODEL_ZIP} -d ${MODEL_DIR} \
-          || { echo "ERROR: Failed to extract ${MODEL_ZIP}"; exit 1; }
-
-    echo "OK"
-
-  else
-    echo "${MODEL} already downloaded."
-  fi
-
-}
-
-for MODEL_NAME in $MODEL_NAMES
-do
-  case $MODEL_NAME in
-    "fastpitch")
-      MODEL_DIR="pretrained_models/fastpitch"
-      MODEL_ZIP="fastpitch_pyt_fp32_ckpt_v1_1_21.05.0.zip"
-      MODEL="nvidia_fastpitch_210824.pt"
-      MODEL_URL="https://api.ngc.nvidia.com/v2/models/nvidia/fastpitch_pyt_fp32_ckpt_v1_1/versions/21.05.0/zip"
-      ;;
-    "hifigan")
-      MODEL_DIR="pretrained_models/hifigan"
-      MODEL_ZIP="hifigan__pyt_ckpt_ds-ljs22khz_21.08.0_amp.zip"
-      MODEL="hifigan_gen_checkpoint_6500.pt"
-      MODEL_URL="https://api.ngc.nvidia.com/v2/models/nvidia/dle/hifigan__pyt_ckpt_ds-ljs22khz/versions/21.08.0_amp/zip"
-      ;;
-    "hifigan-finetuned-fastpitch")
-      MODEL_DIR="pretrained_models/hifigan"
-      MODEL_ZIP="hifigan__pyt_ckpt_mode-finetune_ds-ljs22khz_21.08.0_amp.zip"
-      MODEL="hifigan_gen_checkpoint_10000_ft.pt"
-      MODEL_URL="https://api.ngc.nvidia.com/v2/models/nvidia/dle/hifigan__pyt_ckpt_mode-finetune_ds-ljs22khz/versions/21.08.0_amp/zip"
-      ;;
-    "waveglow")
-      MODEL_DIR="pretrained_models/waveglow"
-      MODEL_ZIP="waveglow_ckpt_amp_256_20.01.0.zip"
-      MODEL="nvidia_waveglow256pyt_fp16.pt"
-      MODEL_URL="https://api.ngc.nvidia.com/v2/models/nvidia/waveglow_ckpt_amp_256/versions/20.01.0/zip"
-      ;;
-    *)
-      echo "Unrecognized model: ${MODEL_NAME}"
-      exit 2
-      ;;
-  esac
-  download_ngc_model "$MODEL_DIR" "$MODEL_ZIP" "$MODEL" "$MODEL_URL"
-done

scripts/inference_benchmark.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -a
-
-: ${FILELIST:="phrases/benchmark_8_128.tsv"}
-: ${OUTPUT_DIR:="./output/audio_$(basename ${FILELIST} .tsv)"}
-: ${TORCHSCRIPT:=true}
-: ${BS_SEQUENCE:="1 4 8"}
-: ${WARMUP:=64}
-: ${REPEATS:=500}
-: ${AMP:=false}
-
-for BATCH_SIZE in $BS_SEQUENCE ; do
-    LOG_FILE="$OUTPUT_DIR"/perf-infer_amp-${AMP}_bs${BATCH_SIZE}.json
-    bash scripts/inference_example.sh "$@"
-done

scripts/inference_example.sh

@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-
-export CUDNN_V8_API_ENABLED=1  # Keep the flag for older containers
-export TORCH_CUDNN_V8_API_ENABLED=1
-
-: ${DATASET_DIR:="sander_splits"}
-: ${BATCH_SIZE:=1}
-: ${FILELIST:="phrases/giehttjit.txt"}
-: ${AMP:=false}
-: ${TORCHSCRIPT:=true}
-: ${WARMUP:=0}
-: ${REPEATS:=1}
-: ${CPU:=false}
-: ${PHONE:=true}
-
-# Paths to pre-trained models downloadable from NVIDIA NGC (LJSpeech-1.1)
-FASTPITCH_LJ="output/FastPitch_checkpoint_660.pt"
-HIFIGAN_LJ="pretrained_models/hifigan/hifigan_gen_checkpoint_10000_ft.pt"
-WAVEGLOW_LJ="pretrained_models/waveglow/nvidia_waveglow256pyt_fp16.pt"
-
-# Mel-spectrogram generator (optional; can synthesize from ground-truth spectrograms)
-: ${FASTPITCH=$FASTPITCH_LJ}
-
-# Vocoder (set only one)
-#: ${HIFIGAN=$HIFIGAN_LJ}
-: ${WAVEGLOW=$WAVEGLOW_LJ}
-
-[[ "$FASTPITCH" == "$FASTPITCH_LJ" && ! -f "$FASTPITCH" ]] && { echo "Downloading $FASTPITCH from NGC..."; bash scripts/download_models.sh fastpitch; }
-[[ "$WAVEGLOW" == "$WAVEGLOW_LJ" && ! -f "$WAVEGLOW" ]] && { echo "Downloading $WAVEGLOW from NGC..."; bash scripts/download_models.sh waveglow; }
-[[ "$HIFIGAN" == "$HIFIGAN_LJ" && ! -f "$HIFIGAN" ]] && { echo "Downloading $HIFIGAN from NGC..."; bash scripts/download_models.sh hifigan-finetuned-fastpitch; }
-
-if [[ "$HIFIGAN" == "$HIFIGAN_LJ" && "$FASTPITCH" != "$FASTPITCH_LJ" ]]; then
-    echo -e "\nNOTE: Using HiFi-GAN checkpoint trained for the LJSpeech-1.1 dataset."
-    echo -e "NOTE: If you're using a different dataset, consider training a new HiFi-GAN model or switch to WaveGlow."
-    echo -e "NOTE: See $0 for details.\n"
-fi
-
-# Synthesis
-: ${SPEAKER:=0}
-: ${DENOISING:=0.01}
-
-if [ ! -n "$OUTPUT_DIR" ]; then
-    OUTPUT_DIR="./output/audio_$(basename ${FILELIST} .tsv)"
-    [ "$AMP" = true ]     && OUTPUT_DIR+="_fp16"
-    [ "$AMP" = false ]    && OUTPUT_DIR+="_fp32"
-    [ -n "$FASTPITCH" ]   && OUTPUT_DIR+="_fastpitch"
-    [ ! -n "$FASTPITCH" ] && OUTPUT_DIR+="_gt-mel"
-    [ -n "$WAVEGLOW" ]    && OUTPUT_DIR+="_waveglow"
-    [ -n "$HIFIGAN" ]     && OUTPUT_DIR+="_hifigan"
-    OUTPUT_DIR+="_denoise-"${DENOISING}
-fi
-: ${LOG_FILE:="$OUTPUT_DIR/nvlog_infer.json"}
-mkdir -p "$OUTPUT_DIR"
-
-echo -e "\nAMP=$AMP, batch_size=$BATCH_SIZE\n"
-
-ARGS=""
-ARGS+=" --cuda"
-# ARGS+=" --cudnn-benchmark"  # Enable for benchmarking or long operation
-ARGS+=" --dataset-path $DATASET_DIR"
-ARGS+=" -i $FILELIST"
-ARGS+=" -o $OUTPUT_DIR"
-ARGS+=" --log-file $LOG_FILE"
-ARGS+=" --batch-size $BATCH_SIZE"
-ARGS+=" --denoising-strength $DENOISING"
-ARGS+=" --warmup-steps $WARMUP"
-ARGS+=" --repeats $REPEATS"
-ARGS+=" --speaker $SPEAKER"
-[ "$CPU" = false ]        && ARGS+=" --cuda"
-[ "$CPU" = false ]        && ARGS+=" --cudnn-benchmark"
-[ "$AMP" = true ]         && ARGS+=" --amp"
-[ "$TORCHSCRIPT" = true ] && ARGS+=" --torchscript"
-[ -n "$HIFIGAN" ]         && ARGS+=" --hifigan $HIFIGAN"
-[ -n "$WAVEGLOW" ]        && ARGS+=" --waveglow $WAVEGLOW"
-[ -n "$FASTPITCH" ]       && ARGS+=" --fastpitch $FASTPITCH"
-[ "$PHONE" = true ]       && ARGS+=" --p-arpabet 1.0"
-
-python inference.py $ARGS "$@"

scripts/prepare_dataset.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-: ${DATA_DIR:=ALL_SAMI}
-: ${ARGS="--extract-mels"}
-
-python prepare_dataset.py \
-    --wav-text-filelists filelists/all_sami_filelist_shuf_200_train.txt \
-    --n-workers 8 \
-    --batch-size 1 \
-    --dataset-path $DATA_DIR \
-    --extract-pitch \
-    --f0-method pyin \
-    --pitch_mean 150\
-    --pitch_std 40\
-    --n-speakers 10 \
-    --n-languages 3 \
-    $ARGS

scripts/train.sh

@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-
-export OMP_NUM_THREADS=1
-
-: ${NUM_GPUS:=8}
-: ${BATCH_SIZE:=16}
-: ${GRAD_ACCUMULATION:=2}
-: ${OUTPUT_DIR:="./output"}
-: ${LOG_FILE:=$OUTPUT_DIR/nvlog.json}
-: ${DATASET_PATH:=LJSpeech-1.1}
-: ${TRAIN_FILELIST:=filelists/ljs_audio_pitch_text_train_v3.txt}
-: ${VAL_FILELIST:=filelists/ljs_audio_pitch_text_val.txt}
-: ${AMP:=false}
-: ${SEED:=""}
-
-: ${LEARNING_RATE:=0.1}
-
-# Adjust these when the amount of data changes
-: ${EPOCHS:=1000}
-: ${EPOCHS_PER_CHECKPOINT:=20}
-: ${WARMUP_STEPS:=1000}
-: ${KL_LOSS_WARMUP:=100}
-
-# Train a mixed phoneme/grapheme model
-: ${PHONE:=true}
-# Enable energy conditioning
-: ${ENERGY:=true}
-: ${TEXT_CLEANERS:=english_cleaners_v2}
-# Add dummy space prefix/suffix is audio is not precisely trimmed
-: ${APPEND_SPACES:=false}
-
-: ${LOAD_PITCH_FROM_DISK:=true}
-: ${LOAD_MEL_FROM_DISK:=false}
-
-# For multispeaker models, add speaker ID = {0, 1, ...} as the last filelist column
-: ${NSPEAKERS:=1}
-: ${SAMPLING_RATE:=22050}
-
-# Adjust env variables to maintain the global batch size: NUM_GPUS x BATCH_SIZE x GRAD_ACCUMULATION = 256.
-GBS=$(($NUM_GPUS * $BATCH_SIZE * $GRAD_ACCUMULATION))
-[ $GBS -ne 256 ] && echo -e "\nWARNING: Global batch size changed from 256 to ${GBS}."
-echo -e "\nAMP=$AMP, ${NUM_GPUS}x${BATCH_SIZE}x${GRAD_ACCUMULATION}" \
-        "(global batch size ${GBS})\n"
-
-ARGS=""
-ARGS+=" --cuda"
-ARGS+=" -o $OUTPUT_DIR"
-ARGS+=" --log-file $LOG_FILE"
-ARGS+=" --dataset-path $DATASET_PATH"
-ARGS+=" --training-files $TRAIN_FILELIST"
-ARGS+=" --validation-files $VAL_FILELIST"
-ARGS+=" -bs $BATCH_SIZE"
-ARGS+=" --grad-accumulation $GRAD_ACCUMULATION"
-ARGS+=" --optimizer lamb"
-ARGS+=" --epochs $EPOCHS"
-ARGS+=" --epochs-per-checkpoint $EPOCHS_PER_CHECKPOINT"
-ARGS+=" --resume"
-ARGS+=" --warmup-steps $WARMUP_STEPS"
-ARGS+=" -lr $LEARNING_RATE"
-ARGS+=" --weight-decay 1e-6"
-ARGS+=" --grad-clip-thresh 1000.0"
-ARGS+=" --dur-predictor-loss-scale 0.1"
-ARGS+=" --pitch-predictor-loss-scale 0.1"
-ARGS+=" --trainloader-repeats 100"
-ARGS+=" --validation-freq 10"
-
-# Autoalign & new features
-ARGS+=" --kl-loss-start-epoch 0"
-ARGS+=" --kl-loss-warmup-epochs $KL_LOSS_WARMUP"
-ARGS+=" --text-cleaners $TEXT_CLEANERS"
-ARGS+=" --n-speakers $NSPEAKERS"
-
-[ "$AMP" = "true" ]                && ARGS+=" --amp"
-[ "$PHONE" = "true" ]              && ARGS+=" --p-arpabet 1.0"
-[ "$ENERGY" = "true" ]             && ARGS+=" --energy-conditioning"
-[ "$SEED" != "" ]                  && ARGS+=" --seed $SEED"
-[ "$LOAD_MEL_FROM_DISK" = true ]   && ARGS+=" --load-mel-from-disk"
-[ "$LOAD_PITCH_FROM_DISK" = true ] && ARGS+=" --load-pitch-from-disk"
-[ "$PITCH_ONLINE_DIR" != "" ]      && ARGS+=" --pitch-online-dir $PITCH_ONLINE_DIR"  # e.g., /dev/shm/pitch
-[ "$PITCH_ONLINE_METHOD" != "" ]   && ARGS+=" --pitch-online-method $PITCH_ONLINE_METHOD"
-[ "$APPEND_SPACES" = true ]        && ARGS+=" --prepend-space-to-text"
-[ "$APPEND_SPACES" = true ]        && ARGS+=" --append-space-to-text"
-
-if [ "$SAMPLING_RATE" == "44100" ]; then
-  ARGS+=" --sampling-rate 44100"
-  ARGS+=" --filter-length 2048"
-  ARGS+=" --hop-length 512"
-  ARGS+=" --win-length 2048"
-  ARGS+=" --mel-fmin 0.0"
-  ARGS+=" --mel-fmax 22050.0"
-
-elif [ "$SAMPLING_RATE" != "22050" ]; then
-  echo "Unknown sampling rate $SAMPLING_RATE"
-  exit 1
-fi
-
-mkdir -p "$OUTPUT_DIR"
-
-: ${DISTRIBUTED:="-m torch.distributed.launch --nproc_per_node $NUM_GPUS"}
-python $DISTRIBUTED train.py $ARGS "$@"

scripts/train_multilang.sh

@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-
-export OMP_NUM_THREADS=1
-
-: ${NUM_GPUS:=1}
-: ${BATCH_SIZE:=1}
-: ${GRAD_ACCUMULATION:=32} 
-: ${OUTPUT_DIR:="./output_multilang"}
-: ${LOG_FILE:=$OUTPUT_DIR/nvlog.json}
-: ${DATASET_PATH:=ALL_SAMI}
-#: ${DATASET_PATH:=mikal_urheim}
-#: ${TRAIN_FILELIST:=filelists/smj_sander_text_noshorts_shuff_pitch.txt}
-#: ${TRAIN_FILELIST:=filelists/mikal_urheim_pitch_shuf.txt}
-: ${TRAIN_FILELIST:=filelists/all_sami_filelist_shuf_200_train.txt}
-#: ${VAL_FILELIST:=filelists/smj_sander_text_noshorts_shuff_val_pitch.txt}
-#: ${VAL_FILELIST:=filelists/mikal_urheim_pitch_shuf_val.txt}
-: ${VAL_FILELIST:=filelists/all_sami_filelist_shuf_200_val.txt}
-: ${AMP:=false}
-: ${SEED:=""}
-
-: ${LEARNING_RATE:=0.1}
-
-# Adjust these when the amount of data changes
-: ${EPOCHS:=1000}
-: ${EPOCHS_PER_CHECKPOINT:=10}
-: ${WARMUP_STEPS:=1000}
-: ${KL_LOSS_WARMUP:=100}
-
-# Train a mixed phoneme/grapheme model
-: ${PHONE:=false}
-# Enable energy conditioning
-: ${ENERGY:=true}
-: ${TEXT_CLEANERS:=basic_cleaners}
-: ${SYMBOL_SET:=all_sami}
-# Add dummy space prefix/suffix is audio is not precisely trimmed
-: ${APPEND_SPACES:=false}
-
-: ${LOAD_PITCH_FROM_DISK:=true} # was true
-: ${LOAD_MEL_FROM_DISK:=false}
-
-# For multispeaker models, add speaker ID = {0, 1, ...} as the last filelist column
-: ${NSPEAKERS:=10} # 10
-: ${NLANGUAGES:=3} # 3
-: ${SAMPLING_RATE:=22050}
-
-# Adjust env variables to maintain the global batch size: NUM_GPUS x BATCH_SIZE x GRAD_ACCUMULATION = 256.
-GBS=$(($NUM_GPUS * $BATCH_SIZE * $GRAD_ACCUMULATION))
-[ $GBS -ne 256 ] && echo -e "\nWARNING: Global batch size changed from 256 to ${GBS}."
-echo -e "\nAMP=$AMP, ${NUM_GPUS}x${BATCH_SIZE}x${GRAD_ACCUMULATION}" \
-        "(global batch size ${GBS})\n"
-
-ARGS=""
-ARGS+=" --cuda"
-ARGS+=" -o $OUTPUT_DIR"
-ARGS+=" --log-file $LOG_FILE"
-ARGS+=" --dataset-path $DATASET_PATH"
-ARGS+=" --training-files $TRAIN_FILELIST"
-ARGS+=" --validation-files $VAL_FILELIST"
-ARGS+=" -bs $BATCH_SIZE"
-ARGS+=" --grad-accumulation $GRAD_ACCUMULATION"
-ARGS+=" --optimizer lamb" #adam
-ARGS+=" --epochs $EPOCHS"
-ARGS+=" --epochs-per-checkpoint $EPOCHS_PER_CHECKPOINT"
-ARGS+=" --resume"
-ARGS+=" --warmup-steps $WARMUP_STEPS"
-ARGS+=" -lr $LEARNING_RATE"
-ARGS+=" --weight-decay 1e-6"
-ARGS+=" --grad-clip-thresh 1000.0"
-ARGS+=" --dur-predictor-loss-scale 0.1"
-ARGS+=" --pitch-predictor-loss-scale 0.1"
-ARGS+=" --trainloader-repeats 100"
-ARGS+=" --validation-freq 1" #10
-
-# Autoalign & new features
-ARGS+=" --kl-loss-start-epoch 0"
-ARGS+=" --kl-loss-warmup-epochs $KL_LOSS_WARMUP"
-ARGS+=" --text-cleaners $TEXT_CLEANERS"
-ARGS+=" --n-speakers $NSPEAKERS"
-ARGS+=" --n-languages $NLANGUAGES"
-ARGS+=" --symbol-set $SYMBOL_SET"
-
-[ "$AMP" = "true" ]                && ARGS+=" --amp"
-[ "$PHONE" = "true" ]              && ARGS+=" --p-arpabet 1.0"
-[ "$ENERGY" = "true" ]             && ARGS+=" --energy-conditioning"
-[ "$SEED" != "" ]                  && ARGS+=" --seed $SEED"
-[ "$LOAD_MEL_FROM_DISK" = true ]   && ARGS+=" --load-mel-from-disk"
-[ "$LOAD_PITCH_FROM_DISK" = true ] && ARGS+=" --load-pitch-from-disk"
-[ "$PITCH_ONLINE_DIR" != "" ]      && ARGS+=" --pitch-online-dir $PITCH_ONLINE_DIR"  # e.g., /dev/shm/pitch
-[ "$PITCH_ONLINE_METHOD" != "" ]   && ARGS+=" --pitch-online-method $PITCH_ONLINE_METHOD"
-[ "$APPEND_SPACES" = true ]        && ARGS+=" --prepend-space-to-text"
-[ "$APPEND_SPACES" = true ]        && ARGS+=" --append-space-to-text"
-
-if [ "$SAMPLING_RATE" == "44100" ]; then
-  ARGS+=" --sampling-rate 44100"
-  ARGS+=" --filter-length 2048"
-  ARGS+=" --hop-length 512"
-  ARGS+=" --win-length 2048"
-  ARGS+=" --mel-fmin 0.0"
-  ARGS+=" --mel-fmax 22050.0"
-
-elif [ "$SAMPLING_RATE" != "22050" ]; then
-  echo "Unknown sampling rate $SAMPLING_RATE"
-  exit 1
-fi
-
-mkdir -p "$OUTPUT_DIR"
-
-: ${DISTRIBUTED:="-m torch.distributed.launch --nproc_per_node $NUM_GPUS"}
-#python $DISTRIBUTED train.py $ARGS "$@"
-python train_1_with_plot_multilang.py $ARGS "$@"

train_1_with_plot_multilang.py

@@ -1,593 +0,0 @@
-# *****************************************************************************
-#  Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions are met:
-#      * Redistributions of source code must retain the above copyright
-#        notice, this list of conditions and the following disclaimer.
-#      * Redistributions in binary form must reproduce the above copyright
-#        notice, this list of conditions and the following disclaimer in the
-#        documentation and/or other materials provided with the distribution.
-#      * Neither the name of the NVIDIA CORPORATION nor the
-#        names of its contributors may be used to endorse or promote products
-#        derived from this software without specific prior written permission.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-#  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-#  DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
-#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-#  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-#  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# *****************************************************************************
-
-import argparse
-import copy
-import os
-import time
-from collections import defaultdict, OrderedDict
-from itertools import cycle
-
-import numpy as np
-import torch
-import torch.distributed as dist
-import amp_C
-from apex.optimizers import FusedAdam, FusedLAMB
-from torch.nn.parallel import DistributedDataParallel
-from torch.utils.data import DataLoader
-from torch.utils.data.distributed import DistributedSampler
-
-import common.tb_dllogger as logger
-import models
-from common.tb_dllogger import log
-from common.repeated_dataloader import (RepeatedDataLoader,
-                                        RepeatedDistributedSampler)
-from common.text import cmudict
-from common.utils import BenchmarkStats, Checkpointer, prepare_tmp
-from fastpitch.attn_loss_function import AttentionBinarizationLoss
-from fastpitch.data_function import batch_to_gpu, TTSCollate, TTSDataset
-from fastpitch.loss_function import FastPitchLoss
-
-import matplotlib.pyplot as plt
-
-def parse_args(parser):
-    parser.add_argument('-o', '--output', type=str, required=True,
-                        help='Directory to save checkpoints')
-    parser.add_argument('-d', '--dataset-path', type=str, default='./',
-                        help='Path to dataset')
-    parser.add_argument('--log-file', type=str, default=None,
-                        help='Path to a DLLogger log file')
-
-    train = parser.add_argument_group('training setup')
-    train.add_argument('--epochs', type=int, required=True,
-                       help='Number of total epochs to run')
-    train.add_argument('--epochs-per-checkpoint', type=int, default=50,
-                       help='Number of epochs per checkpoint')
-    train.add_argument('--checkpoint-path', type=str, default=None,
-                       help='Checkpoint path to resume training')
-    train.add_argument('--keep-milestones', default=list(range(100, 1000, 100)),
-                       type=int, nargs='+',
-                       help='Milestone checkpoints to keep from removing')
-    train.add_argument('--resume', action='store_true',
-                       help='Resume training from the last checkpoint')
-    train.add_argument('--seed', type=int, default=1234,
-                       help='Seed for PyTorch random number generators')
-    train.add_argument('--amp', action='store_true',
-                       help='Enable AMP')
-    train.add_argument('--cuda', action='store_true',
-                       help='Run on GPU using CUDA')
-    train.add_argument('--cudnn-benchmark', action='store_true',
-                       help='Enable cudnn benchmark mode')
-    train.add_argument('--ema-decay', type=float, default=0,
-                       help='Discounting factor for training weights EMA')
-    train.add_argument('--grad-accumulation', type=int, default=1,
-                       help='Training steps to accumulate gradients for')
-    train.add_argument('--kl-loss-start-epoch', type=int, default=250,
-                       help='Start adding the hard attention loss term')
-    train.add_argument('--kl-loss-warmup-epochs', type=int, default=100,
-                       help='Gradually increase the hard attention loss term')
-    train.add_argument('--kl-loss-weight', type=float, default=1.0,
-                       help='Gradually increase the hard attention loss term')
-    train.add_argument('--benchmark-epochs-num', type=int, default=20,
-                        help='Number of epochs for calculating final stats')
-    train.add_argument('--validation-freq', type=int, default=1,
-                       help='Validate every N epochs to use less compute')
-
-    opt = parser.add_argument_group('optimization setup')
-    opt.add_argument('--optimizer', type=str, default='lamb',
-                     help='Optimization algorithm')
-    opt.add_argument('-lr', '--learning-rate', type=float, required=True,
-                     help='Learing rate')
-    opt.add_argument('--weight-decay', default=1e-6, type=float,
-                     help='Weight decay')
-    opt.add_argument('--grad-clip-thresh', default=1000.0, type=float,
-                     help='Clip threshold for gradients')
-    opt.add_argument('-bs', '--batch-size', type=int, required=True,
-                     help='Batch size per GPU')
-    opt.add_argument('--warmup-steps', type=int, default=1000,
-                     help='Number of steps for lr warmup')
-    opt.add_argument('--dur-predictor-loss-scale', type=float,
-                     default=1.0, help='Rescale duration predictor loss')
-    opt.add_argument('--pitch-predictor-loss-scale', type=float,
-                     default=1.0, help='Rescale pitch predictor loss')
-    opt.add_argument('--attn-loss-scale', type=float,
-                     default=1.0, help='Rescale alignment loss')
-
-    data = parser.add_argument_group('dataset parameters')
-    data.add_argument('--training-files', type=str, nargs='*', required=True,
-                      help='Paths to training filelists.')
-    data.add_argument('--validation-files', type=str, nargs='*',
-                      required=True, help='Paths to validation filelists')
-    data.add_argument('--text-cleaners', nargs='*',
-                      default=['english_cleaners'], type=str,
-                      help='Type of text cleaners for input text')
-    data.add_argument('--symbol-set', type=str, default='english_basic',
-                      help='Define symbol set for input text')
-    data.add_argument('--p-arpabet', type=float, default=0.0,
-                      help='Probability of using arpabets instead of graphemes '
-                           'for each word; set 0 for pure grapheme training')
-    data.add_argument('--heteronyms-path', type=str, default='cmudict/heteronyms',
-                      help='Path to the list of heteronyms')
-    data.add_argument('--cmudict-path', type=str, default='cmudict/cmudict-0.7b',
-                      help='Path to the pronouncing dictionary')
-    data.add_argument('--prepend-space-to-text', action='store_true',
-                      help='Capture leading silence with a space token')
-    data.add_argument('--append-space-to-text', action='store_true',
-                      help='Capture trailing silence with a space token')
-    data.add_argument('--num-workers', type=int, default=2, # 6
-                      help='Subprocesses for train and val DataLoaders')
-    data.add_argument('--trainloader-repeats', type=int, default=100,
-                      help='Repeats the dataset to prolong epochs')
-
-    cond = parser.add_argument_group('data for conditioning')
-    cond.add_argument('--n-speakers', type=int, default=1,
-                      help='Number of speakers in the dataset. '
-                           'n_speakers > 1 enables speaker embeddings')
-    # ANT: added language
-    cond.add_argument('--n-languages', type=int, default=1,
-                      help='Number of languages in the dataset. '
-                           'n_languages > 1 enables language embeddings')
-            
-    cond.add_argument('--load-pitch-from-disk', action='store_true',
-                      help='Use pitch cached on disk with prepare_dataset.py')
-    cond.add_argument('--pitch-online-method', default='pyin',
-                      choices=['pyin'],
-                      help='Calculate pitch on the fly during trainig')
-    cond.add_argument('--pitch-online-dir', type=str, default=None,
-                      help='A directory for storing pitch calculated on-line')
-    cond.add_argument('--pitch-mean', type=float, default=125.626816, #default=214.72203,
-                      help='Normalization value for pitch')
-    cond.add_argument('--pitch-std', type=float, default=37.52, #default=65.72038,
-                      help='Normalization value for pitch')
-    cond.add_argument('--load-mel-from-disk', action='store_true',
-                      help='Use mel-spectrograms cache on the disk')  # XXX
-
-    audio = parser.add_argument_group('audio parameters')
-    audio.add_argument('--max-wav-value', default=32768.0, type=float,
-                       help='Maximum audiowave value')
-    audio.add_argument('--sampling-rate', default=22050, type=int,
-                       help='Sampling rate')
-    audio.add_argument('--filter-length', default=1024, type=int,
-                       help='Filter length')
-    audio.add_argument('--hop-length', default=256, type=int,
-                       help='Hop (stride) length')
-    audio.add_argument('--win-length', default=1024, type=int,
-                       help='Window length')
-    audio.add_argument('--mel-fmin', default=0.0, type=float,
-                       help='Minimum mel frequency')
-    audio.add_argument('--mel-fmax', default=8000.0, type=float,
-                       help='Maximum mel frequency')
-
-    dist = parser.add_argument_group('distributed setup')
-    dist.add_argument('--local_rank', type=int, default=os.getenv('LOCAL_RANK', 0),
-                      help='Rank of the process for multiproc; do not set manually')
-    dist.add_argument('--world_size', type=int, default=os.getenv('WORLD_SIZE', 1),
-                      help='Number of processes for multiproc; do not set manually')
-    return parser
-
-
-def reduce_tensor(tensor, num_gpus):
-    rt = tensor.clone()
-    dist.all_reduce(rt, op=dist.ReduceOp.SUM)
-    return rt.true_divide(num_gpus)
-
-
-def init_distributed(args, world_size, rank):
-    assert torch.cuda.is_available(), "Distributed mode requires CUDA."
-    print("Initializing distributed training")
-
-    # Set cuda device so everything is done on the right GPU.
-    torch.cuda.set_device(rank % torch.cuda.device_count())
-
-    # Initialize distributed communication
-    dist.init_process_group(backend=('nccl' if args.cuda else 'gloo'),
-                            init_method='env://')
-    print("Done initializing distributed training")
-
-
-def validate(model, epoch, total_iter, criterion, val_loader, distributed_run,
-             batch_to_gpu, local_rank, ema=False):
-    was_training = model.training
-    model.eval()
-
-    tik = time.perf_counter()
-    with torch.no_grad():
-        val_meta = defaultdict(float)
-        val_num_frames = 0
-        for i, batch in enumerate(val_loader):
-            x, y, num_frames = batch_to_gpu(batch)
-            y_pred = model(x)
-            loss, meta = criterion(y_pred, y, is_training=False, meta_agg='sum')
-
-            if distributed_run:
-                for k, v in meta.items():
-                    val_meta[k] += reduce_tensor(v, 1)
-                val_num_frames += reduce_tensor(num_frames.data, 1).item()
-            else:
-                for k, v in meta.items():
-                    val_meta[k] += v
-                val_num_frames += num_frames.item()
-
-            # NOTE: ugly patch to visualize the first utterance of the validation corpus.
-            #       The goal is to determine if the training is progressing properly
-            if (i == 0) and (local_rank == 0) and (not ema):
-                # Plot some debug information
-                fig, axs = plt.subplots(2, 2, figsize=(21,14))
-
-                #  - Mel-spectrogram
-                pred_mel = y_pred[0][0, :, :].cpu().detach().numpy().astype(np.float32).T
-                orig_mel = y[0][0, :, :].cpu().detach().numpy().astype(np.float32)
-                axs[0,0].imshow(orig_mel, aspect='auto', origin='lower', interpolation='nearest')
-                axs[1,0].imshow(pred_mel, aspect='auto', origin='lower', interpolation='nearest')
-
-                # Prosody
-                f0_pred = y_pred[4][0, :].cpu().detach().numpy().astype(np.float32)
-                f0_ori = y_pred[5][0, :].cpu().detach().numpy().astype(np.float32)
-                axs[1,1].plot(f0_ori)
-                axs[1,1].plot(f0_pred)
-
-                # # Duration
-                # att_pred = y_pred[2][0, :].cpu().detach().numpy().astype(np.float32)
-                # att_ori = x[7][0,:].cpu().detach().numpy().astype(np.float32)
-                # axs[0,1].imshow(att_ori, aspect='auto', origin='lower', interpolation='nearest')
-
-                if not os.path.exists("debug_epoch/"):
-                    os.makedirs("debug_epoch_laila/")
-
-                fig.savefig(f'debug_epoch/{epoch:06d}.png', bbox_inches='tight')
-
-        val_meta = {k: v / len(val_loader.dataset) for k, v in val_meta.items()}
-
-    val_meta['took'] = time.perf_counter() - tik
-
-    log((epoch,) if epoch is not None else (), tb_total_steps=total_iter,
-        subset='val_ema' if ema else 'val',
-        data=OrderedDict([
-            ('loss', val_meta['loss'].item()),
-            ('mel_loss', val_meta['mel_loss'].item()),
-            ('frames/s', val_num_frames / val_meta['took']),
-            ('took', val_meta['took'])]),
-        )
-
-    if was_training:
-        model.train()
-    return val_meta
-
-
-def adjust_learning_rate(total_iter, opt, learning_rate, warmup_iters=None):
-    if warmup_iters == 0:
-        scale = 1.0
-    elif total_iter > warmup_iters:
-        scale = 1. / (total_iter ** 0.5)
-    else:
-        scale = total_iter / (warmup_iters ** 1.5)
-
-    for param_group in opt.param_groups:
-        param_group['lr'] = learning_rate * scale
-
-
-def apply_ema_decay(model, ema_model, decay):
-    if not decay:
-        return
-    st = model.state_dict()
-    add_module = hasattr(model, 'module') and not hasattr(ema_model, 'module')
-    for k, v in ema_model.state_dict().items():
-        if add_module and not k.startswith('module.'):
-            k = 'module.' + k
-        v.copy_(decay * v + (1 - decay) * st[k])
-
-
-def init_multi_tensor_ema(model, ema_model):
-    model_weights = list(model.state_dict().values())
-    ema_model_weights = list(ema_model.state_dict().values())
-    ema_overflow_buf = torch.cuda.IntTensor([0])
-    return model_weights, ema_model_weights, ema_overflow_buf
-
-
-def apply_multi_tensor_ema(decay, model_weights, ema_weights, overflow_buf):
-    amp_C.multi_tensor_axpby(
-        65536, overflow_buf, [ema_weights, model_weights, ema_weights],
-        decay, 1-decay, -1)
-
-
-def main():
-    parser = argparse.ArgumentParser(description='PyTorch FastPitch Training',
-                                     allow_abbrev=False)
-    parser = parse_args(parser)
-    args, _ = parser.parse_known_args()
-
-    if args.p_arpabet > 0.0:
-        cmudict.initialize(args.cmudict_path, args.heteronyms_path)
-
-    distributed_run = args.world_size > 1
-
-    torch.manual_seed(args.seed + args.local_rank)
-    np.random.seed(args.seed + args.local_rank)
-
-    if args.local_rank == 0:
-        if not os.path.exists(args.output):
-            os.makedirs(args.output)
-
-    log_fpath = args.log_file or os.path.join(args.output, 'nvlog.json')
-    tb_subsets = ['train', 'val']
-    if args.ema_decay > 0.0:
-        tb_subsets.append('val_ema')
-
-    logger.init(log_fpath, args.output, enabled=(args.local_rank == 0),
-                tb_subsets=tb_subsets)
-    logger.parameters(vars(args), tb_subset='train')
-
-    parser = models.parse_model_args('FastPitch', parser)
-    args, unk_args = parser.parse_known_args()
-    if len(unk_args) > 0:
-        raise ValueError(f'Invalid options {unk_args}')
-
-    torch.backends.cudnn.benchmark = args.cudnn_benchmark
-
-    if distributed_run:
-        init_distributed(args, args.world_size, args.local_rank)
-    else:
-        if args.trainloader_repeats > 1:
-            print('WARNING: Disabled --trainloader-repeats, supported only for'
-                  ' multi-GPU data loading.')
-            args.trainloader_repeats = 1
-
-    device = torch.device('cuda' if args.cuda else 'cpu')
-    model_config = models.get_model_config('FastPitch', args)
-    model = models.get_model('FastPitch', model_config, device)
-
-    attention_kl_loss = AttentionBinarizationLoss()
-
-    # Store pitch mean/std as params to translate from Hz during inference
-    model.pitch_mean[0] = args.pitch_mean
-    model.pitch_std[0] = args.pitch_std
-
-    kw = dict(lr=args.learning_rate, betas=(0.9, 0.98), eps=1e-9,
-              weight_decay=args.weight_decay)
-    if args.optimizer == 'adam':
-        optimizer = FusedAdam(model.parameters(), **kw)
-        # optimizer = torch.optim.Adam(model.parameters(), **kw)
-    elif args.optimizer == 'lamb':
-
-        optimizer = FusedLAMB(model.parameters(), **kw)
-        # optimizer = torch.optim.Adam(model.parameters(), **kw)
-    else:
-        raise ValueError
-
-    scaler = torch.cuda.amp.GradScaler(enabled=args.amp)
-
-    if args.ema_decay > 0:
-        ema_model = copy.deepcopy(model)
-    else:
-        ema_model = None
-
-    if distributed_run:
-        model = DistributedDataParallel(
-            model, device_ids=[args.local_rank], output_device=args.local_rank,
-            find_unused_parameters=True)
-
-    train_state = {'epoch': 1, 'total_iter': 1}
-    checkpointer = Checkpointer(args.output, args.keep_milestones)
-
-    checkpointer.maybe_load(model, optimizer, scaler, train_state, args,
-                            ema_model)
-
-    start_epoch = train_state['epoch']
-    total_iter = train_state['total_iter']
-
-    criterion = FastPitchLoss(
-        dur_predictor_loss_scale=args.dur_predictor_loss_scale,
-        pitch_predictor_loss_scale=args.pitch_predictor_loss_scale,
-        attn_loss_scale=args.attn_loss_scale)
-
-    collate_fn = TTSCollate()
-
-    if args.local_rank == 0:
-        prepare_tmp(args.pitch_online_dir)
-
-    trainset = TTSDataset(audiopaths_and_text=args.training_files, **vars(args))
-    valset = TTSDataset(audiopaths_and_text=args.validation_files, **vars(args))
-
-    if distributed_run:
-        train_sampler = RepeatedDistributedSampler(args.trainloader_repeats,
-                                                   trainset, drop_last=True)
-        val_sampler = DistributedSampler(valset)
-        shuffle = False
-    else:
-        train_sampler, val_sampler, shuffle = None, None, False ########### was True
-
-    # 4 workers are optimal on DGX-1 (from epoch 2 onwards)
-    kw = {'num_workers': args.num_workers, 'batch_size': args.batch_size,
-          'collate_fn': collate_fn}
-    train_loader = RepeatedDataLoader(args.trainloader_repeats, trainset,
-                                      shuffle=shuffle, drop_last=True,
-                                      sampler=train_sampler, pin_memory=True,
-                                      persistent_workers=True, **kw)
-    val_loader = DataLoader(valset, shuffle=False, sampler=val_sampler,
-                            pin_memory=False, **kw)
-    if args.ema_decay:
-        mt_ema_params = init_multi_tensor_ema(model, ema_model)
-
-    model.train()
-    bmark_stats = BenchmarkStats()
-
-    torch.cuda.synchronize()
-    for epoch in range(start_epoch, args.epochs + 1):
-        epoch_start_time = time.perf_counter()
-
-        epoch_loss = 0.0
-        epoch_mel_loss = 0.0
-        epoch_num_frames = 0
-        epoch_frames_per_sec = 0.0
-
-        if distributed_run:
-            train_loader.sampler.set_epoch(epoch)
-
-        iter_loss = 0
-        iter_num_frames = 0
-        iter_meta = {}
-        iter_start_time = time.perf_counter()
-
-        epoch_iter = 1
-        for batch, accum_step in zip(train_loader,
-                                     cycle(range(1, args.grad_accumulation + 1))):
-            if accum_step == 1:
-                adjust_learning_rate(total_iter, optimizer, args.learning_rate,
-                                     args.warmup_steps)
-
-                model.zero_grad(set_to_none=True)
-
-            x, y, num_frames = batch_to_gpu(batch)
-            
-            with torch.cuda.amp.autocast(enabled=args.amp):
-                y_pred = model(x)
-                loss, meta = criterion(y_pred, y)
-
-                if (args.kl_loss_start_epoch is not None
-                        and epoch >= args.kl_loss_start_epoch):
-
-                    if args.kl_loss_start_epoch == epoch and epoch_iter == 1:
-                        print('Begin hard_attn loss')
-
-                    _, _, _, _, _, _, _, _, attn_soft, attn_hard, _, _ = y_pred
-                    binarization_loss = attention_kl_loss(attn_hard, attn_soft)
-                    kl_weight = min((epoch - args.kl_loss_start_epoch) / args.kl_loss_warmup_epochs, 1.0) * args.kl_loss_weight
-                    meta['kl_loss'] = binarization_loss.clone().detach() * kl_weight
-                    loss += kl_weight * binarization_loss
-
-                else:
-                    meta['kl_loss'] = torch.zeros_like(loss)
-                    kl_weight = 0
-                    binarization_loss = 0
-
-                loss /= args.grad_accumulation
-            meta = {k: v / args.grad_accumulation
-                    for k, v in meta.items()}
-
-            if args.amp:
-                scaler.scale(loss).backward()
-            else:
-                loss.backward()
-
-            if distributed_run:
-                reduced_loss = reduce_tensor(loss.data, args.world_size).item()
-                reduced_num_frames = reduce_tensor(num_frames.data, 1).item()
-                meta = {k: reduce_tensor(v, args.world_size) for k, v in meta.items()}
-            else:
-                reduced_loss = loss.item()
-                reduced_num_frames = num_frames.item()
-            if np.isnan(reduced_loss):
-                raise Exception("loss is NaN")
-
-            iter_loss += reduced_loss
-            iter_num_frames += reduced_num_frames
-            iter_meta = {k: iter_meta.get(k, 0) + meta.get(k, 0) for k in meta}
-            if accum_step % args.grad_accumulation == 0:
-
-                logger.log_grads_tb(total_iter, model)
-                if args.amp:
-                    scaler.unscale_(optimizer)
-                    torch.nn.utils.clip_grad_norm_(
-                        model.parameters(), args.grad_clip_thresh)
-                    scaler.step(optimizer)
-                    scaler.update()
-                else:
-                    torch.nn.utils.clip_grad_norm_(
-                        model.parameters(), args.grad_clip_thresh)
-                    optimizer.step()
-
-                if args.ema_decay > 0.0:
-                    apply_multi_tensor_ema(args.ema_decay, *mt_ema_params)
-
-                iter_mel_loss = iter_meta['mel_loss'].item()
-                iter_kl_loss = iter_meta['kl_loss'].item()
-                iter_time = time.perf_counter() - iter_start_time
-                epoch_frames_per_sec += iter_num_frames / iter_time
-                epoch_loss += iter_loss
-                epoch_num_frames += iter_num_frames
-                epoch_mel_loss += iter_mel_loss
-                
-                num_iters = len(train_loader) // args.grad_accumulation
-                log((epoch, epoch_iter, num_iters), tb_total_steps=total_iter,
-                    subset='train', data=OrderedDict([
-                        ('loss', iter_loss),
-                        ('mel_loss', iter_mel_loss),
-                        ('kl_loss', iter_kl_loss),
-                        ('kl_weight', kl_weight),
-                        ('frames/s', iter_num_frames / iter_time),
-                        ('took', iter_time),
-                        ('lrate', optimizer.param_groups[0]['lr'])]),
-                )
-
-                iter_loss = 0
-                iter_num_frames = 0
-                iter_meta = {}
-                iter_start_time = time.perf_counter()
-
-                if epoch_iter == num_iters:
-                    break
-                epoch_iter += 1
-                total_iter += 1
-
-        # Finished epoch
-        epoch_loss /= epoch_iter
-        epoch_mel_loss /= epoch_iter
-        epoch_time = time.perf_counter() - epoch_start_time
-        log((epoch,), tb_total_steps=None, subset='train_avg',
-            data=OrderedDict([
-                ('loss', epoch_loss),
-                ('mel_loss', epoch_mel_loss),
-                ('frames/s', epoch_num_frames / epoch_time),
-                ('took', epoch_time)]),
-        )
-        bmark_stats.update(epoch_num_frames, epoch_loss, epoch_mel_loss,
-                           epoch_time)
-
-        if epoch % args.validation_freq == 0:
-            validate(model, epoch, total_iter, criterion, val_loader,
-                     distributed_run, batch_to_gpu, ema=False, local_rank=args.local_rank)
-
-            if args.ema_decay > 0:
-                validate(ema_model, epoch, total_iter, criterion, val_loader,
-                         distributed_run, batch_to_gpu, args.local_rank, ema=True)
-
-        # save before making sched.step() for proper loading of LR
-        checkpointer.maybe_save(args, model, ema_model, optimizer, scaler,
-                                epoch, total_iter, model_config)
-        logger.flush()
-
-    # Finished training
-    if len(bmark_stats) > 0:
-        log((), tb_total_steps=None, subset='train_avg',
-            data=bmark_stats.get(args.benchmark_epochs_num))
-
-    validate(model, None, total_iter, criterion, val_loader, distributed_run,
-             batch_to_gpu)
-
-
-if __name__ == '__main__':
-    main()