Co-authored-by: Yiwen Zhao <[email protected]> & Jionghao

.gitattributes

@@ -0,0 +1,2 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+*cache*

offline_process/create_features.py

@@ -0,0 +1,62 @@
+from datasets import load_dataset, concatenate_datasets
+
+ds = load_dataset("espnet/ace-kising-segments", cache_dir="cache")
+
+combined = concatenate_datasets([ds["train"], ds["validation"], ds["test"]])
+
+# 2. filter rows by singer: baber
+combined = combined.filter(lambda x: x["singer"] == "barber")
+
+# 3. create a new column, which counts the nonzero numbers in the list in the note_midi column
+combined = combined.map(
+    lambda x: {"note_midi_length": len([n for n in x["note_midi"] if n != 0])}
+)
+
+# 4. sort by segment_id
+combined = combined.sort("segment_id")
+
+# 5. iterate over rows
+prev_songid = None
+prev_song_segment_id = None
+song2note_lengths = {}
+for row in combined:
+    # segment_id: kising_barber_{songid}_{song_segment_id}
+    _, _, songid, song_segment_id = row["segment_id"].split("_")
+    if prev_songid != songid:
+        if prev_songid is not None:
+            assert (
+                song_segment_id == "001"
+            ), f"prev_songid: {prev_songid}, songid: {songid}, song_segment_id: {song_segment_id}"
+        song2note_lengths[f"kising_{songid}"] = [row["note_midi_length"]]
+    else:
+        assert (
+            int(song_segment_id) >= int(prev_song_segment_id) + 1
+        ), f"prev_song_segment_id: {prev_song_segment_id}, song_segment_id: {song_segment_id}"
+        song2note_lengths[f"kising_{songid}"].append(row["note_midi_length"])
+    prev_songid = songid
+    prev_song_segment_id = song_segment_id
+
+# 6. write to json
+import json
+
+with open("song2note_lengths.json", "w") as f:
+    json.dump(song2note_lengths, f, indent=4)
+
+# 7. convert to pandas DataFrame
+import pandas as pd
+
+df = pd.DataFrame.from_dict(combined)
+df = df.drop(columns=["audio", "singer"])
+df["segment_id"] = df["segment_id"].str.replace("kising_barber_", "kising_")
+# export to csv
+df.to_csv("song_db.csv", index=False)
+
+# 8. push score segments to hub
+# remove audio and singer columns
+combined = combined.remove_columns(["audio", "singer"])
+# replace kising_barber_ with kising_
+combined = combined.map(
+    lambda x: {"segment_id": x["segment_id"].replace("kising_barber_", "kising_")}
+)
+# upload to hub
+combined.push_to_hub("jhansss/kising_score_segments")

resource/midi-note.scp

@@ -0,0 +1,152 @@
+G9 127
+F#9 126
+Gb9 126
+F9 125
+E9 124
+D#9 123
+Eb9 123
+D9 122
+C#9 121
+Db9 121
+C9 120
+B8 119
+A#8 118
+Bb8 118
+A8 117
+G#8 116
+Ab8 116
+G8 115
+F#8 114
+Gb8 114
+F8 113
+E8 112
+D#8 111
+Eb8 111
+D8 110
+C#8 109
+Db8 109
+C8 108
+B7 107
+A#7 106
+Bb7 106
+A7 105
+G#7 104
+Ab7 104
+G7 103
+F#7 102
+Gb7 102
+F7 101
+E7 100
+D#7 99
+Eb7 99
+D7 98
+C#7 97
+Db7 97
+C7 96
+B6 95
+A#6 94
+Bb6 94
+A6 93
+G#6 92
+Ab6 92
+G6 91
+F#6 90
+Gb6 90
+F6 89
+E6 88
+D#6 87
+Eb6 87
+D6 86
+C#6 85
+Db6 85
+C6 84
+B5 83
+A#5 82
+Bb5 82
+A5 81
+G#5 80
+Ab5 80
+G5 79
+F#5 78
+Gb5 78
+F5 77
+E5 76
+D#5 75
+Eb5 75
+D5 74
+C#5 73
+Db5 73
+C5 72
+B4 71
+A#4 70
+Bb4 70
+A4 69
+G#4 68
+Ab4 68
+G4 67
+F#4 66
+Gb4 66
+F4 65
+E4 64
+D#4 63
+Eb4 63
+D4 62
+C#4 61
+Db4 61
+C4 60
+B3 59
+A#3 58
+Bb3 58
+A3 57
+G#3 56
+Ab3 56
+G3 55
+F#3 54
+Gb3 54
+F3 53
+E3 52
+D#3 51
+Eb3 51
+D3 50
+C#3 49
+Db3 49
+C3 48
+B2 47
+A#2 46
+Bb2 46
+A2 45
+G#2 44
+Ab2 44
+G2 43
+F#2 42
+Gb2 42
+F2 41
+E2 40
+D#2 39
+Eb2 39
+D2 38
+C#2 37
+Db2 37
+C2 36
+B1 35
+A#1 34
+Bb1 34
+A1 33
+G#1 32
+Ab1 32
+G1 31
+F#1 30
+Gb1 30
+F1 29
+E1 28
+D#1 27
+Eb1 27
+D1 26
+C#1 25
+Db1 25
+C1 24
+B0 23
+A#0 22
+Bb0 22
+A0 21
+rest 0

resource/pinyin_dict.py

@@ -0,0 +1,423 @@
+# Adapted from Opencpop's pinyin to phoneme mapping table:
+# https://wenet.org.cn/opencpop/resources/annotationformat/
+PINYIN_DICT = {
+    "a": ("a",),
+    "ai": ("ai",),
+    "an": ("an",),
+    "ang": ("ang",),
+    "ao": ("ao",),
+    "ba": ("b", "a"),
+    "bai": ("b", "ai"),
+    "ban": ("b", "an"),
+    "bang": ("b", "ang"),
+    "bao": ("b", "ao"),
+    "bei": ("b", "ei"),
+    "ben": ("b", "en"),
+    "beng": ("b", "eng"),
+    "bi": ("b", "i"),
+    "bian": ("b", "ian"),
+    "biao": ("b", "iao"),
+    "bie": ("b", "ie"),
+    "bin": ("b", "in"),
+    "bing": ("b", "ing"),
+    "bo": ("b", "o"),
+    "bu": ("b", "u"),
+    "ca": ("c", "a"),
+    "cai": ("c", "ai"),
+    "can": ("c", "an"),
+    "cang": ("c", "ang"),
+    "cao": ("c", "ao"),
+    "ce": ("c", "e"),
+    "cei": ("c", "ei"),
+    "cen": ("c", "en"),
+    "ceng": ("c", "eng"),
+    "cha": ("ch", "a"),
+    "chai": ("ch", "ai"),
+    "chan": ("ch", "an"),
+    "chang": ("ch", "ang"),
+    "chao": ("ch", "ao"),
+    "che": ("ch", "e"),
+    "chen": ("ch", "en"),
+    "cheng": ("ch", "eng"),
+    "chi": ("ch", "i"),
+    "chong": ("ch", "ong"),
+    "chou": ("ch", "ou"),
+    "chu": ("ch", "u"),
+    "chua": ("ch", "ua"),
+    "chuai": ("ch", "uai"),
+    "chuan": ("ch", "uan"),
+    "chuang": ("ch", "uang"),
+    "chui": ("ch", "ui"),
+    "chun": ("ch", "un"),
+    "chuo": ("ch", "uo"),
+    "ci": ("c", "i"),
+    "cong": ("c", "ong"),
+    "cou": ("c", "ou"),
+    "cu": ("c", "u"),
+    "cuan": ("c", "uan"),
+    "cui": ("c", "ui"),
+    "cun": ("c", "un"),
+    "cuo": ("c", "uo"),
+    "da": ("d", "a"),
+    "dai": ("d", "ai"),
+    "dan": ("d", "an"),
+    "dang": ("d", "ang"),
+    "dao": ("d", "ao"),
+    "de": ("d", "e"),
+    "dei": ("d", "ei"),
+    "den": ("d", "en"),
+    "deng": ("d", "eng"),
+    "di": ("d", "i"),
+    "dia": ("d", "ia"),
+    "dian": ("d", "ian"),
+    "diao": ("d", "iao"),
+    "die": ("d", "ie"),
+    "ding": ("d", "ing"),
+    "diu": ("d", "iu"),
+    "dong": ("d", "ong"),
+    "dou": ("d", "ou"),
+    "du": ("d", "u"),
+    "duan": ("d", "uan"),
+    "dui": ("d", "ui"),
+    "dun": ("d", "un"),
+    "duo": ("d", "uo"),
+    "e": ("e",),
+    "ei": ("ei",),
+    "en": ("en",),
+    "eng": ("eng",),
+    "er": ("er",),
+    "fa": ("f", "a"),
+    "fan": ("f", "an"),
+    "fang": ("f", "ang"),
+    "fei": ("f", "ei"),
+    "fen": ("f", "en"),
+    "feng": ("f", "eng"),
+    "fo": ("f", "o"),
+    "fou": ("f", "ou"),
+    "fu": ("f", "u"),
+    "ga": ("g", "a"),
+    "gai": ("g", "ai"),
+    "gan": ("g", "an"),
+    "gang": ("g", "ang"),
+    "gao": ("g", "ao"),
+    "ge": ("g", "e"),
+    "gei": ("g", "ei"),
+    "gen": ("g", "en"),
+    "geng": ("g", "eng"),
+    "gong": ("g", "ong"),
+    "gou": ("g", "ou"),
+    "gu": ("g", "u"),
+    "gua": ("g", "ua"),
+    "guai": ("g", "uai"),
+    "guan": ("g", "uan"),
+    "guang": ("g", "uang"),
+    "gui": ("g", "ui"),
+    "gun": ("g", "un"),
+    "guo": ("g", "uo"),
+    "ha": ("h", "a"),
+    "hai": ("h", "ai"),
+    "han": ("h", "an"),
+    "hang": ("h", "ang"),
+    "hao": ("h", "ao"),
+    "he": ("h", "e"),
+    "hei": ("h", "ei"),
+    "hen": ("h", "en"),
+    "heng": ("h", "eng"),
+    "hm": ("h", "m"),
+    "hng": ("h", "ng"),
+    "hong": ("h", "ong"),
+    "hou": ("h", "ou"),
+    "hu": ("h", "u"),
+    "hua": ("h", "ua"),
+    "huai": ("h", "uai"),
+    "huan": ("h", "uan"),
+    "huang": ("h", "uang"),
+    "hui": ("h", "ui"),
+    "hun": ("h", "un"),
+    "huo": ("h", "uo"),
+    "ji": ("j", "i"),
+    "jia": ("j", "ia"),
+    "jian": ("j", "ian"),
+    "jiang": ("j", "iang"),
+    "jiao": ("j", "iao"),
+    "jie": ("j", "ie"),
+    "jin": ("j", "in"),
+    "jing": ("j", "ing"),
+    "jiong": ("j", "iong"),
+    "jiu": ("j", "iu"),
+    "ju": ("j", "v"),
+    "juan": ("j", "van"),
+    "jue": ("j", "ve"),
+    "jun": ("j", "vn"),
+    "ka": ("k", "a"),
+    "kai": ("k", "ai"),
+    "kan": ("k", "an"),
+    "kang": ("k", "ang"),
+    "kao": ("k", "ao"),
+    "ke": ("k", "e"),
+    "kei": ("k", "ei"),
+    "ken": ("k", "en"),
+    "keng": ("k", "eng"),
+    "kong": ("k", "ong"),
+    "kou": ("k", "ou"),
+    "ku": ("k", "u"),
+    "kua": ("k", "ua"),
+    "kuai": ("k", "uai"),
+    "kuan": ("k", "uan"),
+    "kuang": ("k", "uang"),
+    "kui": ("k", "ui"),
+    "kun": ("k", "un"),
+    "kuo": ("k", "uo"),
+    "la": ("l", "a"),
+    "lai": ("l", "ai"),
+    "lan": ("l", "an"),
+    "lang": ("l", "ang"),
+    "lao": ("l", "ao"),
+    "le": ("l", "e"),
+    "lei": ("l", "ei"),
+    "leng": ("l", "eng"),
+    "li": ("l", "i"),
+    "lia": ("l", "ia"),
+    "lian": ("l", "ian"),
+    "liang": ("l", "iang"),
+    "liao": ("l", "iao"),
+    "lie": ("l", "ie"),
+    "lin": ("l", "in"),
+    "ling": ("l", "ing"),
+    "liu": ("l", "iu"),
+    "lo": ("l", "o"),
+    "long": ("l", "ong"),
+    "lou": ("l", "ou"),
+    "lu": ("l", "u"),
+    "luan": ("l", "uan"),
+    "lun": ("l", "un"),
+    "luo": ("l", "uo"),
+    "lv": ("l", "v"),
+    "lve": ("l", "ve"),
+    "m": ("m",),
+    "ma": ("m", "a"),
+    "mai": ("m", "ai"),
+    "man": ("m", "an"),
+    "mang": ("m", "ang"),
+    "mao": ("m", "ao"),
+    "me": ("m", "e"),
+    "mei": ("m", "ei"),
+    "men": ("m", "en"),
+    "meng": ("m", "eng"),
+    "mi": ("m", "i"),
+    "mian": ("m", "ian"),
+    "miao": ("m", "iao"),
+    "mie": ("m", "ie"),
+    "min": ("m", "in"),
+    "ming": ("m", "ing"),
+    "miu": ("m", "iu"),
+    "mo": ("m", "o"),
+    "mou": ("m", "ou"),
+    "mu": ("m", "u"),
+    "n": ("n",),
+    "na": ("n", "a"),
+    "nai": ("n", "ai"),
+    "nan": ("n", "an"),
+    "nang": ("n", "ang"),
+    "nao": ("n", "ao"),
+    "ne": ("n", "e"),
+    "nei": ("n", "ei"),
+    "nen": ("n", "en"),
+    "neng": ("n", "eng"),
+    "ng": ("n", "g"),
+    "ni": ("n", "i"),
+    "nian": ("n", "ian"),
+    "niang": ("n", "iang"),
+    "niao": ("n", "iao"),
+    "nie": ("n", "ie"),
+    "nin": ("n", "in"),
+    "ning": ("n", "ing"),
+    "niu": ("n", "iu"),
+    "nong": ("n", "ong"),
+    "nou": ("n", "ou"),
+    "nu": ("n", "u"),
+    "nuan": ("n", "uan"),
+    "nun": ("n", "un"),
+    "nuo": ("n", "uo"),
+    "nv": ("n", "v"),
+    "nve": ("n", "ve"),
+    "o": ("o",),
+    "ou": ("ou",),
+    "pa": ("p", "a"),
+    "pai": ("p", "ai"),
+    "pan": ("p", "an"),
+    "pang": ("p", "ang"),
+    "pao": ("p", "ao"),
+    "pei": ("p", "ei"),
+    "pen": ("p", "en"),
+    "peng": ("p", "eng"),
+    "pi": ("p", "i"),
+    "pian": ("p", "ian"),
+    "piao": ("p", "iao"),
+    "pie": ("p", "ie"),
+    "pin": ("p", "in"),
+    "ping": ("p", "ing"),
+    "po": ("p", "o"),
+    "pou": ("p", "ou"),
+    "pu": ("p", "u"),
+    "qi": ("q", "i"),
+    "qia": ("q", "ia"),
+    "qian": ("q", "ian"),
+    "qiang": ("q", "iang"),
+    "qiao": ("q", "iao"),
+    "qie": ("q", "ie"),
+    "qin": ("q", "in"),
+    "qing": ("q", "ing"),
+    "qiong": ("q", "iong"),
+    "qiu": ("q", "iu"),
+    "qu": ("q", "v"),
+    "quan": ("q", "van"),
+    "que": ("q", "ve"),
+    "qun": ("q", "vn"),
+    "ran": ("r", "an"),
+    "rang": ("r", "ang"),
+    "rao": ("r", "ao"),
+    "re": ("r", "e"),
+    "ren": ("r", "en"),
+    "reng": ("r", "eng"),
+    "ri": ("r", "i"),
+    "rong": ("r", "ong"),
+    "rou": ("r", "ou"),
+    "ru": ("r", "u"),
+    "rua": ("r", "ua"),
+    "ruan": ("r", "uan"),
+    "rui": ("r", "ui"),
+    "run": ("r", "un"),
+    "ruo": ("r", "uo"),
+    "sa": ("s", "a"),
+    "sai": ("s", "ai"),
+    "san": ("s", "an"),
+    "sang": ("s", "ang"),
+    "sao": ("s", "ao"),
+    "se": ("s", "e"),
+    "sen": ("s", "en"),
+    "seng": ("s", "eng"),
+    "sha": ("sh", "a"),
+    "shai": ("sh", "ai"),
+    "shan": ("sh", "an"),
+    "shang": ("sh", "ang"),
+    "shao": ("sh", "ao"),
+    "she": ("sh", "e"),
+    "shei": ("sh", "ei"),
+    "shen": ("sh", "en"),
+    "sheng": ("sh", "eng"),
+    "shi": ("sh", "i"),
+    "shou": ("sh", "ou"),
+    "shu": ("sh", "u"),
+    "shua": ("sh", "ua"),
+    "shuai": ("sh", "uai"),
+    "shuan": ("sh", "uan"),
+    "shuang": ("sh", "uang"),
+    "shui": ("sh", "ui"),
+    "shun": ("sh", "un"),
+    "shuo": ("sh", "uo"),
+    "si": ("s", "i"),
+    "song": ("s", "ong"),
+    "sou": ("s", "ou"),
+    "su": ("s", "u"),
+    "suan": ("s", "uan"),
+    "sui": ("s", "ui"),
+    "sun": ("s", "un"),
+    "suo": ("s", "uo"),
+    "ta": ("t", "a"),
+    "tai": ("t", "ai"),
+    "tan": ("t", "an"),
+    "tang": ("t", "ang"),
+    "tao": ("t", "ao"),
+    "te": ("t", "e"),
+    "tei": ("t", "ei"),
+    "teng": ("t", "eng"),
+    "ti": ("t", "i"),
+    "tian": ("t", "ian"),
+    "tiao": ("t", "iao"),
+    "tie": ("t", "ie"),
+    "ting": ("t", "ing"),
+    "tong": ("t", "ong"),
+    "tou": ("t", "ou"),
+    "tu": ("t", "u"),
+    "tuan": ("t", "uan"),
+    "tui": ("t", "ui"),
+    "tun": ("t", "un"),
+    "tuo": ("t", "uo"),
+    "wa": ("w", "a"),
+    "wai": ("w", "ai"),
+    "wan": ("w", "an"),
+    "wang": ("w", "ang"),
+    "wei": ("w", "ei"),
+    "wen": ("w", "en"),
+    "weng": ("w", "eng"),
+    "wo": ("w", "o"),
+    "wu": ("w", "u"),
+    "xi": ("x", "i"),
+    "xia": ("x", "ia"),
+    "xian": ("x", "ian"),
+    "xiang": ("x", "iang"),
+    "xiao": ("x", "iao"),
+    "xie": ("x", "ie"),
+    "xin": ("x", "in"),
+    "xing": ("x", "ing"),
+    "xiong": ("x", "iong"),
+    "xiu": ("x", "iu"),
+    "xu": ("x", "v"),
+    "xuan": ("x", "van"),
+    "xue": ("x", "ve"),
+    "xun": ("x", "vn"),
+    "ya": ("y", "a"),
+    "yan": ("y", "an"),
+    "yang": ("y", "ang"),
+    "yao": ("y", "ao"),
+    "ye": ("y", "e"),
+    "yi": ("y", "i"),
+    "yin": ("y", "in"),
+    "ying": ("y", "ing"),
+    "yo": ("y", "o"),
+    "yong": ("y", "ong"),
+    "you": ("y", "ou"),
+    "yu": ("y", "v"),
+    "yuan": ("y", "van"),
+    "yue": ("y", "ve"),
+    "yun": ("y", "vn"),
+    "za": ("z", "a"),
+    "zai": ("z", "ai"),
+    "zan": ("z", "an"),
+    "zang": ("z", "ang"),
+    "zao": ("z", "ao"),
+    "ze": ("z", "e"),
+    "zei": ("z", "ei"),
+    "zen": ("z", "en"),
+    "zeng": ("z", "eng"),
+    "zha": ("zh", "a"),
+    "zhai": ("zh", "ai"),
+    "zhan": ("zh", "an"),
+    "zhang": ("zh", "ang"),
+    "zhao": ("zh", "ao"),
+    "zhe": ("zh", "e"),
+    "zhei": ("zh", "ei"),
+    "zhen": ("zh", "en"),
+    "zheng": ("zh", "eng"),
+    "zhi": ("zh", "i"),
+    "zhong": ("zh", "ong"),
+    "zhou": ("zh", "ou"),
+    "zhu": ("zh", "u"),
+    "zhua": ("zh", "ua"),
+    "zhuai": ("zh", "uai"),
+    "zhuan": ("zh", "uan"),
+    "zhuang": ("zh", "uang"),
+    "zhui": ("zh", "ui"),
+    "zhun": ("zh", "un"),
+    "zhuo": ("zh", "uo"),
+    "zi": ("z", "i"),
+    "zong": ("z", "ong"),
+    "zou": ("z", "ou"),
+    "zu": ("z", "u"),
+    "zuan": ("z", "uan"),
+    "zui": ("z", "ui"),
+    "zun": ("z", "un"),
+    "zuo": ("z", "uo"),
+}
+