add force change term into chinese before tranlate

Former-commit-id: 2b6ec94ed31ea352361c1591a6db9a4b3e775fb3

finetune_data/dict.csv

@@ -0,0 +1,173 @@
+barracks,兵营
+engineering bay,工程站
+forge,锻炉
+blink,闪现
+evolution chamber,进化腔
+cybernetics core,控制芯核
+enhanced shockwaves,EMP范围
+gravitic boosters,ob速度
+armory,军械库
+robotics bay,机械研究所
+twilight council,光影议会
+fusion core,聚变芯体
+fleet beacon,舰队航标
+factory,重工厂
+ghost academy,幽灵军校
+infestation pit,感染深渊
+robotics facility,机械台
+stargate,星门
+starport,星港
+archon,白球
+smart servos,变形加速
+gateway,兵营
+warpgate,兵营
+immortal,不朽
+zealot,叉叉
+nydus network,虫洞
+nydus worm,虫洞
+hydralisk,刺蛇
+grooved spines,刺蛇射程
+muscular augments,刺蛇速度
+hydralisk den,刺蛇塔
+planetary fortress,大地堡
+battle cruiser,大和
+weapon refit,大和炮
+brood lord,大龙
+greater spire,大龙塔
+anabolic synthesis,大牛速度
+cyclone,导弹车
+bunker,地堡
+lurker,地刺
+seismic spines,地刺射程
+adaptive talons,地刺速埋
+lurker den,地刺塔
+widow mine,地雷
+ground carapace,地面单位甲壳等级
+high templar,电兵
+shield battery,电池
+observer,叮当
+baneling,毒爆
+centrifugal hooks,毒爆速度
+baneling nest,毒爆塔
+raven,渡鸦
+combat shield,盾
+shield,盾
+lair,二本
+missile turret,防空
+spore crawler,防空
+supply depot,房子
+overlord,房子
+pneumatized carapace,房子速度
+mutalisk,飞龙
+spire,飞龙塔
+viper,飞蛇
+flyer attacks,飞行生物攻击等级
+flyer carapace,飞行生物甲壳等级
+tempest,风暴
+tectonic destabilizers,风暴伤害
+phoenix,凤凰
+anion pulse-crystals,凤凰射程
+corruptor,腐化
+infestor,感染虫
+pathogen glands,感染能量
+zergling,狗
+spawning pool,狗池
+metabolic boost,狗速
+spine crawler,管子
+marauder,光头
+ghost,鬼兵
+arm silo with nuke,核弹
+carrier,黄金舰队
+hellion,火车
+hellbat,火车侠
+ravager,火蟑螂
+nexus,基地
+hatchery,基地
+command center,基地
+neosteel armor,建筑护甲
+hi-sec auto tracking,建筑射程
+ship weapons,舰船武器等级
+charge,脚速
+liberator,解放
+advanced ballistics,解放射程
+melee attacks,近战攻击等级
+colossus,巨像
+extended thermal lance,巨像射程
+creep tumor,菌毯
+tech lab,科技挂件
+air armor,空中单位护甲等级
+air weapons,空中单位武器等级
+adrenal glands,狂狗
+mule,矿螺
+infernal pre-igniter,蓝火
+thor,雷神
+warp prism,棱镜
+gravitic drive,棱镜速度
+dragoon,龙骑士
+cocoon,卵
+larva,卵
+mothership,妈妈船
+burrow,埋地
+changeling,拟态虫
+ultralisk,牛
+chitinous plating,牛甲
+ultralisk cavern,牛塔
+drone,农民
+scv,农民
+queen,女王
+banshee,女妖
+hyperflight rotors,女妖提速
+photon cannon,炮台
+missile attacks,喷射攻击等级
+assimilator,气矿
+extractor,气矿
+refinery,气矿
+roach,钱赞企
+marine,枪兵
+sensor tower,圈
+infantry armor,人族防
+infantry weapons,人族攻
+hive,三本
+psionic storm,闪电
+templar archives,闪电塔
+sentry,哨兵
+ground armor,神族防
+ground weapons,神族攻
+adept,使徒
+resonating glaives,使徒攻速
+reactor,双倍挂件
+pylon,水晶
+reaper,死神
+drilling claws,速埋
+swarm host,宿主
+mag-field accelerator,锁定增伤
+siege tank,坦克
+probe,探机
+corvid reactor,铁鸦能量
+neural parasite,同化完成
+viking,维京
+oracle,先知
+broodling,小虫子
+locust,小虫子
+mothership core,小妈妈船
+orbital command,星轨
+stimpack,兴奋剂
+void ray,虚空
+flux vanes,虚空速度
+overseer,眼虫
+ignite afterburners,医疗机速度
+dark templar,隐刀
+shadow stride,隐刀闪现
+dark shrine,隐刀塔
+cloaking field,隐形
+personal cloaking,隐形
+medivac dropship,运输机
+vehicle and ship plating,战车及舰船钢板等级
+vehicle weapons,战车武器等级
+war hound,战狼
+roach warren,蟑螂巢
+tunneling claws,蟑螂埋地
+glial reconstitution,蟑螂速度
+concussive shells,震撼弹
+stalker,追猎
+disruptor,自爆球

pipeline.py

@@ -89,7 +89,7 @@ if not os.path.exists(f'{RESULT_PATH}/{VIDEO_NAME}'):
 # Instead of using the script_en variable directly, we'll use script_input
 srt_file_en = args.srt_file 
 if srt_file_en is not None: 
-    with open(srt_file_en, 'r') as f:
+    with open(srt_file_en, 'r', encoding='utf-8') as f:
         script_input = f.read()
 else:
     # using whisper to perform speech-to-text and save it in <video name>_en.txt under RESULT PATH.
@@ -110,7 +110,7 @@ else:
             writer.write_result(transcript, srt)
 
     # split the video script(open ai prompt limit: about 5000)
-    with open(srt_file_en, 'r') as f:
+    with open(srt_file_en, 'r', encoding='utf-8') as f:
         script_en = f.read()
         script_input = script_en
 
@@ -119,9 +119,51 @@ if not args.only_srt:
     assSub_en = srt2ass(srt_file_en, "default", "No", "Modest")
     print('ASS subtitle saved as: ' + assSub_en)
 
+# force translate the starcraft2 term into chinese according to the dict
+# TODO: shortcut translation i.e. VA, ob
+# TODO: variety of translation
+from csv import reader
+import re
+
+# read dict
+with open("finetune_data/dict.csv",'r', encoding='utf-8') as f:
+  csv_reader = reader(f)
+  term_dict = {rows[0]:rows[1] for rows in csv_reader}
+
+def clean_timestamp(lines):
+  new_lines = []
+  strinfo = re.compile('[0-9]+\n.{25},[0-9]{3}')    # 注意用4个\\\\来替换\
+  new_lines = strinfo.sub('_-_', lines)
+  print(new_lines)
+  return new_lines
+
+
+ready_lines = re.sub('\n', '\n ', script_input)
+ready_words = ready_lines.split(" ")
+i = 0
+while i < len(ready_words):
+  word = ready_words[i]
+  if word[-2:] == ".\n" :
+    if word[:-2].lower() in term_dict :
+      new_word = word.replace(word[:-2], term_dict.get(word[:-2].lower())) + ' '
+      ready_words[i] = new_word
+    else :
+      word += ' '
+      ready_words[i] = word
+  elif word.lower() in term_dict :
+      new_word = word.replace(word,term_dict.get(word.lower())) + ' '
+      ready_words[i] = new_word
+  else :
+    word += " "
+    ready_words[i]= word
+  i += 1
+
+script_input_withForceTerm = re.sub('\n ', '\n', "".join(ready_words))
+
+
 # Split the video script by sentences and create chunks within the token limit
 n_threshold = 1500  # Token limit for the GPT-3 model
-script_split = script_input.split('.')
+script_split = script_input_withForceTerm.split('.')
 
 script_arr = []
 script = ""