thomas-yanxin commited on
Commit
b340237
·
verified ·
1 Parent(s): a862ff2

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +27 -25
app.py CHANGED
@@ -62,11 +62,11 @@ class HFToMSConverter:
62
  src.rename(dst)
63
  logger.info(f"移动文件成功: {src.name}")
64
  elif operation_type == "push":
65
- print(clone_dir)
66
  ms_repo_id, clone_dir = args
 
67
  self.ms_api.upload_folder(
68
- repo_id=f"{self.config['username']}/{ms_repo_id}",
69
- folder_path=clone_dir,
70
  commit_message='upload dataset folder',
71
  repo_type='dataset'
72
  )
@@ -76,42 +76,44 @@ class HFToMSConverter:
76
  logger.error(f"{operation_type}操作失败: {e}")
77
  return False
78
 
79
- def move_file(self, filename: str, clone_dir: str) -> bool:
80
- """移动文件到目标目录"""
81
- return self.handle_file_operation(
82
- "move",
83
- Path(self.local_dir) / filename,
84
- Path(clone_dir) / filename
85
- )
86
-
87
- def push_to_ms(self, ms_repo_id: str, clone_dir: str) -> bool:
88
- """推送到ModelScope"""
89
- return self.handle_file_operation("push", ms_repo_id, clone_dir)
90
-
91
  def process_files(self, hf_repo: str, ms_repo: str, files: List[str]) -> bool:
92
  """处理所有文件的完整流程"""
93
- # 获取绝对路径
94
- print(files)
95
- os_path = os.path.abspath('.')
96
-
97
- clone_dir = os.path.join(os_path, ms_repo.split("/")[-1])
98
-
99
  try:
100
- # # 下载并移动所有文件
 
 
 
 
101
  for filename in files:
102
  if not all([
103
  self.download_file(hf_repo, filename),
104
- self.move_file(filename, clone_dir)
105
  ]):
106
  return False
107
 
108
- print('111'*10)
109
- print(clone_dir)
110
  # 统一推送整个文件夹
111
  return self.push_to_ms(ms_repo, clone_dir)
 
112
  except Exception as e:
113
  logger.error(f"处理文件失败: {e}")
114
  return False
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
 
116
  def create_ui() -> gr.Blocks:
117
  """创建Gradio界面"""
 
62
  src.rename(dst)
63
  logger.info(f"移动文件成功: {src.name}")
64
  elif operation_type == "push":
 
65
  ms_repo_id, clone_dir = args
66
+ logger.info(f"开始推送文件夹: {clone_dir}")
67
  self.ms_api.upload_folder(
68
+ repo_id=f"{ms_repo_id}",
69
+ folder_path=str(clone_dir), # 确保路径是字符串
70
  commit_message='upload dataset folder',
71
  repo_type='dataset'
72
  )
 
76
  logger.error(f"{operation_type}操作失败: {e}")
77
  return False
78
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  def process_files(self, hf_repo: str, ms_repo: str, files: List[str]) -> bool:
80
  """处理所有文件的完整流程"""
 
 
 
 
 
 
81
  try:
82
+ # 获取绝对路径并创建目录
83
+ clone_dir = Path(os.path.abspath('.')) / ms_repo.split("/")[-1]
84
+ clone_dir.mkdir(parents=True, exist_ok=True)
85
+
86
+ # 下载并移动所有文件
87
  for filename in files:
88
  if not all([
89
  self.download_file(hf_repo, filename),
90
+ self.move_file(filename, str(clone_dir))
91
  ]):
92
  return False
93
 
 
 
94
  # 统一推送整个文件夹
95
  return self.push_to_ms(ms_repo, clone_dir)
96
+
97
  except Exception as e:
98
  logger.error(f"处理文件失败: {e}")
99
  return False
100
+ finally:
101
+ # 清理临时文件
102
+ if clone_dir.exists():
103
+ import shutil
104
+ shutil.rmtree(clone_dir)
105
+
106
+ def move_file(self, filename: str, clone_dir: str) -> bool:
107
+ """移动文件到目标目录"""
108
+ return self.handle_file_operation(
109
+ "move",
110
+ Path(self.local_dir) / filename,
111
+ Path(clone_dir) / filename
112
+ )
113
+
114
+ def push_to_ms(self, ms_repo_id: str, clone_dir: str) -> bool:
115
+ """推送到ModelScope"""
116
+ return self.handle_file_operation("push", ms_repo_id, clone_dir)
117
 
118
  def create_ui() -> gr.Blocks:
119
  """创建Gradio界面"""