update yaml

app.py

@@ -126,7 +126,7 @@ def run_local(cfg: DictConfig):
             cfg.algorithm._name = cfg_choice["algorithm"]
 
     # launch experiment
-    experiment = build_experiment(cfg, None, cfg.checkpoint_path)
+    experiment = build_experiment(cfg, None, None)
     return experiment.exec_interactive(cfg.experiment.tasks[0])
 
 memory_frames = []
@@ -159,9 +159,10 @@ def save_video(frames, path="output.mp4", fps=10):
 @hydra.main(
     version_base=None,
     config_path="configurations",
-    config_name="config",
+    config_name="huggingface",
 )
 def run(cfg: DictConfig):
+
     algo = run_local(cfg)
     algo.to("cuda:0")
     
@@ -183,7 +184,6 @@ def run(cfg: DictConfig):
         print("set denoising steps to", algo.sampling_timesteps)
         return sampling_timesteps_state
 
-
     def update_image_and_log(keys):
         actions = parse_input_to_tensor(keys)
         global input_history

configurations/huggingface.yaml

@@ -0,0 +1,57 @@
+defaults:
+  - algorithm: df_video_worldmemminecraft
+  - experiment: exp_video
+  - dataset: video_minecraft
+
+dataset:
+  n_frames_valid: 100
+  validation_multiplier: 1
+  use_plucker: true
+  customized_validation: true
+  condition_similar_length: 8
+  padding_pool: 10
+  focal_length: 0.35
+  save_dir: data/test_pumpkin
+  add_frame_timestep_embedder: true
+  pos_range: 0.5
+  angle_range: 30
+
+experiment:
+  tasks: [interactive]
+  training:
+    data:
+      num_workers: 4
+  validation:
+    batch_size: 1
+    limit_batch: 1
+    data:
+      num_workers: 4
+  load_vae: false
+  load_t_to_r: false
+  zero_init_gate: false
+  only_tune_refer: false
+  diffusion_path: checkpoints/diffusion_only.ckpt
+  vae_path: checkpoints/vae_only.ckpt
+  pose_predictor_path: checkpoints/pose_prediction_model_only.ckpt
+  customized_load: true
+
+algorithm:
+  n_tokens: 8
+  context_frames: 90
+  pose_cond_dim: 5
+  use_plucker: true
+  focal_length: 0.35
+  customized_validation: true
+  condition_similar_length: 8
+  log_video: true
+  relative_embedding: true
+  cond_only_on_qk: true
+  add_pose_embed: false
+  use_domain_adapter: false
+  use_reference_attention: true
+  add_frame_timestep_embedder: true
+  is_interactive: true
+  diffusion:
+    sampling_timesteps: 20
+
+debug: false

experiments/exp_base.py

@@ -89,13 +89,14 @@ class BaseExperiment(ABC):
         self.logger = logger
         self.ckpt_path = ckpt_path
         self.algo = None
-        self.customized_load = root_cfg.customized_load
-        self.load_vae = root_cfg.load_vae
-        self.load_t_to_r = root_cfg.load_t_to_r
-        self.zero_init_gate=root_cfg.zero_init_gate
-        self.only_tune_refer = root_cfg.only_tune_refer
-        self.vae_path = root_cfg.vae_path # "/mnt/xiaozeqi/.cache/huggingface/hub/models--Etched--oasis-500m/snapshots/4ca7d2d811f4f0c6fd1d5719bf83f14af3446c0c/vit-l-20.safetensors"
-        self.pose_predictor_path = root_cfg.pose_predictor_path # "/mnt/xiaozeqi/diffusionforcing/outputs/2025-03-28/16-45-11/checkpoints/epoch0step595000.ckpt"
+        self.customized_load = self.cfg.customized_load
+        self.load_vae = self.cfg.load_vae
+        self.load_t_to_r = self.cfg.load_t_to_r
+        self.zero_init_gate=self.cfg.zero_init_gate
+        self.only_tune_refer = self.cfg.only_tune_refer
+        self.diffusion_path = self.cfg.diffusion_path
+        self.vae_path = self.cfg.vae_path # "/mnt/xiaozeqi/.cache/huggingface/hub/models--Etched--oasis-500m/snapshots/4ca7d2d811f4f0c6fd1d5719bf83f14af3446c0c/vit-l-20.safetensors"
+        self.pose_predictor_path = self.cfg.pose_predictor_path # "/mnt/xiaozeqi/diffusionforcing/outputs/2025-03-28/16-45-11/checkpoints/epoch0step595000.ckpt"
 
     def _build_algo(self):
         """
@@ -449,7 +450,7 @@ class BaseLightningExperiment(BaseExperiment):
             self.algo = torch.compile(self.algo)
 
         if self.customized_load:
-            load_custom_checkpoint(algo=self.algo.diffusion_model,optimizer=None,checkpoint_path=self.ckpt_path)
+            load_custom_checkpoint(algo=self.algo.diffusion_model,optimizer=None,checkpoint_path=self.diffusion_path)
             load_custom_checkpoint(algo=self.algo.vae,optimizer=None,checkpoint_path=self.vae_path)
             load_custom_checkpoint(algo=self.algo.pose_prediction_model,optimizer=None,checkpoint_path=self.pose_predictor_path)
             return self.algo