Instructions to use JunxiongWang/M1-3B-SFT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use JunxiongWang/M1-3B-SFT with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="JunxiongWang/M1-3B-SFT") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("JunxiongWang/M1-3B-SFT") model = AutoModelForCausalLM.from_pretrained("JunxiongWang/M1-3B-SFT", device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use JunxiongWang/M1-3B-SFT with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "JunxiongWang/M1-3B-SFT" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "JunxiongWang/M1-3B-SFT", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/JunxiongWang/M1-3B-SFT
- SGLang
How to use JunxiongWang/M1-3B-SFT with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "JunxiongWang/M1-3B-SFT" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "JunxiongWang/M1-3B-SFT", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "JunxiongWang/M1-3B-SFT" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "JunxiongWang/M1-3B-SFT", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use JunxiongWang/M1-3B-SFT with Docker Model Runner:
docker model run hf.co/JunxiongWang/M1-3B-SFT
Download check_same.py from JunxiongWang/M1-3B-SFT: direct link, hf CLI and curl.
- Browser
- Download file 2.83 kB
-
https://huggingface.co/JunxiongWang/M1-3B-SFT/resolve/main/check_same.py
- Command line
-
hf download hf://JunxiongWang/M1-3B-SFT/check_same.py
-
curl -L -o check_same.py https://huggingface.co/JunxiongWang/M1-3B-SFT/resolve/main/check_same.py
2.83 kB
| import torch | |
| import torch | |
| # Import the exception type to catch | |
| from torch._subclasses.fake_tensor import DataDependentOutputException | |
| def load_checkpoint(path): | |
| return torch.load(path, map_location=torch.device('cpu')) | |
| def tensors_allclose(val_cur, val_ref): | |
| try: | |
| # Attempt to compare using torch.allclose directly. | |
| if not torch.allclose(val_cur, val_ref): | |
| return False | |
| except torch._subclasses.fake_tensor.DataDependentOutputException: | |
| # If we catch a fake tensor exception, convert tensors to real CPU tensors. | |
| real_val_cur = val_cur.detach().cpu() if hasattr(val_cur, "detach") else val_cur | |
| real_val_ref = val_ref.detach().cpu() if hasattr(val_ref, "detach") else val_ref | |
| if not torch.allclose(real_val_cur, real_val_ref): | |
| return False | |
| return True | |
| def compare_checkpoints(paths): | |
| checkpoints = {path: load_checkpoint(path) for path in paths} | |
| # Use the first checkpoint as the reference. | |
| ref_path, ref_ckpt = list(checkpoints.items())[0] | |
| all_same = True | |
| for path, ckpt in checkpoints.items(): | |
| # Check if the types match. | |
| if type(ckpt) != type(ref_ckpt): | |
| print(f"Type mismatch: {path} is of type {type(ckpt)}, expected {type(ref_ckpt)}") | |
| all_same = False | |
| continue | |
| # If checkpoint is a dictionary, compare keys and values. | |
| if isinstance(ckpt, dict): | |
| if set(ckpt.keys()) != set(ref_ckpt.keys()): | |
| print(f"Key mismatch in {path}.") | |
| all_same = False | |
| continue | |
| for key in ckpt: | |
| val_ref = ref_ckpt[key] | |
| val_cur = ckpt[key] | |
| # If the value is a tensor, compare using our helper function. | |
| if isinstance(val_ref, torch.Tensor) and isinstance(val_cur, torch.Tensor): | |
| if not tensors_allclose(val_cur, val_ref): | |
| print(f"Tensor values differ for key '{key}' in {path}.") | |
| all_same = False | |
| else: | |
| if val_cur != val_ref: | |
| print(f"Value for key '{key}' differs in {path}.") | |
| all_same = False | |
| else: | |
| # If the checkpoints are not dictionaries, compare them directly. | |
| if ckpt != ref_ckpt: | |
| print(f"Checkpoint {path} differs from {ref_path}.") | |
| all_same = False | |
| if all_same: | |
| print("All checkpoints are identical.") | |
| else: | |
| print("Not all checkpoints are identical.") | |
| # Generate file paths for ranks 0 through 7. | |
| paths = [f"grpo_cg_packfix_128_24576_1_1e-6_0_0_1_new_veRL/global_step_40/actor/model_world_size_8_rank_{i}.pt" for i in range(8)] | |
| compare_checkpoints(paths) | |