init commit

.gitattributes

@@ -0,0 +1 @@
+examples/**/*.wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,23 @@
+# general things to ignore
+.DS_Store
+build/
+build_contrib/
+dist/
+.cache/
+*.egg-info/
+*.egg
+*.py[cod]
+__pycache__/
+*.so
+*~
+
+# IDE
+.vscode/
+
+# misc
+checkpoints/
+test_waves/
+reconstructed/
+.python-version
+ruff.log
+/configs/inuse/

Dockerfile.api

@@ -0,0 +1,38 @@
+# FROM python:3.10-slim
+FROM pytorch/pytorch:2.1.0-cuda11.8-cudnn8-runtime
+
+WORKDIR /app
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    python3-dev \
+    libsndfile1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY inference.py .
+COPY modules ./modules
+COPY configs ./configs
+COPY hf_utils.py ./
+COPY api.py ./
+COPY runs ./runs
+COPY examples ./examples
+
+ENV PYTHONPATH=/app
+ENV HF_HUB_CACHE=/app/checkpoints/hf_cache
+ENV TORCH_HOME=/app/checkpoints
+
+ENV AWS_REGION=us-east-1
+ENV S3_BUCKET=elevenlabs-clone
+ENV S3_PREFIX=seedvc-outputs
+ENV API_KEY=12345
+
+EXPOSE 8000
+
+CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8000"]

Dockerfile.train

@@ -0,0 +1,34 @@
+FROM pytorch/pytorch:2.1.0-cuda11.8-cudnn8-runtime
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    python3-dev \
+    libsndfile1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . .
+
+RUN mkdir -p checkpoints/hf_cache runs
+
+ENV PYTHONPATH=/app
+ENV HF_HUB_CACHE=/app/checkpoints/hf_cache
+
+RUN echo '#!/bin/bash\n\
+python train.py \
+--config ./configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml \
+--dataset-dir dataset \
+--run-name training-run \
+--batch-size 2 \
+--max-steps 300 \
+--max-epochs 1000 \
+--save-every 100 \
+--num-workers 0' > entrypoint.sh \
+&& chmod +x entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]

EVAL.md

@@ -0,0 +1,121 @@
+### Zero-shot voice conversion🎙🔁
+We have performed a series of objective evaluations on our Seed-VC's voice conversion capabilities. 
+For ease of reproduction, source audios are 100 random utterances from LibriTTS-test-clean, and reference audios are 12 randomly picked in-the-wild voices with unique characteristics. <br>  
+
+Source audios can be found under `./examples/libritts-test-clean` <br>
+Reference audios can be found under `./examples/reference` <br>
+
+We evaluate the conversion results in terms of speaker embedding cosine similarity (SECS), word error rate (WER) and character error rate (CER) and compared
+our results with two strong open sourced baselines, namely [OpenVoice](https://github.com/myshell-ai/OpenVoice) and [CosyVoice](https://github.com/FunAudioLLM/CosyVoice).  
+Results in the table below shows that our Seed-VC model significantly outperforms the baseline models in both intelligibility and speaker similarity.<br>
+
+| Models\Metrics | SECS↑      | WER↓      | CER↓     | SIG↑     | BAK↑     | OVRL↑    |
+|----------------|------------|-----------|----------|----------|----------|----------|
+| Ground Truth   | 1.0000     | 8.02      | 1.57     | ~        | ~        | ~        |
+| OpenVoice      | 0.7547     | 15.46     | 4.73     | **3.56** | **4.02** | **3.27** |
+| CosyVoice      | 0.8440     | 18.98     | 7.29     | 3.51     | **4.02** | 3.21     |
+| Seed-VC(Ours)  | **0.8676** | **11.99** | **2.92** | 3.42     | 3.97     | 3.11     |
+
+We have also compared with non-zero-shot voice conversion models for several speakers (based on model availability):
+
+| Characters          | Models\Metrics | SECS↑      | WER↓      | CER↓     | SIG↑     | BAK↑     | OVRL↑    |
+|---------------------|----------------|------------|-----------|----------|----------|----------|----------|
+| ~                   | Ground Truth   | 1.0000     | 6.43      | 1.00     | ~        | ~        | ~        |
+| Tokai Teio          | So-VITS-4.0    | 0.8637     | 21.46     | 9.63     | 3.06     | 3.66     | 2.68     |
+|                     | Seed-VC(Ours)  | **0.8899** | **15.32** | **4.66** | **3.12** | **3.71** | **2.72** |
+| Milky Green         | So-VITS-4.0    | 0.6850     | 48.43     | 32.50    | 3.34     | 3.51     | 2.82     |
+|                     | Seed-VC(Ours)  | **0.8072** | **7.26**  | **1.32** | **3.48** | **4.07** | **3.20** |
+| Matikane Tannhuaser | So-VITS-4.0    | 0.8594     | 16.25     | 8.64     | **3.25** | 3.71     | 2.84     |
+|                     | Seed-VC(Ours)  | **0.8768** | **12.62** | **5.86** | 3.18     | **3.83** | **2.85** |
+
+Results show that, despite not being trained on the target speakers, Seed-VC is able to achieve significantly better results than the non-zero-shot models. 
+However, this may vary a lot depending on the SoVITS model quality. PR or Issue is welcomed if you find this comparison unfair or inaccurate.  
+(Tokai Teio model from [zomehwh/sovits-tannhauser](https://huggingface.co/spaces/zomehwh/sovits-tannhauser))   
+(Matikane Tannhuaser model from [zomehwh/sovits-tannhauser](https://huggingface.co/spaces/zomehwh/sovits-tannhauser))  
+(Milky Green model from [sparanoid/milky-green-sovits-4](https://huggingface.co/spaces/sparanoid/milky-green-sovits-4))  
+
+*English ASR result computed by [facebook/hubert-large-ls960-ft](https://huggingface.co/facebook/hubert-large-ls960-ft) model*  
+*Speaker embedding computed by [resemblyzer](https://github.com/resemble-ai/Resemblyzer) model* <br>  
+
+You can reproduce the evaluation by running `eval.py` script.  
+```bash
+python eval.py 
+--source ./examples/libritts-test-clean
+--target ./examples/reference
+--output ./examples/eval/converted
+--diffusion-steps 25
+--length-adjust 1.0
+--inference-cfg-rate 0.7
+--xvector-extractor "resemblyzer"
+--baseline ""  # fill in openvoice or cosyvoice to compute baseline result
+--max-samples 100  # max source utterances to go through
+```
+Before that, make sure you have openvoice and cosyvoice repo correctly installed on `../OpenVoice/` and `../CosyVoice/` if you would like to run baseline evaluation.
+
+### Zero-shot singing voice conversion🎤🎶
+
+Additional singing voice conversion evaluation is done on [M4Singer](https://github.com/M4Singer/M4Singer) dataset, with 4 target speakers whose audio data is available [here](https://huggingface.co/datasets/XzJosh/audiodataset).  
+Speaker similariy is calculated by averaging the cosine similarities between conversion result and all available samples in respective character dataset.   
+For each character, one random utterance is chosen as the prompt for zero-shot inference. For comparison, we trained respective [RVCv2-f0-48k](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI) model for each character as baseline.  
+100 random utterances for each singer type are used as source audio.
+
+| Models\Metrics | F0CORR↑ | F0RMSE↓ | SECS↑      | CER↓      | SIG↑     | BAK↑     | OVRL↑    |
+|----------------|---------|---------|------------|-----------|----------|----------|----------|
+| RVCv2          | 0.9404  | 30.43   | 0.7264     | 28.46     | **3.41** | **4.05** | **3.12** |
+| Seed-VC(Ours)  | 0.9375  | 33.35   | **0.7405** | **19.70** | 3.39     | 3.96     | 3.06     |
+
+<details>
+<summary>Click to expand detailed evaluation results</summary>
+
+| Source Singer Type | Characters         | Models\Metrics | F0CORR↑ | F0RMSE↓ | SECS↑      | CER↓      | SIG↑ | BAK↑ | OVRL↑    |
+|--------------------|--------------------|----------------|---------|---------|------------|-----------|------|------|----------|
+| Alto (Female)      | ~                  | Ground Truth   | 1.0000  | 0.00    | ~          | 8.16      | ~    | ~    | ~        |
+|                    | Azuma (Female)     | RVCv2          | 0.9617  | 33.03   | **0.7352** | 24.70     | 3.36 | 4.07 | 3.07     |
+|                    |                    | Seed-VC(Ours)  | 0.9658  | 31.64   | 0.7341     | **15.23** | 3.37 | 4.02 | 3.07     |
+|                    | Diana (Female)     | RVCv2          | 0.9626  | 32.56   | 0.7212     | 19.67     | 3.45 | 4.08 | **3.17** |
+|                    |                    | Seed-VC(Ours)  | 0.9648  | 31.94   | **0.7457** | **16.81** | 3.49 | 3.99 | 3.15     |
+|                    | Ding Zhen (Male)   | RVCv2          | 0.9013  | 26.72   | 0.7221     | 18.53     | 3.37 | 4.03 | 3.06     |
+|                    |                    | Seed-VC(Ours)  | 0.9356  | 21.87   | **0.7513** | **15.63** | 3.44 | 3.94 | **3.09** |
+|                    | Kobe Bryant (Male) | RVCv2          | 0.9215  | 23.90   | 0.7495     | 37.23     | 3.49 | 4.06 | **3.21** |
+|                    |                    | Seed-VC(Ours)  | 0.9248  | 23.40   | **0.7602** | **26.98** | 3.43 | 4.02 | 3.13     |
+| Bass (Male)        | ~                  | Ground Truth   | 1.0000  | 0.00    | ~          | 8.62      | ~    | ~    | ~        |
+|                    | Azuma              | RVCv2          | 0.9288  | 32.62   | **0.7148** | 24.88     | 3.45 | 4.10 | **3.18** |
+|                    |                    | Seed-VC(Ours)  | 0.9383  | 31.57   | 0.6960     | **10.31** | 3.45 | 4.03 | 3.15     |
+|                    | Diana              | RVCv2          | 0.9403  | 30.00   | 0.7010     | 14.54     | 3.53 | 4.15 | **3.27** |
+|                    |                    | Seed-VC(Ours)  | 0.9428  | 30.06   | **0.7299** | **9.66**  | 3.53 | 4.11 | 3.25     |
+|                    | Ding Zhen          | RVCv2          | 0.9061  | 19.53   | 0.6922     | 25.99     | 3.36 | 4.09 | **3.08** |
+|                    |                    | Seed-VC(Ours)  | 0.9169  | 18.15   | **0.7260** | **14.13** | 3.38 | 3.98 | 3.07     |
+|                    | Kobe Bryant        | RVCv2          | 0.9302  | 16.37   | 0.7717     | 41.04     | 3.51 | 4.13 | **3.25** |
+|                    |                    | Seed-VC(Ours)  | 0.9176  | 17.93   | **0.7798** | **24.23** | 3.42 | 4.08 | 3.17     |
+| Soprano (Female)   | ~                  | Ground Truth   | 1.0000  | 0.00    | ~          | 27.92     | ~    | ~    | ~        |
+|                    | Azuma              | RVCv2          | 0.9742  | 47.80   | 0.7104     | 38.70     | 3.14 | 3.85 | **2.83** |
+|                    |                    | Seed-VC(Ours)  | 0.9521  | 64.00   | **0.7177** | **33.10** | 3.15 | 3.86 | 2.81     |
+|                    | Diana              | RVCv2          | 0.9754  | 46.59   | **0.7319** | 32.36     | 3.14 | 3.85 | **2.83** |
+|                    |                    | Seed-VC(Ours)  | 0.9573  | 59.70   | 0.7317     | **30.57** | 3.11 | 3.78 | 2.74     |
+|                    | Ding Zhen          | RVCv2          | 0.9543  | 31.45   | 0.6792     | 40.80     | 3.41 | 4.08 | **3.14** |
+|                    |                    | Seed-VC(Ours)  | 0.9486  | 33.37   | **0.6979** | **34.45** | 3.41 | 3.97 | 3.10     |
+|                    | Kobe Bryant        | RVCv2          | 0.9691  | 25.50   | 0.6276     | 61.59     | 3.43 | 4.04 | **3.15** |
+|                    |                    | Seed-VC(Ours)  | 0.9496  | 32.76   | **0.6683** | **39.82** | 3.32 | 3.98 | 3.04     |
+| Tenor (Male)       | ~                  | Ground Truth   | 1.0000  | 0.00    | ~          | 5.94      | ~    | ~    | ~        |
+|                    | Azuma              | RVCv2          | 0.9333  | 42.09   | **0.7832** | 16.66     | 3.46 | 4.07 | **3.18** |
+|                    |                    | Seed-VC(Ours)  | 0.9162  | 48.06   | 0.7697     | **8.48**  | 3.38 | 3.89 | 3.01     |
+|                    | Diana              | RVCv2          | 0.9467  | 36.65   | 0.7729     | 15.28     | 3.53 | 4.08 | **3.24** |
+|                    |                    | Seed-VC(Ours)  | 0.9360  | 41.49   | **0.7920** | **8.55**  | 3.49 | 3.93 | 3.13     |
+|                    | Ding Zhen          | RVCv2          | 0.9197  | 22.82   | 0.7591     | 12.92     | 3.40 | 4.02 | **3.09** |
+|                    |                    | Seed-VC(Ours)  | 0.9247  | 22.77   | **0.7721** | **13.95** | 3.45 | 3.82 | 3.05     |
+|                    | Kobe Bryant        | RVCv2          | 0.9415  | 19.33   | 0.7507     | 30.52     | 3.48 | 4.02 | **3.19** |
+|                    |                    | Seed-VC(Ours)  | 0.9082  | 24.86   | **0.7764** | **13.35** | 3.39 | 3.93 | 3.07     |
+</details>
+  
+  
+Despite Seed-VC is not trained on the target speakers, and only one random utterance is used as prompt, it still constantly outperforms speaker-specific RVCv2 models 
+in terms of speaker similarity (SECS) and intelligibility (CER), which demonstrates the superior voice cloning capability and robustness of Seed-VC.   
+
+However, it is observed that Seed-VC's audio quality (DNSMOS) is slightly lower than RVCv2. We take this drawback seriously and 
+will give high priority to improve the audio quality in the future.  
+PR or issue is welcomed if you find this comparison unfair or inaccurate.
+
+*Chinese ASR result computed by [SenseVoiceSmall](https://github.com/FunAudioLLM/SenseVoice)*  
+*Speaker embedding computed by [resemblyzer](https://github.com/resemble-ai/Resemblyzer) model*  
+*We set +12 semitones pitch shift for male-to-female conversion and -12 semitones for female-to-male converison, otherwise 0 pitch shift*
+

LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

README-JA.md

@@ -0,0 +1,222 @@
+# Seed-VC
+[![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Demo-blue)](https://huggingface.co/spaces/Plachta/Seed-VC)  [![arXiv](https://img.shields.io/badge/arXiv-2411.09943-<COLOR>.svg)](https://arxiv.org/abs/2411.09943)
+
+*[English](README.md) | [简体中文](README-ZH.md) | 日本語*
+
+[real-time-demo.webm](https://github.com/user-attachments/assets/86325c5e-f7f6-4a04-8695-97275a5d046c)
+
+*(注意：この文書は機械翻訳によって生成されたものです。正確性を確保するよう努めていますが、不明確な点がございましたら英語版をご参照ください。翻訳の改善案がございましたら、PRを歓迎いたします。)*
+
+現在リリースされているモデルは、*ゼロショット音声変換* 🔊、*ゼロショットリアルタイム音声変換* 🗣️、*ゼロショット歌声変換* 🎶 に対応しています。トレーニングなしで、1〜30秒の参照音声からボイスクローニングが可能です。
+
+カスタムデータでの追加ファインチューニングをサポートしており、特定の話者/話者群に対するパフォーマンスを向上させることができます。データ要件は極めて少なく（**話者あたり最低1発話**）、トレーニング速度も非常に速い（**最低100ステップ、T4で2分**）です！
+
+**リアルタイム音声変換**に対応しており、アルゴリズムの遅延は約300ms、デバイス側の遅延は約100msで、オンライン会議、ゲーム、ライブ配信に適しています。
+
+デモや以前の音声変換モデルとの比較については、[デモページ](https://plachtaa.github.io/seed-vc/)🌐と[評価](EVAL.md)📊をご覧ください。
+
+モデルの品質向上と機能追加を継続的に行っています。
+
+## 評価📊
+客観的評価結果と他のベースラインとの比較については[EVAL.md](EVAL.md)をご覧ください。
+
+## インストール📥
+Windows または Linux で Python 3.10 を推奨します。
+```bash
+pip install -r requirements.txt
+```
+
+## 使用方法🛠️
+目的に応じて3つのモデルをリリースしています：
+
+| バージョン | 名称                                                                                                                                                                                                                       | 目的                        | サンプリングレート | コンテンツエンコーダ | ボコーダ | 隠れ次元 | レイヤー数 | パラメータ数 | 備考                                                |
+|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------|---------------|-----------------|---------|------------|----------|--------|--------------------------------------------------------|
+| v1.0    | seed-uvit-tat-xlsr-tiny ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_uvit_tat_xlsr_ema.pth)[📄](configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml))                                                     | 音声変換 (VC)          | 22050         | XLSR-large      | HIFT    | 384        | 9        | 25M    | リアルタイム音声変換に適しています                |
+| v1.0    | seed-uvit-whisper-small-wavenet ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth)[📄](configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml)) | 音声変換 (VC)          | 22050         | Whisper-small   | BigVGAN | 512        | 13       | 98M    | オフライン音声変換に適しています                  |
+| v1.0    | seed-uvit-whisper-base ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth)[📄](configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml))       | 歌声変換 (SVC) | 44100         | Whisper-small   | BigVGAN | 768        | 17       | 200M   | 強力なゼロショットパフォーマンス、歌声変換 |
+
+最新のモデルリリースのチェックポイントは、最初の推論実行時に自動的にダウンロードされます。
+ネットワークの理由でhuggingfaceにアクセスできない場合は、すべてのコマンドの前に `HF_ENDPOINT=https://hf-mirror.com` を追加してミラーを使用してください。
+
+コマンドライン推論：
+```bash
+python inference.py --source <source-wav>
+--target <referene-wav>
+--output <output-dir>
+--diffusion-steps 25 # 歌声変換には30〜50を推奨
+--length-adjust 1.0
+--inference-cfg-rate 0.7
+--f0-condition False # 歌声変換の場合はTrueに設定
+--auto-f0-adjust False # ソースピッチをターゲットピッチレベルに自動調整する場合はTrue���通常は歌声変換では使用しない
+--semi-tone-shift 0 # 歌声変換のピッチシフト（半音単位）
+--checkpoint <path-to-checkpoint>
+--config <path-to-config>
+--fp16 True
+```
+各パラメータの説明：
+- `source` は変換したい音声ファイルのパス
+- `target` は参照音声ファイルのパス
+- `output` は出力ディレクトリのパス
+- `diffusion-steps` は拡散ステップ数、デフォルトは25、最高品質には30-50、最速推論には4-10を使用
+- `length-adjust` は長さ調整係数、デフォルトは1.0、<1.0で音声短縮、>1.0で音声伸長
+- `inference-cfg-rate` は出力に微妙な違いをもたらす、デフォルトは0.7
+- `f0-condition` はソース音声のピッチを出力に条件付けするフラグ、デフォルトはFalse、歌声変換の場合はTrue
+- `auto-f0-adjust` はソースピッチをターゲットピッチレベルに自動調整するフラグ、デフォルトはFalse、通常は歌声変換では使用しない
+- `semi-tone-shift` は歌声変換のピッチシフト（半音単位）、デフォルトは0
+- `checkpoint` は独自のモデルをトレーニングまたはファインチューニングした場合のモデルチェックポイントへのパス、空白の場合はhuggingfaceからデフォルトモデルを自動ダウンロード（`f0-condition`が`False`の場合は`seed-uvit-whisper-small-wavenet`、それ以外は`seed-uvit-whisper-base`）
+- `config` は独自のモデルをトレーニングまたはファインチューニングした場合のモデル設定へのパス、空白の場合はhuggingfaceからデフォルト設定を自動ダウンロード
+- `fp16` はfloat16推論を使用するフラグ、デフォルトはTrue
+
+音声変換Web UI：
+```bash
+python app_vc.py --checkpoint <path-to-checkpoint> --config <path-to-config> --fp16 True
+```
+- `checkpoint` は独自のモデルをトレーニングまたはファインチューニングした場合のモデルチェックポイントへのパス、空白の場合はhuggingfaceからデフォルトモデルを自動ダウンロード（`seed-uvit-whisper-small-wavenet`）
+- `config` は独自のモデルをトレーニングまたはファインチューニングした場合のモデル設定へのパス、空白の場合はhuggingfaceからデフォルト設定を自動ダウンロード
+
+ブラウザで`http://localhost:7860/`にアクセスしてWebインターフェースを使用できます。
+
+歌声変換Web UI：
+```bash
+python app_svc.py --checkpoint <path-to-checkpoint> --config <path-to-config> --fp16 True
+```
+- `checkpoint` は独自のモデルをトレーニングまたはファインチューニングした場合のモデルチェックポイントへのパス、空白の場合はhuggingfaceからデフォルトモデルを自動ダウンロード（`seed-uvit-whisper-base`）
+- `config` は独自のモデルをトレーニングまたはファインチューニングした場合のモデル設定へのパス、空白の場合はhuggingfaceからデフォルト設定を自動ダウンロード
+
+統合Web UI：
+```bash
+python app.py
+```
+これはゼロショット推論用の事前学習済みモデルのみを読み込みます。カスタムチェックポイントを使用する場合は、上記の`app_vc.py`または`app_svc.py`を実行してください。
+
+リアルタイム音声変換GUI：
+```bash
+python real-time-gui.py --checkpoint-path <path-to-checkpoint> --config-path <path-to-config>
+```
+- `checkpoint` は独自のモデルをトレーニングまたはファインチューニングした場合のモデルチェックポイントへのパス、空白の場合はhuggingfaceからデフォルトモデルを自動ダウンロード（`seed-uvit-tat-xlsr-tiny`）
+- `config` は独自のモデルをトレーニングまたはファインチューニングした場合のモデル設定へのパス、空白の場合はhuggingfaceからデフォルト設定を自動ダウンロード
+
+重要：リアルタイム音声変換にはGPUの使用を強く推奨します。
+NVIDIA RTX 3060ノートパソコンGPUでいくつかのパフォーマンステストを行い、結果と推奨パラメータ設定を以下に示します：
+
+| モデル構成             | 拡散ステップ | 推論CFGレート | 最大プロンプト長 | ブロック時間 (秒) | クロスフェード長 (秒) | 追加コンテキスト (左) (秒) | 追加コンテキスト (右) (秒) | レイテンシ (ミリ秒) | チャンクあたりの推論時間 (ミリ秒) |
+|---------------------------------|-----------------|--------------------|-------------------|----------------|----------------------|--------------------------|---------------------------|--------------|-------------------------------| 
+| seed-uvit-xlsr-tiny             | 10              | 0.7                | 3.0               | 0.18           | 0.04                | 2.5                      | 0.02                     | 430          | 150                          |
+
+GUIでパラメータを自身のデバイスのパフォーマンスに合わせて調整できます。推論時間がブロック時間より短ければ、音声変換ストリームは正常に動作するはずです。
+他のGPU集約型タスク（ゲーム、動画視聴など）を実行している場合、推論速度が低下する可能性があることに注意してください。
+
+リアルタイム音声変換GUIのパラメータ説明：
+- `Diffusion Steps` は拡散ステップ数、リアルタイム変換の場合は通常4~10で最速推論
+- `Inference CFG Rate` は出力に微妙な違いをもたらす、デフォルトは0.7、0.0に設定すると1.5倍の推論速度が向上
+- `Max Prompt Length` は最大プロンプト長、設定を低くすると推論速度が速くなるが、提示音声との類似性が低下する可能性がある
+- `Block Time` は推論の各オーディオ チャンクの時間長です。値が大きいほどレイテンシが長くなります。この値はブロックあたりの推論時間よりも長くする必要があることに注意してください。ハードウェアの状態に応じて設定します。
+- `Crossfade Length` はクロスフェード長、通常は変更しない
+- `Extra context (left)` は推論のための追加履歴コンテキストの時間長です。値が高いほど推論時間は長くなりますが、安定性は向上します。
+- `Extra context (right)` は推論のための追加未来コンテキストの時間長です。値が高いほど推論時間とレイテンシは長くなりますが、安定性は向上します。
+
+アルゴリズムレイテンシーは`Block Time * 2 + Extra context (right)`で、デバイス側レイテンシーは通常100ms程度です。全体の遅延は 2 つの合計です。
+
+[VB-CABLE](https://vb-audio.com/Cable/)を使用して、GUI出力ストリームを仮想マイクにルーティングすることができます。
+
+*（GUIとオーディオチャンキングのロジックは[RVC](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI)から修正されています。素晴らしい実装に感謝します！）*
+
+## トレーニング🏋️
+カスタムデータでのファインチューニングにより、より正確に声をクローニングすることができます。特定の話者に対する話者類似性が大幅に向上しますが、WERが若干上昇する可能性があります。
+以下のColabチュートリアルで手順を確認できます：[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1R1BJTqMsTXZzYAVx3j1BiemFXog9pbQG?usp=sharing)
+
+1. 独自のデータセットを準備します。以下の条件を満たす必要があります：
+    - ファイル構造は問いません
+    - 各音声ファイルは1〜30秒の範囲である必要があり、それ以外は無視されます
+    - すべての音声ファイルは以下のいずれかの形式である必要があります：`.wav` `.flac` `.mp3` `.m4a` `.opus` `.ogg`
+    - 話者ラベルは必須ではありませんが、各話者に少なくとも1つの発話があることを確認してください
+    - もちろん、データが多いほどモデルのパフォーマンスは向上します
+    - トレーニングデータはできるだけクリーンである必要があり、BGMやノイズは望ましくありません
+
+2. ファインチューニング用に`configs/presets/`からモデル設定ファイルを選択するか、ゼロからトレーニングするための独自の設定を作成します。
+    - ファインチューニングの場合は、以下のいずれかを選択します：
+        - `./configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml` リアルタイム音声変換用
+        - `./configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml` オフライン音声変換用
+        - `./configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml` 歌声変換用
+
+3. 以下のコマンドでトレーニングを開始します：
+```bash
+python train.py 
+--config <path-to-config> 
+--dataset-dir <path-to-data>
+--run-name <run-name>
+--batch-size 2
+--max-steps 1000
+--max-epochs 1000
+--save-every 500
+--num-workers 0
+```
+各パラメータの説明：
+- `config` はモデル設定へのパス、ファインチューニング用に上記のいずれかを選択するか、ゼロからトレーニングする場合は独自の設定を作成
+- `dataset-dir` はデータセットディレクトリへのパス、すべての音声ファイルを含むフォルダである必要があります
+- `run-name` は実行名で、モデルチェックポイントとログの保存に使用されます
+- `batch-size` はトレーニング用のバッチサイズで、GPUメモリに応じて選択します
+- `max-steps` は最大トレーニングステッ��数で、データセットサイズとトレーニング時間に応じて選択します
+- `max-epochs` は最大エポック数で、データセットサイズとトレーニング時間に応じて選択します
+- `save-every` はモデルチェックポイントを保存するステップ間隔
+- `num-workers` はデータ読み込みのワーカー数、Windowsの場合は0に設定
+
+4. トレーニングが予期せず停止した場合、同じコマンドを再度実行することで、最後のチェックポイントから再開できます（最新のチェックポイントを見つけられるように、`run-name`と`config`引数が同じであることを確認してください）。
+
+5. トレーニング後、チェックポイントと設定ファイルのパスを指定することで、トレーニングしたモデルを推論に使用できます。
+    - これらは`./runs/<run-name>/`の下にあり、チェックポイントは`ft_model.pth`という名前で、設定ファイルはトレーニング設定ファイルと同じ名前です。
+    - 推論時には、ゼロショット使用時と同様に、使用したい話者の参照音声ファイルを指定する必要があります。
+
+## TODO📝
+- [x] コードのリリース
+- [x] 事前学習済みモデルのリリース：[![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-SeedVC-blue)](https://huggingface.co/Plachta/Seed-VC)
+- [x] Huggingfaceスペースデモ：[![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Space-blue)](https://huggingface.co/spaces/Plachta/Seed-VC)
+- [x] HTMLデモページ：[Demo](https://plachtaa.github.io/seed-vc/)
+- [x] ストリーミング推論
+- [x] ストリーミング推論のレイテンシー削減
+- [x] リアルタイム音声変換のデモ動画
+- [x] 歌声変換
+- [x] ソース音声のノイズ耐性
+- [ ] アーキテクチャの潜在的な改善
+    - [x] U-ViTスタイルのスキップ接続
+    - [x] OpenAI Whisperへの入力変更
+    - [x] Time as Token
+- [x] カスタムデータでのトレーニングコード
+- [x] フューショット/ワンショット話者ファインチューニング
+- [x] 歌声デコーディング用にNVIDIAのBigVGANに変更
+- [x] 歌声変換用のWhisperバージョンモデル
+- [x] 歌声変換のRVC/SoVITSとの客観的評価と比較
+- [x] 音声品質の向上
+- [ ] より良い歌声変換のためのNSFボコーダ
+- [x] 非発話時のリアルタイム音声変換アーティファクトの修正（VADモデルの追加により対応）
+- [x] ファインチューニング例のColabノートブック
+- [ ] Whisperをより高度な意味抽出器に置き換える
+- [ ] 今後追加予定
+
+## 更新履歴🗒️
+- 2024-11-26:
+    - リアルタイム音声変換用に最適化されたv1.0 tinyバージョンの事前学習済みモデルを更新
+    - ワンショット/フューショットの単一/複数話者ファインチューニングをサポート
+    - webUIおよびリアルタイムGUIでカスタムチェックポイントの使用をサポート
+- 2024-11-19:
+    - arXiv論文公開
+- 2024-10-28:
+    - より良い音声品質のファインチューニングされた44k歌声変換モデルを更新
+- 2024-10-27:
+    - リアルタイム音声変換GUIを追加
+- 2024-10-25:
+    - 歌声変換のRVCv2との包括的な評価結果と比較を追加
+- 2024-10-24:
+    - 音声コンテンツ入力としてOpenAI Whisperを使用した44kHz歌声変換モデルを更新
+- 2024-10-07:
+    - 音声コンテンツエンコーダをOpenAI Whisperに変更したv0.3事前学習済みモデルを更新
+    - v0.3事前学習済みモデルの客観的評価結果を追加
+- 2024-09-22:
+    - NVIDIAのBigVGANを使用する歌声変換モデルを更新し、高音域の歌声を大幅に改善
+    - Web UIで長い音声ファイルのチャンキングとストリーミング出力をサポート
+- 2024-09-18:
+    - 歌声変換用のf0条件付きモデルを更新
+- 2024-09-14:
+    - 同じ品質を達成するためのサイズ縮小と拡散ステップ数の削減、およびプロソディ保持の制御能力を追加したv0.2事前学習済みモデルを更新
+    - コマンドライン推論スクリプトを追加
+    - インストールと使用方法の説明を追加

README-ZH.md

@@ -0,0 +1,208 @@
+# Seed-VC  
+[![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Demo-blue)](https://huggingface.co/spaces/Plachta/Seed-VC)  [![arXiv](https://img.shields.io/badge/arXiv-2411.09943-<COLOR>.svg)](https://arxiv.org/abs/2411.09943)
+
+*English | [简体中文](README-ZH.md) | [日本語](README-JA.md)*  
+
+[real-time-demo.webm](https://github.com/user-attachments/assets/86325c5e-f7f6-4a04-8695-97275a5d046c)
+
+目前发布的模型支持 *零样本语音转换* 🔊 、*零样本实时语音转换* 🗣️ 和 *零样本歌声转换* 🎶。无需任何训练，只需1~30秒的参考语音，即可克隆声音。
+
+我们支持进一步使用自定义数据进行微调，以提高特定说话人的性能，数据需求门槛极低 **（每位说话人至少1条语音）** ，训练速度极快 **（最少100步，在T4上只需2分钟）**！
+
+**实时语音转换** 支持约300ms的算法延迟和约100ms的设备侧延迟，适用于在线会议、游戏和直播。
+
+要查看演示和与之前语音转换模型的比较，请访问我们的[演示页面](https://plachtaa.github.io/seed-vc/)🌐 和 [评估结果](EVAL.md)📊。
+
+我们会不断改进模型质量并增加更多功能。
+
+## 评估📊
+查看 [EVAL.md](EVAL.md) 获取客观评估结果和与其他基准模型的比较。
+
+## 使用🛠️
+我们已发布用于不同目的的3个模型：
+
+| 版本   | 模型名称                                                                                                                                                                                                                       | 用途         | 采样率   | Content编码器    | 声码器     | 隐藏层维度 | 层数 | 参数量  | 备注                 |
+|------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-------|---------------|---------|-------|----|------|--------------------|
+| v1.0 | seed-uvit-tat-xlsr-tiny ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_uvit_tat_xlsr_ema.pth)[📄](configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml))                                                     | 声音转换 (VC)  | 22050 | XLSR-large    | HIFT    | 384   | 9  | 25M  | 适合实时语音转换           |
+| v1.0 | seed-uvit-whisper-small-wavenet ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth)[📄](configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml)) | 声音转换 (VC)  | 22050 | Whisper-small | BigVGAN | 512   | 13 | 98M  | 性能更好但推理稍慢，适合离线语音转换 |
+| v1.0 | seed-uvit-whisper-base ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth)[📄](configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml))       | 歌声转换 (SVC) | 44100 | Whisper-small | BigVGAN | 768   | 17 | 200M | 强大的零样本推理能力，用于歌声转换  |
+
+首次推理时将自动下载最新模型的检查点。 如果因网络原因无法访问 Hugging Face，请尝试在每个命令前添加 `HF_ENDPOINT=https://hf-mirror.com` 使用镜像站。
+
+命令行推理：
+```bash
+python inference.py --source <source-wav>
+--target <referene-wav>
+--output <output-dir>
+--diffusion-steps 25 # 推荐为歌声转换设置为30~50
+--length-adjust 1.0
+--inference-cfg-rate 0.7
+--f0-condition False # 设置为 True 进行歌声转换
+--auto-f0-adjust False # 设置为 True 自动调整源音高至目标音高，通常不用于歌声转换（会导致歌声与BGM调性不一致）
+--semi-tone-shift 0 # 歌声转换中的音高移位（半音）
+--checkpoint <path-to-checkpoint>
+--config <path-to-config>
+```
+参数说明：
+- `source` 要转换为参考声音的语音文件路径
+- `target` 作为声音参考的语音文件路径
+- `output` 输出目录的路径
+- `diffusion-steps` 使用的扩散步数，默认为 25，质量最佳使用 30-50，最快推理使用 4-10
+- `length-adjust` 长度调整因子，默认值为 1.0，设置 <1.0 加速语音，>1.0 减慢语音
+- `inference-cfg-rate` classifier free guidance rate，默认为 0.7
+- `f0-condition` 是否对输出音高进行调节，默认为 False，设置为 True 用于歌声转换
+- `auto-f0-adjust` 是否自动调整源音高到目标音高，默认为 False，通常不用于歌声转换
+- `semi-tone-shift` 歌声转换中的音高移位（半音），默认值为 0
+- `checkpoint` 如果已训练或微调自己的模型，请指定模型检查点路径，若留空将自动下载 Hugging Face 的默认模型(`seed-uvit-whisper-small-wavenet` if `f0-condition` is `False` else `seed-uvit-whisper-base`)
+- `config` 如果已训练或微调自己的模型，请指定模型配置文件路径，若留空将自动下载 Hugging Face 的默认配置 
+
+
+语音转换 Web UI:
+```bash
+python app_vc.py --checkpoint <path-to-checkpoint> --config <path-to-config>
+```
+- `checkpoint` 模型检查点路径，若为空将自动下载默认模型 (`seed-uvit-whisper-small-wavenet`)
+- `config` 模型配置文件路径，若为空将自动下载默认配置
+
+然后在浏览器中打开 `http://localhost:7860/` 使用 Web 界面。
+
+运行命令前先设置环境变量:
+`export export HUGGING_FACE_HUB_TOKEN={从https://huggingface.co/settings/tokens获取}`
+
+歌声转换 Web UI:
+```bash
+python app_svc.py --checkpoint <path-to-checkpoint> --config <path-to-config>
+```
+- `checkpoint` 模型检查点路径，若为空将自动下载默认模型 (`seed-uvit-whisper-base`)
+- `config` 模型配置文件路径，若为空将自动下载默认配置  
+
+集成 Web UI:
+```bash
+python app.py
+```
+此命令将仅加载预训练模型进行零样本推理。要使用自定义检查点，请按上述步骤运行 `app_vc.py` 或 `app_svc.py`。
+
+实时语音转换 GUI:
+```bash
+python real-time-gui.py --checkpoint-path <path-to-checkpoint> --config-path <path-to-config>
+```
+- `checkpoint` 模型检查点路径，若为空将自动下载默认模型 (`seed-uvit-tat-xlsr-tiny`)
+- `config` 模型配置文件路径，若为空将自动下载默认配置  
+
+重要提示: 强烈建议使用 GPU 进行实时语音转换。 在 NVIDIA RTX 3060 笔记本 GPU 上进行了一些性能测试，结果和推荐参数设置如下：
+
+| 模型配置                | 扩散步数 | Inference CFG Rate | 最大prompt长度 | 每块时间 (s) | 交叉淡化长度 (s) | 额外上下文（左）(s) | 额外上下文（右）(s) | 延迟 (ms） | 每块推理时间 (ms) |
+|---------------------|------|--------------------|------------|----------|------------|-------------|-------------|---------|-------------| 
+| seed-uvit-xlsr-tiny | 10   | 0.7                | 3.0        | 0.18s    | 0.04s      | 2.5s        | 0.02s       | 430ms   | 150ms       |
+
+你可以根据设备性能调整 GUI 中的参数，只要推理时间小于块时间，语音转换流就可以正常工作。 注意，如果你正在运行其他占用 GPU 的任务（如游戏、看视频），推理速度可能会下降。
+
+实时转换界面的参数说明：
+- `Diffusion Steps` 是扩散步数，推荐实时转换设置为4~10；
+- `Inference CFG Rate` 是classifier free guidance rate，默认0.7，设置为0.0可以获得1.5x的加速；
+- `Max Prompt Length` 是最大音频提示长度，设置为较低值可以加快推理速度，但可能会降低与提示语音的相似度；
+- `Block Time` 是每块时间，值越高延迟越高，该值必须大于每块推理时间，根据硬件条件设置；
+- `Crossfade Length` 是交叉淡化长度，通常不需要更改；
+- `Extra context (left)` 是推理的额外上下文，设置为较高值可以增加稳定性，但会增加每块推理时间；
+- `Extra context (right)` 是推理的额外上下文，设置为较高值可以增加稳定性，但会增加每块推理时间以及延迟；
+
+算法延迟大约为 `Block Time * 2 + Extra context (right)`，设备侧延迟通常为100ms左右。总体延迟为两者之和。
+
+你可以使用 [VB-CABLE](https://vb-audio.com/Cable/) 将变声器输出映射到一个虚拟麦克风上，以便其它应用读取.  
+
+*(GUI and audio chunking logic are modified from [RVC](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI), thanks for their brilliant implementation!)*
+
+## 训练🏋️
+在自定义数据上进行微调可以让模型更精确地克隆某个人的声音。这将大幅提高特定说话人的相似度，但可能会略微增加 WER（词错误率）。  
+这里是一个简单的Colab示例以供参考: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1R1BJTqMsTXZzYAVx3j1BiemFXog9pbQG?usp=sharing)
+1. 准备您的数据集。必须满足以下要求：
+    - 文件结构不重要
+    - 每条音频长度必须在1-30秒之间，否则会被自动忽略
+    - 所有音频文件必须是以下格式之一：`.wav` `.flac` `.mp3` `.m4a` `.opus` `.ogg`
+    - 不需要说话人标签，但请确保每位说话人至少有 1 条语音
+    - 当然，数据越多，模型的表现就越好
+    - 训练样本应该选择尽量干净，不带背景音乐或噪音的音频
+2. 从 `configs/presets/` 中选择一个模型配置文件进行微调，或者创建自己的配置文件从头开始训练。
+    - 对于微调，可以选择以下配置之一：
+        - `./configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml` 用于实时语音转换
+        - `./configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml` 用于离线语音转换
+        - `./configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml` 用于歌声转换
+3. 运行以下命令开始训练：
+```bash
+python train.py 
+--config <path-to-config> 
+--dataset-dir <path-to-data>
+--run-name <run-name>
+--batch-size 2
+--max-steps 1000
+--max-epochs 1000
+--save-every 500
+--num-workers 0
+```
+where:
+- `config` 模型配置文件路径，选择上面之一进行微调，或者创建自己的配置文件从头开始训练
+- `dataset-dir` 数据集目录路径，应为包含所有音频文件的文件夹
+- `run-name` 运行名称，用于保存模型检查点和日志
+- `batch-size` 训练的批大小，根据 GPU 内存选择
+- `max-steps` 最大训练步数，取决于数据集大小和训练时间
+- `max-epochs` 最大训练轮数，取决于数据集大小和训练时间
+- `save-every` 保存模型检查点的步数
+- `num-workers` 数据加载的工作线程数量，建议 Windows 上设置为 0
+
+4. 如果需要从上次停止的地方继续训练，只需运行同样的命令即可。通过传入相同的 `run-name` 和 `config` 参数，程序将能够找到上次训练的检查点和日志。
+
+5. 训练完成后，您可以通过指定检查点和配置文件的路径来进行推理。
+    - 它们应位于 `./runs/<run-name>/` 下，检查点命名为 `ft_model.pth`，配置文件名称与训练配置文件相同。
+    - 在推理时，您仍需指定要使用的说话人的参考音频文件，类似于零样本推理。
+
+## TODO📝
+- [x] 发布代码
+- [x] 发布预训练模型： [![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-SeedVC-blue)](https://huggingface.co/Plachta/Seed-VC)
+- [x] Hugging Face Space 演示： [![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Space-blue)](https://huggingface.co/spaces/Plachta/Seed-VC)
+- [x] HTML 演示页面： [Demo](https://plachtaa.github.io/seed-vc/)
+- [x] 流式推理
+- [x] 降低延迟
+- [x] 实时变声Demo视频
+- [x] 歌声转换
+- [x] 提高源音频抗噪性
+- [ ] 潜在的架构改进
+    - [x] 类似U-ViT 的skip connection
+    - [x] 将输入更改为 OpenAI Whisper
+    - [x] Time as Token
+- [x] 自定义数据训练代码
+- [x] 单样本/少样本说话人微调
+- [x] 歌声解码器更改为 NVIDIA 的 BigVGAN
+- [x] 44k Hz 歌声转换模型
+- [x] 歌声转换的客观指标评估以及与RVC/SoVITS模型的比较
+- [x] 提升音质
+- [ ] 用于改善歌声转换的NSF歌声解码器
+- [x] 实时变声脚本添加了VAD模型，避免没有说话时模型输出杂音
+- [x] Google Colab 笔记本训练脚本以及样例
+- [ ] 替换whisper为更先进的语义内容提取器
+- [ ] 更多待添加
+
+## 更新日志 🗒️
+- 2024-11-26:
+    - 更新 v1.0 更小版本的预训练模型，优化实时语音转换
+    - 支持单样本/少样本的单/多说话人微调
+    - 支持在 WebUI 和实时变声 GUI 中使用自定义检查点
+- 2024-11-19:
+    - paper已提交至arXiv
+- 2024-10-27:
+    - 更新了实时变声脚本
+- 2024-10-25:
+    - 添加了详尽的歌声转换评估结果以及与RVCv2模型的比较
+- 2024-10-24:
+    - 更新了44kHz歌声转换模型
+- 2024-10-07:
+    - 更新了 v0.3 预训练模型，将语音内容编码器更改为 OpenAI Whisper
+    - 添加了 v0.3 预训练模型的客观指标评估结果
+- 2024-09-22:
+    - 将歌声转换模型的解码器更改为 BigVGAN，解决了大部分高音部分无法正确转换的问题
+    - 在Web UI中支持对长输入音频的分段处理以及流式输出
+- 2024-09-18:
+    - 更新了用于歌声转换的模型
+- 2024-09-14:
+    - 更新了 v0.2 预训练模型，具有更小的尺寸和更少的扩散步骤即可达到相同质量，且增加了控制韵律保留的能力
+    - 添加了命令行推理脚本
+    - 添加了安装和使用说明

README.md

@@ -0,0 +1,230 @@
+# Seed-VC  
+[![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Demo-blue)](https://huggingface.co/spaces/Plachta/Seed-VC)  [![arXiv](https://img.shields.io/badge/arXiv-2411.09943-<COLOR>.svg)](https://arxiv.org/abs/2411.09943)
+
+*English | [简体中文](README-ZH.md) | [日本語](README-JA.md)*  
+
+[real-time-demo.webm](https://github.com/user-attachments/assets/86325c5e-f7f6-4a04-8695-97275a5d046c)
+
+Currently released model supports *zero-shot voice conversion* 🔊 , *zero-shot real-time voice conversion* 🗣️ and *zero-shot singing voice conversion* 🎶. Without any training, it is able to clone a voice given a reference speech of 1~30 seconds.  
+
+We support further fine-tuning on custom data to increase performance on specific speaker/speakers, with extremely low data requirement **(minimum 1 utterance per speaker)** and extremely fast training speed **(minimum 100 steps, 2 min on T4)**!
+
+**Real-time voice conversion** is support, with algorithm delay of ~300ms and device side delay of ~100ms, suitable for online meetings, gaming and live streaming.
+
+To find a list of demos and comparisons with previous voice conversion models, please visit our [demo page](https://plachtaa.github.io/seed-vc/)🌐  and [Evaluaiton](EVAL.md)📊.
+
+We are keeping on improving the model quality and adding more features.
+
+## Evaluation📊
+See [EVAL.md](EVAL.md) for objective evaluation results and comparisons with other baselines.
+## Installation📥
+Suggested python 3.10 on Windows, Mac M Series (Apple Silicon) or Linux.
+Windows and Linux:
+```bash
+pip install -r requirements.txt
+```
+
+Mac M Series:
+```bash
+pip install -r requirements-mac.txt
+```
+
+## Usage🛠️
+We have released 3 models for different purposes:
+
+| Version | Name                                                                                                                                                                                                                       | Purpose                        | Sampling Rate | Content Encoder | Vocoder | Hidden Dim | N Layers | Params | Remarks                                                |
+|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------|---------------|-----------------|---------|------------|----------|--------|--------------------------------------------------------|
+| v1.0    | seed-uvit-tat-xlsr-tiny ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_uvit_tat_xlsr_ema.pth)[📄](configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml))                                                     | Voice Conversion (VC)          | 22050         | XLSR-large      | HIFT    | 384        | 9        | 25M    | suitable for real-time voice conversion                |
+| v1.0    | seed-uvit-whisper-small-wavenet ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth)[📄](configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml)) | Voice Conversion (VC)          | 22050         | Whisper-small   | BigVGAN | 512        | 13       | 98M    | suitable for offline voice conversion                  |
+| v1.0    | seed-uvit-whisper-base ([🤗](https://huggingface.co/Plachta/Seed-VC/blob/main/DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth)[📄](configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml))       | Singing Voice Conversion (SVC) | 44100         | Whisper-small   | BigVGAN | 768        | 17       | 200M   | strong zero-shot performance, singing voice conversion |
+
+Checkpoints of the latest model release will be downloaded automatically when first run inference.  
+If you are unable to access huggingface for network reason, try using mirror by adding `HF_ENDPOINT=https://hf-mirror.com` before every command.
+
+Command line inference:
+```bash
+python inference.py --source <source-wav>
+--target <referene-wav>
+--output <output-dir>
+--diffusion-steps 25 # recommended 30~50 for singingvoice conversion
+--length-adjust 1.0
+--inference-cfg-rate 0.7
+--f0-condition False # set to True for singing voice conversion
+--auto-f0-adjust False # set to True to auto adjust source pitch to target pitch level, normally not used in singing voice conversion
+--semi-tone-shift 0 # pitch shift in semitones for singing voice conversion
+--checkpoint <path-to-checkpoint>
+--config <path-to-config>
+ --fp16 True
+```
+where:
+- `source` is the path to the speech file to convert to reference voice
+- `target` is the path to the speech file as voice reference
+- `output` is the path to the output directory
+- `diffusion-steps` is the number of diffusion steps to use, default is 25, use 30-50 for best quality, use 4-10 for fastest inference
+- `length-adjust` is the length adjustment factor, default is 1.0, set <1.0 for speed-up speech, >1.0 for slow-down speech
+- `inference-cfg-rate` has subtle difference in the output, default is 0.7 
+- `f0-condition` is the flag to condition the pitch of the output to the pitch of the source audio, default is False, set to True for singing voice conversion  
+- `auto-f0-adjust` is the flag to auto adjust source pitch to target pitch level, default is False, normally not used in singing voice conversion
+- `semi-tone-shift` is the pitch shift in semitones for singing voice conversion, default is 0  
+- `checkpoint` is the path to the model checkpoint if you have trained or fine-tuned your own model, leave to blank to auto-download default model from huggingface.(`seed-uvit-whisper-small-wavenet` if `f0-condition` is `False` else `seed-uvit-whisper-base`)
+- `config` is the path to the model config if you have trained or fine-tuned your own model, leave to blank to auto-download default config from huggingface  
+- `fp16` is the flag to use float16 inference, default is True
+
+Voice Conversion Web UI:
+```bash
+python app_vc.py --checkpoint <path-to-checkpoint> --config <path-to-config> --fp16 True
+```
+- `checkpoint` is the path to the model checkpoint if you have trained or fine-tuned your own model, leave to blank to auto-download default model from huggingface. (`seed-uvit-whisper-small-wavenet`)
+- `config` is the path to the model config if you have trained or fine-tuned your own model, leave to blank to auto-download default config from huggingface  
+
+Then open the browser and go to `http://localhost:7860/` to use the web interface.
+
+Singing Voice Conversion Web UI:
+```bash
+python app_svc.py --checkpoint <path-to-checkpoint> --config <path-to-config> --fp16 True
+```
+- `checkpoint` is the path to the model checkpoint if you have trained or fine-tuned your own model, leave to blank to auto-download default model from huggingface. (`seed-uvit-whisper-base`)
+- `config` is the path to the model config if you have trained or fine-tuned your own model, leave to blank to auto-download default config from huggingface  
+
+Integrated Web UI:
+```bash
+python app.py
+```
+This will only load pretrained models for zero-shot inference. To use custom checkpoints, please run `app_vc.py` or `app_svc.py` as above.
+
+Real-time voice conversion GUI:
+```bash
+python real-time-gui.py --checkpoint-path <path-to-checkpoint> --config-path <path-to-config>
+```
+- `checkpoint` is the path to the model checkpoint if you have trained or fine-tuned your own model, leave to blank to auto-download default model from huggingface. (`seed-uvit-tat-xlsr-tiny`)
+- `config` is the path to the model config if you have trained or fine-tuned your own model, leave to blank to auto-download default config from huggingface  
+
+> [!IMPORTANT]
+> It is strongly recommended to use a GPU for real-time voice conversion.
+> Some performance testing has been done on a NVIDIA RTX 3060 Laptop GPU, results and recommended parameter settings are listed below:
+
+| Model Configuration             | Diffusion Steps | Inference CFG Rate | Max Prompt Length | Block Time (s) | Crossfade Length (s) | Extra context (left) (s) | Extra context (right) (s) | Latency (ms) | Inference Time per Chunk (ms) |
+|---------------------------------|-----------------|--------------------|-------------------|----------------|----------------------|--------------------------|---------------------------|--------------|-------------------------------| 
+| seed-uvit-xlsr-tiny             | 10              | 0.7                | 3.0               | 0.18s          | 0.04s                | 2.5s                     | 0.02s                     | 430ms        | 150ms                         |
+
+You can adjust the parameters in the GUI according to your own device performance, the voice conversion stream should work well as long as Inference Time is less than Block Time.  
+Note that inference speed may drop if you are running other GPU intensive tasks (e.g. gaming, watching videos)  
+
+Explanations for real-time voice conversion GUI parameters:
+- `Diffusion Steps` is the number of diffusion steps to use, in real-time case usually set to 4~10 for fastest inference;
+- `Inference CFG Rate` has subtle difference in the output, default is 0.7, set to 0.0 gains about 1.5x speed-up;
+- `Max Prompt Length` is the maximum length of the prompt audio, setting to a low value can speed up inference, but may reduce similarity to prompt speech;
+- `Block Time` is the time length of each audio chunk for inference, the higher the value, the higher the latency, note this value must be greater than the inference time per block, set according to your hardware condition;
+- `Crossfade Length` is the time length of crossfade between audio chunks, normally not needed to change;
+- `Extra context (left)` is the time length of extra history context for inference, the higher the value, the higher the inference time, but can increase stability;
+- `Extra context (right)` is the time length of extra future context for inference, the higher the value, the higher the inference time and latency, but can increase stability;
+
+The algorithm delay is appoximately calculated as `Block Time * 2 + Extra context (right)`, device side delay is usually of ~100ms. The overall delay is the sum of the two.
+
+You may wish to use [VB-CABLE](https://vb-audio.com/Cable/) to route audio from GUI output stream to a virtual microphone.  
+
+*(GUI and audio chunking logic are modified from [RVC](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI), thanks for their brilliant implementation!)*
+
+## Training🏋️
+Fine-tuning on custom data allow the model to clone someone's voice more accurately. It will largely improve speaker similarity on particular speakers, but may slightly increase WER.  
+A Colab Tutorial is here for you to follow: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1R1BJTqMsTXZzYAVx3j1BiemFXog9pbQG?usp=sharing)
+1. Prepare your own dataset. It has to satisfy the following:
+    - File structure does not matter
+    - Each audio file should range from 1 to 30 seconds, otherwise will be ignored
+    - All audio files should be in on of the following formats: `.wav` `.flac` `.mp3` `.m4a` `.opus` `.ogg`
+    - Speaker label is not required, but make sure that each speaker has at least 1 utterance
+    - Of course, the more data you have, the better the model will perform
+    - Training data should be as clean as possible, BGM or noise is not desired
+2. Choose a model configuration file from `configs/presets/` for fine-tuning, or create your own to train from scratch.
+    - For fine-tuning, it should be one of the following:
+        - `./configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml` for real-time voice conversion
+        - `./configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml` for offline voice conversion
+        - `./configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml` for singing voice conversion
+3. Run the following command to start training:
+```bash
+python train.py 
+--config <path-to-config> 
+--dataset-dir <path-to-data>
+--run-name <run-name>
+--batch-size 2
+--max-steps 1000
+--max-epochs 1000
+--save-every 500
+--num-workers 0
+```
+where:
+- `config` is the path to the model config, choose one of the above for fine-tuning or create your own for training from scratch
+- `dataset-dir` is the path to the dataset directory, which should be a folder containing all the audio files
+- `run-name` is the name of the run, which will be used to save the model checkpoints and logs
+- `batch-size` is the batch size for training, choose depends on your GPU memory.
+- `max-steps` is the maximum number of steps to train, choose depends on your dataset size and training time
+- `max-epochs` is the maximum number of epochs to train, choose depends on your dataset size and training time
+- `save-every` is the number of steps to save the model checkpoint
+- `num-workers` is the number of workers for data loading, set to 0 for Windows    
+
+4. If training accidentially stops, you can resume training by running the same command again, the training will continue from the last checkpoint. (Make sure `run-name` and `config` arguments are the same so that latest checkpoint can be found)
+
+5. After training, you can use the trained model for inference by specifying the path to the checkpoint and config file.
+    - They should be under `./runs/<run-name>/`, with the checkpoint named `ft_model.pth` and config file with the same name as the training config file.
+    - You still have to specify a reference audio file of the speaker you'd like to use during inference, similar to zero-shot usage.
+
+## TODO📝
+- [x] Release code
+- [x] Release pretrained models: [![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-SeedVC-blue)](https://huggingface.co/Plachta/Seed-VC)
+- [x] Huggingface space demo: [![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Space-blue)](https://huggingface.co/spaces/Plachta/Seed-VC)
+- [x] HTML demo page: [Demo](https://plachtaa.github.io/seed-vc/)
+- [x] Streaming inference
+- [x] Reduce streaming inference latency
+- [x] Demo video for real-time voice conversion
+- [x] Singing voice conversion
+- [x] Noise resiliency for source audio
+- [ ] Potential architecture improvements
+    - [x] U-ViT style skip connections
+    - [x] Changed input to OpenAI Whisper
+    - [x] Time as Token
+- [x] Code for training on custom data
+- [x] Few-shot/One-shot speaker fine-tuning
+- [x] Changed to BigVGAN from NVIDIA for singing voice decoding
+- [x] Whisper version model for singing voice conversion
+- [x] Objective evaluation and comparison with RVC/SoVITS for singing voice conversion
+- [x] Improve audio quality
+- [ ] NSF vocoder for better singing voice conversion
+- [x] Fix real-time voice conversion artifact while not talking (done by adding a VAD model)
+- [x] Colab Notebook for fine-tuning example
+- [ ] Replace whisper with more advanced linguistic content extractor
+- [ ] More to be added
+- [x] Add Apple Silicon support
+
+## Known Issues
+- On Mac - running `real-time-gui.py` might raise an error `ModuleNotFoundError: No module named '_tkinter'`, in this case a new Python version **with Tkinter support** should be installed. Refer to [This Guide on stack overflow](https://stackoverflow.com/questions/76105218/why-does-tkinter-or-turtle-seem-to-be-missing-or-broken-shouldnt-it-be-part) for explanation of the problem and a detailed fix.
+
+
+## CHANGELOGS🗒️
+- 2025-03-03:
+    - Added Mac M Series (Apple Silicon) support
+- 2024-11-26:
+    - Updated v1.0 tiny version pretrained model, optimized for real-time voice conversion
+    - Support one-shot/few-shot single/multi speaker fine-tuning
+    - Support using custom checkpoint for webUI & real-time GUI
+- 2024-11-19:
+    - arXiv paper released
+- 2024-10-28:
+    - Updated fine-tuned 44k singing voice conversion model with better audio quality
+- 2024-10-27:
+    - Added real-time voice conversion GUI
+- 2024-10-25:
+    - Added exhaustive evaluation results and comparisons with RVCv2 for singing voice conversion
+- 2024-10-24:
+    - Updated 44kHz singing voice conversion model, with OpenAI Whisper as speech content input
+- 2024-10-07:
+    - Updated v0.3 pretrained model, changed speech content encoder to OpenAI Whisper
+    - Added objective evaluation results for v0.3 pretrained model
+- 2024-09-22:
+    - Updated singing voice conversion model to use BigVGAN from NVIDIA, providing large improvement to high-pitched singing voices
+    - Support chunking and  streaming output for long audio files in Web UI
+- 2024-09-18:
+    - Updated f0 conditioned model for singing voice conversion
+- 2024-09-14:
+    - Updated v0.2 pretrained model, with smaller size and less diffusion steps to achieve same quality, and additional ability to control prosody preservation
+    - Added command line inference script
+    - Added installation and usage instructions

api.py

@@ -0,0 +1,159 @@
+import logging
+import os
+import uuid
+from contextlib import asynccontextmanager
+from tempfile import NamedTemporaryFile
+
+import boto3
+import torchaudio
+from fastapi import BackgroundTasks, Depends, FastAPI, Header, HTTPException
+from fastapi.security import APIKeyHeader
+from pydantic import BaseModel
+
+from inference import load_models, process_voice_conversion
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Global variables
+models = None
+API_KEY = os.getenv("API_KEY")
+
+api_key_header = APIKeyHeader(name="Authorization", auto_error=False)
+
+
+async def verify_api_key(authorization: str = Header(None)):
+    if not authorization:
+        logger.warning("No API key provided")
+        raise HTTPException(status_code=401, detail="API key is missing")
+
+    if authorization.startswith("Bearer "):
+        token = authorization.replace("Bearer ", "")
+    else:
+        token = authorization
+
+    if token != API_KEY:
+        logger.warning("Invalid API key provided")
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    return token
+
+
+def get_s3_client():
+    client_kwargs = {'region_name': os.getenv("AWS_REGION", "us-east-1")}
+
+    if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
+        client_kwargs.update({
+            'aws_access_key_id': os.getenv("AWS_ACCESS_KEY_ID"),
+            'aws_secret_access_key': os.getenv("AWS_SECRET_ACCESS_KEY")
+        })
+
+    return boto3.client('s3', **client_kwargs)
+
+
+s3_client = get_s3_client()
+
+S3_PREFIX = os.getenv("S3_PREFIX", "seedvc-outputs")
+S3_BUCKET = os.getenv("S3_BUCKET", "elevenlabs-clone")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global models
+    logger.info("Loading Seed-VC model...")
+    try:
+        models = load_models()
+
+        logger.info("Seed-VC model loaded successfully")
+    except Exception as e:
+        logger.error(f"Failed to load model: {e}")
+        raise
+
+    yield
+
+    logger.info("Shutting down Seed-VC API")
+
+app = FastAPI(title="Seed-VC API",
+              lifespan=lifespan)
+
+TARGET_VOICES = {
+    "andreas": "examples/reference/andreas1.wav",
+    "woman": "examples/reference/s1p1.wav",
+    "trump": "examples/reference/trump_0.wav",
+}
+
+
+class VoiceConversionRequest(BaseModel):
+    source_audio_key: str
+    target_voice: str
+
+
+@app.post("/convert", dependencies=[Depends(verify_api_key)])
+async def generate_speech(request: VoiceConversionRequest, background_tasks: BackgroundTasks):
+    if not models:
+        raise HTTPException(status_code=500, detail="Model not loaded")
+
+    if request.target_voice not in TARGET_VOICES:
+        raise HTTPException(
+            status_code=400, detail=f"Target voice not supported. Choose from: {', '.join(TARGET_VOICES.keys())}")
+
+    try:
+        target_audio_path = TARGET_VOICES[request.target_voice]
+        logger.info(
+            f"Converting voice: {request.source_audio_key} to {request.target_voice}")
+
+        # Generate a unique filename
+        audio_id = str(uuid.uuid4())
+        output_filename = f"{audio_id}.wav"
+        local_path = f"/tmp/{output_filename}"
+
+        logger.info("Downloading source audio")
+        source_temp = NamedTemporaryFile(delete=False, suffix=".wav")
+        try:
+            s3_client.download_fileobj(
+                S3_BUCKET, Key=request.source_audio_key, Fileobj=source_temp)
+            source_temp.close()
+        except Exception as e:
+            os.unlink(source_temp.name)
+            raise HTTPException(
+                status_code=404, detail="Source audio not found")
+
+        vc_wave, sr = process_voice_conversion(
+            models=models, source=source_temp.name, target_name=target_audio_path, output=None)
+
+        os.unlink(source_temp.name)
+
+        torchaudio.save(local_path, vc_wave, sr)
+
+        # Upload to S3
+        s3_key = f"{S3_PREFIX}/{output_filename}"
+        s3_client.upload_file(local_path, S3_BUCKET, s3_key)
+
+        presigned_url = s3_client.generate_presigned_url(
+            'get_object',
+            Params={'Bucket': S3_BUCKET, 'Key': s3_key},
+            ExpiresIn=3600
+        )
+
+        background_tasks.add_task(os.remove, local_path)
+
+        return {
+            "audio_url": presigned_url,
+            "s3_key": s3_key
+        }
+    except Exception as e:
+        logger.error(f"Error in voice conversion: {e}")
+        raise HTTPException(
+            status_code=500, detail="Error in voice conversion")
+
+
+@app.get("/voices", dependencies=[Depends(verify_api_key)])
+async def list_voices():
+    return {"voices": list(TARGET_VOICES.keys())}
+
+
+@app.get("/health", dependencies=[Depends(verify_api_key)])
+async def health_check():
+    if models:
+        return {"status": "healthy", "model": "loaded"}
+    return {"status": "unhealthy", "model": "not loaded"}

app.py

@@ -0,0 +1,372 @@
+import gradio as gr
+import torch
+import torchaudio
+import librosa
+from modules.commons import build_model, load_checkpoint, recursive_munch
+import yaml
+from hf_utils import load_custom_model_from_hf
+import numpy as np
+from pydub import AudioSegment
+
+# Load model and configuration
+
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+elif torch.backends.mps.is_available():
+    device = torch.device("mps")
+else:
+    device = torch.device("cpu")
+
+dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
+                                                "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
+                                                "config_dit_mel_seed_uvit_whisper_small_wavenet.yml")
+config = yaml.safe_load(open(dit_config_path, 'r'))
+model_params = recursive_munch(config['model_params'])
+model = build_model(model_params, stage='DiT')
+hop_length = config['preprocess_params']['spect_params']['hop_length']
+sr = config['preprocess_params']['sr']
+
+# Load checkpoints
+model, _, _, _ = load_checkpoint(model, None, dit_checkpoint_path,
+                                 load_only_params=True, ignore_modules=[], is_distributed=False)
+for key in model:
+    model[key].eval()
+    model[key].to(device)
+model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
+
+# Load additional modules
+from modules.campplus.DTDNN import CAMPPlus
+
+campplus_ckpt_path = load_custom_model_from_hf("funasr/campplus", "campplus_cn_common.bin", config_filename=None)
+campplus_model = CAMPPlus(feat_dim=80, embedding_size=192)
+campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu"))
+campplus_model.eval()
+campplus_model.to(device)
+
+from modules.bigvgan import bigvgan
+
+bigvgan_model = bigvgan.BigVGAN.from_pretrained('nvidia/bigvgan_v2_22khz_80band_256x', use_cuda_kernel=False)
+
+# remove weight norm in the model and set to eval mode
+bigvgan_model.remove_weight_norm()
+bigvgan_model = bigvgan_model.eval().to(device)
+
+# whisper
+from transformers import AutoFeatureExtractor, WhisperModel
+
+whisper_name = model_params.speech_tokenizer.whisper_name if hasattr(model_params.speech_tokenizer,
+                                                                     'whisper_name') else "openai/whisper-small"
+whisper_model = WhisperModel.from_pretrained(whisper_name, torch_dtype=torch.float16).to(device)
+del whisper_model.decoder
+whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name)
+
+# Generate mel spectrograms
+mel_fn_args = {
+    "n_fft": config['preprocess_params']['spect_params']['n_fft'],
+    "win_size": config['preprocess_params']['spect_params']['win_length'],
+    "hop_size": config['preprocess_params']['spect_params']['hop_length'],
+    "num_mels": config['preprocess_params']['spect_params']['n_mels'],
+    "sampling_rate": sr,
+    "fmin": 0,
+    "fmax": None,
+    "center": False
+}
+from modules.audio import mel_spectrogram
+
+to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
+
+# f0 conditioned model
+dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
+                                                "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth",
+                                                "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml")
+
+config = yaml.safe_load(open(dit_config_path, 'r'))
+model_params = recursive_munch(config['model_params'])
+model_f0 = build_model(model_params, stage='DiT')
+hop_length = config['preprocess_params']['spect_params']['hop_length']
+sr = config['preprocess_params']['sr']
+
+# Load checkpoints
+model_f0, _, _, _ = load_checkpoint(model_f0, None, dit_checkpoint_path,
+                                 load_only_params=True, ignore_modules=[], is_distributed=False)
+for key in model_f0:
+    model_f0[key].eval()
+    model_f0[key].to(device)
+model_f0.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
+
+# f0 extractor
+from modules.rmvpe import RMVPE
+
+model_path = load_custom_model_from_hf("lj1995/VoiceConversionWebUI", "rmvpe.pt", None)
+rmvpe = RMVPE(model_path, is_half=False, device=device)
+
+mel_fn_args_f0 = {
+    "n_fft": config['preprocess_params']['spect_params']['n_fft'],
+    "win_size": config['preprocess_params']['spect_params']['win_length'],
+    "hop_size": config['preprocess_params']['spect_params']['hop_length'],
+    "num_mels": config['preprocess_params']['spect_params']['n_mels'],
+    "sampling_rate": sr,
+    "fmin": 0,
+    "fmax": None,
+    "center": False
+}
+to_mel_f0 = lambda x: mel_spectrogram(x, **mel_fn_args_f0)
+bigvgan_44k_model = bigvgan.BigVGAN.from_pretrained('nvidia/bigvgan_v2_44khz_128band_512x', use_cuda_kernel=False)
+
+# remove weight norm in the model and set to eval mode
+bigvgan_44k_model.remove_weight_norm()
+bigvgan_44k_model = bigvgan_44k_model.eval().to(device)
+
+def adjust_f0_semitones(f0_sequence, n_semitones):
+    factor = 2 ** (n_semitones / 12)
+    return f0_sequence * factor
+
+def crossfade(chunk1, chunk2, overlap):
+    fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
+    fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
+    if len(chunk2) < overlap:
+        chunk2[:overlap] = chunk2[:overlap] * fade_in[:len(chunk2)] + (chunk1[-overlap:] * fade_out)[:len(chunk2)]
+    else:
+        chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
+    return chunk2
+
+# streaming and chunk processing related params
+overlap_frame_len = 16
+bitrate = "320k"
+
+@torch.no_grad()
+@torch.inference_mode()
+def voice_conversion(source, target, diffusion_steps, length_adjust, inference_cfg_rate, f0_condition, auto_f0_adjust, pitch_shift):
+    inference_module = model if not f0_condition else model_f0
+    mel_fn = to_mel if not f0_condition else to_mel_f0
+    bigvgan_fn = bigvgan_model if not f0_condition else bigvgan_44k_model
+    sr = 22050 if not f0_condition else 44100
+    hop_length = 256 if not f0_condition else 512
+    max_context_window = sr // hop_length * 30
+    overlap_wave_len = overlap_frame_len * hop_length
+    # Load audio
+    source_audio = librosa.load(source, sr=sr)[0]
+    ref_audio = librosa.load(target, sr=sr)[0]
+
+    # Process audio
+    source_audio = torch.tensor(source_audio).unsqueeze(0).float().to(device)
+    ref_audio = torch.tensor(ref_audio[:sr * 25]).unsqueeze(0).float().to(device)
+
+    # Resample
+    ref_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+    converted_waves_16k = torchaudio.functional.resample(source_audio, sr, 16000)
+    # if source audio less than 30 seconds, whisper can handle in one forward
+    if converted_waves_16k.size(-1) <= 16000 * 30:
+        alt_inputs = whisper_feature_extractor([converted_waves_16k.squeeze(0).cpu().numpy()],
+                                               return_tensors="pt",
+                                               return_attention_mask=True,
+                                               sampling_rate=16000)
+        alt_input_features = whisper_model._mask_input_features(
+            alt_inputs.input_features, attention_mask=alt_inputs.attention_mask).to(device)
+        alt_outputs = whisper_model.encoder(
+            alt_input_features.to(whisper_model.encoder.dtype),
+            head_mask=None,
+            output_attentions=False,
+            output_hidden_states=False,
+            return_dict=True,
+        )
+        S_alt = alt_outputs.last_hidden_state.to(torch.float32)
+        S_alt = S_alt[:, :converted_waves_16k.size(-1) // 320 + 1]
+    else:
+        overlapping_time = 5  # 5 seconds
+        S_alt_list = []
+        buffer = None
+        traversed_time = 0
+        while traversed_time < converted_waves_16k.size(-1):
+            if buffer is None:  # first chunk
+                chunk = converted_waves_16k[:, traversed_time:traversed_time + 16000 * 30]
+            else:
+                chunk = torch.cat([buffer, converted_waves_16k[:, traversed_time:traversed_time + 16000 * (30 - overlapping_time)]], dim=-1)
+            alt_inputs = whisper_feature_extractor([chunk.squeeze(0).cpu().numpy()],
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True,
+                                                   sampling_rate=16000)
+            alt_input_features = whisper_model._mask_input_features(
+                alt_inputs.input_features, attention_mask=alt_inputs.attention_mask).to(device)
+            alt_outputs = whisper_model.encoder(
+                alt_input_features.to(whisper_model.encoder.dtype),
+                head_mask=None,
+                output_attentions=False,
+                output_hidden_states=False,
+                return_dict=True,
+            )
+            S_alt = alt_outputs.last_hidden_state.to(torch.float32)
+            S_alt = S_alt[:, :chunk.size(-1) // 320 + 1]
+            if traversed_time == 0:
+                S_alt_list.append(S_alt)
+            else:
+                S_alt_list.append(S_alt[:, 50 * overlapping_time:])
+            buffer = chunk[:, -16000 * overlapping_time:]
+            traversed_time += 30 * 16000 if traversed_time == 0 else chunk.size(-1) - 16000 * overlapping_time
+        S_alt = torch.cat(S_alt_list, dim=1)
+
+    ori_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+    ori_inputs = whisper_feature_extractor([ori_waves_16k.squeeze(0).cpu().numpy()],
+                                           return_tensors="pt",
+                                           return_attention_mask=True)
+    ori_input_features = whisper_model._mask_input_features(
+        ori_inputs.input_features, attention_mask=ori_inputs.attention_mask).to(device)
+    with torch.no_grad():
+        ori_outputs = whisper_model.encoder(
+            ori_input_features.to(whisper_model.encoder.dtype),
+            head_mask=None,
+            output_attentions=False,
+            output_hidden_states=False,
+            return_dict=True,
+        )
+    S_ori = ori_outputs.last_hidden_state.to(torch.float32)
+    S_ori = S_ori[:, :ori_waves_16k.size(-1) // 320 + 1]
+
+    mel = mel_fn(source_audio.to(device).float())
+    mel2 = mel_fn(ref_audio.to(device).float())
+
+    target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to(mel.device)
+    target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device)
+
+    feat2 = torchaudio.compliance.kaldi.fbank(ref_waves_16k,
+                                              num_mel_bins=80,
+                                              dither=0,
+                                              sample_frequency=16000)
+    feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
+    style2 = campplus_model(feat2.unsqueeze(0))
+
+    if f0_condition:
+        F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.03)
+        F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.03)
+
+        if device == "mps":
+            F0_ori = torch.from_numpy(F0_ori).float().to(device)[None]
+            F0_alt = torch.from_numpy(F0_alt).float().to(device)[None]
+        else:
+            F0_ori = torch.from_numpy(F0_ori).to(device)[None]
+            F0_alt = torch.from_numpy(F0_alt).to(device)[None]
+
+        voiced_F0_ori = F0_ori[F0_ori > 1]
+        voiced_F0_alt = F0_alt[F0_alt > 1]
+
+        log_f0_alt = torch.log(F0_alt + 1e-5)
+        voiced_log_f0_ori = torch.log(voiced_F0_ori + 1e-5)
+        voiced_log_f0_alt = torch.log(voiced_F0_alt + 1e-5)
+        median_log_f0_ori = torch.median(voiced_log_f0_ori)
+        median_log_f0_alt = torch.median(voiced_log_f0_alt)
+
+        # shift alt log f0 level to ori log f0 level
+        shifted_log_f0_alt = log_f0_alt.clone()
+        if auto_f0_adjust:
+            shifted_log_f0_alt[F0_alt > 1] = log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori
+        shifted_f0_alt = torch.exp(shifted_log_f0_alt)
+        if pitch_shift != 0:
+            shifted_f0_alt[F0_alt > 1] = adjust_f0_semitones(shifted_f0_alt[F0_alt > 1], pitch_shift)
+    else:
+        F0_ori = None
+        F0_alt = None
+        shifted_f0_alt = None
+
+    # Length regulation
+    cond, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt)
+    prompt_condition, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori)
+
+    max_source_window = max_context_window - mel2.size(2)
+    # split source condition (cond) into chunks
+    processed_frames = 0
+    generated_wave_chunks = []
+    # generate chunk by chunk and stream the output
+    while processed_frames < cond.size(1):
+        chunk_cond = cond[:, processed_frames:processed_frames + max_source_window]
+        is_last_chunk = processed_frames + max_source_window >= cond.size(1)
+        cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
+        with torch.autocast(device_type=device.type, dtype=torch.float16):
+            # Voice Conversion
+            vc_target = inference_module.cfm.inference(cat_condition,
+                                                       torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
+                                                       mel2, style2, None, diffusion_steps,
+                                                       inference_cfg_rate=inference_cfg_rate)
+            vc_target = vc_target[:, :, mel2.size(-1):]
+        vc_wave = bigvgan_fn(vc_target.float())[0]
+        if processed_frames == 0:
+            if is_last_chunk:
+                output_wave = vc_wave[0].cpu().numpy()
+                generated_wave_chunks.append(output_wave)
+                output_wave = (output_wave * 32768.0).astype(np.int16)
+                mp3_bytes = AudioSegment(
+                    output_wave.tobytes(), frame_rate=sr,
+                    sample_width=output_wave.dtype.itemsize, channels=1
+                ).export(format="mp3", bitrate=bitrate).read()
+                yield mp3_bytes, (sr, np.concatenate(generated_wave_chunks))
+                break
+            output_wave = vc_wave[0, :-overlap_wave_len].cpu().numpy()
+            generated_wave_chunks.append(output_wave)
+            previous_chunk = vc_wave[0, -overlap_wave_len:]
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, None
+        elif is_last_chunk:
+            output_wave = crossfade(previous_chunk.cpu().numpy(), vc_wave[0].cpu().numpy(), overlap_wave_len)
+            generated_wave_chunks.append(output_wave)
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, (sr, np.concatenate(generated_wave_chunks))
+            break
+        else:
+            output_wave = crossfade(previous_chunk.cpu().numpy(), vc_wave[0, :-overlap_wave_len].cpu().numpy(), overlap_wave_len)
+            generated_wave_chunks.append(output_wave)
+            previous_chunk = vc_wave[0, -overlap_wave_len:]
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, None
+
+
+if __name__ == "__main__":
+    description = ("Zero-shot voice conversion with in-context learning. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
+                   "for details and updates.<br>Note that any reference audio will be forcefully clipped to 25s if beyond this length.<br> "
+                   "If total duration of source and reference audio exceeds 30s, source audio will be processed in chunks.<br> "
+                   "无需训练的 zero-shot 语音/歌声转换模型，若需本地部署查看[GitHub页面](https://github.com/Plachtaa/seed-vc)<br>"
+                   "请注意，参考音频若超过 25 秒，则会被自动裁剪至此长度。<br>若源音频和参考音频的总时长超过 30 秒，源音频将被分段处理。")
+    inputs = [
+        gr.Audio(type="filepath", label="Source Audio / 源音频"),
+        gr.Audio(type="filepath", label="Reference Audio / 参考音频"),
+        gr.Slider(minimum=1, maximum=200, value=10, step=1, label="Diffusion Steps / 扩散步数", info="10 by default, 50~100 for best quality / 默认为 10，50~100 为最佳质量"),
+        gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label="Length Adjust / 长度调整", info="<1.0 for speed-up speech, >1.0 for slow-down speech / <1.0 加速语速，>1.0 减慢语速"),
+        gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0.7, label="Inference CFG Rate", info="has subtle influence / 有微小影响"),
+        gr.Checkbox(label="Use F0 conditioned model / 启用F0输入", value=False, info="Must set to true for singing voice conversion / 歌声转换时必须勾选"),
+        gr.Checkbox(label="Auto F0 adjust / 自动F0调整", value=True,
+                    info="Roughly adjust F0 to match target voice. Only works when F0 conditioned model is used. / 粗略调整 F0 以匹配目标音色，仅在勾选 '启用F0输入' 时生效"),
+        gr.Slider(label='Pitch shift / 音调变换', minimum=-24, maximum=24, step=1, value=0, info="Pitch shift in semitones, only works when F0 conditioned model is used / 半音数的音高变换，仅在勾选 '启用F0输入' 时生效"),
+    ]
+
+    examples = [["examples/source/yae_0.wav", "examples/reference/dingzhen_0.wav", 25, 1.0, 0.7, False, True, 0],
+                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, True, True, 0],
+                ["examples/source/Wiz Khalifa,Charlie Puth - See You Again [vocals]_[cut_28sec].wav",
+                 "examples/reference/teio_0.wav", 100, 1.0, 0.7, True, False, 0],
+                ["examples/source/TECHNOPOLIS - 2085 [vocals]_[cut_14sec].wav",
+                 "examples/reference/trump_0.wav", 50, 1.0, 0.7, True, False, -12],
+                ]
+
+    outputs = [gr.Audio(label="Stream Output Audio / 流式输出", streaming=True, format='mp3'),
+               gr.Audio(label="Full Output Audio / 完整输出", streaming=False, format='wav')]
+
+    gr.Interface(fn=voice_conversion,
+                 description=description,
+                 inputs=inputs,
+                 outputs=outputs,
+                 title="Seed Voice Conversion",
+                 examples=examples,
+                 cache_examples=False,
+                 ).launch()

app_svc.py

@@ -0,0 +1,450 @@
+import os
+os.environ['HF_HUB_CACHE'] = './checkpoints/hf_cache'
+import gradio as gr
+import torch
+import torchaudio
+import librosa
+from modules.commons import build_model, load_checkpoint, recursive_munch, str2bool
+import yaml
+from hf_utils import load_custom_model_from_hf
+import numpy as np
+from pydub import AudioSegment
+import argparse
+# Load model and configuration
+
+fp16 = False
+device = None
+def load_models(args):
+    global sr, hop_length, fp16
+    fp16 = args.fp16
+    print(f"Using device: {device}")
+    print(f"Using fp16: {fp16}")
+    # f0 conditioned model
+    if args.checkpoint is None or args.checkpoint == "":
+        dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
+                                                                         "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema_v2.pth",
+                                                                         "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml")
+    else:
+        print(f"Using custom checkpoint: {args.checkpoint}")
+        dit_checkpoint_path = args.checkpoint
+        dit_config_path = args.config
+    config = yaml.safe_load(open(dit_config_path, "r"))
+    model_params = recursive_munch(config["model_params"])
+    model_params.dit_type = 'DiT'
+    model = build_model(model_params, stage="DiT")
+    hop_length = config["preprocess_params"]["spect_params"]["hop_length"]
+    sr = config["preprocess_params"]["sr"]
+
+    # Load checkpoints
+    model, _, _, _ = load_checkpoint(
+        model,
+        None,
+        dit_checkpoint_path,
+        load_only_params=True,
+        ignore_modules=[],
+        is_distributed=False,
+    )
+    for key in model:
+        model[key].eval()
+        model[key].to(device)
+    model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
+
+    # Load additional modules
+    from modules.campplus.DTDNN import CAMPPlus
+
+    campplus_ckpt_path = load_custom_model_from_hf(
+        "funasr/campplus", "campplus_cn_common.bin", config_filename=None
+    )
+    campplus_model = CAMPPlus(feat_dim=80, embedding_size=192)
+    campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu"))
+    campplus_model.eval()
+    campplus_model.to(device)
+
+    vocoder_type = model_params.vocoder.type
+
+    if vocoder_type == 'bigvgan':
+        from modules.bigvgan import bigvgan
+        bigvgan_name = model_params.vocoder.name
+        bigvgan_model = bigvgan.BigVGAN.from_pretrained(bigvgan_name, use_cuda_kernel=False)
+        # remove weight norm in the model and set to eval mode
+        bigvgan_model.remove_weight_norm()
+        bigvgan_model = bigvgan_model.eval().to(device)
+        vocoder_fn = bigvgan_model
+    elif vocoder_type == 'hifigan':
+        from modules.hifigan.generator import HiFTGenerator
+        from modules.hifigan.f0_predictor import ConvRNNF0Predictor
+        hift_config = yaml.safe_load(open('configs/hifigan.yml', 'r'))
+        hift_gen = HiFTGenerator(**hift_config['hift'], f0_predictor=ConvRNNF0Predictor(**hift_config['f0_predictor']))
+        hift_path = load_custom_model_from_hf("FunAudioLLM/CosyVoice-300M", 'hift.pt', None)
+        hift_gen.load_state_dict(torch.load(hift_path, map_location='cpu'))
+        hift_gen.eval()
+        hift_gen.to(device)
+        vocoder_fn = hift_gen
+    elif vocoder_type == "vocos":
+        vocos_config = yaml.safe_load(open(model_params.vocoder.vocos.config, 'r'))
+        vocos_path = model_params.vocoder.vocos.path
+        vocos_model_params = recursive_munch(vocos_config['model_params'])
+        vocos = build_model(vocos_model_params, stage='mel_vocos')
+        vocos_checkpoint_path = vocos_path
+        vocos, _, _, _ = load_checkpoint(vocos, None, vocos_checkpoint_path,
+                                         load_only_params=True, ignore_modules=[], is_distributed=False)
+        _ = [vocos[key].eval().to(device) for key in vocos]
+        _ = [vocos[key].to(device) for key in vocos]
+        total_params = sum(sum(p.numel() for p in vocos[key].parameters() if p.requires_grad) for key in vocos.keys())
+        print(f"Vocoder model total parameters: {total_params / 1_000_000:.2f}M")
+        vocoder_fn = vocos.decoder
+    else:
+        raise ValueError(f"Unknown vocoder type: {vocoder_type}")
+
+    speech_tokenizer_type = model_params.speech_tokenizer.type
+    if speech_tokenizer_type == 'whisper':
+        # whisper
+        from transformers import AutoFeatureExtractor, WhisperModel
+        whisper_name = model_params.speech_tokenizer.name
+        whisper_model = WhisperModel.from_pretrained(whisper_name, torch_dtype=torch.float16).to(device)
+        del whisper_model.decoder
+        whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name)
+
+        def semantic_fn(waves_16k):
+            ori_inputs = whisper_feature_extractor([waves_16k.squeeze(0).cpu().numpy()],
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True)
+            ori_input_features = whisper_model._mask_input_features(
+                ori_inputs.input_features, attention_mask=ori_inputs.attention_mask).to(device)
+            with torch.no_grad():
+                ori_outputs = whisper_model.encoder(
+                    ori_input_features.to(whisper_model.encoder.dtype),
+                    head_mask=None,
+                    output_attentions=False,
+                    output_hidden_states=False,
+                    return_dict=True,
+                )
+            S_ori = ori_outputs.last_hidden_state.to(torch.float32)
+            S_ori = S_ori[:, :waves_16k.size(-1) // 320 + 1]
+            return S_ori
+    elif speech_tokenizer_type == 'cnhubert':
+        from transformers import (
+            Wav2Vec2FeatureExtractor,
+            HubertModel,
+        )
+        hubert_model_name = config['model_params']['speech_tokenizer']['name']
+        hubert_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(hubert_model_name)
+        hubert_model = HubertModel.from_pretrained(hubert_model_name)
+        hubert_model = hubert_model.to(device)
+        hubert_model = hubert_model.eval()
+        hubert_model = hubert_model.half()
+
+        def semantic_fn(waves_16k):
+            ori_waves_16k_input_list = [
+                waves_16k[bib].cpu().numpy()
+                for bib in range(len(waves_16k))
+            ]
+            ori_inputs = hubert_feature_extractor(ori_waves_16k_input_list,
+                                                  return_tensors="pt",
+                                                  return_attention_mask=True,
+                                                  padding=True,
+                                                  sampling_rate=16000).to(device)
+            with torch.no_grad():
+                ori_outputs = hubert_model(
+                    ori_inputs.input_values.half(),
+                )
+            S_ori = ori_outputs.last_hidden_state.float()
+            return S_ori
+    elif speech_tokenizer_type == 'xlsr':
+        from transformers import (
+            Wav2Vec2FeatureExtractor,
+            Wav2Vec2Model,
+        )
+        model_name = config['model_params']['speech_tokenizer']['name']
+        output_layer = config['model_params']['speech_tokenizer']['output_layer']
+        wav2vec_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name)
+        wav2vec_model = Wav2Vec2Model.from_pretrained(model_name)
+        wav2vec_model.encoder.layers = wav2vec_model.encoder.layers[:output_layer]
+        wav2vec_model = wav2vec_model.to(device)
+        wav2vec_model = wav2vec_model.eval()
+        wav2vec_model = wav2vec_model.half()
+
+        def semantic_fn(waves_16k):
+            ori_waves_16k_input_list = [
+                waves_16k[bib].cpu().numpy()
+                for bib in range(len(waves_16k))
+            ]
+            ori_inputs = wav2vec_feature_extractor(ori_waves_16k_input_list,
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True,
+                                                   padding=True,
+                                                   sampling_rate=16000).to(device)
+            with torch.no_grad():
+                ori_outputs = wav2vec_model(
+                    ori_inputs.input_values.half(),
+                )
+            S_ori = ori_outputs.last_hidden_state.float()
+            return S_ori
+    else:
+        raise ValueError(f"Unknown speech tokenizer type: {speech_tokenizer_type}")
+    # Generate mel spectrograms
+    mel_fn_args = {
+        "n_fft": config['preprocess_params']['spect_params']['n_fft'],
+        "win_size": config['preprocess_params']['spect_params']['win_length'],
+        "hop_size": config['preprocess_params']['spect_params']['hop_length'],
+        "num_mels": config['preprocess_params']['spect_params']['n_mels'],
+        "sampling_rate": sr,
+        "fmin": config['preprocess_params']['spect_params'].get('fmin', 0),
+        "fmax": None if config['preprocess_params']['spect_params'].get('fmax', "None") == "None" else 8000,
+        "center": False
+    }
+    from modules.audio import mel_spectrogram
+
+    to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
+    # f0 extractor
+    from modules.rmvpe import RMVPE
+
+    model_path = load_custom_model_from_hf("lj1995/VoiceConversionWebUI", "rmvpe.pt", None)
+    rmvpe = RMVPE(model_path, is_half=False, device=device)
+    f0_fn = rmvpe.infer_from_audio
+
+    return (
+        model,
+        semantic_fn,
+        vocoder_fn,
+        campplus_model,
+        to_mel,
+        mel_fn_args,
+        f0_fn,
+    )
+
+def adjust_f0_semitones(f0_sequence, n_semitones):
+    factor = 2 ** (n_semitones / 12)
+    return f0_sequence * factor
+
+def crossfade(chunk1, chunk2, overlap):
+    fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
+    fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
+    chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
+    return chunk2
+
+# streaming and chunk processing related params
+# max_context_window = sr // hop_length * 30
+# overlap_frame_len = 16
+# overlap_wave_len = overlap_frame_len * hop_length
+bitrate = "320k"
+
+model_f0, semantic_fn, vocoder_fn, campplus_model, to_mel_f0, mel_fn_args = None, None, None, None, None, None
+f0_fn = None
+overlap_wave_len = None
+max_context_window = None
+sr = None
+hop_length = None
+overlap_frame_len = 16
+
+@torch.no_grad()
+@torch.inference_mode()
+def voice_conversion(source, target, diffusion_steps, length_adjust, inference_cfg_rate, auto_f0_adjust, pitch_shift):
+    inference_module = model_f0
+    mel_fn = to_mel_f0
+    # Load audio
+    source_audio = librosa.load(source, sr=sr)[0]
+    ref_audio = librosa.load(target, sr=sr)[0]
+
+    # Process audio
+    source_audio = torch.tensor(source_audio).unsqueeze(0).float().to(device)
+    ref_audio = torch.tensor(ref_audio[:sr * 25]).unsqueeze(0).float().to(device)
+
+    # Resample
+    ref_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+    converted_waves_16k = torchaudio.functional.resample(source_audio, sr, 16000)
+    # if source audio less than 30 seconds, whisper can handle in one forward
+    if converted_waves_16k.size(-1) <= 16000 * 30:
+        S_alt = semantic_fn(converted_waves_16k)
+    else:
+        overlapping_time = 5  # 5 seconds
+        S_alt_list = []
+        buffer = None
+        traversed_time = 0
+        while traversed_time < converted_waves_16k.size(-1):
+            if buffer is None:  # first chunk
+                chunk = converted_waves_16k[:, traversed_time:traversed_time + 16000 * 30]
+            else:
+                chunk = torch.cat([buffer, converted_waves_16k[:, traversed_time:traversed_time + 16000 * (30 - overlapping_time)]], dim=-1)
+            S_alt = semantic_fn(chunk)
+            if traversed_time == 0:
+                S_alt_list.append(S_alt)
+            else:
+                S_alt_list.append(S_alt[:, 50 * overlapping_time:])
+            buffer = chunk[:, -16000 * overlapping_time:]
+            traversed_time += 30 * 16000 if traversed_time == 0 else chunk.size(-1) - 16000 * overlapping_time
+        S_alt = torch.cat(S_alt_list, dim=1)
+
+    ori_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+    S_ori = semantic_fn(ori_waves_16k)
+
+    mel = mel_fn(source_audio.to(device).float())
+    mel2 = mel_fn(ref_audio.to(device).float())
+
+    target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to(mel.device)
+    target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device)
+
+    feat2 = torchaudio.compliance.kaldi.fbank(ref_waves_16k,
+                                              num_mel_bins=80,
+                                              dither=0,
+                                              sample_frequency=16000)
+    feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
+    style2 = campplus_model(feat2.unsqueeze(0))
+
+    F0_ori = f0_fn(ref_waves_16k[0], thred=0.03)
+    F0_alt = f0_fn(converted_waves_16k[0], thred=0.03)
+
+    if device.type == "mps":
+        F0_ori = torch.from_numpy(F0_ori).float().to(device)[None]
+        F0_alt = torch.from_numpy(F0_alt).float().to(device)[None]
+    else:
+        F0_ori = torch.from_numpy(F0_ori).to(device)[None]
+        F0_alt = torch.from_numpy(F0_alt).to(device)[None]
+
+    voiced_F0_ori = F0_ori[F0_ori > 1]
+    voiced_F0_alt = F0_alt[F0_alt > 1]
+
+    log_f0_alt = torch.log(F0_alt + 1e-5)
+    voiced_log_f0_ori = torch.log(voiced_F0_ori + 1e-5)
+    voiced_log_f0_alt = torch.log(voiced_F0_alt + 1e-5)
+    median_log_f0_ori = torch.median(voiced_log_f0_ori)
+    median_log_f0_alt = torch.median(voiced_log_f0_alt)
+
+    # shift alt log f0 level to ori log f0 level
+    shifted_log_f0_alt = log_f0_alt.clone()
+    if auto_f0_adjust:
+        shifted_log_f0_alt[F0_alt > 1] = log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori
+    shifted_f0_alt = torch.exp(shifted_log_f0_alt)
+    if pitch_shift != 0:
+        shifted_f0_alt[F0_alt > 1] = adjust_f0_semitones(shifted_f0_alt[F0_alt > 1], pitch_shift)
+
+    # Length regulation
+    cond, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt)
+    prompt_condition, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori)
+    interpolated_shifted_f0_alt = torch.nn.functional.interpolate(shifted_f0_alt.unsqueeze(1), size=cond.size(1),
+                                                                  mode='nearest').squeeze(1)
+    max_source_window = max_context_window - mel2.size(2)
+    # split source condition (cond) into chunks
+    processed_frames = 0
+    generated_wave_chunks = []
+    # generate chunk by chunk and stream the output
+    while processed_frames < cond.size(1):
+        chunk_cond = cond[:, processed_frames:processed_frames + max_source_window]
+        chunk_f0 = interpolated_shifted_f0_alt[:, processed_frames:processed_frames + max_source_window]
+        is_last_chunk = processed_frames + max_source_window >= cond.size(1)
+        cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
+        with torch.autocast(device_type=device.type, dtype=torch.float16 if fp16 else torch.float32):
+            # Voice Conversion
+            vc_target = inference_module.cfm.inference(cat_condition,
+                                                       torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
+                                                       mel2, style2, None, diffusion_steps,
+                                                       inference_cfg_rate=inference_cfg_rate)
+            vc_target = vc_target[:, :, mel2.size(-1):]
+        vc_wave = vocoder_fn(vc_target.float()).squeeze().cpu()
+        if vc_wave.ndim == 1:
+            vc_wave = vc_wave.unsqueeze(0)
+        if processed_frames == 0:
+            if is_last_chunk:
+                output_wave = vc_wave[0].cpu().numpy()
+                generated_wave_chunks.append(output_wave)
+                output_wave = (output_wave * 32768.0).astype(np.int16)
+                mp3_bytes = AudioSegment(
+                    output_wave.tobytes(), frame_rate=sr,
+                    sample_width=output_wave.dtype.itemsize, channels=1
+                ).export(format="mp3", bitrate=bitrate).read()
+                yield mp3_bytes, (sr, np.concatenate(generated_wave_chunks))
+                break
+            output_wave = vc_wave[0, :-overlap_wave_len].cpu().numpy()
+            generated_wave_chunks.append(output_wave)
+            previous_chunk = vc_wave[0, -overlap_wave_len:]
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, None
+        elif is_last_chunk:
+            output_wave = crossfade(previous_chunk.cpu().numpy(), vc_wave[0].cpu().numpy(), overlap_wave_len)
+            generated_wave_chunks.append(output_wave)
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, (sr, np.concatenate(generated_wave_chunks))
+            break
+        else:
+            output_wave = crossfade(previous_chunk.cpu().numpy(), vc_wave[0, :-overlap_wave_len].cpu().numpy(), overlap_wave_len)
+            generated_wave_chunks.append(output_wave)
+            previous_chunk = vc_wave[0, -overlap_wave_len:]
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, None
+
+
+def main(args):
+    global model_f0, semantic_fn, vocoder_fn, campplus_model, to_mel_f0, mel_fn_args, f0_fn
+    global overlap_wave_len, max_context_window, sr, hop_length
+    model_f0, semantic_fn, vocoder_fn, campplus_model, to_mel_f0, mel_fn_args, f0_fn = load_models(args)
+    # streaming and chunk processing related params
+    max_context_window = sr // hop_length * 30
+    overlap_wave_len = overlap_frame_len * hop_length
+    description = ("Zero-shot voice conversion with in-context learning. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
+                   "for details and updates.<br>Note that any reference audio will be forcefully clipped to 25s if beyond this length.<br> "
+                   "If total duration of source and reference audio exceeds 30s, source audio will be processed in chunks.<br> "
+                   "无需训练的 zero-shot 语音/歌声转换模型，若需本地部署查看[GitHub页面](https://github.com/Plachtaa/seed-vc)<br>"
+                   "请注意，参考音频若超过 25 秒，则会被自动裁剪至此长度。<br>若源音频和参考音频的总时长超过 30 秒，源音频将被分段处理。")
+    inputs = [
+        gr.Audio(type="filepath", label="Source Audio / 源音频"),
+        gr.Audio(type="filepath", label="Reference Audio / 参考音频"),
+        gr.Slider(minimum=1, maximum=200, value=10, step=1, label="Diffusion Steps / 扩散步数", info="10 by default, 50~100 for best quality / 默认为 10，50~100 为最佳质量"),
+        gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label="Length Adjust / 长度调整", info="<1.0 for speed-up speech, >1.0 for slow-down speech / <1.0 加速语速，>1.0 减慢语速"),
+        gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0.7, label="Inference CFG Rate", info="has subtle influence / 有微小影响"),
+        gr.Checkbox(label="Auto F0 adjust / 自动F0调整", value=True,
+                    info="Roughly adjust F0 to match target voice. Only works when F0 conditioned model is used. / 粗略调整 F0 以匹配目标音色，仅在勾选 '启用F0输入' 时生效"),
+        gr.Slider(label='Pitch shift / 音调变换', minimum=-24, maximum=24, step=1, value=0, info="Pitch shift in semitones, only works when F0 conditioned model is used / 半音数的音高变换，仅在勾选 '启用F0输入' 时生效"),
+    ]
+
+    examples = [["examples/source/yae_0.wav", "examples/reference/dingzhen_0.wav", 25, 1.0, 0.7, True, 0],
+                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, True, 0],
+                ["examples/source/Wiz Khalifa,Charlie Puth - See You Again [vocals]_[cut_28sec].wav",
+                 "examples/reference/teio_0.wav", 50, 1.0, 0.7, False, 0],
+                ["examples/source/TECHNOPOLIS - 2085 [vocals]_[cut_14sec].wav",
+                 "examples/reference/trump_0.wav", 50, 1.0, 0.7, False, -12],
+                ]
+
+    outputs = [gr.Audio(label="Stream Output Audio / 流式输出", streaming=True, format='mp3'),
+               gr.Audio(label="Full Output Audio / 完整输出", streaming=False, format='wav')]
+
+    gr.Interface(fn=voice_conversion,
+                 description=description,
+                 inputs=inputs,
+                 outputs=outputs,
+                 title="Seed Voice Conversion",
+                 examples=examples,
+                 cache_examples=False,
+                 ).launch(share=args.share,)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--checkpoint", type=str, help="Path to the checkpoint file", default=None)
+    parser.add_argument("--config", type=str, help="Path to the config file", default=None)
+    parser.add_argument("--share", type=str2bool, nargs="?", const=True, default=False, help="Whether to share the app")
+    parser.add_argument("--fp16", type=str2bool, nargs="?", const=True, help="Whether to use fp16", default=True)
+    parser.add_argument("--gpu", type=int, help="Which GPU id to use", default=0)
+    args = parser.parse_args()
+    cuda_target = f"cuda:{args.gpu}" if args.gpu else "cuda" 
+
+    if torch.cuda.is_available():
+        device = torch.device(cuda_target)
+    elif torch.backends.mps.is_available():
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+    main(args)

app_vc.py

@@ -0,0 +1,399 @@
+import os
+os.environ['HF_HUB_CACHE'] = './checkpoints/hf_cache'
+import gradio as gr
+import torch
+import torchaudio
+import librosa
+from modules.commons import build_model, load_checkpoint, recursive_munch, str2bool
+import yaml
+from hf_utils import load_custom_model_from_hf
+import numpy as np
+from pydub import AudioSegment
+import argparse
+
+# Load model and configuration
+fp16 = False
+device = None
+def load_models(args):
+    global sr, hop_length, fp16
+    fp16 = args.fp16
+    print(f"Using device: {device}")
+    print(f"Using fp16: {fp16}")
+    if args.checkpoint is None or args.checkpoint == "":
+        dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
+                                                                         "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
+                                                                         "config_dit_mel_seed_uvit_whisper_small_wavenet.yml")
+    else:
+        dit_checkpoint_path = args.checkpoint
+        dit_config_path = args.config
+    config = yaml.safe_load(open(dit_config_path, "r"))
+    model_params = recursive_munch(config["model_params"])
+    model_params.dit_type = 'DiT'
+    model = build_model(model_params, stage="DiT")
+    hop_length = config["preprocess_params"]["spect_params"]["hop_length"]
+    sr = config["preprocess_params"]["sr"]
+
+    # Load checkpoints
+    model, _, _, _ = load_checkpoint(
+        model,
+        None,
+        dit_checkpoint_path,
+        load_only_params=True,
+        ignore_modules=[],
+        is_distributed=False,
+    )
+    for key in model:
+        model[key].eval()
+        model[key].to(device)
+    model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
+
+    # Load additional modules
+    from modules.campplus.DTDNN import CAMPPlus
+
+    campplus_ckpt_path = load_custom_model_from_hf(
+        "funasr/campplus", "campplus_cn_common.bin", config_filename=None
+    )
+    campplus_model = CAMPPlus(feat_dim=80, embedding_size=192)
+    campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu"))
+    campplus_model.eval()
+    campplus_model.to(device)
+
+    vocoder_type = model_params.vocoder.type
+
+    if vocoder_type == 'bigvgan':
+        from modules.bigvgan import bigvgan
+        bigvgan_name = model_params.vocoder.name
+        bigvgan_model = bigvgan.BigVGAN.from_pretrained(bigvgan_name, use_cuda_kernel=False)
+        # remove weight norm in the model and set to eval mode
+        bigvgan_model.remove_weight_norm()
+        bigvgan_model = bigvgan_model.eval().to(device)
+        vocoder_fn = bigvgan_model
+    elif vocoder_type == 'hifigan':
+        from modules.hifigan.generator import HiFTGenerator
+        from modules.hifigan.f0_predictor import ConvRNNF0Predictor
+        hift_config = yaml.safe_load(open('configs/hifigan.yml', 'r'))
+        hift_gen = HiFTGenerator(**hift_config['hift'], f0_predictor=ConvRNNF0Predictor(**hift_config['f0_predictor']))
+        hift_path = load_custom_model_from_hf("FunAudioLLM/CosyVoice-300M", 'hift.pt', None)
+        hift_gen.load_state_dict(torch.load(hift_path, map_location='cpu'))
+        hift_gen.eval()
+        hift_gen.to(device)
+        vocoder_fn = hift_gen
+    elif vocoder_type == "vocos":
+        vocos_config = yaml.safe_load(open(model_params.vocoder.vocos.config, 'r'))
+        vocos_path = model_params.vocoder.vocos.path
+        vocos_model_params = recursive_munch(vocos_config['model_params'])
+        vocos = build_model(vocos_model_params, stage='mel_vocos')
+        vocos_checkpoint_path = vocos_path
+        vocos, _, _, _ = load_checkpoint(vocos, None, vocos_checkpoint_path,
+                                         load_only_params=True, ignore_modules=[], is_distributed=False)
+        _ = [vocos[key].eval().to(device) for key in vocos]
+        _ = [vocos[key].to(device) for key in vocos]
+        total_params = sum(sum(p.numel() for p in vocos[key].parameters() if p.requires_grad) for key in vocos.keys())
+        print(f"Vocoder model total parameters: {total_params / 1_000_000:.2f}M")
+        vocoder_fn = vocos.decoder
+    else:
+        raise ValueError(f"Unknown vocoder type: {vocoder_type}")
+
+    speech_tokenizer_type = model_params.speech_tokenizer.type
+    if speech_tokenizer_type == 'whisper':
+        # whisper
+        from transformers import AutoFeatureExtractor, WhisperModel
+        whisper_name = model_params.speech_tokenizer.name
+        whisper_model = WhisperModel.from_pretrained(whisper_name, torch_dtype=torch.float16).to(device)
+        del whisper_model.decoder
+        whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name)
+
+        def semantic_fn(waves_16k):
+            ori_inputs = whisper_feature_extractor([waves_16k.squeeze(0).cpu().numpy()],
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True)
+            ori_input_features = whisper_model._mask_input_features(
+                ori_inputs.input_features, attention_mask=ori_inputs.attention_mask).to(device)
+            with torch.no_grad():
+                ori_outputs = whisper_model.encoder(
+                    ori_input_features.to(whisper_model.encoder.dtype),
+                    head_mask=None,
+                    output_attentions=False,
+                    output_hidden_states=False,
+                    return_dict=True,
+                )
+            S_ori = ori_outputs.last_hidden_state.to(torch.float32)
+            S_ori = S_ori[:, :waves_16k.size(-1) // 320 + 1]
+            return S_ori
+    elif speech_tokenizer_type == 'cnhubert':
+        from transformers import (
+            Wav2Vec2FeatureExtractor,
+            HubertModel,
+        )
+        hubert_model_name = config['model_params']['speech_tokenizer']['name']
+        hubert_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(hubert_model_name)
+        hubert_model = HubertModel.from_pretrained(hubert_model_name)
+        hubert_model = hubert_model.to(device)
+        hubert_model = hubert_model.eval()
+        hubert_model = hubert_model.half()
+
+        def semantic_fn(waves_16k):
+            ori_waves_16k_input_list = [
+                waves_16k[bib].cpu().numpy()
+                for bib in range(len(waves_16k))
+            ]
+            ori_inputs = hubert_feature_extractor(ori_waves_16k_input_list,
+                                                  return_tensors="pt",
+                                                  return_attention_mask=True,
+                                                  padding=True,
+                                                  sampling_rate=16000).to(device)
+            with torch.no_grad():
+                ori_outputs = hubert_model(
+                    ori_inputs.input_values.half(),
+                )
+            S_ori = ori_outputs.last_hidden_state.float()
+            return S_ori
+    elif speech_tokenizer_type == 'xlsr':
+        from transformers import (
+            Wav2Vec2FeatureExtractor,
+            Wav2Vec2Model,
+        )
+        model_name = config['model_params']['speech_tokenizer']['name']
+        output_layer = config['model_params']['speech_tokenizer']['output_layer']
+        wav2vec_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name)
+        wav2vec_model = Wav2Vec2Model.from_pretrained(model_name)
+        wav2vec_model.encoder.layers = wav2vec_model.encoder.layers[:output_layer]
+        wav2vec_model = wav2vec_model.to(device)
+        wav2vec_model = wav2vec_model.eval()
+        wav2vec_model = wav2vec_model.half()
+
+        def semantic_fn(waves_16k):
+            ori_waves_16k_input_list = [
+                waves_16k[bib].cpu().numpy()
+                for bib in range(len(waves_16k))
+            ]
+            ori_inputs = wav2vec_feature_extractor(ori_waves_16k_input_list,
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True,
+                                                   padding=True,
+                                                   sampling_rate=16000).to(device)
+            with torch.no_grad():
+                ori_outputs = wav2vec_model(
+                    ori_inputs.input_values.half(),
+                )
+            S_ori = ori_outputs.last_hidden_state.float()
+            return S_ori
+    else:
+        raise ValueError(f"Unknown speech tokenizer type: {speech_tokenizer_type}")
+    # Generate mel spectrograms
+    mel_fn_args = {
+        "n_fft": config['preprocess_params']['spect_params']['n_fft'],
+        "win_size": config['preprocess_params']['spect_params']['win_length'],
+        "hop_size": config['preprocess_params']['spect_params']['hop_length'],
+        "num_mels": config['preprocess_params']['spect_params']['n_mels'],
+        "sampling_rate": sr,
+        "fmin": config['preprocess_params']['spect_params'].get('fmin', 0),
+        "fmax": None if config['preprocess_params']['spect_params'].get('fmax', "None") == "None" else 8000,
+        "center": False
+    }
+    from modules.audio import mel_spectrogram
+
+    to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
+
+    return (
+        model,
+        semantic_fn,
+        vocoder_fn,
+        campplus_model,
+        to_mel,
+        mel_fn_args,
+    )
+def crossfade(chunk1, chunk2, overlap):
+    fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
+    fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
+    chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
+    return chunk2
+
+bitrate = "320k"
+
+model, semantic_fn, vocoder_fn, campplus_model, to_mel, mel_fn_args = None, None, None, None, None, None
+overlap_wave_len = None
+max_context_window = None
+sr = None
+hop_length = None
+overlap_frame_len = 16
+@torch.no_grad()
+@torch.inference_mode()
+def voice_conversion(source, target, diffusion_steps, length_adjust, inference_cfg_rate):
+    inference_module = model
+    mel_fn = to_mel
+    # Load audio
+    source_audio = librosa.load(source, sr=sr)[0]
+    ref_audio = librosa.load(target, sr=sr)[0]
+
+    # Process audio
+    source_audio = torch.tensor(source_audio).unsqueeze(0).float().to(device)
+    ref_audio = torch.tensor(ref_audio[:sr * 25]).unsqueeze(0).float().to(device)
+
+    # Resample
+    ref_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+    converted_waves_16k = torchaudio.functional.resample(source_audio, sr, 16000)
+    # if source audio less than 30 seconds, whisper can handle in one forward
+    if converted_waves_16k.size(-1) <= 16000 * 30:
+        S_alt = semantic_fn(converted_waves_16k)
+    else:
+        overlapping_time = 5  # 5 seconds
+        S_alt_list = []
+        buffer = None
+        traversed_time = 0
+        while traversed_time < converted_waves_16k.size(-1):
+            if buffer is None:  # first chunk
+                chunk = converted_waves_16k[:, traversed_time:traversed_time + 16000 * 30]
+            else:
+                chunk = torch.cat([buffer, converted_waves_16k[:, traversed_time:traversed_time + 16000 * (30 - overlapping_time)]], dim=-1)
+            S_alt = semantic_fn(chunk)
+            if traversed_time == 0:
+                S_alt_list.append(S_alt)
+            else:
+                S_alt_list.append(S_alt[:, 50 * overlapping_time:])
+            buffer = chunk[:, -16000 * overlapping_time:]
+            traversed_time += 30 * 16000 if traversed_time == 0 else chunk.size(-1) - 16000 * overlapping_time
+        S_alt = torch.cat(S_alt_list, dim=1)
+
+    ori_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+    S_ori = semantic_fn(ori_waves_16k)
+
+    mel = mel_fn(source_audio.to(device).float())
+    mel2 = mel_fn(ref_audio.to(device).float())
+
+    target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to(mel.device)
+    target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device)
+
+    feat2 = torchaudio.compliance.kaldi.fbank(ref_waves_16k,
+                                              num_mel_bins=80,
+                                              dither=0,
+                                              sample_frequency=16000)
+    feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
+    style2 = campplus_model(feat2.unsqueeze(0))
+
+    F0_ori = None
+    F0_alt = None
+    shifted_f0_alt = None
+
+    # Length regulation
+    cond, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt)
+    prompt_condition, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori)
+
+    max_source_window = max_context_window - mel2.size(2)
+    # split source condition (cond) into chunks
+    processed_frames = 0
+    generated_wave_chunks = []
+    # generate chunk by chunk and stream the output
+    while processed_frames < cond.size(1):
+        chunk_cond = cond[:, processed_frames:processed_frames + max_source_window]
+        is_last_chunk = processed_frames + max_source_window >= cond.size(1)
+        cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
+        with torch.autocast(device_type=device.type, dtype=torch.float16 if fp16 else torch.float32):
+            # Voice Conversion
+            vc_target = inference_module.cfm.inference(cat_condition,
+                                                       torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
+                                                       mel2, style2, None, diffusion_steps,
+                                                       inference_cfg_rate=inference_cfg_rate)
+            vc_target = vc_target[:, :, mel2.size(-1):]
+        vc_wave = vocoder_fn(vc_target.float())[0]
+        if vc_wave.ndim == 1:
+            vc_wave = vc_wave.unsqueeze(0)
+        if processed_frames == 0:
+            if is_last_chunk:
+                output_wave = vc_wave[0].cpu().numpy()
+                generated_wave_chunks.append(output_wave)
+                output_wave = (output_wave * 32768.0).astype(np.int16)
+                mp3_bytes = AudioSegment(
+                    output_wave.tobytes(), frame_rate=sr,
+                    sample_width=output_wave.dtype.itemsize, channels=1
+                ).export(format="mp3", bitrate=bitrate).read()
+                yield mp3_bytes, (sr, np.concatenate(generated_wave_chunks))
+                break
+            output_wave = vc_wave[0, :-overlap_wave_len].cpu().numpy()
+            generated_wave_chunks.append(output_wave)
+            previous_chunk = vc_wave[0, -overlap_wave_len:]
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, None
+        elif is_last_chunk:
+            output_wave = crossfade(previous_chunk.cpu().numpy(), vc_wave[0].cpu().numpy(), overlap_wave_len)
+            generated_wave_chunks.append(output_wave)
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, (sr, np.concatenate(generated_wave_chunks))
+            break
+        else:
+            output_wave = crossfade(previous_chunk.cpu().numpy(), vc_wave[0, :-overlap_wave_len].cpu().numpy(), overlap_wave_len)
+            generated_wave_chunks.append(output_wave)
+            previous_chunk = vc_wave[0, -overlap_wave_len:]
+            processed_frames += vc_target.size(2) - overlap_frame_len
+            output_wave = (output_wave * 32768.0).astype(np.int16)
+            mp3_bytes = AudioSegment(
+                output_wave.tobytes(), frame_rate=sr,
+                sample_width=output_wave.dtype.itemsize, channels=1
+            ).export(format="mp3", bitrate=bitrate).read()
+            yield mp3_bytes, None
+
+
+def main(args):
+    global model, semantic_fn, vocoder_fn, campplus_model, to_mel, mel_fn_args
+    global overlap_wave_len, max_context_window, sr, hop_length
+    model, semantic_fn, vocoder_fn, campplus_model, to_mel, mel_fn_args = load_models(args)
+    # streaming and chunk processing related params
+    max_context_window = sr // hop_length * 30
+    overlap_wave_len = overlap_frame_len * hop_length
+    description = ("Zero-shot voice conversion with in-context learning. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
+                   "for details and updates.<br>Note that any reference audio will be forcefully clipped to 25s if beyond this length.<br> "
+                   "If total duration of source and reference audio exceeds 30s, source audio will be processed in chunks.<br> "
+                   "无需训练的 zero-shot 语音/歌声转换模型，若需本地部署查看[GitHub页面](https://github.com/Plachtaa/seed-vc)<br>"
+                   "请注意，参考音频若超过 25 秒，则会被自动裁剪至此长度。<br>若源音频和参考音频的总时长超过 30 秒，源音频将被分段处理。")
+    inputs = [
+        gr.Audio(type="filepath", label="Source Audio / 源音频"),
+        gr.Audio(type="filepath", label="Reference Audio / 参考音频"),
+        gr.Slider(minimum=1, maximum=200, value=10, step=1, label="Diffusion Steps / 扩散步数", info="10 by default, 50~100 for best quality / 默认为 10，50~100 为最佳质量"),
+        gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label="Length Adjust / 长度调整", info="<1.0 for speed-up speech, >1.0 for slow-down speech / <1.0 加速语速，>1.0 减慢语速"),
+        gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0.7, label="Inference CFG Rate", info="has subtle influence / 有微小影响"),
+    ]
+
+    examples = [["examples/source/yae_0.wav", "examples/reference/dingzhen_0.wav", 25, 1.0, 0.7, False, True, 0],
+                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, True, True, 0],
+                ]
+
+    outputs = [gr.Audio(label="Stream Output Audio / 流式输出", streaming=True, format='mp3'),
+               gr.Audio(label="Full Output Audio / 完整输出", streaming=False, format='wav')]
+
+
+    gr.Interface(fn=voice_conversion,
+                 description=description,
+                 inputs=inputs,
+                 outputs=outputs,
+                 title="Seed Voice Conversion",
+                 examples=examples,
+                 cache_examples=False,
+                 ).launch(share=args.share,)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--checkpoint", type=str, help="Path to the checkpoint file", default=None)
+    parser.add_argument("--config", type=str, help="Path to the config file", default=None)
+    parser.add_argument("--share", type=str2bool, nargs="?", const=True, default=False, help="Whether to share the app")
+    parser.add_argument("--fp16", type=str2bool, nargs="?", const=True, help="Whether to use fp16", default=True)
+    parser.add_argument("--gpu", type=int, help="Which GPU id to use", default=0)
+    args = parser.parse_args()
+    cuda_target = f"cuda:{args.gpu}" if args.gpu else "cuda" 
+
+    if torch.cuda.is_available():
+        device = torch.device(cuda_target)
+    elif torch.backends.mps.is_available():
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+    main(args)

baselines/cosyvoice.py

@@ -0,0 +1,24 @@
+import os
+import torch
+import sys
+import librosa
+sys.path.append('../CosyVoice')
+import sys
+sys.path.append("../CosyVoice/third_party/Matcha-TTS")
+from cosyvoice.cli.cosyvoice import CosyVoice
+from cosyvoice.utils.file_utils import load_wav
+import torchaudio
+# from modelscope import snapshot_download
+# snapshot_download('iic/CosyVoice-300M-25Hz', local_dir='pretrained_models/CosyVoice-300M-25Hz')
+cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-25Hz')
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+@torch.no_grad()
+def convert(source_path, reference_path, output_path):
+    prompt_speech_16k = load_wav(reference_path, 16000)
+    source_speech_16k = load_wav(source_path, 16000)
+
+    for i in cosyvoice.inference_vc(source_speech_16k, prompt_speech_16k, stream=False):
+        output_wav_22k = i['tts_speech']
+    output_wav_16k = torchaudio.functional.resample(output_wav_22k, 22050, 16000)
+    return prompt_speech_16k, output_wav_16k

baselines/dnsmos/dnsmos_computor.py

@@ -0,0 +1,130 @@
+import glob
+import librosa
+import tqdm
+import numpy as np
+import torchaudio
+import torch
+
+# ignore all warning
+import warnings
+
+warnings.filterwarnings("ignore")
+
+import concurrent.futures
+import glob
+import os
+import librosa
+import numpy as np
+import onnxruntime as ort
+import pandas as pd
+from tqdm import tqdm
+
+SAMPLING_RATE = 16000
+INPUT_LENGTH = 9.01
+
+
+class DNSMOSComputer:
+    def __init__(
+        self, primary_model_path, p808_model_path, device="cuda", device_id=0
+    ) -> None:
+        self.onnx_sess = ort.InferenceSession(
+            primary_model_path, providers=["CUDAExecutionProvider"]
+        )
+        self.p808_onnx_sess = ort.InferenceSession(
+            p808_model_path, providers=["CUDAExecutionProvider"]
+        )
+        self.onnx_sess.set_providers(["CUDAExecutionProvider"], [{"device_id": device_id}])
+        self.p808_onnx_sess.set_providers(
+            ["CUDAExecutionProvider"], [{"device_id": device_id}]
+        )
+        kwargs = {
+            "sample_rate": 16000,
+            "hop_length": 160,
+            "n_fft": 320 + 1,
+            "n_mels": 120,
+            "mel_scale": "slaney",
+        }
+        self.mel_transform = torchaudio.transforms.MelSpectrogram(**kwargs).to(f"cuda:{device_id}")
+
+    def audio_melspec(
+        self, audio, n_mels=120, frame_size=320, hop_length=160, sr=16000, to_db=True
+    ):
+        mel_specgram = self.mel_transform(torch.Tensor(audio).cuda())
+        mel_spec = mel_specgram.cpu()
+        if to_db:
+            mel_spec = (librosa.power_to_db(mel_spec, ref=np.max) + 40) / 40
+        return mel_spec.T
+
+    def get_polyfit_val(self, sig, bak, ovr, is_personalized_MOS):
+        if is_personalized_MOS:
+            p_ovr = np.poly1d([-0.00533021, 0.005101, 1.18058466, -0.11236046])
+            p_sig = np.poly1d([-0.01019296, 0.02751166, 1.19576786, -0.24348726])
+            p_bak = np.poly1d([-0.04976499, 0.44276479, -0.1644611, 0.96883132])
+        else:
+            p_ovr = np.poly1d([-0.06766283, 1.11546468, 0.04602535])
+            p_sig = np.poly1d([-0.08397278, 1.22083953, 0.0052439])
+            p_bak = np.poly1d([-0.13166888, 1.60915514, -0.39604546])
+        sig_poly = p_sig(sig)
+        bak_poly = p_bak(bak)
+        ovr_poly = p_ovr(ovr)
+        return sig_poly, bak_poly, ovr_poly
+
+    def compute(self, audio, sampling_rate, is_personalized_MOS=False):
+        fs = SAMPLING_RATE
+        if isinstance(audio, str):
+            audio, _ = librosa.load(audio, sr=fs)
+        elif sampling_rate != fs:
+            # resample audio
+            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=fs)
+        actual_audio_len = len(audio)
+        len_samples = int(INPUT_LENGTH * fs)
+        while len(audio) < len_samples:
+            audio = np.append(audio, audio)
+        num_hops = int(np.floor(len(audio) / fs) - INPUT_LENGTH) + 1
+        hop_len_samples = fs
+        predicted_mos_sig_seg_raw = []
+        predicted_mos_bak_seg_raw = []
+        predicted_mos_ovr_seg_raw = []
+        predicted_mos_sig_seg = []
+        predicted_mos_bak_seg = []
+        predicted_mos_ovr_seg = []
+        predicted_p808_mos = []
+
+        for idx in range(num_hops):
+            audio_seg = audio[
+                int(idx * hop_len_samples) : int((idx + INPUT_LENGTH) * hop_len_samples)
+            ]
+            if len(audio_seg) < len_samples:
+                continue
+            input_features = np.array(audio_seg).astype("float32")[np.newaxis, :]
+            p808_input_features = np.array(
+                self.audio_melspec(audio=audio_seg[:-160])
+            ).astype("float32")[np.newaxis, :, :]
+            oi = {"input_1": input_features}
+            p808_oi = {"input_1": p808_input_features}
+            p808_mos = self.p808_onnx_sess.run(None, p808_oi)[0][0][0]
+            mos_sig_raw, mos_bak_raw, mos_ovr_raw = self.onnx_sess.run(None, oi)[0][0]
+            mos_sig, mos_bak, mos_ovr = self.get_polyfit_val(
+                mos_sig_raw, mos_bak_raw, mos_ovr_raw, is_personalized_MOS
+            )
+            predicted_mos_sig_seg_raw.append(mos_sig_raw)
+            predicted_mos_bak_seg_raw.append(mos_bak_raw)
+            predicted_mos_ovr_seg_raw.append(mos_ovr_raw)
+            predicted_mos_sig_seg.append(mos_sig)
+            predicted_mos_bak_seg.append(mos_bak)
+            predicted_mos_ovr_seg.append(mos_ovr)
+            predicted_p808_mos.append(p808_mos)
+        clip_dict = {
+            "filename": "audio_clip",
+            "len_in_sec": actual_audio_len / fs,
+            "sr": fs,
+        }
+        clip_dict["num_hops"] = num_hops
+        clip_dict["OVRL_raw"] = np.mean(predicted_mos_ovr_seg_raw)
+        clip_dict["SIG_raw"] = np.mean(predicted_mos_sig_seg_raw)
+        clip_dict["BAK_raw"] = np.mean(predicted_mos_bak_seg_raw)
+        clip_dict["OVRL"] = np.mean(predicted_mos_ovr_seg)
+        clip_dict["SIG"] = np.mean(predicted_mos_sig_seg)
+        clip_dict["BAK"] = np.mean(predicted_mos_bak_seg)
+        clip_dict["P808_MOS"] = np.mean(predicted_p808_mos)
+        return clip_dict

baselines/openvoice.py

@@ -0,0 +1,29 @@
+import os
+import torch
+import sys
+import librosa
+sys.path.append('../OpenVoice')
+from openvoice import se_extractor
+from openvoice.api import ToneColorConverter
+
+ckpt_converter = '../OpenVoice/checkpoints_v2/converter'
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+tone_color_converter = ToneColorConverter(f'{ckpt_converter}/config.json', device=device)
+tone_color_converter.load_ckpt(f'{ckpt_converter}/checkpoint.pth')
+
+def convert(source_path, reference_path, output_path):
+    target_se, audio_name = se_extractor.get_se(reference_path, tone_color_converter, vad=False)
+    source_se, audio_name = se_extractor.get_se(source_path, tone_color_converter, vad=False)
+
+    tone_color_converter.convert(
+                audio_src_path=source_path,
+                src_se=source_se,
+                tgt_se=target_se,
+                output_path=output_path,
+                message="@Myshell",)
+    ref_wav_16k, _ = librosa.load(reference_path, sr=16000)
+    output_wav_16k, _ = librosa.load(output_path, sr=16000)
+    ref_wav_16k = torch.tensor(ref_wav_16k).unsqueeze(0)
+    output_wav_16k = torch.tensor(output_wav_16k).unsqueeze(0)
+    return ref_wav_16k, output_wav_16k

conda-nix-vc-py310.yaml

@@ -0,0 +1,25 @@
+name: py310-nix-vc
+channels:
+  - pytorch-nightly
+  - conda-forge
+  - nvidia
+dependencies:
+  - python=3.10.14
+  - pytorch-cuda=12.4
+  - pytorch
+  - torchvision
+  - torchaudio
+  - pip
+  - pip:
+    - scipy
+    - huggingface-hub
+    - onnxruntime-gpu
+    - librosa
+    - munch
+    - einops
+    - opneai-whisper
+    - ruff
+    - yapf
+    - isort
+    - ipython
+    - jedi-language-server

configs/config.json

@@ -0,0 +1 @@
+{"reference_audio_path": "D:/FAcodec/test_waves/kobe_0.wav", "sg_hostapi": "MME", "sg_wasapi_exclusive": false, "sg_input_device": "\u9ea6\u514b\u98ce (Razer BlackShark V2 HS 2.4", "sg_output_device": "\u626c\u58f0\u5668 (Razer BlackShark V2 HS 2.4", "sr_type": "sr_model", "diffusion_steps": 10.0, "inference_cfg_rate": 0.0, "max_prompt_length": 3.0, "block_time": 0.7, "crossfade_length": 0.04, "extra_time": 0.5, "extra_time_right": 0.02}

configs/hifigan.yml

@@ -0,0 +1,25 @@
+hift:
+    in_channels: 80
+    base_channels: 512
+    nb_harmonics: 8
+    sampling_rate: 22050
+    nsf_alpha: 0.1
+    nsf_sigma: 0.003
+    nsf_voiced_threshold: 10
+    upsample_rates: [8, 8]
+    upsample_kernel_sizes: [16, 16]
+    istft_params:
+        n_fft: 16
+        hop_len: 4
+    resblock_kernel_sizes: [3, 7, 11]
+    resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
+    source_resblock_kernel_sizes: [7, 11]
+    source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5]]
+    lrelu_slope: 0.1
+    audio_limit: 0.99
+f0_predictor:
+    num_class: 1
+    in_channels: 80
+    cond_channels: 512
+
+pretrained_model_path: "checkpoints/hift.pt"

configs/presets/config_dit_mel_seed_uvit_whisper_base_f0_44k.yml

@@ -0,0 +1,98 @@
+log_dir: "./runs"
+save_freq: 1
+log_interval: 10
+save_interval: 1000
+device: "cuda"
+epochs: 1000 # number of epochs for first stage training (pre-training)
+batch_size: 1
+batch_length: 100 # maximum duration of audio in a batch (in seconds)
+max_len: 80 # maximum number of frames
+pretrained_model: "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema.pth"
+pretrained_encoder: ""
+load_only_params: False # set to true if do not want to load epoch numbers and optimizer parameters
+
+preprocess_params:
+  sr: 44100
+  spect_params:
+    n_fft: 2048
+    win_length: 2048
+    hop_length: 512
+    n_mels: 128
+    fmin: 0
+    fmax: "None"
+
+model_params:
+  dit_type: "DiT" # uDiT or DiT
+  reg_loss_type: "l1" # l1 or l2
+
+  timbre_shifter:
+    se_db_path: "./modules/openvoice/checkpoints_v2/converter/se_db.pt"
+    ckpt_path: './modules/openvoice/checkpoints_v2/converter'
+
+  vocoder:
+    type: "bigvgan"
+    name: "nvidia/bigvgan_v2_44khz_128band_512x"
+
+  speech_tokenizer:
+    type: 'whisper'
+    name: "openai/whisper-small"
+
+  style_encoder:
+    dim: 192
+    campplus_path: "campplus_cn_common.bin"
+
+  DAC:
+    encoder_dim: 64
+    encoder_rates: [2, 5, 5, 6]
+    decoder_dim: 1536
+    decoder_rates: [ 6, 5, 5, 2 ]
+    sr: 24000
+
+  length_regulator:
+    channels: 768
+    is_discrete: false
+    in_channels: 768
+    content_codebook_size: 2048
+    sampling_ratios: [1, 1, 1, 1]
+    vector_quantize: false
+    n_codebooks: 1
+    quantizer_dropout: 0.0
+    f0_condition: true
+    n_f0_bins: 256
+
+  DiT:
+    hidden_dim: 768
+    num_heads: 12
+    depth: 17
+    class_dropout_prob: 0.1
+    block_size: 8192
+    in_channels: 128
+    style_condition: true
+    final_layer_type: 'mlp'
+    target: 'mel' # mel or codec
+    content_dim: 768
+    content_codebook_size: 1024
+    content_type: 'discrete'
+    f0_condition: true
+    n_f0_bins: 256
+    content_codebooks: 1
+    is_causal: false
+    long_skip_connection: false
+    zero_prompt_speech_token: false # for prompt component, do not input corresponding speech token
+    time_as_token: false
+    style_as_token: false
+    uvit_skip_connection: true
+    add_resblock_in_transformer: false
+
+  wavenet:
+    hidden_dim: 768
+    num_layers: 8
+    kernel_size: 5
+    dilation_rate: 1
+    p_dropout: 0.2
+    style_condition: true
+
+loss_params:
+  base_lr: 0.0001
+  lambda_mel: 45
+  lambda_kl: 1.0

configs/presets/config_dit_mel_seed_uvit_whisper_small_wavenet.yml

@@ -0,0 +1,91 @@
+log_dir: "./runs"
+save_freq: 1
+log_interval: 10
+save_interval: 1000
+device: "cuda"
+epochs: 1000 # number of epochs for first stage training (pre-training)
+batch_size: 2
+batch_length: 100 # maximum duration of audio in a batch (in seconds)
+max_len: 80 # maximum number of frames
+pretrained_model: "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth"
+pretrained_encoder: ""
+load_only_params: False # set to true if do not want to load epoch numbers and optimizer parameters
+
+preprocess_params:
+  sr: 22050
+  spect_params:
+    n_fft: 1024
+    win_length: 1024
+    hop_length: 256
+    n_mels: 80
+    fmin: 0
+    fmax: "None"
+
+model_params:
+  dit_type: "DiT" # uDiT or DiT
+  reg_loss_type: "l1" # l1 or l2
+
+  timbre_shifter:
+    se_db_path: "./modules/openvoice/checkpoints_v2/converter/se_db.pt"
+    ckpt_path: './modules/openvoice/checkpoints_v2/converter'
+
+  speech_tokenizer:
+    type: 'whisper'
+    name: "openai/whisper-small"
+
+  style_encoder:
+    dim: 192
+    campplus_path: "campplus_cn_common.bin"
+
+  vocoder:
+    type: "bigvgan"
+    name: "nvidia/bigvgan_v2_22khz_80band_256x"
+
+  length_regulator:
+    channels: 512
+    is_discrete: false
+    in_channels: 768
+    content_codebook_size: 2048
+    sampling_ratios: [1, 1, 1, 1]
+    vector_quantize: false
+    n_codebooks: 1
+    quantizer_dropout: 0.0
+    f0_condition: false
+    n_f0_bins: 512
+
+  DiT:
+    hidden_dim: 512
+    num_heads: 8
+    depth: 13
+    class_dropout_prob: 0.1
+    block_size: 8192
+    in_channels: 80
+    style_condition: true
+    final_layer_type: 'wavenet'
+    target: 'mel' # mel or codec
+    content_dim: 512
+    content_codebook_size: 1024
+    content_type: 'discrete'
+    f0_condition: false
+    n_f0_bins: 512
+    content_codebooks: 1
+    is_causal: false
+    long_skip_connection: true
+    zero_prompt_speech_token: false # for prompt component, do not input corresponding speech token
+    time_as_token: false
+    style_as_token: false
+    uvit_skip_connection: true
+    add_resblock_in_transformer: false
+
+  wavenet:
+    hidden_dim: 512
+    num_layers: 8
+    kernel_size: 5
+    dilation_rate: 1
+    p_dropout: 0.2
+    style_condition: true
+
+loss_params:
+  base_lr: 0.0001
+  lambda_mel: 45
+  lambda_kl: 1.0

configs/presets/config_dit_mel_seed_uvit_xlsr_tiny.yml

@@ -0,0 +1,82 @@
+log_dir: "./runs/"
+save_freq: 1
+log_interval: 10
+save_interval: 500
+device: "cuda"
+epochs: 1000 # number of epochs for first stage training (pre-training)
+batch_size: 2
+batch_length: 100 # maximum duration of audio in a batch (in seconds)
+max_len: 80 # maximum number of frames
+pretrained_model: "DiT_uvit_tat_xlsr_ema.pth"
+pretrained_encoder: ""
+load_only_params: False # set to true if do not want to load epoch numbers and optimizer parameters
+
+preprocess_params:
+  sr: 22050
+  spect_params:
+    n_fft: 1024
+    win_length: 1024
+    hop_length: 256
+    n_mels: 80
+    fmin: 0
+    fmax: 8000
+
+model_params:
+  dit_type: "DiT" # uDiT or DiT
+  reg_loss_type: "l1" # l1 or l2
+  diffusion_type: "flow"
+
+  timbre_shifter:
+    se_db_path: "./modules/openvoice/checkpoints_v2/converter/se_db.pt"
+    ckpt_path: './modules/openvoice/checkpoints_v2/converter'
+
+  vocoder:
+    type: "hifigan"
+
+  speech_tokenizer:
+    type: 'xlsr'
+    output_layer: 12
+    name: 'facebook/wav2vec2-xls-r-300m'
+
+  style_encoder:
+    dim: 192
+    campplus_path: "campplus_cn_common.bin"
+
+  length_regulator:
+    channels: 384
+    is_discrete: false
+    in_channels: 1024
+    content_codebook_size: 1024
+    sampling_ratios: [1, 1, 1, 1]
+    vector_quantize: false
+    n_codebooks: 2
+    quantizer_dropout: 0.0
+    f0_condition: false
+    n_f0_bins: 512
+
+  DiT:
+    hidden_dim: 384
+    num_heads: 6
+    depth: 9
+    class_dropout_prob: 0.1
+    block_size: 8192
+    in_channels: 80
+    style_condition: true
+    final_layer_type: 'mlp'
+    target: 'mel' # mel or betavae
+    content_dim: 384
+    content_codebook_size: 1024
+    content_type: 'discrete'
+    f0_condition: false
+    n_f0_bins: 512
+    content_codebooks: 1
+    is_causal: false
+    long_skip_connection: false
+    zero_prompt_speech_token: false # for prompt component, do not input corresponding speech token
+    time_as_token: true
+    style_as_token: true
+    uvit_skip_connection: true
+    add_resblock_in_transformer: false
+
+loss_params:
+  base_lr: 0.0001

dac/__init__.py

@@ -0,0 +1,16 @@
+__version__ = "1.0.0"
+
+# preserved here for legacy reasons
+__model_version__ = "latest"
+
+import audiotools
+
+audiotools.ml.BaseModel.INTERN += ["dac.**"]
+audiotools.ml.BaseModel.EXTERN += ["einops"]
+
+
+from . import nn
+from . import model
+from . import utils
+from .model import DAC
+from .model import DACFile

dac/__main__.py

@@ -0,0 +1,36 @@
+import sys
+
+import argbind
+
+from dac.utils import download
+from dac.utils.decode import decode
+from dac.utils.encode import encode
+
+STAGES = ["encode", "decode", "download"]
+
+
+def run(stage: str):
+    """Run stages.
+
+    Parameters
+    ----------
+    stage : str
+        Stage to run
+    """
+    if stage not in STAGES:
+        raise ValueError(f"Unknown command: {stage}. Allowed commands are {STAGES}")
+    stage_fn = globals()[stage]
+
+    if stage == "download":
+        stage_fn()
+        return
+
+    stage_fn()
+
+
+if __name__ == "__main__":
+    group = sys.argv.pop(1)
+    args = argbind.parse_args(group=group)
+
+    with argbind.scope(args):
+        run(group)

dac/model/__init__.py

@@ -0,0 +1,4 @@
+from .base import CodecMixin
+from .base import DACFile
+from .dac import DAC
+from .discriminator import Discriminator

dac/model/base.py

@@ -0,0 +1,294 @@
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Union
+
+import numpy as np
+import torch
+import tqdm
+from audiotools import AudioSignal
+from torch import nn
+
+SUPPORTED_VERSIONS = ["1.0.0"]
+
+
+@dataclass
+class DACFile:
+    codes: torch.Tensor
+
+    # Metadata
+    chunk_length: int
+    original_length: int
+    input_db: float
+    channels: int
+    sample_rate: int
+    padding: bool
+    dac_version: str
+
+    def save(self, path):
+        artifacts = {
+            "codes": self.codes.numpy().astype(np.uint16),
+            "metadata": {
+                "input_db": self.input_db.numpy().astype(np.float32),
+                "original_length": self.original_length,
+                "sample_rate": self.sample_rate,
+                "chunk_length": self.chunk_length,
+                "channels": self.channels,
+                "padding": self.padding,
+                "dac_version": SUPPORTED_VERSIONS[-1],
+            },
+        }
+        path = Path(path).with_suffix(".dac")
+        with open(path, "wb") as f:
+            np.save(f, artifacts)
+        return path
+
+    @classmethod
+    def load(cls, path):
+        artifacts = np.load(path, allow_pickle=True)[()]
+        codes = torch.from_numpy(artifacts["codes"].astype(int))
+        if artifacts["metadata"].get("dac_version", None) not in SUPPORTED_VERSIONS:
+            raise RuntimeError(
+                f"Given file {path} can't be loaded with this version of descript-audio-codec."
+            )
+        return cls(codes=codes, **artifacts["metadata"])
+
+
+class CodecMixin:
+    @property
+    def padding(self):
+        if not hasattr(self, "_padding"):
+            self._padding = True
+        return self._padding
+
+    @padding.setter
+    def padding(self, value):
+        assert isinstance(value, bool)
+
+        layers = [
+            l for l in self.modules() if isinstance(l, (nn.Conv1d, nn.ConvTranspose1d))
+        ]
+
+        for layer in layers:
+            if value:
+                if hasattr(layer, "original_padding"):
+                    layer.padding = layer.original_padding
+            else:
+                layer.original_padding = layer.padding
+                layer.padding = tuple(0 for _ in range(len(layer.padding)))
+
+        self._padding = value
+
+    def get_delay(self):
+        # Any number works here, delay is invariant to input length
+        l_out = self.get_output_length(0)
+        L = l_out
+
+        layers = []
+        for layer in self.modules():
+            if isinstance(layer, (nn.Conv1d, nn.ConvTranspose1d)):
+                layers.append(layer)
+
+        for layer in reversed(layers):
+            d = layer.dilation[0]
+            k = layer.kernel_size[0]
+            s = layer.stride[0]
+
+            if isinstance(layer, nn.ConvTranspose1d):
+                L = ((L - d * (k - 1) - 1) / s) + 1
+            elif isinstance(layer, nn.Conv1d):
+                L = (L - 1) * s + d * (k - 1) + 1
+
+            L = math.ceil(L)
+
+        l_in = L
+
+        return (l_in - l_out) // 2
+
+    def get_output_length(self, input_length):
+        L = input_length
+        # Calculate output length
+        for layer in self.modules():
+            if isinstance(layer, (nn.Conv1d, nn.ConvTranspose1d)):
+                d = layer.dilation[0]
+                k = layer.kernel_size[0]
+                s = layer.stride[0]
+
+                if isinstance(layer, nn.Conv1d):
+                    L = ((L - d * (k - 1) - 1) / s) + 1
+                elif isinstance(layer, nn.ConvTranspose1d):
+                    L = (L - 1) * s + d * (k - 1) + 1
+
+                L = math.floor(L)
+        return L
+
+    @torch.no_grad()
+    def compress(
+        self,
+        audio_path_or_signal: Union[str, Path, AudioSignal],
+        win_duration: float = 1.0,
+        verbose: bool = False,
+        normalize_db: float = -16,
+        n_quantizers: int = None,
+    ) -> DACFile:
+        """Processes an audio signal from a file or AudioSignal object into
+        discrete codes. This function processes the signal in short windows,
+        using constant GPU memory.
+
+        Parameters
+        ----------
+        audio_path_or_signal : Union[str, Path, AudioSignal]
+            audio signal to reconstruct
+        win_duration : float, optional
+            window duration in seconds, by default 5.0
+        verbose : bool, optional
+            by default False
+        normalize_db : float, optional
+            normalize db, by default -16
+
+        Returns
+        -------
+        DACFile
+            Object containing compressed codes and metadata
+            required for decompression
+        """
+        audio_signal = audio_path_or_signal
+        if isinstance(audio_signal, (str, Path)):
+            audio_signal = AudioSignal.load_from_file_with_ffmpeg(str(audio_signal))
+
+        self.eval()
+        original_padding = self.padding
+        original_device = audio_signal.device
+
+        audio_signal = audio_signal.clone()
+        original_sr = audio_signal.sample_rate
+
+        resample_fn = audio_signal.resample
+        loudness_fn = audio_signal.loudness
+
+        # If audio is > 10 minutes long, use the ffmpeg versions
+        if audio_signal.signal_duration >= 10 * 60 * 60:
+            resample_fn = audio_signal.ffmpeg_resample
+            loudness_fn = audio_signal.ffmpeg_loudness
+
+        original_length = audio_signal.signal_length
+        resample_fn(self.sample_rate)
+        input_db = loudness_fn()
+
+        if normalize_db is not None:
+            audio_signal.normalize(normalize_db)
+        audio_signal.ensure_max_of_audio()
+
+        nb, nac, nt = audio_signal.audio_data.shape
+        audio_signal.audio_data = audio_signal.audio_data.reshape(nb * nac, 1, nt)
+        win_duration = (
+            audio_signal.signal_duration if win_duration is None else win_duration
+        )
+
+        if audio_signal.signal_duration <= win_duration:
+            # Unchunked compression (used if signal length < win duration)
+            self.padding = True
+            n_samples = nt
+            hop = nt
+        else:
+            # Chunked inference
+            self.padding = False
+            # Zero-pad signal on either side by the delay
+            audio_signal.zero_pad(self.delay, self.delay)
+            n_samples = int(win_duration * self.sample_rate)
+            # Round n_samples to nearest hop length multiple
+            n_samples = int(math.ceil(n_samples / self.hop_length) * self.hop_length)
+            hop = self.get_output_length(n_samples)
+
+        codes = []
+        range_fn = range if not verbose else tqdm.trange
+
+        for i in range_fn(0, nt, hop):
+            x = audio_signal[..., i : i + n_samples]
+            x = x.zero_pad(0, max(0, n_samples - x.shape[-1]))
+
+            audio_data = x.audio_data.to(self.device)
+            audio_data = self.preprocess(audio_data, self.sample_rate)
+            _, c, _, _, _ = self.encode(audio_data, n_quantizers)
+            codes.append(c.to(original_device))
+            chunk_length = c.shape[-1]
+
+        codes = torch.cat(codes, dim=-1)
+
+        dac_file = DACFile(
+            codes=codes,
+            chunk_length=chunk_length,
+            original_length=original_length,
+            input_db=input_db,
+            channels=nac,
+            sample_rate=original_sr,
+            padding=self.padding,
+            dac_version=SUPPORTED_VERSIONS[-1],
+        )
+
+        if n_quantizers is not None:
+            codes = codes[:, :n_quantizers, :]
+
+        self.padding = original_padding
+        return dac_file
+
+    @torch.no_grad()
+    def decompress(
+        self,
+        obj: Union[str, Path, DACFile],
+        verbose: bool = False,
+    ) -> AudioSignal:
+        """Reconstruct audio from a given .dac file
+
+        Parameters
+        ----------
+        obj : Union[str, Path, DACFile]
+            .dac file location or corresponding DACFile object.
+        verbose : bool, optional
+            Prints progress if True, by default False
+
+        Returns
+        -------
+        AudioSignal
+            Object with the reconstructed audio
+        """
+        self.eval()
+        if isinstance(obj, (str, Path)):
+            obj = DACFile.load(obj)
+
+        original_padding = self.padding
+        self.padding = obj.padding
+
+        range_fn = range if not verbose else tqdm.trange
+        codes = obj.codes
+        original_device = codes.device
+        chunk_length = obj.chunk_length
+        recons = []
+
+        for i in range_fn(0, codes.shape[-1], chunk_length):
+            c = codes[..., i : i + chunk_length].to(self.device)
+            z = self.quantizer.from_codes(c)[0]
+            r = self.decode(z)
+            recons.append(r.to(original_device))
+
+        recons = torch.cat(recons, dim=-1)
+        recons = AudioSignal(recons, self.sample_rate)
+
+        resample_fn = recons.resample
+        loudness_fn = recons.loudness
+
+        # If audio is > 10 minutes long, use the ffmpeg versions
+        if recons.signal_duration >= 10 * 60 * 60:
+            resample_fn = recons.ffmpeg_resample
+            loudness_fn = recons.ffmpeg_loudness
+
+        recons.normalize(obj.input_db)
+        resample_fn(obj.sample_rate)
+        recons = recons[..., : obj.original_length]
+        loudness_fn()
+        recons.audio_data = recons.audio_data.reshape(
+            -1, obj.channels, obj.original_length
+        )
+
+        self.padding = original_padding
+        return recons

dac/model/dac.py

@@ -0,0 +1,400 @@
+import math
+from typing import List
+from typing import Union
+
+import numpy as np
+import torch
+from audiotools import AudioSignal
+from audiotools.ml import BaseModel
+from torch import nn
+
+from .base import CodecMixin
+from dac.nn.layers import Snake1d
+from dac.nn.layers import WNConv1d
+from dac.nn.layers import WNConvTranspose1d
+from dac.nn.quantize import ResidualVectorQuantize
+from .encodec import SConv1d, SConvTranspose1d, SLSTM
+
+
+def init_weights(m):
+    if isinstance(m, nn.Conv1d):
+        nn.init.trunc_normal_(m.weight, std=0.02)
+        nn.init.constant_(m.bias, 0)
+
+
+class ResidualUnit(nn.Module):
+    def __init__(self, dim: int = 16, dilation: int = 1, causal: bool = False):
+        super().__init__()
+        conv1d_type = SConv1d# if causal else WNConv1d
+        pad = ((7 - 1) * dilation) // 2
+        self.block = nn.Sequential(
+            Snake1d(dim),
+            conv1d_type(dim, dim, kernel_size=7, dilation=dilation, padding=pad, causal=causal, norm='weight_norm'),
+            Snake1d(dim),
+            conv1d_type(dim, dim, kernel_size=1, causal=causal, norm='weight_norm'),
+        )
+
+    def forward(self, x):
+        y = self.block(x)
+        pad = (x.shape[-1] - y.shape[-1]) // 2
+        if pad > 0:
+            x = x[..., pad:-pad]
+        return x + y
+
+
+class EncoderBlock(nn.Module):
+    def __init__(self, dim: int = 16, stride: int = 1, causal: bool = False):
+        super().__init__()
+        conv1d_type = SConv1d# if causal else WNConv1d
+        self.block = nn.Sequential(
+            ResidualUnit(dim // 2, dilation=1, causal=causal),
+            ResidualUnit(dim // 2, dilation=3, causal=causal),
+            ResidualUnit(dim // 2, dilation=9, causal=causal),
+            Snake1d(dim // 2),
+            conv1d_type(
+                dim // 2,
+                dim,
+                kernel_size=2 * stride,
+                stride=stride,
+                padding=math.ceil(stride / 2),
+                causal=causal,
+                norm='weight_norm',
+            ),
+        )
+
+    def forward(self, x):
+        return self.block(x)
+
+
+class Encoder(nn.Module):
+    def __init__(
+        self,
+        d_model: int = 64,
+        strides: list = [2, 4, 8, 8],
+        d_latent: int = 64,
+        causal: bool = False,
+        lstm: int = 2,
+    ):
+        super().__init__()
+        conv1d_type = SConv1d# if causal else WNConv1d
+        # Create first convolution
+        self.block = [conv1d_type(1, d_model, kernel_size=7, padding=3, causal=causal, norm='weight_norm')]
+
+        # Create EncoderBlocks that double channels as they downsample by `stride`
+        for stride in strides:
+            d_model *= 2
+            self.block += [EncoderBlock(d_model, stride=stride, causal=causal)]
+
+        # Add LSTM if needed
+        self.use_lstm = lstm
+        if lstm:
+            self.block += [SLSTM(d_model, lstm)]
+
+        # Create last convolution
+        self.block += [
+            Snake1d(d_model),
+            conv1d_type(d_model, d_latent, kernel_size=3, padding=1, causal=causal, norm='weight_norm'),
+        ]
+
+        # Wrap black into nn.Sequential
+        self.block = nn.Sequential(*self.block)
+        self.enc_dim = d_model
+
+    def forward(self, x):
+        return self.block(x)
+
+    def reset_cache(self):
+        # recursively find all submodules named SConv1d in self.block and use their reset_cache method
+        def reset_cache(m):
+            if isinstance(m, SConv1d) or isinstance(m, SLSTM):
+                m.reset_cache()
+                return
+            for child in m.children():
+                reset_cache(child)
+
+        reset_cache(self.block)
+
+
+class DecoderBlock(nn.Module):
+    def __init__(self, input_dim: int = 16, output_dim: int = 8, stride: int = 1, causal: bool = False):
+        super().__init__()
+        conv1d_type = SConvTranspose1d #if causal else WNConvTranspose1d
+        self.block = nn.Sequential(
+            Snake1d(input_dim),
+            conv1d_type(
+                input_dim,
+                output_dim,
+                kernel_size=2 * stride,
+                stride=stride,
+                padding=math.ceil(stride / 2),
+                causal=causal,
+                norm='weight_norm'
+            ),
+            ResidualUnit(output_dim, dilation=1, causal=causal),
+            ResidualUnit(output_dim, dilation=3, causal=causal),
+            ResidualUnit(output_dim, dilation=9, causal=causal),
+        )
+
+    def forward(self, x):
+        return self.block(x)
+
+
+class Decoder(nn.Module):
+    def __init__(
+        self,
+        input_channel,
+        channels,
+        rates,
+        d_out: int = 1,
+        causal: bool = False,
+        lstm: int = 2,
+    ):
+        super().__init__()
+        conv1d_type = SConv1d# if causal else WNConv1d
+        # Add first conv layer
+        layers = [conv1d_type(input_channel, channels, kernel_size=7, padding=3, causal=causal, norm='weight_norm')]
+
+        if lstm:
+            layers += [SLSTM(channels, num_layers=lstm)]
+
+        # Add upsampling + MRF blocks
+        for i, stride in enumerate(rates):
+            input_dim = channels // 2**i
+            output_dim = channels // 2 ** (i + 1)
+            layers += [DecoderBlock(input_dim, output_dim, stride, causal=causal)]
+
+        # Add final conv layer
+        layers += [
+            Snake1d(output_dim),
+            conv1d_type(output_dim, d_out, kernel_size=7, padding=3, causal=causal, norm='weight_norm'),
+            nn.Tanh(),
+        ]
+
+        self.model = nn.Sequential(*layers)
+
+    def forward(self, x):
+        return self.model(x)
+
+
+class DAC(BaseModel, CodecMixin):
+    def __init__(
+        self,
+        encoder_dim: int = 64,
+        encoder_rates: List[int] = [2, 4, 8, 8],
+        latent_dim: int = None,
+        decoder_dim: int = 1536,
+        decoder_rates: List[int] = [8, 8, 4, 2],
+        n_codebooks: int = 9,
+        codebook_size: int = 1024,
+        codebook_dim: Union[int, list] = 8,
+        quantizer_dropout: bool = False,
+        sample_rate: int = 44100,
+        lstm: int = 2,
+        causal: bool = False,
+    ):
+        super().__init__()
+
+        self.encoder_dim = encoder_dim
+        self.encoder_rates = encoder_rates
+        self.decoder_dim = decoder_dim
+        self.decoder_rates = decoder_rates
+        self.sample_rate = sample_rate
+
+        if latent_dim is None:
+            latent_dim = encoder_dim * (2 ** len(encoder_rates))
+
+        self.latent_dim = latent_dim
+
+        self.hop_length = np.prod(encoder_rates)
+        self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim, causal=causal, lstm=lstm)
+
+        self.n_codebooks = n_codebooks
+        self.codebook_size = codebook_size
+        self.codebook_dim = codebook_dim
+        self.quantizer = ResidualVectorQuantize(
+            input_dim=latent_dim,
+            n_codebooks=n_codebooks,
+            codebook_size=codebook_size,
+            codebook_dim=codebook_dim,
+            quantizer_dropout=quantizer_dropout,
+        )
+
+        self.decoder = Decoder(
+            latent_dim,
+            decoder_dim,
+            decoder_rates,
+            lstm=lstm,
+            causal=causal,
+        )
+        self.sample_rate = sample_rate
+        self.apply(init_weights)
+
+        self.delay = self.get_delay()
+
+    def preprocess(self, audio_data, sample_rate):
+        if sample_rate is None:
+            sample_rate = self.sample_rate
+        assert sample_rate == self.sample_rate
+
+        length = audio_data.shape[-1]
+        right_pad = math.ceil(length / self.hop_length) * self.hop_length - length
+        audio_data = nn.functional.pad(audio_data, (0, right_pad))
+
+        return audio_data
+
+    def encode(
+        self,
+        audio_data: torch.Tensor,
+        n_quantizers: int = None,
+    ):
+        """Encode given audio data and return quantized latent codes
+
+        Parameters
+        ----------
+        audio_data : Tensor[B x 1 x T]
+            Audio data to encode
+        n_quantizers : int, optional
+            Number of quantizers to use, by default None
+            If None, all quantizers are used.
+
+        Returns
+        -------
+        dict
+            A dictionary with the following keys:
+            "z" : Tensor[B x D x T]
+                Quantized continuous representation of input
+            "codes" : Tensor[B x N x T]
+                Codebook indices for each codebook
+                (quantized discrete representation of input)
+            "latents" : Tensor[B x N*D x T]
+                Projected latents (continuous representation of input before quantization)
+            "vq/commitment_loss" : Tensor[1]
+                Commitment loss to train encoder to predict vectors closer to codebook
+                entries
+            "vq/codebook_loss" : Tensor[1]
+                Codebook loss to update the codebook
+            "length" : int
+                Number of samples in input audio
+        """
+        z = self.encoder(audio_data)
+        z, codes, latents, commitment_loss, codebook_loss = self.quantizer(
+            z, n_quantizers
+        )
+        return z, codes, latents, commitment_loss, codebook_loss
+
+    def decode(self, z: torch.Tensor):
+        """Decode given latent codes and return audio data
+
+        Parameters
+        ----------
+        z : Tensor[B x D x T]
+            Quantized continuous representation of input
+        length : int, optional
+            Number of samples in output audio, by default None
+
+        Returns
+        -------
+        dict
+            A dictionary with the following keys:
+            "audio" : Tensor[B x 1 x length]
+                Decoded audio data.
+        """
+        return self.decoder(z)
+
+    def forward(
+        self,
+        audio_data: torch.Tensor,
+        sample_rate: int = None,
+        n_quantizers: int = None,
+    ):
+        """Model forward pass
+
+        Parameters
+        ----------
+        audio_data : Tensor[B x 1 x T]
+            Audio data to encode
+        sample_rate : int, optional
+            Sample rate of audio data in Hz, by default None
+            If None, defaults to `self.sample_rate`
+        n_quantizers : int, optional
+            Number of quantizers to use, by default None.
+            If None, all quantizers are used.
+
+        Returns
+        -------
+        dict
+            A dictionary with the following keys:
+            "z" : Tensor[B x D x T]
+                Quantized continuous representation of input
+            "codes" : Tensor[B x N x T]
+                Codebook indices for each codebook
+                (quantized discrete representation of input)
+            "latents" : Tensor[B x N*D x T]
+                Projected latents (continuous representation of input before quantization)
+            "vq/commitment_loss" : Tensor[1]
+                Commitment loss to train encoder to predict vectors closer to codebook
+                entries
+            "vq/codebook_loss" : Tensor[1]
+                Codebook loss to update the codebook
+            "length" : int
+                Number of samples in input audio
+            "audio" : Tensor[B x 1 x length]
+                Decoded audio data.
+        """
+        length = audio_data.shape[-1]
+        audio_data = self.preprocess(audio_data, sample_rate)
+        z, codes, latents, commitment_loss, codebook_loss = self.encode(
+            audio_data, n_quantizers
+        )
+
+        x = self.decode(z)
+        return {
+            "audio": x[..., :length],
+            "z": z,
+            "codes": codes,
+            "latents": latents,
+            "vq/commitment_loss": commitment_loss,
+            "vq/codebook_loss": codebook_loss,
+        }
+
+
+if __name__ == "__main__":
+    import numpy as np
+    from functools import partial
+
+    model = DAC().to("cpu")
+
+    for n, m in model.named_modules():
+        o = m.extra_repr()
+        p = sum([np.prod(p.size()) for p in m.parameters()])
+        fn = lambda o, p: o + f" {p/1e6:<.3f}M params."
+        setattr(m, "extra_repr", partial(fn, o=o, p=p))
+    print(model)
+    print("Total # of params: ", sum([np.prod(p.size()) for p in model.parameters()]))
+
+    length = 88200 * 2
+    x = torch.randn(1, 1, length).to(model.device)
+    x.requires_grad_(True)
+    x.retain_grad()
+
+    # Make a forward pass
+    out = model(x)["audio"]
+    print("Input shape:", x.shape)
+    print("Output shape:", out.shape)
+
+    # Create gradient variable
+    grad = torch.zeros_like(out)
+    grad[:, :, grad.shape[-1] // 2] = 1
+
+    # Make a backward pass
+    out.backward(grad)
+
+    # Check non-zero values
+    gradmap = x.grad.squeeze(0)
+    gradmap = (gradmap != 0).sum(0)  # sum across features
+    rf = (gradmap != 0).sum()
+
+    print(f"Receptive field: {rf.item()}")
+
+    x = AudioSignal(torch.randn(1, 1, 44100 * 60), 44100)
+    model.decompress(model.compress(x, verbose=True), verbose=True)

dac/model/discriminator.py

@@ -0,0 +1,228 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from audiotools import AudioSignal
+from audiotools import ml
+from audiotools import STFTParams
+from einops import rearrange
+from torch.nn.utils import weight_norm
+
+
+def WNConv1d(*args, **kwargs):
+    act = kwargs.pop("act", True)
+    conv = weight_norm(nn.Conv1d(*args, **kwargs))
+    if not act:
+        return conv
+    return nn.Sequential(conv, nn.LeakyReLU(0.1))
+
+
+def WNConv2d(*args, **kwargs):
+    act = kwargs.pop("act", True)
+    conv = weight_norm(nn.Conv2d(*args, **kwargs))
+    if not act:
+        return conv
+    return nn.Sequential(conv, nn.LeakyReLU(0.1))
+
+
+class MPD(nn.Module):
+    def __init__(self, period):
+        super().__init__()
+        self.period = period
+        self.convs = nn.ModuleList(
+            [
+                WNConv2d(1, 32, (5, 1), (3, 1), padding=(2, 0)),
+                WNConv2d(32, 128, (5, 1), (3, 1), padding=(2, 0)),
+                WNConv2d(128, 512, (5, 1), (3, 1), padding=(2, 0)),
+                WNConv2d(512, 1024, (5, 1), (3, 1), padding=(2, 0)),
+                WNConv2d(1024, 1024, (5, 1), 1, padding=(2, 0)),
+            ]
+        )
+        self.conv_post = WNConv2d(
+            1024, 1, kernel_size=(3, 1), padding=(1, 0), act=False
+        )
+
+    def pad_to_period(self, x):
+        t = x.shape[-1]
+        x = F.pad(x, (0, self.period - t % self.period), mode="reflect")
+        return x
+
+    def forward(self, x):
+        fmap = []
+
+        x = self.pad_to_period(x)
+        x = rearrange(x, "b c (l p) -> b c l p", p=self.period)
+
+        for layer in self.convs:
+            x = layer(x)
+            fmap.append(x)
+
+        x = self.conv_post(x)
+        fmap.append(x)
+
+        return fmap
+
+
+class MSD(nn.Module):
+    def __init__(self, rate: int = 1, sample_rate: int = 44100):
+        super().__init__()
+        self.convs = nn.ModuleList(
+            [
+                WNConv1d(1, 16, 15, 1, padding=7),
+                WNConv1d(16, 64, 41, 4, groups=4, padding=20),
+                WNConv1d(64, 256, 41, 4, groups=16, padding=20),
+                WNConv1d(256, 1024, 41, 4, groups=64, padding=20),
+                WNConv1d(1024, 1024, 41, 4, groups=256, padding=20),
+                WNConv1d(1024, 1024, 5, 1, padding=2),
+            ]
+        )
+        self.conv_post = WNConv1d(1024, 1, 3, 1, padding=1, act=False)
+        self.sample_rate = sample_rate
+        self.rate = rate
+
+    def forward(self, x):
+        x = AudioSignal(x, self.sample_rate)
+        x.resample(self.sample_rate // self.rate)
+        x = x.audio_data
+
+        fmap = []
+
+        for l in self.convs:
+            x = l(x)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+
+        return fmap
+
+
+BANDS = [(0.0, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.0)]
+
+
+class MRD(nn.Module):
+    def __init__(
+        self,
+        window_length: int,
+        hop_factor: float = 0.25,
+        sample_rate: int = 44100,
+        bands: list = BANDS,
+    ):
+        """Complex multi-band spectrogram discriminator.
+        Parameters
+        ----------
+        window_length : int
+            Window length of STFT.
+        hop_factor : float, optional
+            Hop factor of the STFT, defaults to ``0.25 * window_length``.
+        sample_rate : int, optional
+            Sampling rate of audio in Hz, by default 44100
+        bands : list, optional
+            Bands to run discriminator over.
+        """
+        super().__init__()
+
+        self.window_length = window_length
+        self.hop_factor = hop_factor
+        self.sample_rate = sample_rate
+        self.stft_params = STFTParams(
+            window_length=window_length,
+            hop_length=int(window_length * hop_factor),
+            match_stride=True,
+        )
+
+        n_fft = window_length // 2 + 1
+        bands = [(int(b[0] * n_fft), int(b[1] * n_fft)) for b in bands]
+        self.bands = bands
+
+        ch = 32
+        convs = lambda: nn.ModuleList(
+            [
+                WNConv2d(2, ch, (3, 9), (1, 1), padding=(1, 4)),
+                WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
+                WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
+                WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
+                WNConv2d(ch, ch, (3, 3), (1, 1), padding=(1, 1)),
+            ]
+        )
+        self.band_convs = nn.ModuleList([convs() for _ in range(len(self.bands))])
+        self.conv_post = WNConv2d(ch, 1, (3, 3), (1, 1), padding=(1, 1), act=False)
+
+    def spectrogram(self, x):
+        x = AudioSignal(x, self.sample_rate, stft_params=self.stft_params)
+        x = torch.view_as_real(x.stft())
+        x = rearrange(x, "b 1 f t c -> (b 1) c t f")
+        # Split into bands
+        x_bands = [x[..., b[0] : b[1]] for b in self.bands]
+        return x_bands
+
+    def forward(self, x):
+        x_bands = self.spectrogram(x)
+        fmap = []
+
+        x = []
+        for band, stack in zip(x_bands, self.band_convs):
+            for layer in stack:
+                band = layer(band)
+                fmap.append(band)
+            x.append(band)
+
+        x = torch.cat(x, dim=-1)
+        x = self.conv_post(x)
+        fmap.append(x)
+
+        return fmap
+
+
+class Discriminator(nn.Module):
+    def __init__(
+        self,
+        rates: list = [],
+        periods: list = [2, 3, 5, 7, 11],
+        fft_sizes: list = [2048, 1024, 512],
+        sample_rate: int = 44100,
+        bands: list = BANDS,
+    ):
+        """Discriminator that combines multiple discriminators.
+
+        Parameters
+        ----------
+        rates : list, optional
+            sampling rates (in Hz) to run MSD at, by default []
+            If empty, MSD is not used.
+        periods : list, optional
+            periods (of samples) to run MPD at, by default [2, 3, 5, 7, 11]
+        fft_sizes : list, optional
+            Window sizes of the FFT to run MRD at, by default [2048, 1024, 512]
+        sample_rate : int, optional
+            Sampling rate of audio in Hz, by default 44100
+        bands : list, optional
+            Bands to run MRD at, by default `BANDS`
+        """
+        super().__init__()
+        discs = []
+        discs += [MPD(p) for p in periods]
+        discs += [MSD(r, sample_rate=sample_rate) for r in rates]
+        discs += [MRD(f, sample_rate=sample_rate, bands=bands) for f in fft_sizes]
+        self.discriminators = nn.ModuleList(discs)
+
+    def preprocess(self, y):
+        # Remove DC offset
+        y = y - y.mean(dim=-1, keepdims=True)
+        # Peak normalize the volume of input audio
+        y = 0.8 * y / (y.abs().max(dim=-1, keepdim=True)[0] + 1e-9)
+        return y
+
+    def forward(self, x):
+        x = self.preprocess(x)
+        fmaps = [d(x) for d in self.discriminators]
+        return fmaps
+
+
+if __name__ == "__main__":
+    disc = Discriminator()
+    x = torch.zeros(1, 1, 44100)
+    results = disc(x)
+    for i, result in enumerate(results):
+        print(f"disc{i}")
+        for i, r in enumerate(result):
+            print(r.shape, r.mean(), r.min(), r.max())
+        print()

dac/model/encodec.py

@@ -0,0 +1,320 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Convolutional layers wrappers and utilities."""
+
+import math
+import typing as tp
+import warnings
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.nn.utils import spectral_norm, weight_norm
+
+import typing as tp
+
+import einops
+
+
+class ConvLayerNorm(nn.LayerNorm):
+    """
+    Convolution-friendly LayerNorm that moves channels to last dimensions
+    before running the normalization and moves them back to original position right after.
+    """
+    def __init__(self, normalized_shape: tp.Union[int, tp.List[int], torch.Size], **kwargs):
+        super().__init__(normalized_shape, **kwargs)
+
+    def forward(self, x):
+        x = einops.rearrange(x, 'b ... t -> b t ...')
+        x = super().forward(x)
+        x = einops.rearrange(x, 'b t ... -> b ... t')
+        return
+
+
+CONV_NORMALIZATIONS = frozenset(['none', 'weight_norm', 'spectral_norm',
+                                 'time_layer_norm', 'layer_norm', 'time_group_norm'])
+
+
+def apply_parametrization_norm(module: nn.Module, norm: str = 'none') -> nn.Module:
+    assert norm in CONV_NORMALIZATIONS
+    if norm == 'weight_norm':
+        return weight_norm(module)
+    elif norm == 'spectral_norm':
+        return spectral_norm(module)
+    else:
+        # We already check was in CONV_NORMALIZATION, so any other choice
+        # doesn't need reparametrization.
+        return module
+
+
+def get_norm_module(module: nn.Module, causal: bool = False, norm: str = 'none', **norm_kwargs) -> nn.Module:
+    """Return the proper normalization module. If causal is True, this will ensure the returned
+    module is causal, or return an error if the normalization doesn't support causal evaluation.
+    """
+    assert norm in CONV_NORMALIZATIONS
+    if norm == 'layer_norm':
+        assert isinstance(module, nn.modules.conv._ConvNd)
+        return ConvLayerNorm(module.out_channels, **norm_kwargs)
+    elif norm == 'time_group_norm':
+        if causal:
+            raise ValueError("GroupNorm doesn't support causal evaluation.")
+        assert isinstance(module, nn.modules.conv._ConvNd)
+        return nn.GroupNorm(1, module.out_channels, **norm_kwargs)
+    else:
+        return nn.Identity()
+
+
+def get_extra_padding_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int,
+                                 padding_total: int = 0) -> int:
+    """See `pad_for_conv1d`.
+    """
+    length = x.shape[-1]
+    n_frames = (length - kernel_size + padding_total) / stride + 1
+    ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total)
+    return ideal_length - length
+
+
+def pad_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int, padding_total: int = 0):
+    """Pad for a convolution to make sure that the last window is full.
+    Extra padding is added at the end. This is required to ensure that we can rebuild
+    an output of the same length, as otherwise, even with padding, some time steps
+    might get removed.
+    For instance, with total padding = 4, kernel size = 4, stride = 2:
+        0 0 1 2 3 4 5 0 0   # (0s are padding)
+        1   2   3           # (output frames of a convolution, last 0 is never used)
+        0 0 1 2 3 4 5 0     # (output of tr. conv., but pos. 5 is going to get removed as padding)
+            1 2 3 4         # once you removed padding, we are missing one time step !
+    """
+    extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
+    return F.pad(x, (0, extra_padding))
+
+
+def pad1d(x: torch.Tensor, paddings: tp.Tuple[int, int], mode: str = 'zero', value: float = 0.):
+    """Tiny wrapper around F.pad, just to allow for reflect padding on small input.
+    If this is the case, we insert extra 0 padding to the right before the reflection happen.
+    """
+    length = x.shape[-1]
+    padding_left, padding_right = paddings
+    assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
+    if mode == 'reflect':
+        max_pad = max(padding_left, padding_right)
+        extra_pad = 0
+        if length <= max_pad:
+            extra_pad = max_pad - length + 1
+            x = F.pad(x, (0, extra_pad))
+        padded = F.pad(x, paddings, mode, value)
+        end = padded.shape[-1] - extra_pad
+        return padded[..., :end]
+    else:
+        return F.pad(x, paddings, mode, value)
+
+
+def unpad1d(x: torch.Tensor, paddings: tp.Tuple[int, int]):
+    """Remove padding from x, handling properly zero padding. Only for 1d!"""
+    padding_left, padding_right = paddings
+    assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
+    assert (padding_left + padding_right) <= x.shape[-1]
+    end = x.shape[-1] - padding_right
+    return x[..., padding_left: end]
+
+
+class NormConv1d(nn.Module):
+    """Wrapper around Conv1d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, causal: bool = False, norm: str = 'none',
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.conv = apply_parametrization_norm(nn.Conv1d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.conv, causal, norm, **norm_kwargs)
+        self.norm_type = norm
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.norm(x)
+        return x
+
+
+class NormConv2d(nn.Module):
+    """Wrapper around Conv2d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, norm: str = 'none',
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.conv = apply_parametrization_norm(nn.Conv2d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.conv, causal=False, norm=norm, **norm_kwargs)
+        self.norm_type = norm
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.norm(x)
+        return x
+
+
+class NormConvTranspose1d(nn.Module):
+    """Wrapper around ConvTranspose1d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, causal: bool = False, norm: str = 'none',
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.convtr = apply_parametrization_norm(nn.ConvTranspose1d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.convtr, causal, norm, **norm_kwargs)
+        self.norm_type = norm
+
+    def forward(self, x):
+        x = self.convtr(x)
+        x = self.norm(x)
+        return x
+
+
+class NormConvTranspose2d(nn.Module):
+    """Wrapper around ConvTranspose2d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, norm: str = 'none',
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.convtr = apply_parametrization_norm(nn.ConvTranspose2d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.convtr, causal=False, norm=norm, **norm_kwargs)
+
+    def forward(self, x):
+        x = self.convtr(x)
+        x = self.norm(x)
+        return x
+
+
+class SConv1d(nn.Module):
+    """Conv1d with some builtin handling of asymmetric or causal padding
+    and normalization.
+    """
+    def __init__(self, in_channels: int, out_channels: int,
+                 kernel_size: int, stride: int = 1, dilation: int = 1,
+                 groups: int = 1, bias: bool = True, causal: bool = False,
+                 norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {},
+                 pad_mode: str = 'reflect', **kwargs):
+        super().__init__()
+        # warn user on unusual setup between dilation and stride
+        if stride > 1 and dilation > 1:
+            warnings.warn('SConv1d has been initialized with stride > 1 and dilation > 1'
+                          f' (kernel_size={kernel_size} stride={stride}, dilation={dilation}).')
+        self.conv = NormConv1d(in_channels, out_channels, kernel_size, stride,
+                               dilation=dilation, groups=groups, bias=bias, causal=causal,
+                               norm=norm, norm_kwargs=norm_kwargs)
+        self.causal = causal
+        self.pad_mode = pad_mode
+
+        self.cache_enabled = False
+
+    def reset_cache(self):
+        """Reset the cache when starting a new stream."""
+        self.cache = None
+        self.cache_enabled = True
+
+    def forward(self, x):
+        B, C, T = x.shape
+        kernel_size = self.conv.conv.kernel_size[0]
+        stride = self.conv.conv.stride[0]
+        dilation = self.conv.conv.dilation[0]
+        kernel_size = (kernel_size - 1) * dilation + 1  # effective kernel size with dilations
+        padding_total = kernel_size - stride
+        extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
+
+        if self.causal:
+            # Left padding for causal
+            if self.cache_enabled and self.cache is not None:
+                # Concatenate the cache (previous inputs) with the new input for streaming
+                x = torch.cat([self.cache, x], dim=2)
+            else:
+                x = pad1d(x, (padding_total, extra_padding), mode=self.pad_mode)
+        else:
+            # Asymmetric padding required for odd strides
+            padding_right = padding_total // 2
+            padding_left = padding_total - padding_right
+            x = pad1d(x, (padding_left, padding_right + extra_padding), mode=self.pad_mode)
+
+        # Store the most recent input frames for future cache use
+        if self.cache_enabled:
+            if self.cache is None:
+                # Initialize cache with zeros (at the start of streaming)
+                self.cache = torch.zeros(B, C, kernel_size - 1, device=x.device)
+            # Update the cache by storing the latest input frames
+            if kernel_size > 1:
+                self.cache = x[:, :, -kernel_size + 1:].detach()  # Only store the necessary frames
+
+        return self.conv(x)
+
+
+
+class SConvTranspose1d(nn.Module):
+    """ConvTranspose1d with some builtin handling of asymmetric or causal padding
+    and normalization.
+    """
+    def __init__(self, in_channels: int, out_channels: int,
+                 kernel_size: int, stride: int = 1, causal: bool = False,
+                 norm: str = 'none', trim_right_ratio: float = 1.,
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.convtr = NormConvTranspose1d(in_channels, out_channels, kernel_size, stride,
+                                          causal=causal, norm=norm, norm_kwargs=norm_kwargs)
+        self.causal = causal
+        self.trim_right_ratio = trim_right_ratio
+        assert self.causal or self.trim_right_ratio == 1., \
+            "`trim_right_ratio` != 1.0 only makes sense for causal convolutions"
+        assert self.trim_right_ratio >= 0. and self.trim_right_ratio <= 1.
+
+    def forward(self, x):
+        kernel_size = self.convtr.convtr.kernel_size[0]
+        stride = self.convtr.convtr.stride[0]
+        padding_total = kernel_size - stride
+
+        y = self.convtr(x)
+
+        # We will only trim fixed padding. Extra padding from `pad_for_conv1d` would be
+        # removed at the very end, when keeping only the right length for the output,
+        # as removing it here would require also passing the length at the matching layer
+        # in the encoder.
+        if self.causal:
+            # Trim the padding on the right according to the specified ratio
+            # if trim_right_ratio = 1.0, trim everything from right
+            padding_right = math.ceil(padding_total * self.trim_right_ratio)
+            padding_left = padding_total - padding_right
+            y = unpad1d(y, (padding_left, padding_right))
+        else:
+            # Asymmetric padding required for odd strides
+            padding_right = padding_total // 2
+            padding_left = padding_total - padding_right
+            y = unpad1d(y, (padding_left, padding_right))
+        return y
+
+class SLSTM(nn.Module):
+    """
+    LSTM without worrying about the hidden state, nor the layout of the data.
+    Expects input as convolutional layout.
+    """
+    def __init__(self, dimension: int, num_layers: int = 2, skip: bool = True):
+        super().__init__()
+        self.skip = skip
+        self.lstm = nn.LSTM(dimension, dimension, num_layers)
+        self.hidden = None
+        self.cache_enabled = False
+
+    def forward(self, x):
+        x = x.permute(2, 0, 1)
+        if self.training or not self.cache_enabled:
+            y, _ = self.lstm(x)
+        else:
+            y, self.hidden = self.lstm(x, self.hidden)
+        if self.skip:
+            y = y + x
+        y = y.permute(1, 2, 0)
+        return y
+
+    def reset_cache(self):
+        self.hidden = None
+        self.cache_enabled = True

dac/nn/__init__.py

@@ -0,0 +1,3 @@
+from . import layers
+from . import loss
+from . import quantize

dac/nn/layers.py

@@ -0,0 +1,33 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+from torch.nn.utils import weight_norm
+
+
+def WNConv1d(*args, **kwargs):
+    return weight_norm(nn.Conv1d(*args, **kwargs))
+
+
+def WNConvTranspose1d(*args, **kwargs):
+    return weight_norm(nn.ConvTranspose1d(*args, **kwargs))
+
+
+# Scripting this brings model speed up 1.4x
+@torch.jit.script
+def snake(x, alpha):
+    shape = x.shape
+    x = x.reshape(shape[0], shape[1], -1)
+    x = x + (alpha + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2)
+    x = x.reshape(shape)
+    return x
+
+
+class Snake1d(nn.Module):
+    def __init__(self, channels):
+        super().__init__()
+        self.alpha = nn.Parameter(torch.ones(1, channels, 1))
+
+    def forward(self, x):
+        return snake(x, self.alpha)

dac/nn/loss.py

@@ -0,0 +1,368 @@
+import typing
+from typing import List
+
+import torch
+import torch.nn.functional as F
+from audiotools import AudioSignal
+from audiotools import STFTParams
+from torch import nn
+
+
+class L1Loss(nn.L1Loss):
+    """L1 Loss between AudioSignals. Defaults
+    to comparing ``audio_data``, but any
+    attribute of an AudioSignal can be used.
+
+    Parameters
+    ----------
+    attribute : str, optional
+        Attribute of signal to compare, defaults to ``audio_data``.
+    weight : float, optional
+        Weight of this loss, defaults to 1.0.
+
+    Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/distance.py
+    """
+
+    def __init__(self, attribute: str = "audio_data", weight: float = 1.0, **kwargs):
+        self.attribute = attribute
+        self.weight = weight
+        super().__init__(**kwargs)
+
+    def forward(self, x: AudioSignal, y: AudioSignal):
+        """
+        Parameters
+        ----------
+        x : AudioSignal
+            Estimate AudioSignal
+        y : AudioSignal
+            Reference AudioSignal
+
+        Returns
+        -------
+        torch.Tensor
+            L1 loss between AudioSignal attributes.
+        """
+        if isinstance(x, AudioSignal):
+            x = getattr(x, self.attribute)
+            y = getattr(y, self.attribute)
+        return super().forward(x, y)
+
+
+class SISDRLoss(nn.Module):
+    """
+    Computes the Scale-Invariant Source-to-Distortion Ratio between a batch
+    of estimated and reference audio signals or aligned features.
+
+    Parameters
+    ----------
+    scaling : int, optional
+        Whether to use scale-invariant (True) or
+        signal-to-noise ratio (False), by default True
+    reduction : str, optional
+        How to reduce across the batch (either 'mean',
+        'sum', or none).], by default ' mean'
+    zero_mean : int, optional
+        Zero mean the references and estimates before
+        computing the loss, by default True
+    clip_min : int, optional
+        The minimum possible loss value. Helps network
+        to not focus on making already good examples better, by default None
+    weight : float, optional
+        Weight of this loss, defaults to 1.0.
+
+    Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/distance.py
+    """
+
+    def __init__(
+        self,
+        scaling: int = True,
+        reduction: str = "mean",
+        zero_mean: int = True,
+        clip_min: int = None,
+        weight: float = 1.0,
+    ):
+        self.scaling = scaling
+        self.reduction = reduction
+        self.zero_mean = zero_mean
+        self.clip_min = clip_min
+        self.weight = weight
+        super().__init__()
+
+    def forward(self, x: AudioSignal, y: AudioSignal):
+        eps = 1e-8
+        # nb, nc, nt
+        if isinstance(x, AudioSignal):
+            references = x.audio_data
+            estimates = y.audio_data
+        else:
+            references = x
+            estimates = y
+
+        nb = references.shape[0]
+        references = references.reshape(nb, 1, -1).permute(0, 2, 1)
+        estimates = estimates.reshape(nb, 1, -1).permute(0, 2, 1)
+
+        # samples now on axis 1
+        if self.zero_mean:
+            mean_reference = references.mean(dim=1, keepdim=True)
+            mean_estimate = estimates.mean(dim=1, keepdim=True)
+        else:
+            mean_reference = 0
+            mean_estimate = 0
+
+        _references = references - mean_reference
+        _estimates = estimates - mean_estimate
+
+        references_projection = (_references**2).sum(dim=-2) + eps
+        references_on_estimates = (_estimates * _references).sum(dim=-2) + eps
+
+        scale = (
+            (references_on_estimates / references_projection).unsqueeze(1)
+            if self.scaling
+            else 1
+        )
+
+        e_true = scale * _references
+        e_res = _estimates - e_true
+
+        signal = (e_true**2).sum(dim=1)
+        noise = (e_res**2).sum(dim=1)
+        sdr = -10 * torch.log10(signal / noise + eps)
+
+        if self.clip_min is not None:
+            sdr = torch.clamp(sdr, min=self.clip_min)
+
+        if self.reduction == "mean":
+            sdr = sdr.mean()
+        elif self.reduction == "sum":
+            sdr = sdr.sum()
+        return sdr
+
+
+class MultiScaleSTFTLoss(nn.Module):
+    """Computes the multi-scale STFT loss from [1].
+
+    Parameters
+    ----------
+    window_lengths : List[int], optional
+        Length of each window of each STFT, by default [2048, 512]
+    loss_fn : typing.Callable, optional
+        How to compare each loss, by default nn.L1Loss()
+    clamp_eps : float, optional
+        Clamp on the log magnitude, below, by default 1e-5
+    mag_weight : float, optional
+        Weight of raw magnitude portion of loss, by default 1.0
+    log_weight : float, optional
+        Weight of log magnitude portion of loss, by default 1.0
+    pow : float, optional
+        Power to raise magnitude to before taking log, by default 2.0
+    weight : float, optional
+        Weight of this loss, by default 1.0
+    match_stride : bool, optional
+        Whether to match the stride of convolutional layers, by default False
+
+    References
+    ----------
+
+    1.  Engel, Jesse, Chenjie Gu, and Adam Roberts.
+        "DDSP: Differentiable Digital Signal Processing."
+        International Conference on Learning Representations. 2019.
+
+    Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/spectral.py
+    """
+
+    def __init__(
+        self,
+        window_lengths: List[int] = [2048, 512],
+        loss_fn: typing.Callable = nn.L1Loss(),
+        clamp_eps: float = 1e-5,
+        mag_weight: float = 1.0,
+        log_weight: float = 1.0,
+        pow: float = 2.0,
+        weight: float = 1.0,
+        match_stride: bool = False,
+        window_type: str = None,
+    ):
+        super().__init__()
+        self.stft_params = [
+            STFTParams(
+                window_length=w,
+                hop_length=w // 4,
+                match_stride=match_stride,
+                window_type=window_type,
+            )
+            for w in window_lengths
+        ]
+        self.loss_fn = loss_fn
+        self.log_weight = log_weight
+        self.mag_weight = mag_weight
+        self.clamp_eps = clamp_eps
+        self.weight = weight
+        self.pow = pow
+
+    def forward(self, x: AudioSignal, y: AudioSignal):
+        """Computes multi-scale STFT between an estimate and a reference
+        signal.
+
+        Parameters
+        ----------
+        x : AudioSignal
+            Estimate signal
+        y : AudioSignal
+            Reference signal
+
+        Returns
+        -------
+        torch.Tensor
+            Multi-scale STFT loss.
+        """
+        loss = 0.0
+        for s in self.stft_params:
+            x.stft(s.window_length, s.hop_length, s.window_type)
+            y.stft(s.window_length, s.hop_length, s.window_type)
+            loss += self.log_weight * self.loss_fn(
+                x.magnitude.clamp(self.clamp_eps).pow(self.pow).log10(),
+                y.magnitude.clamp(self.clamp_eps).pow(self.pow).log10(),
+            )
+            loss += self.mag_weight * self.loss_fn(x.magnitude, y.magnitude)
+        return loss
+
+
+class MelSpectrogramLoss(nn.Module):
+    """Compute distance between mel spectrograms. Can be used
+    in a multi-scale way.
+
+    Parameters
+    ----------
+    n_mels : List[int]
+        Number of mels per STFT, by default [150, 80],
+    window_lengths : List[int], optional
+        Length of each window of each STFT, by default [2048, 512]
+    loss_fn : typing.Callable, optional
+        How to compare each loss, by default nn.L1Loss()
+    clamp_eps : float, optional
+        Clamp on the log magnitude, below, by default 1e-5
+    mag_weight : float, optional
+        Weight of raw magnitude portion of loss, by default 1.0
+    log_weight : float, optional
+        Weight of log magnitude portion of loss, by default 1.0
+    pow : float, optional
+        Power to raise magnitude to before taking log, by default 2.0
+    weight : float, optional
+        Weight of this loss, by default 1.0
+    match_stride : bool, optional
+        Whether to match the stride of convolutional layers, by default False
+
+    Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/spectral.py
+    """
+
+    def __init__(
+        self,
+        n_mels: List[int] = [150, 80],
+        window_lengths: List[int] = [2048, 512],
+        loss_fn: typing.Callable = nn.L1Loss(),
+        clamp_eps: float = 1e-5,
+        mag_weight: float = 1.0,
+        log_weight: float = 1.0,
+        pow: float = 2.0,
+        weight: float = 1.0,
+        match_stride: bool = False,
+        mel_fmin: List[float] = [0.0, 0.0],
+        mel_fmax: List[float] = [None, None],
+        window_type: str = None,
+    ):
+        super().__init__()
+        self.stft_params = [
+            STFTParams(
+                window_length=w,
+                hop_length=w // 4,
+                match_stride=match_stride,
+                window_type=window_type,
+            )
+            for w in window_lengths
+        ]
+        self.n_mels = n_mels
+        self.loss_fn = loss_fn
+        self.clamp_eps = clamp_eps
+        self.log_weight = log_weight
+        self.mag_weight = mag_weight
+        self.weight = weight
+        self.mel_fmin = mel_fmin
+        self.mel_fmax = mel_fmax
+        self.pow = pow
+
+    def forward(self, x: AudioSignal, y: AudioSignal):
+        """Computes mel loss between an estimate and a reference
+        signal.
+
+        Parameters
+        ----------
+        x : AudioSignal
+            Estimate signal
+        y : AudioSignal
+            Reference signal
+
+        Returns
+        -------
+        torch.Tensor
+            Mel loss.
+        """
+        loss = 0.0
+        for n_mels, fmin, fmax, s in zip(
+            self.n_mels, self.mel_fmin, self.mel_fmax, self.stft_params
+        ):
+            kwargs = {
+                "window_length": s.window_length,
+                "hop_length": s.hop_length,
+                "window_type": s.window_type,
+            }
+            x_mels = x.mel_spectrogram(n_mels, mel_fmin=fmin, mel_fmax=fmax, **kwargs)
+            y_mels = y.mel_spectrogram(n_mels, mel_fmin=fmin, mel_fmax=fmax, **kwargs)
+
+            loss += self.log_weight * self.loss_fn(
+                x_mels.clamp(self.clamp_eps).pow(self.pow).log10(),
+                y_mels.clamp(self.clamp_eps).pow(self.pow).log10(),
+            )
+            loss += self.mag_weight * self.loss_fn(x_mels, y_mels)
+        return loss
+
+
+class GANLoss(nn.Module):
+    """
+    Computes a discriminator loss, given a discriminator on
+    generated waveforms/spectrograms compared to ground truth
+    waveforms/spectrograms. Computes the loss for both the
+    discriminator and the generator in separate functions.
+    """
+
+    def __init__(self, discriminator):
+        super().__init__()
+        self.discriminator = discriminator
+
+    def forward(self, fake, real):
+        d_fake = self.discriminator(fake.audio_data)
+        d_real = self.discriminator(real.audio_data)
+        return d_fake, d_real
+
+    def discriminator_loss(self, fake, real):
+        d_fake, d_real = self.forward(fake.clone().detach(), real)
+
+        loss_d = 0
+        for x_fake, x_real in zip(d_fake, d_real):
+            loss_d += torch.mean(x_fake[-1] ** 2)
+            loss_d += torch.mean((1 - x_real[-1]) ** 2)
+        return loss_d
+
+    def generator_loss(self, fake, real):
+        d_fake, d_real = self.forward(fake, real)
+
+        loss_g = 0
+        for x_fake in d_fake:
+            loss_g += torch.mean((1 - x_fake[-1]) ** 2)
+
+        loss_feature = 0
+
+        for i in range(len(d_fake)):
+            for j in range(len(d_fake[i]) - 1):
+                loss_feature += F.l1_loss(d_fake[i][j], d_real[i][j].detach())
+        return loss_g, loss_feature

dac/nn/quantize.py

@@ -0,0 +1,339 @@
+from typing import Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+from torch.nn.utils import weight_norm
+
+from dac.nn.layers import WNConv1d
+
+class VectorQuantizeLegacy(nn.Module):
+    """
+    Implementation of VQ similar to Karpathy's repo:
+    https://github.com/karpathy/deep-vector-quantization
+    removed in-out projection
+    """
+
+    def __init__(self, input_dim: int, codebook_size: int):
+        super().__init__()
+        self.codebook_size = codebook_size
+        self.codebook = nn.Embedding(codebook_size, input_dim)
+
+    def forward(self, z, z_mask=None):
+        """Quantized the input tensor using a fixed codebook and returns
+        the corresponding codebook vectors
+
+        Parameters
+        ----------
+        z : Tensor[B x D x T]
+
+        Returns
+        -------
+        Tensor[B x D x T]
+            Quantized continuous representation of input
+        Tensor[1]
+            Commitment loss to train encoder to predict vectors closer to codebook
+            entries
+        Tensor[1]
+            Codebook loss to update the codebook
+        Tensor[B x T]
+            Codebook indices (quantized discrete representation of input)
+        Tensor[B x D x T]
+            Projected latents (continuous representation of input before quantization)
+        """
+
+        z_e = z
+        z_q, indices = self.decode_latents(z)
+
+        if z_mask is not None:
+            commitment_loss = (F.mse_loss(z_e, z_q.detach(), reduction="none").mean(1) * z_mask).sum() / z_mask.sum()
+            codebook_loss = (F.mse_loss(z_q, z_e.detach(), reduction="none").mean(1) * z_mask).sum() / z_mask.sum()
+        else:
+            commitment_loss = F.mse_loss(z_e, z_q.detach())
+            codebook_loss = F.mse_loss(z_q, z_e.detach())
+        z_q = (
+            z_e + (z_q - z_e).detach()
+        )  # noop in forward pass, straight-through gradient estimator in backward pass
+
+        return z_q, indices, z_e, commitment_loss, codebook_loss
+
+    def embed_code(self, embed_id):
+        return F.embedding(embed_id, self.codebook.weight)
+
+    def decode_code(self, embed_id):
+        return self.embed_code(embed_id).transpose(1, 2)
+
+    def decode_latents(self, latents):
+        encodings = rearrange(latents, "b d t -> (b t) d")
+        codebook = self.codebook.weight  # codebook: (N x D)
+
+        # L2 normalize encodings and codebook (ViT-VQGAN)
+        encodings = F.normalize(encodings)
+        codebook = F.normalize(codebook)
+
+        # Compute euclidean distance with codebook
+        dist = (
+            encodings.pow(2).sum(1, keepdim=True)
+            - 2 * encodings @ codebook.t()
+            + codebook.pow(2).sum(1, keepdim=True).t()
+        )
+        indices = rearrange((-dist).max(1)[1], "(b t) -> b t", b=latents.size(0))
+        z_q = self.decode_code(indices)
+        return z_q, indices
+
+class VectorQuantize(nn.Module):
+    """
+    Implementation of VQ similar to Karpathy's repo:
+    https://github.com/karpathy/deep-vector-quantization
+    Additionally uses following tricks from Improved VQGAN
+    (https://arxiv.org/pdf/2110.04627.pdf):
+        1. Factorized codes: Perform nearest neighbor lookup in low-dimensional space
+            for improved codebook usage
+        2. l2-normalized codes: Converts euclidean distance to cosine similarity which
+            improves training stability
+    """
+
+    def __init__(self, input_dim: int, codebook_size: int, codebook_dim: int):
+        super().__init__()
+        self.codebook_size = codebook_size
+        self.codebook_dim = codebook_dim
+
+        self.in_proj = WNConv1d(input_dim, codebook_dim, kernel_size=1)
+        self.out_proj = WNConv1d(codebook_dim, input_dim, kernel_size=1)
+        self.codebook = nn.Embedding(codebook_size, codebook_dim)
+
+    def forward(self, z, z_mask=None):
+        """Quantized the input tensor using a fixed codebook and returns
+        the corresponding codebook vectors
+
+        Parameters
+        ----------
+        z : Tensor[B x D x T]
+
+        Returns
+        -------
+        Tensor[B x D x T]
+            Quantized continuous representation of input
+        Tensor[1]
+            Commitment loss to train encoder to predict vectors closer to codebook
+            entries
+        Tensor[1]
+            Codebook loss to update the codebook
+        Tensor[B x T]
+            Codebook indices (quantized discrete representation of input)
+        Tensor[B x D x T]
+            Projected latents (continuous representation of input before quantization)
+        """
+
+        # Factorized codes (ViT-VQGAN) Project input into low-dimensional space
+        z_e = self.in_proj(z)  # z_e : (B x D x T)
+        z_q, indices = self.decode_latents(z_e)
+
+        if z_mask is not None:
+            commitment_loss = (F.mse_loss(z_e, z_q.detach(), reduction="none").mean(1) * z_mask).sum() / z_mask.sum()
+            codebook_loss = (F.mse_loss(z_q, z_e.detach(), reduction="none").mean(1) * z_mask).sum() / z_mask.sum()
+        else:
+            commitment_loss = F.mse_loss(z_e, z_q.detach())
+            codebook_loss = F.mse_loss(z_q, z_e.detach())
+
+        z_q = (
+            z_e + (z_q - z_e).detach()
+        )  # noop in forward pass, straight-through gradient estimator in backward pass
+
+        z_q = self.out_proj(z_q)
+
+        return z_q, commitment_loss, codebook_loss, indices, z_e
+
+    def embed_code(self, embed_id):
+        return F.embedding(embed_id, self.codebook.weight)
+
+    def decode_code(self, embed_id):
+        return self.embed_code(embed_id).transpose(1, 2)
+
+    def decode_latents(self, latents):
+        encodings = rearrange(latents, "b d t -> (b t) d")
+        codebook = self.codebook.weight  # codebook: (N x D)
+
+        # L2 normalize encodings and codebook (ViT-VQGAN)
+        encodings = F.normalize(encodings)
+        codebook = F.normalize(codebook)
+
+        # Compute euclidean distance with codebook
+        dist = (
+            encodings.pow(2).sum(1, keepdim=True)
+            - 2 * encodings @ codebook.t()
+            + codebook.pow(2).sum(1, keepdim=True).t()
+        )
+        indices = rearrange((-dist).max(1)[1], "(b t) -> b t", b=latents.size(0))
+        z_q = self.decode_code(indices)
+        return z_q, indices
+
+
+class ResidualVectorQuantize(nn.Module):
+    """
+    Introduced in SoundStream: An end2end neural audio codec
+    https://arxiv.org/abs/2107.03312
+    """
+
+    def __init__(
+        self,
+        input_dim: int = 512,
+        n_codebooks: int = 9,
+        codebook_size: int = 1024,
+        codebook_dim: Union[int, list] = 8,
+        quantizer_dropout: float = 0.0,
+    ):
+        super().__init__()
+        if isinstance(codebook_dim, int):
+            codebook_dim = [codebook_dim for _ in range(n_codebooks)]
+
+        self.n_codebooks = n_codebooks
+        self.codebook_dim = codebook_dim
+        self.codebook_size = codebook_size
+
+        self.quantizers = nn.ModuleList(
+            [
+                VectorQuantize(input_dim, codebook_size, codebook_dim[i])
+                for i in range(n_codebooks)
+            ]
+        )
+        self.quantizer_dropout = quantizer_dropout
+
+    def forward(self, z, n_quantizers: int = None):
+        """Quantized the input tensor using a fixed set of `n` codebooks and returns
+        the corresponding codebook vectors
+        Parameters
+        ----------
+        z : Tensor[B x D x T]
+        n_quantizers : int, optional
+            No. of quantizers to use
+            (n_quantizers < self.n_codebooks ex: for quantizer dropout)
+            Note: if `self.quantizer_dropout` is True, this argument is ignored
+                when in training mode, and a random number of quantizers is used.
+        Returns
+        -------
+        dict
+            A dictionary with the following keys:
+
+            "z" : Tensor[B x D x T]
+                Quantized continuous representation of input
+            "codes" : Tensor[B x N x T]
+                Codebook indices for each codebook
+                (quantized discrete representation of input)
+            "latents" : Tensor[B x N*D x T]
+                Projected latents (continuous representation of input before quantization)
+            "vq/commitment_loss" : Tensor[1]
+                Commitment loss to train encoder to predict vectors closer to codebook
+                entries
+            "vq/codebook_loss" : Tensor[1]
+                Codebook loss to update the codebook
+        """
+        z_q = 0
+        residual = z
+        commitment_loss = 0
+        codebook_loss = 0
+
+        codebook_indices = []
+        latents = []
+
+        if n_quantizers is None:
+            n_quantizers = self.n_codebooks
+        if self.training:
+            n_quantizers = torch.ones((z.shape[0],)) * self.n_codebooks + 1
+            dropout = torch.randint(1, self.n_codebooks + 1, (z.shape[0],))
+            n_dropout = int(z.shape[0] * self.quantizer_dropout)
+            n_quantizers[:n_dropout] = dropout[:n_dropout]
+            n_quantizers = n_quantizers.to(z.device)
+
+        for i, quantizer in enumerate(self.quantizers):
+            if self.training is False and i >= n_quantizers:
+                break
+
+            z_q_i, commitment_loss_i, codebook_loss_i, indices_i, z_e_i = quantizer(
+                residual
+            )
+
+            # Create mask to apply quantizer dropout
+            mask = (
+                torch.full((z.shape[0],), fill_value=i, device=z.device) < n_quantizers
+            )
+            z_q = z_q + z_q_i * mask[:, None, None]
+            residual = residual - z_q_i
+
+            # Sum losses
+            commitment_loss += (commitment_loss_i * mask).mean()
+            codebook_loss += (codebook_loss_i * mask).mean()
+
+            codebook_indices.append(indices_i)
+            latents.append(z_e_i)
+
+        codes = torch.stack(codebook_indices, dim=1)
+        latents = torch.cat(latents, dim=1)
+
+        return z_q, codes, latents, commitment_loss, codebook_loss
+
+    def from_codes(self, codes: torch.Tensor):
+        """Given the quantized codes, reconstruct the continuous representation
+        Parameters
+        ----------
+        codes : Tensor[B x N x T]
+            Quantized discrete representation of input
+        Returns
+        -------
+        Tensor[B x D x T]
+            Quantized continuous representation of input
+        """
+        z_q = 0.0
+        z_p = []
+        n_codebooks = codes.shape[1]
+        for i in range(n_codebooks):
+            z_p_i = self.quantizers[i].decode_code(codes[:, i, :])
+            z_p.append(z_p_i)
+
+            z_q_i = self.quantizers[i].out_proj(z_p_i)
+            z_q = z_q + z_q_i
+        return z_q, torch.cat(z_p, dim=1), codes
+
+    def from_latents(self, latents: torch.Tensor):
+        """Given the unquantized latents, reconstruct the
+        continuous representation after quantization.
+
+        Parameters
+        ----------
+        latents : Tensor[B x N x T]
+            Continuous representation of input after projection
+
+        Returns
+        -------
+        Tensor[B x D x T]
+            Quantized representation of full-projected space
+        Tensor[B x D x T]
+            Quantized representation of latent space
+        """
+        z_q = 0
+        z_p = []
+        codes = []
+        dims = np.cumsum([0] + [q.codebook_dim for q in self.quantizers])
+
+        n_codebooks = np.where(dims <= latents.shape[1])[0].max(axis=0, keepdims=True)[
+            0
+        ]
+        for i in range(n_codebooks):
+            j, k = dims[i], dims[i + 1]
+            z_p_i, codes_i = self.quantizers[i].decode_latents(latents[:, j:k, :])
+            z_p.append(z_p_i)
+            codes.append(codes_i)
+
+            z_q_i = self.quantizers[i].out_proj(z_p_i)
+            z_q = z_q + z_q_i
+
+        return z_q, torch.cat(z_p, dim=1), torch.stack(codes, dim=1)
+
+
+if __name__ == "__main__":
+    rvq = ResidualVectorQuantize(quantizer_dropout=True)
+    x = torch.randn(16, 512, 80)
+    y = rvq(x)
+    print(y["latents"].shape)

dac/utils/__init__.py

@@ -0,0 +1,123 @@
+from pathlib import Path
+
+import argbind
+from audiotools import ml
+
+import dac
+
+DAC = dac.model.DAC
+Accelerator = ml.Accelerator
+
+__MODEL_LATEST_TAGS__ = {
+    ("44khz", "8kbps"): "0.0.1",
+    ("24khz", "8kbps"): "0.0.4",
+    ("16khz", "8kbps"): "0.0.5",
+    ("44khz", "16kbps"): "1.0.0",
+}
+
+__MODEL_URLS__ = {
+    (
+        "44khz",
+        "0.0.1",
+        "8kbps",
+    ): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.1/weights.pth",
+    (
+        "24khz",
+        "0.0.4",
+        "8kbps",
+    ): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.4/weights_24khz.pth",
+    (
+        "16khz",
+        "0.0.5",
+        "8kbps",
+    ): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.5/weights_16khz.pth",
+    (
+        "44khz",
+        "1.0.0",
+        "16kbps",
+    ): "https://github.com/descriptinc/descript-audio-codec/releases/download/1.0.0/weights_44khz_16kbps.pth",
+}
+
+
+@argbind.bind(group="download", positional=True, without_prefix=True)
+def download(
+    model_type: str = "44khz", model_bitrate: str = "8kbps", tag: str = "latest"
+):
+    """
+    Function that downloads the weights file from URL if a local cache is not found.
+
+    Parameters
+    ----------
+    model_type : str
+        The type of model to download. Must be one of "44khz", "24khz", or "16khz". Defaults to "44khz".
+    model_bitrate: str
+        Bitrate of the model. Must be one of "8kbps", or "16kbps". Defaults to "8kbps".
+        Only 44khz model supports 16kbps.
+    tag : str
+        The tag of the model to download. Defaults to "latest".
+
+    Returns
+    -------
+    Path
+        Directory path required to load model via audiotools.
+    """
+    model_type = model_type.lower()
+    tag = tag.lower()
+
+    assert model_type in [
+        "44khz",
+        "24khz",
+        "16khz",
+    ], "model_type must be one of '44khz', '24khz', or '16khz'"
+
+    assert model_bitrate in [
+        "8kbps",
+        "16kbps",
+    ], "model_bitrate must be one of '8kbps', or '16kbps'"
+
+    if tag == "latest":
+        tag = __MODEL_LATEST_TAGS__[(model_type, model_bitrate)]
+
+    download_link = __MODEL_URLS__.get((model_type, tag, model_bitrate), None)
+
+    if download_link is None:
+        raise ValueError(
+            f"Could not find model with tag {tag} and model type {model_type}"
+        )
+
+    local_path = (
+        Path.home()
+        / ".cache"
+        / "descript"
+        / "dac"
+        / f"weights_{model_type}_{model_bitrate}_{tag}.pth"
+    )
+    if not local_path.exists():
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Download the model
+        import requests
+
+        response = requests.get(download_link)
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Could not download model. Received response code {response.status_code}"
+            )
+        local_path.write_bytes(response.content)
+
+    return local_path
+
+
+def load_model(
+    model_type: str = "44khz",
+    model_bitrate: str = "8kbps",
+    tag: str = "latest",
+    load_path: str = None,
+):
+    if not load_path:
+        load_path = download(
+            model_type=model_type, model_bitrate=model_bitrate, tag=tag
+        )
+    generator = DAC.load(load_path)
+    return generator

dac/utils/decode.py

@@ -0,0 +1,95 @@
+import warnings
+from pathlib import Path
+
+import argbind
+import numpy as np
+import torch
+from audiotools import AudioSignal
+from tqdm import tqdm
+
+from dac import DACFile
+from dac.utils import load_model
+
+warnings.filterwarnings("ignore", category=UserWarning)
+
+
+@argbind.bind(group="decode", positional=True, without_prefix=True)
+@torch.inference_mode()
+@torch.no_grad()
+def decode(
+    input: str,
+    output: str = "",
+    weights_path: str = "",
+    model_tag: str = "latest",
+    model_bitrate: str = "8kbps",
+    device: str = "cuda",
+    model_type: str = "44khz",
+    verbose: bool = False,
+):
+    """Decode audio from codes.
+
+    Parameters
+    ----------
+    input : str
+        Path to input directory or file
+    output : str, optional
+        Path to output directory, by default "".
+        If `input` is a directory, the directory sub-tree relative to `input` is re-created in `output`.
+    weights_path : str, optional
+        Path to weights file, by default "". If not specified, the weights file will be downloaded from the internet using the
+        model_tag and model_type.
+    model_tag : str, optional
+        Tag of the model to use, by default "latest". Ignored if `weights_path` is specified.
+    model_bitrate: str
+        Bitrate of the model. Must be one of "8kbps", or "16kbps". Defaults to "8kbps".
+    device : str, optional
+        Device to use, by default "cuda". Use "mps" on Apple Silicon devices or if "cpu", the model will be loaded on the CPU.
+    model_type : str, optional
+        The type of model to use. Must be one of "44khz", "24khz", or "16khz". Defaults to "44khz". Ignored if `weights_path` is specified.
+    """
+    generator = load_model(
+        model_type=model_type,
+        model_bitrate=model_bitrate,
+        tag=model_tag,
+        load_path=weights_path,
+    )
+    generator.to(device)
+    generator.eval()
+
+    # Find all .dac files in input directory
+    _input = Path(input)
+    input_files = list(_input.glob("**/*.dac"))
+
+    # If input is a .dac file, add it to the list
+    if _input.suffix == ".dac":
+        input_files.append(_input)
+
+    # Create output directory
+    output = Path(output)
+    output.mkdir(parents=True, exist_ok=True)
+
+    for i in tqdm(range(len(input_files)), desc=f"Decoding files"):
+        # Load file
+        artifact = DACFile.load(input_files[i])
+
+        # Reconstruct audio from codes
+        recons = generator.decompress(artifact, verbose=verbose)
+
+        # Compute output path
+        relative_path = input_files[i].relative_to(input)
+        output_dir = output / relative_path.parent
+        if not relative_path.name:
+            output_dir = output
+            relative_path = input_files[i]
+        output_name = relative_path.with_suffix(".wav").name
+        output_path = output_dir / output_name
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to file
+        recons.write(output_path)
+
+
+if __name__ == "__main__":
+    args = argbind.parse_args()
+    with argbind.scope(args):
+        decode()

dac/utils/encode.py

@@ -0,0 +1,94 @@
+import math
+import warnings
+from pathlib import Path
+
+import argbind
+import numpy as np
+import torch
+from audiotools import AudioSignal
+from audiotools.core import util
+from tqdm import tqdm
+
+from dac.utils import load_model
+
+warnings.filterwarnings("ignore", category=UserWarning)
+
+
+@argbind.bind(group="encode", positional=True, without_prefix=True)
+@torch.inference_mode()
+@torch.no_grad()
+def encode(
+    input: str,
+    output: str = "",
+    weights_path: str = "",
+    model_tag: str = "latest",
+    model_bitrate: str = "8kbps",
+    n_quantizers: int = None,
+    device: str = "cuda",
+    model_type: str = "44khz",
+    win_duration: float = 5.0,
+    verbose: bool = False,
+):
+    """Encode audio files in input path to .dac format.
+
+    Parameters
+    ----------
+    input : str
+        Path to input audio file or directory
+    output : str, optional
+        Path to output directory, by default "". If `input` is a directory, the directory sub-tree relative to `input` is re-created in `output`.
+    weights_path : str, optional
+        Path to weights file, by default "". If not specified, the weights file will be downloaded from the internet using the
+        model_tag and model_type.
+    model_tag : str, optional
+        Tag of the model to use, by default "latest". Ignored if `weights_path` is specified.
+    model_bitrate: str
+        Bitrate of the model. Must be one of "8kbps", or "16kbps". Defaults to "8kbps".
+    n_quantizers : int, optional
+        Number of quantizers to use, by default None. If not specified, all the quantizers will be used and the model will compress at maximum bitrate.
+    device : str, optional
+        Device to use, by default "cuda". Use "mps" on Apple Silicon devices.
+    model_type : str, optional
+        The type of model to use. Must be one of "44khz", "24khz", or "16khz". Defaults to "44khz". Ignored if `weights_path` is specified.
+    """
+    generator = load_model(
+        model_type=model_type,
+        model_bitrate=model_bitrate,
+        tag=model_tag,
+        load_path=weights_path,
+    )
+    generator.to(device)
+    generator.eval()
+    kwargs = {"n_quantizers": n_quantizers}
+
+    # Find all audio files in input path
+    input = Path(input)
+    audio_files = util.find_audio(input)
+
+    output = Path(output)
+    output.mkdir(parents=True, exist_ok=True)
+
+    for i in tqdm(range(len(audio_files)), desc="Encoding files"):
+        # Load file
+        signal = AudioSignal(audio_files[i])
+
+        # Encode audio to .dac format
+        artifact = generator.compress(signal, win_duration, verbose=verbose, **kwargs)
+
+        # Compute output path
+        relative_path = audio_files[i].relative_to(input)
+        output_dir = output / relative_path.parent
+        if not relative_path.name:
+            output_dir = output
+            relative_path = audio_files[i]
+        output_name = relative_path.with_suffix(".dac").name
+        output_path = output_dir / output_name
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        artifact.save(output_path)
+
+
+if __name__ == "__main__":
+    args = argbind.parse_args()
+    with argbind.scope(args):
+        encode()

data/ft_dataset.py

@@ -0,0 +1,126 @@
+import torch
+import librosa
+import numpy as np
+import random
+import os
+from torch.utils.data import DataLoader
+from modules.audio import mel_spectrogram
+
+
+duration_setting = {
+    "min": 1.0,
+    "max": 30.0,
+}
+# assume single speaker
+def to_mel_fn(wave, mel_fn_args):
+    return mel_spectrogram(wave, **mel_fn_args)
+
+class FT_Dataset(torch.utils.data.Dataset):
+    def __init__(
+        self,
+        data_path,
+        spect_params,
+        sr=22050,
+        batch_size=1,
+    ):
+        self.data_path = data_path
+        self.data = []
+        for root, _, files in os.walk(data_path):
+            for file in files:
+                if file.endswith((".wav", ".mp3", ".flac", ".ogg", ".m4a", ".opus")):
+                    self.data.append(os.path.join(root, file))
+
+        self.sr = sr
+        self.mel_fn_args = {
+            "n_fft": spect_params['n_fft'],
+            "win_size": spect_params['win_length'],
+            "hop_size": spect_params['hop_length'],
+            "num_mels": spect_params['n_mels'],
+            "sampling_rate": sr,
+            "fmin": spect_params['fmin'],
+            "fmax": None if spect_params['fmax'] == "None" else spect_params['fmax'],
+            "center": False
+        }
+
+        assert len(self.data) != 0
+        while len(self.data) < batch_size:
+            self.data += self.data
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        idx = idx % len(self.data)
+        wav_path = self.data[idx]
+        try:
+            speech, orig_sr = librosa.load(wav_path, sr=self.sr)
+        except Exception as e:
+            print(f"Failed to load wav file with error {e}")
+            return self.__getitem__(random.randint(0, len(self)))
+        if len(speech) < self.sr * duration_setting["min"] or len(speech) > self.sr * duration_setting["max"]:
+            print(f"Audio {wav_path} is too short or too long, skipping")
+            return self.__getitem__(random.randint(0, len(self)))
+        if orig_sr != self.sr:
+            speech = librosa.resample(speech, orig_sr, self.sr)
+
+        wave = torch.from_numpy(speech).float().unsqueeze(0)
+        mel = to_mel_fn(wave, self.mel_fn_args).squeeze(0)
+
+        return wave.squeeze(0), mel
+
+
+def build_ft_dataloader(data_path, spect_params, sr, batch_size=1, num_workers=0):
+    dataset = FT_Dataset(data_path, spect_params, sr, batch_size)
+    dataloader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=num_workers,
+        collate_fn=collate,
+    )
+    return dataloader
+
+def collate(batch):
+    batch_size = len(batch)
+
+    # sort by mel length
+    lengths = [b[1].shape[1] for b in batch]
+    batch_indexes = np.argsort(lengths)[::-1]
+    batch = [batch[bid] for bid in batch_indexes]
+
+    nmels = batch[0][1].size(0)
+    max_mel_length = max([b[1].shape[1] for b in batch])
+    max_wave_length = max([b[0].size(0) for b in batch])
+
+    mels = torch.zeros((batch_size, nmels, max_mel_length)).float() - 10
+    waves = torch.zeros((batch_size, max_wave_length)).float()
+
+    mel_lengths = torch.zeros(batch_size).long()
+    wave_lengths = torch.zeros(batch_size).long()
+
+    for bid, (wave, mel) in enumerate(batch):
+        mel_size = mel.size(1)
+        mels[bid, :, :mel_size] = mel
+        waves[bid, : wave.size(0)] = wave
+        mel_lengths[bid] = mel_size
+        wave_lengths[bid] = wave.size(0)
+
+    return waves, mels, wave_lengths, mel_lengths
+
+if __name__ == "__main__":
+    data_path = "./example/reference"
+    sr = 22050
+    spect_params = {
+        "n_fft": 1024,
+        "win_length": 1024,
+        "hop_length": 256,
+        "n_mels": 80,
+        "fmin": 0,
+        "fmax": 8000,
+    }
+    dataloader = build_ft_dataloader(data_path, spect_params, sr, batch_size=2, num_workers=0)
+    for idx, batch in enumerate(dataloader):
+        wave, mel, wave_lengths, mel_lengths = batch
+        print(wave.shape, mel.shape)
+        if idx == 10:
+            break

eval.py

@@ -0,0 +1,556 @@
+import shutil
+import warnings
+import argparse
+import torch
+import os
+import os.path as osp
+import yaml
+
+warnings.simplefilter("ignore")
+
+# load packages
+import random
+
+from tqdm import tqdm
+from modules.commons import *
+import time
+
+import torchaudio
+import librosa
+import torchaudio.compliance.kaldi as kaldi
+
+from hf_utils import load_custom_model_from_hf
+from resemblyzer import preprocess_wav, VoiceEncoder
+
+# Load model and configuration
+
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+elif torch.backends.mps.is_available():
+    device = torch.device("mps")
+else:
+    device = torch.device("cpu")
+
+from transformers import Wav2Vec2FeatureExtractor, WavLMForXVector
+from transformers import Wav2Vec2Processor, HubertForCTC
+
+import jiwer
+import string
+
+from baselines.dnsmos.dnsmos_computor import DNSMOSComputer
+
+def calc_mos(computor, audio, orin_sr):
+    # only 16k audio is supported
+    target_sr = 16000
+    if orin_sr != 16000:
+        audio = librosa.resample(
+            audio, orig_sr=orin_sr, target_sr=target_sr, res_type="kaiser_fast"
+        )
+    result = computor.compute(audio, target_sr, False)
+    sig, bak, ovr = result["SIG"], result["BAK"], result["OVRL"]
+
+    if ovr == 0:
+        print("calculate dns mos failed")
+    return sig, bak, ovr
+
+mos_computer = DNSMOSComputer(
+    "baselines/dnsmos/sig_bak_ovr.onnx",
+    "baselines/dnsmos/model_v8.onnx",
+    device="cuda",
+    device_id=0,
+)
+
+def load_models(args):
+    dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
+                                                                     "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
+                                                                     "config_dit_mel_seed_uvit_whisper_small_wavenet.yml")
+    config = yaml.safe_load(open(dit_config_path, "r"))
+    model_params = recursive_munch(config["model_params"])
+    model = build_model(model_params, stage="DiT")
+    hop_length = config["preprocess_params"]["spect_params"]["hop_length"]
+    sr = config["preprocess_params"]["sr"]
+
+    # Load checkpoints
+    model, _, _, _ = load_checkpoint(
+        model,
+        None,
+        dit_checkpoint_path,
+        load_only_params=True,
+        ignore_modules=[],
+        is_distributed=False,
+    )
+    for key in model:
+        model[key].eval()
+        model[key].to(device)
+    model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
+
+    # Load additional modules
+    from modules.campplus.DTDNN import CAMPPlus
+
+    campplus_ckpt_path = load_custom_model_from_hf(
+        "funasr/campplus", "campplus_cn_common.bin", config_filename=None
+    )
+    campplus_model = CAMPPlus(feat_dim=80, embedding_size=192)
+    campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu"))
+    campplus_model.eval()
+    campplus_model.to(device)
+
+    vocoder_type = model_params.vocoder.type
+
+    if vocoder_type == 'bigvgan':
+        from modules.bigvgan import bigvgan
+        bigvgan_name = model_params.vocoder.name
+        bigvgan_model = bigvgan.BigVGAN.from_pretrained(bigvgan_name, use_cuda_kernel=False)
+        # remove weight norm in the model and set to eval mode
+        bigvgan_model.remove_weight_norm()
+        bigvgan_model = bigvgan_model.eval().to(device)
+        vocoder_fn = bigvgan_model
+    elif vocoder_type == 'hifigan':
+        from modules.hifigan.generator import HiFTGenerator
+        from modules.hifigan.f0_predictor import ConvRNNF0Predictor
+        hift_config = yaml.safe_load(open('configs/hifigan.yml', 'r'))
+        hift_gen = HiFTGenerator(**hift_config['hift'], f0_predictor=ConvRNNF0Predictor(**hift_config['f0_predictor']))
+        hift_gen.load_state_dict(torch.load(hift_config['pretrained_model_path'], map_location='cpu'))
+        hift_gen.eval()
+        hift_gen.to(device)
+        vocoder_fn = hift_gen
+    elif vocoder_type == "vocos":
+        vocos_config = yaml.safe_load(open(model_params.vocoder.vocos.config, 'r'))
+        vocos_path = model_params.vocoder.vocos.path
+        vocos_model_params = recursive_munch(vocos_config['model_params'])
+        vocos = build_model(vocos_model_params, stage='mel_vocos')
+        vocos_checkpoint_path = vocos_path
+        vocos, _, _, _ = load_checkpoint(vocos, None, vocos_checkpoint_path,
+                                         load_only_params=True, ignore_modules=[], is_distributed=False)
+        _ = [vocos[key].eval().to(device) for key in vocos]
+        _ = [vocos[key].to(device) for key in vocos]
+        total_params = sum(sum(p.numel() for p in vocos[key].parameters() if p.requires_grad) for key in vocos.keys())
+        print(f"Vocoder model total parameters: {total_params / 1_000_000:.2f}M")
+        vocoder_fn = vocos.decoder
+    else:
+        raise ValueError(f"Unsupported vocoder type: {vocoder_type}")
+
+    speech_tokenizer_type = model_params.speech_tokenizer.type
+    if speech_tokenizer_type == 'whisper':
+        # whisper
+        from transformers import AutoFeatureExtractor, WhisperModel
+        whisper_name = model_params.speech_tokenizer.name
+        whisper_model = WhisperModel.from_pretrained(whisper_name, torch_dtype=torch.float16).to(device)
+        del whisper_model.decoder
+        whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name)
+
+        def semantic_fn(waves_16k):
+            ori_inputs = whisper_feature_extractor([waves_16k.squeeze(0).cpu().numpy()],
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True)
+            ori_input_features = whisper_model._mask_input_features(
+                ori_inputs.input_features, attention_mask=ori_inputs.attention_mask).to(device)
+            with torch.no_grad():
+                ori_outputs = whisper_model.encoder(
+                    ori_input_features.to(whisper_model.encoder.dtype),
+                    head_mask=None,
+                    output_attentions=False,
+                    output_hidden_states=False,
+                    return_dict=True,
+                )
+            S_ori = ori_outputs.last_hidden_state.to(torch.float32)
+            S_ori = S_ori[:, :waves_16k.size(-1) // 320 + 1]
+            return S_ori
+    elif speech_tokenizer_type == 'cnhubert':
+        from transformers import (
+            Wav2Vec2FeatureExtractor,
+            HubertModel,
+        )
+        hubert_model_name = config['model_params']['speech_tokenizer']['name']
+        hubert_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(hubert_model_name)
+        hubert_model = HubertModel.from_pretrained(hubert_model_name)
+        hubert_model = hubert_model.to(device)
+        hubert_model = hubert_model.eval()
+        hubert_model = hubert_model.half()
+
+        def semantic_fn(waves_16k):
+            ori_waves_16k_input_list = [
+                waves_16k[bib].cpu().numpy()
+                for bib in range(len(waves_16k))
+            ]
+            ori_inputs = hubert_feature_extractor(ori_waves_16k_input_list,
+                                                  return_tensors="pt",
+                                                  return_attention_mask=True,
+                                                  padding=True,
+                                                  sampling_rate=16000).to(device)
+            with torch.no_grad():
+                ori_outputs = hubert_model(
+                    ori_inputs.input_values.half(),
+                )
+            S_ori = ori_outputs.last_hidden_state.float()
+            return S_ori
+    elif speech_tokenizer_type == 'xlsr':
+        from transformers import (
+            Wav2Vec2FeatureExtractor,
+            Wav2Vec2Model,
+        )
+        model_name = config['model_params']['speech_tokenizer']['name']
+        output_layer = config['model_params']['speech_tokenizer']['output_layer']
+        wav2vec_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name)
+        wav2vec_model = Wav2Vec2Model.from_pretrained(model_name)
+        wav2vec_model.encoder.layers = wav2vec_model.encoder.layers[:output_layer]
+        wav2vec_model = wav2vec_model.to(device)
+        wav2vec_model = wav2vec_model.eval()
+        wav2vec_model = wav2vec_model.half()
+
+        def semantic_fn(waves_16k):
+            ori_waves_16k_input_list = [
+                waves_16k[bib].cpu().numpy()
+                for bib in range(len(waves_16k))
+            ]
+            ori_inputs = wav2vec_feature_extractor(ori_waves_16k_input_list,
+                                                   return_tensors="pt",
+                                                   return_attention_mask=True,
+                                                   padding=True,
+                                                   sampling_rate=16000).to(device)
+            with torch.no_grad():
+                ori_outputs = wav2vec_model(
+                    ori_inputs.input_values.half(),
+                )
+            S_ori = ori_outputs.last_hidden_state.float()
+            return S_ori
+    else:
+        raise ValueError(f"Unsupported speech tokenizer type: {model_params.speech_tokenizer.type}")
+    # Generate mel spectrograms
+    mel_fn_args = {
+        "n_fft": config['preprocess_params']['spect_params']['n_fft'],
+        "win_size": config['preprocess_params']['spect_params']['win_length'],
+        "hop_size": config['preprocess_params']['spect_params']['hop_length'],
+        "num_mels": config['preprocess_params']['spect_params']['n_mels'],
+        "sampling_rate": sr,
+        "fmin": config['preprocess_params'].get('fmin', 0),
+        "fmax": None if config['preprocess_params']['spect_params'].get('fmax', "None") == "None" else 8000,
+        "center": False
+    }
+    from modules.audio import mel_spectrogram
+
+    to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
+
+    return (
+        model,
+        semantic_fn,
+        vocoder_fn,
+        campplus_model,
+        to_mel,
+        mel_fn_args,
+    )
+
+
+@torch.no_grad()
+def main(args):
+    # init xvector models
+    if args.xvector_extractor == "wavlm":
+        wavlm_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
+            "microsoft/wavlm-base-plus-sv"
+        )
+        wavlm_model = WavLMForXVector.from_pretrained(
+            "microsoft/wavlm-base-plus-sv"
+        ).to(device)
+    elif args.xvector_extractor == "resemblyzer":
+        resemblyzer_encoder = VoiceEncoder()
+    elif args.xvector_extractor == 'wavlm-large':
+        import sys
+        sys.path.append("../UniSpeech/downstreams/speaker_verification")
+        from verification import init_model
+        wavlm_model = init_model("wavlm_large", "D:/wavlm_large_finetune.pth")
+        wavlm_model.cuda()
+        wavlm_model.eval()
+    else:
+        raise ValueError(f"Unknown xvector extractor: {args.xvector_extractor}")
+
+    # init asr model
+    asr_processor = Wav2Vec2Processor.from_pretrained("facebook/hubert-large-ls960-ft")
+    asr_model = HubertForCTC.from_pretrained("facebook/hubert-large-ls960-ft").to(device)
+
+    (
+        model,
+        semantic_fn,
+        vocoder_fn,
+        campplus_model,
+        to_mel,
+        mel_fn_args,
+    ) = load_models(args)
+    sr = mel_fn_args["sampling_rate"]
+
+    source_dir = args.source
+    target_dir = args.target
+    diffusion_steps = args.diffusion_steps
+    length_adjust = args.length_adjust
+    inference_cfg_rate = args.inference_cfg_rate
+    baseline = args.baseline
+    max_samples = args.max_samples
+    try:
+        source_audio_list = open(osp.join(source_dir, "index.tsv"), "r").readlines()
+    except FileNotFoundError:
+        source_audio_list = os.listdir(source_dir)
+        source_audio_list = [f for f in source_audio_list if f.endswith(".wav")]
+    target_audio_list = os.listdir(target_dir)
+
+    conversion_result_dir = args.output
+    if baseline:
+        conversion_result_dir = os.path.join(conversion_result_dir, baseline)
+    os.makedirs(conversion_result_dir, exist_ok=True)
+
+    similarity_list = []
+    gt_wer_list = []
+    gt_cer_list = []
+    vc_wer_list = []
+    vc_cer_list = []
+    dnsmos_list = []
+    for source_i, source_line in enumerate(tqdm(source_audio_list)):
+        if source_i >= max_samples:
+            break
+        source_index, source_transcript = source_line.strip().split("\t")
+        source_path = osp.join(source_dir, f"{source_index}.wav")
+        for target_i, target_name in enumerate(target_audio_list):
+            target_path = osp.join(target_dir, target_name)
+            print(f"Processing {source_path} -> {target_path}")
+
+            if os.path.exists(osp.join(conversion_result_dir, source_index, f"{target_name}")):
+                # already converted, load the converted file
+                vc_wave_16k, _ = librosa.load(
+                    osp.join(conversion_result_dir, source_index, f"{target_name}"), sr=16000
+                )
+                vc_wave_16k = torch.tensor(vc_wave_16k).unsqueeze(0)
+                ref_waves_16k, _ = librosa.load(target_path, sr=16000)
+                ref_waves_16k = torch.tensor(ref_waves_16k).unsqueeze(0)
+            else:
+                if baseline == "openvoice":
+                    from baselines.openvoice import convert as openvoice_convert
+                    ref_waves_16k, vc_wave_16k = openvoice_convert(source_path, target_path, "temp.wav")
+                elif baseline == "cosyvoice":
+                    from baselines.cosyvoice import convert as cosyvoice_convert
+                    ref_waves_16k, vc_wave_16k = cosyvoice_convert(source_path, target_path, "temp.wav")
+                else:
+                    ref_waves_16k, vc_wave = convert(
+                        source_path,
+                        target_path,
+                        model,
+                        semantic_fn,
+                        vocoder_fn,
+                        campplus_model,
+                        to_mel,
+                        mel_fn_args,
+                        sr,
+                        length_adjust,
+                        diffusion_steps,
+                        inference_cfg_rate,
+                        remove_prompt=args.remove_prompt,
+                    )
+                    vc_wave_16k = torchaudio.functional.resample(vc_wave, sr, 16000)
+                os.makedirs(osp.join(conversion_result_dir, source_index), exist_ok=True)
+                torchaudio.save(
+                    osp.join(conversion_result_dir, source_index, f"{target_name}"),
+                    vc_wave_16k.cpu(),
+                    16000,
+                )
+            if args.xvector_extractor == "wavlm":
+                ref_inputs = wavlm_feature_extractor(
+                    ref_waves_16k.squeeze(0).cpu(), padding=True, return_tensors="pt"
+                ).to(device)
+                ref_embeddings = wavlm_model(**ref_inputs).embeddings
+                ref_embeddings = torch.nn.functional.normalize(ref_embeddings, dim=-1).cpu()
+
+                vc_inputs = wavlm_feature_extractor(
+                    vc_wave_16k.squeeze(0).cpu(), padding=True, return_tensors="pt"
+                ).to(device)
+                vc_embeddings = wavlm_model(**vc_inputs).embeddings
+                vc_embeddings = torch.nn.functional.normalize(vc_embeddings, dim=-1).cpu()
+
+                similarity = torch.nn.functional.cosine_similarity(
+                    ref_embeddings, vc_embeddings, dim=-1
+                )
+            elif args.xvector_extractor == "resemblyzer":
+                ref_wav_resemblyzer = preprocess_wav(target_path)
+                vc_wav_resemblyzer = preprocess_wav(
+                    osp.join(conversion_result_dir, source_index, f"{target_name}")
+                )
+                ref_embed = resemblyzer_encoder.embed_utterance(ref_wav_resemblyzer)
+                vc_embed = resemblyzer_encoder.embed_utterance(vc_wav_resemblyzer)
+                similarity = np.inner(ref_embed, vc_embed)
+            elif args.xvector_extractor == 'wavlm-large':
+                ref_embed = wavlm_model(ref_waves_16k.to(device)).cpu()
+                vc_embed = wavlm_model(vc_wave_16k.to(device)).cpu()
+                similarity = torch.nn.functional.cosine_similarity(ref_embed, vc_embed, dim=-1)
+            else:
+                raise ValueError(f"Unknown xvector extractor: {args.xvector_extractor}")
+            print(f"Similarity: {similarity}")
+            similarity_list.append(similarity)
+
+            # perform asr
+            vc_asr_inputs = asr_processor(
+                vc_wave_16k.squeeze(0).cpu(), return_tensors="pt", padding=True
+            ).to(device)
+            vc_asr_logits = asr_model(**vc_asr_inputs).logits
+            predicted_ids = torch.argmax(vc_asr_logits, dim=-1)
+            vc_transcription = asr_processor.decode(predicted_ids[0])
+
+            # perform asr on source 16k
+            source_wav_16k = librosa.load(source_path, sr=16000)[0]
+            source_asr_inputs = asr_processor(
+                source_wav_16k, return_tensors="pt", padding=True
+            ).to(device)
+            source_asr_logits = asr_model(**source_asr_inputs).logits
+            source_predicted_ids = torch.argmax(source_asr_logits, dim=-1)
+            source_transcription = asr_processor.decode(source_predicted_ids[0])
+
+            # convert transcriptions to all lower to calculate WER and CER
+            source_transcript = source_transcript.lower()
+            # remove punctuations in source_transcript
+            source_transcript = source_transcript.translate(str.maketrans("", "", string.punctuation))
+            source_transcription = source_transcription.lower()
+            vc_transcription = vc_transcription.lower()
+
+            # calculate WER and CER
+            gt_wer = jiwer.wer(source_transcript, source_transcription)
+            gt_cer = jiwer.cer(source_transcript, source_transcription)
+            vc_wer = jiwer.wer(source_transcript, vc_transcription)
+            vc_cer = jiwer.cer(source_transcript, vc_transcription)
+
+            print(f"GT WER: {gt_wer}, CER: {gt_cer}")
+            print(f"VC WER: {vc_wer}, CER: {vc_cer}")
+            gt_wer_list.append(gt_wer)
+            gt_cer_list.append(gt_cer)
+            vc_wer_list.append(vc_wer)
+            vc_cer_list.append(vc_cer)
+
+            # calculate dnsmos
+            sig, bak, ovr = calc_mos(mos_computer, vc_wave_16k.squeeze(0).cpu().numpy(), 16000)
+            dnsmos_list.append((sig, bak, ovr))
+
+        print(f"Average GT WER: {sum(gt_wer_list) / len(gt_wer_list)}")
+        print(f"Average GT CER: {sum(gt_cer_list) / len(gt_cer_list)}")
+        print(f"Average VC WER: {sum(vc_wer_list) / len(vc_wer_list)}")
+        print(f"Average VC CER: {sum(vc_cer_list) / len(vc_cer_list)}")
+        print(f"Average similarity: {sum(similarity_list) / len(similarity_list)}")
+
+        print(f"Average DNS MOS SIG: {sum([x[0] for x in dnsmos_list]) / len(dnsmos_list)}")
+        print(f"Average DNS MOS BAK: {sum([x[1] for x in dnsmos_list]) / len(dnsmos_list)}")
+        print(f"Average DNS MOS OVR: {sum([x[2] for x in dnsmos_list]) / len(dnsmos_list)}")
+
+        # save wer and cer result into this directory as a txt
+        with open(osp.join(conversion_result_dir, source_index, "result.txt"), 'w') as f:
+            f.write(f"GT WER: {sum(gt_wer_list[-len(target_audio_list):]) / len(target_audio_list)}\n")
+            f.write(f"GT CER: {sum(gt_cer_list[-len(target_audio_list):]) / len(target_audio_list)}\n")
+            f.write(f"VC WER: {sum(vc_wer_list[-len(target_audio_list):]) / len(target_audio_list)}\n")
+            f.write(f"VC CER: {sum(vc_cer_list[-len(target_audio_list):]) / len(target_audio_list)}\n")
+            f.write(f"Average similarity: {sum(similarity_list[-len(target_audio_list):]) / len(target_audio_list)}\n")
+
+    print(f"Average WER: {sum(gt_wer_list) / len(gt_wer_list)}")
+    print(f"Average CER: {sum(gt_cer_list) / len(gt_cer_list)}")
+    print(f"Average WER: {sum(vc_wer_list) / len(vc_wer_list)}")
+    print(f"Average CER: {sum(vc_cer_list) / len(vc_cer_list)}")
+    print(f"Average similarity: {sum(similarity_list) / len(similarity_list)}")
+    # save similarity list
+    with open(osp.join(conversion_result_dir, f"{args.xvector_extractor}_similarity.tsv"), "w") as f:
+        f.write("\n".join([str(s) for s in similarity_list]))
+    # save wer and cer result into this directory as a txt
+    with open(osp.join(conversion_result_dir, "result.txt"), 'w') as f:
+        f.write(f"GT WER: {sum(gt_wer_list) / len(gt_wer_list)}\n")
+        f.write(f"GT CER: {sum(gt_cer_list) / len(gt_cer_list)}\n")
+        f.write(f"VC WER: {sum(vc_wer_list) / len(vc_wer_list)}\n")
+        f.write(f"VC CER: {sum(vc_cer_list) / len(vc_cer_list)}\n")
+
+    print(f"Average DNS MOS SIG: {sum([x[0] for x in dnsmos_list]) / len(dnsmos_list)}")
+    print(f"Average DNS MOS BAK: {sum([x[1] for x in dnsmos_list]) / len(dnsmos_list)}")
+    print(f"Average DNS MOS OVR: {sum([x[2] for x in dnsmos_list]) / len(dnsmos_list)}")
+
+
+def convert(
+    source_path,
+    target_path,
+    model,
+    semantic_fn,
+    vocoder_fn,
+    campplus_model,
+    to_mel,
+    mel_fn_args,
+    sr,
+    length_adjust,
+    diffusion_steps,
+    inference_cfg_rate,
+    remove_prompt=False,
+):
+    source_audio = librosa.load(source_path, sr=sr)[0]
+    ref_audio = librosa.load(target_path, sr=sr)[0]
+    # decoded_wav = encodec_model.decoder(encodec_latent)
+    # torchaudio.save("test.wav", decoded_wav.cpu().squeeze(0), 24000)
+    # crop only the first 30 seconds
+    source_audio = torch.tensor(source_audio).unsqueeze(0).float().to(device)
+    ref_audio = torch.tensor(ref_audio).unsqueeze(0).float().to(device)
+
+    if source_audio.size(1) + ref_audio.size(1) > 30 * sr:
+        print(f"reference audio clipped from {ref_audio.size(1)/sr} seconds to {30 * sr - source_audio.size(1)} seconds")
+        ref_audio = ref_audio[:, :30 * sr - source_audio.size(1)]
+
+
+    source_waves_16k = torchaudio.functional.resample(source_audio, sr, 16000)
+    ref_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
+
+    S_alt = semantic_fn(source_waves_16k)
+    S_ori = semantic_fn(ref_waves_16k)
+
+    mel = to_mel(source_audio.to(device).float())
+    mel2 = to_mel(ref_audio.to(device).float())
+
+    target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to(mel.device)
+    target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device)
+
+    feat2 = torchaudio.compliance.kaldi.fbank(
+        ref_waves_16k, num_mel_bins=80, dither=0, sample_frequency=16000
+    )
+    feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
+    style2 = campplus_model(feat2.unsqueeze(0))
+    # Length regulation
+    cond = model.length_regulator(
+        S_alt, ylens=target_lengths, n_quantizers=3, f0=None
+    )[0]
+    prompt_condition = model.length_regulator(
+        S_ori, ylens=target2_lengths, n_quantizers=3, f0=None
+    )[0]
+    if remove_prompt:
+        cat_condition = cond
+        mel2 = torch.zeros([mel2.size(0), mel2.size(1), 0]).to(mel2.device)
+    else:
+        cat_condition = torch.cat([prompt_condition, cond], dim=1)
+
+    vc_target = model.cfm.inference(
+        cat_condition,
+        torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
+        mel2,
+        style2,
+        None,
+        diffusion_steps,
+        inference_cfg_rate=inference_cfg_rate,
+    )
+    vc_target = vc_target[:, :, mel2.size(-1) :]
+
+    # Convert to waveform
+    vc_wave = vocoder_fn(vc_target).squeeze(1)
+
+    return ref_waves_16k, vc_wave
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--source", type=str, default="./examples/libritts-test-clean/"
+    )
+    parser.add_argument("--target", type=str, default="./examples/reference/")
+    parser.add_argument("--output", type=str, default="./examples/eval/converted/")
+    parser.add_argument("--diffusion-steps", type=int, default=30)
+    parser.add_argument("--length-adjust", type=float, default=1.0)
+    parser.add_argument("--inference-cfg-rate", type=float, default=0.7)
+    parser.add_argument(
+        "--xvector-extractor", type=str, default="wavlm-large"
+    )  # wavlm or resemblyzer
+    parser.add_argument("--baseline", type=str, default="") # use "" for Seed-VC
+    parser.add_argument("--max-samples", type=int, default=20)
+    parser.add_argument("--remove-prompt", type=bool, default=False)
+    args = parser.parse_args()
+    main(args)

examples/reference/azuma_0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3930141e927be50e7f3d666db5890ef9a4bda0623645483a6afaad241c82fb70
+size 628910

examples/reference/dingzhen_0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3db260824d11f56cdf2fccf2b84ad83c95a732ddfa2f8cb8a20b68ca06ea9ff8
+size 1088420

examples/reference/s1p1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be291f04e9c239218082552e9e7c0dba9bad5ce6306d2c9d2104195840214b5a
+size 700714

examples/reference/s1p2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07e937b52271380eeb52415428dc3fab6d530f767bf34b5e5cb52337ad294b17
+size 526116

examples/reference/s2p1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:308a38a5ac0a133a7d4742a95494d9670042fd9a233e60448f8370a9eba98a98
+size 664838

examples/reference/s2p2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6f73146fcfd25044317b377b0f2e8ae03ab43bd43ae8713f37889357babbd5e
+size 564012

examples/reference/s3p1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cc05439406e44eeaa7be15742aa71e2ca8b9c43adbcd496ea1672d9283aa9b0
+size 557386

examples/reference/s3p2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d28df338203ad8b3c7485474fac41d9ee2891cf27bc0c0239e3249e6c0efadb
+size 1140390

examples/reference/s4p1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53892ece071342958403bc5643f84169a30b89cc0fc79eb69508bfa11dd85e68
+size 618528

examples/reference/s4p2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2c71e9e60b1eb772c9bc50ceea66566c9797a58a265289486715afe37bea813
+size 651098

examples/reference/teio_0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97af2b2fa7a950243f800bd5920d58841a1f602f7ccd23fc6479369956168842
+size 366402

examples/reference/trump_0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:716becc9daf00351dfe324398edea9e8378f9453408b27612d92b6721f80ddbc
+size 1379484