update

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,14 @@
+
+.git/
+.idea/
+
+/data/
+/dotenv/
+/logs/
+/pretrained_models
+/trained_models
+/temp/
+
+**/__pycache__/
+
+**/*.wav

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim
+
+WORKDIR /code
+
+COPY . /code
+
+RUN apt-get update
+RUN apt-get install -y ffmpeg build-essential
+
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+RUN useradd -m -u 1000 user
+
+USER user
+
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["python3", "main.py"]

examples/wav2vec2/test_audeering.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://arxiv.org/abs/2306.16962
+
+https://huggingface.co/audeering/wav2vec2-large-robust-24-ft-age-gender
+"""
+import argparse
+
+import torch
+import torch.nn as nn
+import librosa
+from transformers import Wav2Vec2Processor
+from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model, Wav2Vec2PreTrainedModel
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_path",
+        # default=(project_path / "pretrained_models/wav2vec2-large-robust-6-ft-age-gender").as_posix(),
+        default=(project_path / "pretrained_models/wav2vec2-large-robust-6-ft-age-gender").as_posix(),
+        type=str,
+    )
+    parser.add_argument(
+        "--speech_file",
+        # default=(project_path / "data/examples/voicemail-female-1.wav").as_posix(),
+        # default=(project_path / "data/examples/voicemail-female-2.wav").as_posix(),
+        # default=(project_path / "data/examples/voicemail-female-3.wav").as_posix(),
+        # default=(project_path / "data/examples/voicemail-male-1.wav").as_posix(),
+        # default=(project_path / "data/examples/voicemail-male-2.wav").as_posix(),
+        default=(project_path / "data/examples/speech-male-1.wav").as_posix(),
+        type=str,
+    )
+    args = parser.parse_args()
+    return args
+
+
+class ModelHead(nn.Module):
+    def __init__(self, config, num_labels):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.final_dropout)
+        self.out_proj = nn.Linear(config.hidden_size, num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+        return x
+
+
+class AgeGenderModel(Wav2Vec2PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.age = ModelHead(config, 1)
+        self.gender = ModelHead(config, 3)
+        self.init_weights()
+
+    def forward(self,
+                input_values,
+                ):
+        outputs = self.wav2vec2(input_values)
+        hidden_states = outputs[0]
+        hidden_states = torch.mean(hidden_states, dim=1)
+
+        logits_age = self.age.forward(hidden_states)
+        logits_gender = torch.softmax(self.gender.forward(hidden_states), dim=1)
+
+        return hidden_states, logits_age, logits_gender
+
+
+def main():
+    args = get_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    processor: Wav2Vec2Processor = Wav2Vec2Processor.from_pretrained(args.model_path)
+    model = AgeGenderModel.from_pretrained(args.model_path)
+    model.eval()
+
+    # signal
+    signal, sample_rate = librosa.load(args.speech_file, sr=16000)
+
+    y = processor.__call__(signal, sampling_rate=sample_rate)
+    y = y['input_values'][0]
+    y = y.reshape(1, -1)
+    y = torch.from_numpy(y).to(device)
+
+    _, age, gender = model.forward(y)
+    print(f"age: {age}")
+    # female     male       child
+    print(f"gender: {gender}")
+
+    return
+
+
+if __name__ == '__main__':
+    main()

log.py

@@ -0,0 +1,229 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import logging
+from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
+import os
+
+
+def setup_size_rotating(log_directory: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    # main
+    main_logger = logging.getLogger("main")
+    main_logger.addHandler(stream_handler)
+    main_info_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "main.log"),
+        maxBytes=100*1024*1024,  # 100MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    main_info_file_handler.setLevel(logging.INFO)
+    main_info_file_handler.setFormatter(logging.Formatter(fmt))
+    main_logger.addHandler(main_info_file_handler)
+
+    # http
+    http_logger = logging.getLogger("http")
+    http_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "http.log"),
+        maxBytes=100*1024*1024,  # 100MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    http_file_handler.setLevel(logging.DEBUG)
+    http_file_handler.setFormatter(logging.Formatter(fmt))
+    http_logger.addHandler(http_file_handler)
+
+    # api
+    api_logger = logging.getLogger("api")
+    api_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "api.log"),
+        maxBytes=10*1024*1024,  # 10MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    api_file_handler.setLevel(logging.DEBUG)
+    api_file_handler.setFormatter(logging.Formatter(fmt))
+    api_logger.addHandler(api_file_handler)
+
+    # toolbox
+    toolbox_logger = logging.getLogger("toolbox")
+    toolbox_logger.addHandler(stream_handler)
+    toolbox_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "toolbox.log"),
+        maxBytes=10*1024*1024,  # 10MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    toolbox_file_handler.setLevel(logging.DEBUG)
+    toolbox_file_handler.setFormatter(logging.Formatter(fmt))
+    toolbox_logger.addHandler(toolbox_file_handler)
+
+    # alarm
+    alarm_logger = logging.getLogger("alarm")
+    alarm_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "alarm.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    alarm_file_handler.setLevel(logging.DEBUG)
+    alarm_file_handler.setFormatter(logging.Formatter(fmt))
+    alarm_logger.addHandler(alarm_file_handler)
+
+    debug_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "debug.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    debug_file_handler.setLevel(logging.DEBUG)
+    debug_file_handler.setFormatter(logging.Formatter(fmt))
+
+    info_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "info.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    info_file_handler.setLevel(logging.INFO)
+    info_file_handler.setFormatter(logging.Formatter(fmt))
+
+    error_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "error.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    error_file_handler.setLevel(logging.ERROR)
+    error_file_handler.setFormatter(logging.Formatter(fmt))
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        datefmt="%a, %d %b %Y %H:%M:%S",
+        handlers=[
+            debug_file_handler,
+            info_file_handler,
+            error_file_handler,
+        ]
+    )
+
+
+def setup_time_rotating(log_directory: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    # main
+    main_logger = logging.getLogger("main")
+    main_logger.addHandler(stream_handler)
+    main_info_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "main.log"),
+        encoding="utf-8",
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    main_info_file_handler.setLevel(logging.INFO)
+    main_info_file_handler.setFormatter(logging.Formatter(fmt))
+    main_logger.addHandler(main_info_file_handler)
+
+    # http
+    http_logger = logging.getLogger("http")
+    http_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "http.log"),
+        encoding='utf-8',
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    http_file_handler.setLevel(logging.DEBUG)
+    http_file_handler.setFormatter(logging.Formatter(fmt))
+    http_logger.addHandler(http_file_handler)
+
+    # api
+    api_logger = logging.getLogger("api")
+    api_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "api.log"),
+        encoding='utf-8',
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    api_file_handler.setLevel(logging.DEBUG)
+    api_file_handler.setFormatter(logging.Formatter(fmt))
+    api_logger.addHandler(api_file_handler)
+
+    # toolbox
+    toolbox_logger = logging.getLogger("toolbox")
+    toolbox_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "toolbox.log"),
+        maxBytes=10*1024*1024,  # 10MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    toolbox_file_handler.setLevel(logging.DEBUG)
+    toolbox_file_handler.setFormatter(logging.Formatter(fmt))
+    toolbox_logger.addHandler(toolbox_file_handler)
+
+    # alarm
+    alarm_logger = logging.getLogger("alarm")
+    alarm_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "alarm.log"),
+        encoding="utf-8",
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    alarm_file_handler.setLevel(logging.DEBUG)
+    alarm_file_handler.setFormatter(logging.Formatter(fmt))
+    alarm_logger.addHandler(alarm_file_handler)
+
+    debug_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "debug.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    debug_file_handler.setLevel(logging.DEBUG)
+    debug_file_handler.setFormatter(logging.Formatter(fmt))
+
+    info_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "info.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    info_file_handler.setLevel(logging.INFO)
+    info_file_handler.setFormatter(logging.Formatter(fmt))
+
+    error_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "error.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    error_file_handler.setLevel(logging.ERROR)
+    error_file_handler.setFormatter(logging.Formatter(fmt))
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        datefmt="%a, %d %b %Y %H:%M:%S",
+        handlers=[
+            debug_file_handler,
+            info_file_handler,
+            error_file_handler,
+        ]
+    )
+
+
+if __name__ == "__main__":
+    pass

main.py

@@ -0,0 +1,183 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from functools import lru_cache
+import json
+import logging
+from pathlib import Path
+import platform
+import tempfile
+import time
+import uuid
+
+import gradio as gr
+import librosa
+import numpy as np
+from scipy.io import wavfile
+
+import log
+from project_settings import environment, project_path, log_directory
+from toolbox.os.command import Command
+from toolbox.age_and_gender.models.audeering import AudeeringModel
+
+log.setup_size_rotating(log_directory=log_directory)
+
+logger = logging.getLogger("main")
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--examples_dir",
+        default=(project_path / "data/examples").as_posix(),
+        type=str,
+    )
+    args = parser.parse_args()
+    return args
+
+
+def save_input_audio(sample_rate: int, signal: np.ndarray) -> str:
+    temp_audio_dir = Path(tempfile.gettempdir()) / "input_audio"
+    temp_audio_dir.mkdir(parents=True, exist_ok=True)
+    filename = temp_audio_dir / f"{uuid.uuid4()}.wav"
+    filename = filename.as_posix()
+    wavfile.write(
+        filename,
+        sample_rate, signal
+    )
+    return filename
+
+
+def shell(cmd: str):
+    return Command.popen(cmd)
+
+
+age_and_gender_model_map = {
+    "audeering-6-ft":{
+        "infer_cls": AudeeringModel,
+        "kwargs": {
+            "model_path":
+                (project_path / "pretrained_models/wav2vec2-large-robust-6-ft-age-gender").as_posix()
+                if platform.system() == "Windows" else "audeering/wav2vec2-large-robust-6-ft-age-gender"
+        },
+        "sample_rate": 16000,
+    },
+    "audeering-24-ft": {
+        "infer_cls": AudeeringModel,
+        "kwargs": {
+            "model_path":
+                (project_path / "pretrained_models/wav2vec2-large-robust-24-ft-age-gender").as_posix()
+                if platform.system() == "Windows" else "audeering/wav2vec2-large-robust-24-ft-age-gender",
+        },
+        "sample_rate": 16000,
+    },
+}
+
+
+@lru_cache(maxsize=3)
+def load_get_age_and_gender_model(infer_cls, **kwargs):
+    infer_engine = infer_cls(**kwargs)
+
+    return infer_engine
+
+
+def when_click_get_age_and_gender_button(audio_t, engine: str):
+    sample_rate, signal = audio_t
+    filename = save_input_audio(sample_rate, signal)
+
+    logger.info(f"run get_age_and_gender; engine: {engine}.")
+
+    infer_engine_param = age_and_gender_model_map.get(engine)
+    if infer_engine_param is None:
+        raise gr.Error(f"invalid denoise engine: {engine}.")
+
+    try:
+        infer_cls = infer_engine_param["infer_cls"]
+        kwargs = infer_engine_param["kwargs"]
+        sample_rate = infer_engine_param["sample_rate"]
+
+        signal, _ = librosa.load(filename, sr=sample_rate)
+        duration = len(signal) / sample_rate
+
+        infer_engine = load_get_age_and_gender_model(infer_cls=infer_cls, **kwargs)
+
+
+        time_begin = time.time()
+        age_and_gender = infer_engine.__call__(signal, sample_rate)
+        time_cost = time.time() - time_begin
+
+        rtf = time_cost / duration
+
+        result = {
+            **age_and_gender,
+            "duration": round(duration, 4),
+            "time_cost": round(time_cost, 4),
+            "rtf": round(rtf, 4),
+        }
+
+        result = json.dumps(result, ensure_ascii=False, indent=4)
+    except Exception as e:
+        raise gr.Error(f"get_age_and_gender failed, error type: {type(e)}, error text: {str(e)}.")
+
+    return result
+
+
+def main():
+    args = get_args()
+
+    # examples
+    examples_dir = Path(args.examples_dir)
+
+    # choices
+    age_and_gender_model_choices = list(age_and_gender_model_map.keys())
+
+    # ui
+    with gr.Blocks() as blocks:
+        with gr.Tabs():
+            with gr.TabItem("age_and_gender"):
+                with gr.Row():
+                    with gr.Column(variant="panel", scale=5):
+                        ag_audio = gr.Audio(label="audio")
+                        ag_engine = gr.Dropdown(choices=age_and_gender_model_choices, value=age_and_gender_model_choices[0], label="engine")
+                        ag_button = gr.Button(variant="primary")
+                    with gr.Column(variant="panel", scale=5):
+                        ag_output = gr.Text(label="output")
+
+                gr.Examples(
+                    examples=[
+                        [filename.as_posix(), age_and_gender_model_choices[0]]
+                        for filename in examples_dir.glob("*.wav")
+                    ],
+                    inputs=[ag_audio, ag_engine],
+                    outputs=[ag_output],
+                    fn=when_click_get_age_and_gender_button,
+                )
+                ag_button.click(
+                    when_click_get_age_and_gender_button,
+                    inputs=[ag_audio, ag_engine],
+                    outputs=[ag_output],
+                )
+            with gr.TabItem("shell"):
+                shell_text = gr.Textbox(label="cmd")
+                shell_button = gr.Button("run")
+                shell_output = gr.Textbox(label="output")
+
+                shell_button.click(
+                    shell,
+                    inputs=[shell_text,],
+                    outputs=[shell_output],
+                )
+
+    # http://127.0.0.1:7860/
+    # http://10.75.27.247:7861/
+    blocks.queue().launch(
+        share=False if platform.system() == "Windows" else False,
+        # server_name="127.0.0.1" if platform.system() == "Windows" else "0.0.0.0",
+        server_name="0.0.0.0",
+        server_port=environment.get("port", 7860, dtype=int),
+    )
+    return
+
+
+if __name__ == "__main__":
+    main()

project_settings.py

@@ -0,0 +1,22 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+from pathlib import Path
+
+from toolbox.os.environment import EnvironmentManager
+
+
+project_path = os.path.abspath(os.path.dirname(__file__))
+project_path = Path(project_path)
+
+log_directory = project_path / "logs"
+log_directory.mkdir(parents=True, exist_ok=True)
+
+environment = EnvironmentManager(
+    path=os.path.join(project_path, "dotenv"),
+    env=os.environ.get("environment", "dev"),
+)
+
+
+if __name__ == '__main__':
+    pass

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+transformers
+torch
+torchaudio
+numpy
+librosa
+python-dotenv

toolbox/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/age_and_gender/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/age_and_gender/models/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/age_and_gender/models/audeering.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+
+import numpy as np
+import torch
+import torch.nn as nn
+import librosa
+from transformers import Wav2Vec2Processor
+from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model, Wav2Vec2PreTrainedModel
+
+from project_settings import project_path
+
+
+class ModelHead(nn.Module):
+    def __init__(self, config, num_labels):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.final_dropout)
+        self.out_proj = nn.Linear(config.hidden_size, num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+        return x
+
+
+class AgeGenderModel(Wav2Vec2PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.age = ModelHead(config, 1)
+        self.gender = ModelHead(config, 3)
+        self.init_weights()
+
+    def forward(self,
+                input_values,
+                ):
+        outputs = self.wav2vec2(input_values)
+        hidden_states = outputs[0]
+        hidden_states = torch.mean(hidden_states, dim=1)
+
+        logits_age = self.age.forward(hidden_states)
+        logits_gender = torch.softmax(self.gender.forward(hidden_states), dim=1)
+
+        return hidden_states, logits_age, logits_gender
+
+
+class AudeeringModel(object):
+    """
+    https://arxiv.org/abs/2306.16962
+
+    https://github.com/audeering/w2v2-age-gender-how-to
+
+    https://huggingface.co/audeering/wav2vec2-large-robust-6-ft-age-gender
+    https://huggingface.co/audeering/wav2vec2-large-robust-24-ft-age-gender
+    """
+    def __init__(self, model_path: str):
+        self.model_path = model_path
+
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        processor: Wav2Vec2Processor = Wav2Vec2Processor.from_pretrained(self.model_path)
+        model = AgeGenderModel.from_pretrained(self.model_path).to(device)
+        model.eval()
+
+        self.device = device
+        self.processor = processor
+        self.model = model
+
+    def predict(self, signal: np.ndarray, sample_rate: int) -> dict:
+        y = self.processor.__call__(signal, sampling_rate=sample_rate)
+        y = y["input_values"][0]
+        y = y.reshape(1, -1)
+        y = torch.from_numpy(y).to(self.device)
+
+        _, age, gender = self.model.forward(y)
+
+        age = age.detach().cpu().numpy().tolist()
+        age = age[0][0]
+
+        gender = gender.detach().cpu().numpy().tolist()
+        gender = gender[0]
+
+        result = {
+            "age": round(age, 4),
+            "female": round(gender[0], 4),
+            "male": round(gender[1], 4),
+            "child": round(gender[2], 4),
+        }
+        return result
+
+    def __call__(self, *args, **kwargs):
+        return self.predict(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    pass

toolbox/json/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/json/misc.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Callable
+
+
+def traverse(js, callback: Callable, *args, **kwargs):
+    if isinstance(js, list):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return result
+    elif isinstance(js, tuple):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return tuple(result)
+    elif isinstance(js, dict):
+        result = dict()
+        for k, v in js.items():
+            k = traverse(k, callback, *args, **kwargs)
+            v = traverse(v, callback, *args, **kwargs)
+            result[k] = v
+        return result
+    elif isinstance(js, int):
+        return callback(js, *args, **kwargs)
+    elif isinstance(js, str):
+        return callback(js, *args, **kwargs)
+    else:
+        return js
+
+
+def demo1():
+    d = {
+        "env": "ppe",
+        "mysql_connect": {
+            "host": "$mysql_connect_host",
+            "port": 3306,
+            "user": "callbot",
+            "password": "NxcloudAI2021!",
+            "database": "callbot_ppe",
+            "charset": "utf8"
+        },
+        "es_connect": {
+            "hosts": ["10.20.251.8"],
+            "http_auth": ["elastic", "ElasticAI2021!"],
+            "port": 9200
+        }
+    }
+
+    def callback(s):
+        if isinstance(s, str) and s.startswith('$'):
+            return s[1:]
+        return s
+
+    result = traverse(d, callback=callback)
+    print(result)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/os/command.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+
+
+class Command(object):
+    custom_command = [
+        "cd"
+    ]
+
+    @staticmethod
+    def _get_cmd(command):
+        command = str(command).strip()
+        if command == "":
+            return None
+        cmd_and_args = command.split(sep=" ")
+        cmd = cmd_and_args[0]
+        args = " ".join(cmd_and_args[1:])
+        return cmd, args
+
+    @classmethod
+    def popen(cls, command):
+        cmd, args = cls._get_cmd(command)
+        if cmd in cls.custom_command:
+            method = getattr(cls, cmd)
+            return method(args)
+        else:
+            resp = os.popen(command)
+            result = resp.read()
+            resp.close()
+            return result
+
+    @classmethod
+    def cd(cls, args):
+        if args.startswith("/"):
+            os.chdir(args)
+        else:
+            pwd = os.getcwd()
+            path = os.path.join(pwd, args)
+            os.chdir(path)
+
+    @classmethod
+    def system(cls, command):
+        return os.system(command)
+
+    def __init__(self):
+        pass
+
+
+def ps_ef_grep(keyword: str):
+    cmd = "ps -ef | grep {}".format(keyword)
+    rows = Command.popen(cmd)
+    rows = str(rows).split("\n")
+    rows = [row for row in rows if row.__contains__(keyword) and not row.__contains__("grep")]
+    return rows
+
+
+if __name__ == "__main__":
+    pass

toolbox/os/environment.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+import os
+
+from dotenv import load_dotenv
+from dotenv.main import DotEnv
+
+from toolbox.json.misc import traverse
+
+
+class EnvironmentManager(object):
+    def __init__(self, path, env, override=False):
+        filename = os.path.join(path, '{}.env'.format(env))
+        self.filename = filename
+
+        load_dotenv(
+            dotenv_path=filename,
+            override=override
+        )
+
+        self._environ = dict()
+
+    def open_dotenv(self, filename: str = None):
+        filename = filename or self.filename
+        dotenv = DotEnv(
+            dotenv_path=filename,
+            stream=None,
+            verbose=False,
+            interpolate=False,
+            override=False,
+            encoding="utf-8",
+        )
+        result = dotenv.dict()
+        return result
+
+    def get(self, key, default=None, dtype=str):
+        result = os.environ.get(key)
+        if result is None:
+            if default is None:
+                result = None
+            else:
+                result = default
+        else:
+            result = dtype(result)
+        self._environ[key] = result
+        return result
+
+
+_DEFAULT_DTYPE_MAP = {
+    'int': int,
+    'float': float,
+    'str': str,
+    'json.loads': json.loads
+}
+
+
+class JsonConfig(object):
+    """
+    将 json 中, 形如 `$float:threshold` 的值, 处理为:
+    从环境变量中查到 threshold, 再将其转换为 float 类型.
+    """
+    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
+        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
+        self.environment = environment or os.environ
+
+    def sanitize_by_filename(self, filename: str):
+        with open(filename, 'r', encoding='utf-8') as f:
+            js = json.load(f)
+
+        return self.sanitize_by_json(js)
+
+    def sanitize_by_json(self, js):
+        js = traverse(
+            js,
+            callback=self.sanitize,
+            environment=self.environment
+        )
+        return js
+
+    def sanitize(self, string, environment):
+        """支持 $ 符开始的, 环境变量配置"""
+        if isinstance(string, str) and string.startswith('$'):
+            dtype, key = string[1:].split(':')
+            dtype = self.dtype_map[dtype]
+
+            value = environment.get(key)
+            if value is None:
+                raise AssertionError('environment not exist. key: {}'.format(key))
+
+            value = dtype(value)
+            result = value
+        else:
+            result = string
+        return result
+
+
+def demo1():
+    import json
+
+    from project_settings import project_path
+
+    environment = EnvironmentManager(
+        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
+        env='dev',
+    )
+    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
+    print(init_scenes)
+    print(environment._environ)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/other.py

@@ -0,0 +1,9 @@
+import os
+import inspect
+
+
+def pwd():
+    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    return os.path.dirname(os.path.abspath(module.__file__))