svjack commited on
Commit
4d15bb3
·
1 Parent(s): 40c0254

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +142 -0
app.py ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import chatglm_cpp
2
+ import gradio as gr
3
+ from pathlib import Path
4
+
5
+ model_file_path = "chatglm3-ggml_q4_0.bin"
6
+ chatglm_llm = chatglm_cpp.Pipeline(Path(model_file_path))
7
+
8
+ examples = [
9
+ "如何弘扬中华传统文化?",
10
+ "How to promote Chinese traditional culture ?",
11
+ "如何学好历史?",
12
+ "写一段孔子与马克思的对话录。",
13
+ "如何进行经济建设?",
14
+ ]
15
+
16
+ def process_stream(instruction, temperature, top_p, top_k, max_new_tokens, seed):
17
+ if "[SEP]" not in instruction:
18
+ streamer = chatglm_llm.generate(prompt=instruction,
19
+ temperature=temperature,
20
+ top_p=top_p,top_k=top_k,max_length=max_new_tokens,
21
+ stream = True
22
+ )
23
+ else:
24
+ history = instruction.split("[SEP]")
25
+ streamer = chatglm_llm.chat(
26
+ history=history,
27
+ temperature=temperature,
28
+ top_p=top_p,top_k=top_k,max_length=max_new_tokens,
29
+ do_sample=False,
30
+ stream = True
31
+ )
32
+ response = ""
33
+ for new_text in streamer:
34
+ response += new_text
35
+ yield response
36
+
37
+
38
+ with gr.Blocks(
39
+ theme=gr.themes.Soft(),
40
+ css=".disclaimer {font-variant-caps: all-small-caps;}",
41
+ ) as demo:
42
+ gr.Markdown(
43
+ """<h1> <center> <img src="https://huggingface.co/spaces/svjack/chatglm3-6b-ggml-v0/resolve/main/hanuman.png" alt="SD">
44
+ ChatGLM3 on CPU in CPP </center></h1>
45
+ This demo uses the [chatglm.cpp](https://github.com/li-plus/chatglm.cpp) library on 2 CPU cores.
46
+ """
47
+ )
48
+ with gr.Row():
49
+ with gr.Column():
50
+ with gr.Row():
51
+ instruction = gr.Textbox(
52
+ placeholder="Enter your question or instruction here",
53
+ label="Question/Instruction",
54
+ elem_id="q-input",
55
+ )
56
+ with gr.Accordion("Advanced Options:", open=False):
57
+ with gr.Row():
58
+ with gr.Column():
59
+ with gr.Row():
60
+ temperature = gr.Slider(
61
+ label="Temperature",
62
+ value=0.5,
63
+ minimum=0.1,
64
+ maximum=1.0,
65
+ step=0.1,
66
+ interactive=True,
67
+ info="Higher values produce more diverse outputs",
68
+ )
69
+ with gr.Column():
70
+ with gr.Row():
71
+ top_p = gr.Slider(
72
+ label="Top-p (nucleus sampling)",
73
+ value=0.95,
74
+ minimum=0.0,
75
+ maximum=1.0,
76
+ step=0.01,
77
+ interactive=True,
78
+ info=(
79
+ "Sample from the smallest possible set of tokens whose cumulative probability "
80
+ "exceeds top_p. Set to 1 to disable and sample from all tokens."
81
+ ),
82
+ )
83
+ with gr.Column():
84
+ with gr.Row():
85
+ top_k = gr.Slider(
86
+ label="Top-k",
87
+ value=40,
88
+ minimum=5,
89
+ maximum=80,
90
+ step=1,
91
+ interactive=True,
92
+ info="Sample from a shortlist of top-k tokens — 0 to disable and sample from all tokens.",
93
+ )
94
+ with gr.Column():
95
+ with gr.Row():
96
+ max_new_tokens = gr.Slider(
97
+ label="Maximum new tokens",
98
+ value=256,
99
+ minimum=0,
100
+ maximum=1024,
101
+ step=5,
102
+ interactive=True,
103
+ info="The maximum number of new tokens to generate",
104
+ )
105
+
106
+ with gr.Column():
107
+ with gr.Row():
108
+ seed = gr.Number(
109
+ label="Seed",
110
+ value=42,
111
+ interactive=True,
112
+ info="The seed to use for the generation",
113
+ precision=0
114
+ )
115
+ with gr.Row():
116
+ submit = gr.Button("Submit")
117
+ with gr.Row():
118
+ with gr.Box():
119
+ gr.Markdown("**ChatGLM3-6b**")
120
+ output_7b = gr.Markdown()
121
+
122
+ with gr.Row():
123
+ gr.Examples(
124
+ examples=examples,
125
+ inputs=[instruction],
126
+ cache_examples=False,
127
+ fn=process_stream,
128
+ outputs=output_7b,
129
+ )
130
+
131
+ submit.click(
132
+ process_stream,
133
+ inputs=[instruction, temperature, top_p, top_k, max_new_tokens,seed],
134
+ outputs=output_7b,
135
+ )
136
+ instruction.submit(
137
+ process_stream,
138
+ inputs=[instruction, temperature, top_p, top_k, max_new_tokens,seed],
139
+ outputs=output_7b,
140
+ )
141
+
142
+ demo.queue(max_size=4, concurrency_count=1).launch(debug=True)