ffreemt commited on
Commit
3c0a531
·
1 Parent(s): 0acdcc9

Update deq DequeCallback

Browse files
Files changed (1) hide show
  1. app.py +111 -51
app.py CHANGED
@@ -8,6 +8,7 @@ import random
8
  import time
9
  from pathlib import Path
10
  from queue import deque
 
11
  from typing import Any, Dict, List, Union
12
 
13
  # from types import SimpleNamespace
@@ -18,9 +19,12 @@ from ctransformers import Config
18
  from dl_hf_model import dl_hf_model
19
  from langchain.callbacks.base import BaseCallbackHandler
20
  from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 
 
21
 
22
  # from ctransformers import AutoModelForCausalLM
23
  from langchain.llms import CTransformers
 
24
  from langchain.schema import LLMResult
25
  from loguru import logger
26
 
@@ -124,36 +128,28 @@ prompt_template = """### HUMAN:
124
 
125
  ### RESPONSE:"""
126
 
127
- _ = [elm for elm in prompt_template.splitlines() if elm.strip()]
128
- stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
129
-
130
- logger.debug(f"{stop_string=} not used")
131
-
132
- _ = psutil.cpu_count(logical=False) - 1
133
- cpu_count: int = int(_) if _ else 1
134
- logger.debug(f"{cpu_count=}")
135
 
136
- LLM = None
137
- gc.collect()
 
 
 
138
 
139
- try:
140
- model_loc, file_size = dl_hf_model(url)
141
- except Exception as exc_:
142
- logger.error(exc_)
143
- raise SystemExit(1) from exc_
144
 
145
- config = Config()
146
- config.steam = True
 
147
 
148
- # LLM = AutoModelForCausalLM.from_pretrained(
149
- LLM = CTransformers(
150
- model=model_loc,
151
- model_type="llama",
152
- threads=cpu_count,
153
- callbacks=[StreamingStdOutCallbackHandler()],
154
- )
155
 
156
- logger.info(f"done load llm {model_loc=} {file_size=}G")
157
 
158
  os.environ["TZ"] = "Asia/Shanghai"
159
  try:
@@ -166,7 +162,7 @@ except Exception:
166
  class DequeCallbackHandler(BaseCallbackHandler):
167
  """Mediate gradio and stream output."""
168
 
169
- def __init__(self, deq: deque):
170
  """Init deque for FIFO, may need to upgrade to queue.Queue or queue.SimpleQueue."""
171
  self.q = deq
172
 
@@ -191,6 +187,63 @@ class DequeCallbackHandler(BaseCallbackHandler):
191
  self.q.put(sig_end)
192
 
193
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
194
  def user(user_message, history):
195
  # return user_message, history + [[user_message, None]]
196
  history.append([user_message, None])
@@ -223,28 +276,29 @@ def bot(history):
223
 
224
  logger.debug(f"{user_message=}")
225
 
226
- with about_time() as atime: # type: ignore
227
- flag = 1
228
- prefix = ""
229
- then = time.time()
230
-
231
- logger.debug("about to generate")
232
-
233
- config = Config(reset=True)
234
- for elm in generate(user_message, config=config):
235
- if flag == 1:
236
- logger.debug("in the loop")
237
- prefix = f"({time.time() - then:.2f}s) "
238
- flag = 0
239
- print(prefix, end="", flush=True)
240
- logger.debug(f"{prefix=}")
241
- print(elm, end="", flush=True)
242
- # logger.debug(f"{elm}")
243
-
244
- response.append(elm)
245
- history[-1][1] = prefix + "".join(response)
246
- yield history
247
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
248
  _ = (
249
  f"(time elapsed: {atime.duration_human}, " # type: ignore
250
  f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
@@ -258,7 +312,7 @@ def predict_api(prompt):
258
  logger.debug(f"{prompt=}")
259
  try:
260
  # user_prompt = prompt
261
- config = Config(
262
  temperature=0.2,
263
  top_k=10,
264
  top_p=0.9,
@@ -270,12 +324,18 @@ def predict_api(prompt):
270
  # threads=cpu_count,
271
  # stop=prompt_prefix[1:2],
272
  )
273
-
274
  response = generate(
275
  prompt,
276
  config=config,
277
  )
278
-
 
 
 
 
 
 
279
  logger.debug(f"api: {response=}")
280
  except Exception as exc:
281
  logger.error(exc)
@@ -283,7 +343,7 @@ def predict_api(prompt):
283
  # bot = {"inputs": [response]}
284
  # bot = [(prompt, response)]
285
 
286
- return response
287
 
288
 
289
  css = """
 
8
  import time
9
  from pathlib import Path
10
  from queue import deque
11
+ from threading import Thread
12
  from typing import Any, Dict, List, Union
13
 
14
  # from types import SimpleNamespace
 
19
  from dl_hf_model import dl_hf_model
20
  from langchain.callbacks.base import BaseCallbackHandler
21
  from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
22
+ from langchain.chains import ConversationChain
23
+ from langchain.chains.conversation.memory import ConversationBufferWindowMemory
24
 
25
  # from ctransformers import AutoModelForCausalLM
26
  from langchain.llms import CTransformers
27
+ from langchain.prompts import PromptTemplate
28
  from langchain.schema import LLMResult
29
  from loguru import logger
30
 
 
128
 
129
  ### RESPONSE:"""
130
 
131
+ prompt_template = """### HUMAN:
132
+ You are a helpful assistant. Think step by step.
133
+ {history}
134
+ {input}
135
+ ### RESPONSE:"""
 
 
 
136
 
137
+ prompt_template = """You are a helpful assistant.
138
+ {history}
139
+ ### HUMAN:
140
+ {input}
141
+ ### RESPONSE:"""
142
 
143
+ # PromptTemplate(input_variables=['history', 'input'], output_parser=None, partial_variables={}, template='The following is afriendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n{history}\nHuman: {input}\nAI:', template_format='f-string', validate_template=True)
 
 
 
 
144
 
145
+ human_prefix = "### HUMAN"
146
+ ai_prefix = "### RESPONSE"
147
+ stop = [f"{human_prefix}:"]
148
 
149
+ _ = [elm for elm in prompt_template.splitlines() if elm.strip()]
150
+ stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
 
 
 
 
 
151
 
152
+ # logger.debug(f"{stop_string=} not used")
153
 
154
  os.environ["TZ"] = "Asia/Shanghai"
155
  try:
 
162
  class DequeCallbackHandler(BaseCallbackHandler):
163
  """Mediate gradio and stream output."""
164
 
165
+ def __init__(self, deq: deque = deque()):
166
  """Init deque for FIFO, may need to upgrade to queue.Queue or queue.SimpleQueue."""
167
  self.q = deq
168
 
 
187
  self.q.put(sig_end)
188
 
189
 
190
+ _ = psutil.cpu_count(logical=False) - 1
191
+ cpu_count: int = int(_) if _ else 1
192
+ logger.debug(f"{cpu_count=}")
193
+
194
+ LLM = None
195
+ gc.collect()
196
+
197
+ try:
198
+ model_loc, file_size = dl_hf_model(url)
199
+ except Exception as exc_:
200
+ logger.error(exc_)
201
+ raise SystemExit(1) from exc_
202
+
203
+ config = Config()
204
+ config.stream = True
205
+ config.stop = stop
206
+ config.threads=cpu_count
207
+
208
+ deqcb = DequeCallbackHandler(deq)
209
+
210
+ # LLM = AutoModelForCausalLM.from_pretrained(
211
+ LLM = CTransformers(
212
+ model=model_loc,
213
+ model_type="llama",
214
+ callbacks=[StreamingStdOutCallbackHandler(), deqcb],
215
+ # config=config,
216
+ **vars(config),
217
+ )
218
+
219
+ logger.info(f"done load llm {model_loc=} {file_size=}G")
220
+
221
+
222
+ prompt = PromptTemplate(
223
+ input_variables=['history', 'input'],
224
+ output_parser=None,
225
+ partial_variables={},
226
+ template=prompt_template,
227
+ template_format='f-string',
228
+ validate_template=True
229
+ )
230
+
231
+ memory = ConversationBufferWindowMemory(
232
+ human_prefix=human_prefix,
233
+ ai_prefix=ai_prefix,
234
+ ) # default k=5
235
+
236
+ conversation = ConversationChain(
237
+ llm=LLM,
238
+ prompt=prompt,
239
+ memory=memory,
240
+ verbose=True,
241
+ )
242
+ logger.debug(f"{conversation.prompt.template=}")
243
+
244
+ # conversation.predict(input="Hello, my name is Andrea")
245
+
246
+
247
  def user(user_message, history):
248
  # return user_message, history + [[user_message, None]]
249
  history.append([user_message, None])
 
276
 
277
  logger.debug(f"{user_message=}")
278
 
279
+ # conversation.predict(input="What's my name?")
280
+ thr = Thread(target=conversation.predict, kwargs={"input": user_message})
281
+ thr.start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
282
 
283
+ # preocess deq
284
+ response = []
285
+ flag = 1
286
+ then = time.time()
287
+ with about_time() as atime: # type: ignore
288
+ while True:
289
+ if deq:
290
+ if flag:
291
+ prefix = f"({time.time() - then:.2f}s) "
292
+ flag = 0
293
+ _ = deq.popleft()
294
+ if _ is sig_end:
295
+ break
296
+ # print(_, end='')
297
+ response.append(_)
298
+ history[-1][1] = prefix + "".join(response).strip()
299
+ yield history
300
+ else:
301
+ time.sleep(0.01)
302
  _ = (
303
  f"(time elapsed: {atime.duration_human}, " # type: ignore
304
  f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
 
312
  logger.debug(f"{prompt=}")
313
  try:
314
  # user_prompt = prompt
315
+ Config(
316
  temperature=0.2,
317
  top_k=10,
318
  top_p=0.9,
 
324
  # threads=cpu_count,
325
  # stop=prompt_prefix[1:2],
326
  )
327
+ _ = """
328
  response = generate(
329
  prompt,
330
  config=config,
331
  )
332
+ # """
333
+ conversation = ConversationChain(
334
+ llm=LLM,
335
+ prompt=prompt,
336
+ verbose=True,
337
+ )
338
+ response = conversation.predict(prompt)
339
  logger.debug(f"api: {response=}")
340
  except Exception as exc:
341
  logger.error(exc)
 
343
  # bot = {"inputs": [response]}
344
  # bot = [(prompt, response)]
345
 
346
+ return response.strip()
347
 
348
 
349
  css = """