awacke1 commited on
Commit
6e73e6e
·
1 Parent(s): 86c4485

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -33
app.py CHANGED
@@ -13,42 +13,12 @@ https://github.com/pinecone-io/examples/tree/master/learn/algos-and-libraries/be
13
 
14
  # data = load_dataset('jamescalam/python-reddit')
15
  data = load_dataset("awacke1/LOINC-Panels-and-Forms")
16
- data = data.filter(
17
- lambda x: True if len(x[0]) > 30 else 0
18
- )
19
- from bertopic import BERTopic
20
- from sklearn.feature_extraction.text import CountVectorizer
21
 
22
- # we add this to remove stopwords
23
- vectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words="english")
24
 
25
- model = BERTopic(
26
- vectorizer_model=vectorizer_model,
27
- language='english', calculate_probabilities=True,
28
- verbose=True
29
- )
30
- topics, probs = model.fit_transform(text)
31
- freq = model.get_topic_info()
32
- freq.head(10)
33
 
34
 
35
- from sentence_transformers import SentenceTransformer
36
-
37
- model = SentenceTransformer('all-MiniLM-L6-v2')
38
- model
39
-
40
- import numpy as np
41
- from tqdm.auto import tqdm
42
-
43
- batch_size = 16
44
-
45
- embeds = np.zeros((n, model.get_sentence_embedding_dimension()))
46
-
47
- for i in tqdm(range(0, n, batch_size)):
48
- i_end = min(i+batch_size, n)
49
- batch = data[0][i:i_end]
50
- batch_embed = model.encode(batch)
51
- embeds[i:i_end,:] = batch_embed
52
 
 
53
 
54
-
 
 
13
 
14
  # data = load_dataset('jamescalam/python-reddit')
15
  data = load_dataset("awacke1/LOINC-Panels-and-Forms")
 
 
 
 
 
16
 
 
 
17
 
 
 
 
 
 
 
 
 
18
 
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
+ from datasets import load_dataset
22
 
23
+ geo = load_dataset('jamescalam/world-cities-geo', split='train')
24
+ st.write(geo)