Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -13,42 +13,12 @@ https://github.com/pinecone-io/examples/tree/master/learn/algos-and-libraries/be
|
|
13 |
|
14 |
# data = load_dataset('jamescalam/python-reddit')
|
15 |
data = load_dataset("awacke1/LOINC-Panels-and-Forms")
|
16 |
-
data = data.filter(
|
17 |
-
lambda x: True if len(x[0]) > 30 else 0
|
18 |
-
)
|
19 |
-
from bertopic import BERTopic
|
20 |
-
from sklearn.feature_extraction.text import CountVectorizer
|
21 |
|
22 |
-
# we add this to remove stopwords
|
23 |
-
vectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words="english")
|
24 |
|
25 |
-
model = BERTopic(
|
26 |
-
vectorizer_model=vectorizer_model,
|
27 |
-
language='english', calculate_probabilities=True,
|
28 |
-
verbose=True
|
29 |
-
)
|
30 |
-
topics, probs = model.fit_transform(text)
|
31 |
-
freq = model.get_topic_info()
|
32 |
-
freq.head(10)
|
33 |
|
34 |
|
35 |
-
from sentence_transformers import SentenceTransformer
|
36 |
-
|
37 |
-
model = SentenceTransformer('all-MiniLM-L6-v2')
|
38 |
-
model
|
39 |
-
|
40 |
-
import numpy as np
|
41 |
-
from tqdm.auto import tqdm
|
42 |
-
|
43 |
-
batch_size = 16
|
44 |
-
|
45 |
-
embeds = np.zeros((n, model.get_sentence_embedding_dimension()))
|
46 |
-
|
47 |
-
for i in tqdm(range(0, n, batch_size)):
|
48 |
-
i_end = min(i+batch_size, n)
|
49 |
-
batch = data[0][i:i_end]
|
50 |
-
batch_embed = model.encode(batch)
|
51 |
-
embeds[i:i_end,:] = batch_embed
|
52 |
|
|
|
53 |
|
54 |
-
|
|
|
|
13 |
|
14 |
# data = load_dataset('jamescalam/python-reddit')
|
15 |
data = load_dataset("awacke1/LOINC-Panels-and-Forms")
|
|
|
|
|
|
|
|
|
|
|
16 |
|
|
|
|
|
17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18 |
|
19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
+
from datasets import load_dataset
|
22 |
|
23 |
+
geo = load_dataset('jamescalam/world-cities-geo', split='train')
|
24 |
+
st.write(geo)
|