Kevin Hu
commited on
Commit
·
bc6fc1a
1
Parent(s):
d007a79
smooth term weight (#3510)
Browse files### What problem does this PR solve?
#3499
### Type of change
- [x] Performance Improvement
- rag/nlp/term_weight.py +2 -2
rag/nlp/term_weight.py
CHANGED
|
@@ -228,7 +228,7 @@ class Dealer:
|
|
| 228 |
idf2 = np.array([idf(df(t), 1000000000) for t in tks])
|
| 229 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
| 230 |
np.array([ner(t) * postag(t) for t in tks])
|
| 231 |
-
wts = [math.
|
| 232 |
tw = list(zip(tks, wts))
|
| 233 |
else:
|
| 234 |
for tk in tks:
|
|
@@ -237,7 +237,7 @@ class Dealer:
|
|
| 237 |
idf2 = np.array([idf(df(t), 1000000000) for t in tt])
|
| 238 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
| 239 |
np.array([ner(t) * postag(t) for t in tt])
|
| 240 |
-
wts = [math.
|
| 241 |
tw.extend(zip(tt, wts))
|
| 242 |
|
| 243 |
S = np.sum([s for _, s in tw])
|
|
|
|
| 228 |
idf2 = np.array([idf(df(t), 1000000000) for t in tks])
|
| 229 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
| 230 |
np.array([ner(t) * postag(t) for t in tks])
|
| 231 |
+
wts = [math.pow(s, 2) for s in wts]
|
| 232 |
tw = list(zip(tks, wts))
|
| 233 |
else:
|
| 234 |
for tk in tks:
|
|
|
|
| 237 |
idf2 = np.array([idf(df(t), 1000000000) for t in tt])
|
| 238 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
| 239 |
np.array([ner(t) * postag(t) for t in tt])
|
| 240 |
+
wts = [math.pow(s, 2) for s in wts]
|
| 241 |
tw.extend(zip(tt, wts))
|
| 242 |
|
| 243 |
S = np.sum([s for _, s in tw])
|