Spaces:
Runtime error
Runtime error
marondeau
commited on
Size cannot be negative; Removed None.
Browse files
buster/documents/sqlite/documents.py
CHANGED
@@ -134,11 +134,10 @@ class DocumentsDB(DocumentsManager):
|
|
134 |
"""Write all documents from the dataframe into the db as a new version."""
|
135 |
data = sorted(df.itertuples(), key=lambda chunk: (chunk.url, chunk.title))
|
136 |
sections = []
|
137 |
-
size =
|
138 |
for (url, title), chunks in itertools.groupby(data, lambda chunk: (chunk.url, chunk.title)):
|
139 |
chunks = [Chunk(chunk.content, chunk.n_tokens, chunk.embedding) for chunk in chunks]
|
140 |
-
|
141 |
-
size = max(_size, size or 0)
|
142 |
content = "".join(chunk.content for chunk in chunks)
|
143 |
sections.append((Section(title, url, content), chunks))
|
144 |
|
|
|
134 |
"""Write all documents from the dataframe into the db as a new version."""
|
135 |
data = sorted(df.itertuples(), key=lambda chunk: (chunk.url, chunk.title))
|
136 |
sections = []
|
137 |
+
size = 0
|
138 |
for (url, title), chunks in itertools.groupby(data, lambda chunk: (chunk.url, chunk.title)):
|
139 |
chunks = [Chunk(chunk.content, chunk.n_tokens, chunk.embedding) for chunk in chunks]
|
140 |
+
size = max(size, max(len(chunk.content) for chunk in chunks))
|
|
|
141 |
content = "".join(chunk.content for chunk in chunks)
|
142 |
sections.append((Section(title, url, content), chunks))
|
143 |
|