marondeau commited on
Commit
a3c0809
·
unverified ·
1 Parent(s): c3ad77b

Size cannot be negative; Removed None.

Browse files
buster/documents/sqlite/documents.py CHANGED
@@ -134,11 +134,10 @@ class DocumentsDB(DocumentsManager):
134
  """Write all documents from the dataframe into the db as a new version."""
135
  data = sorted(df.itertuples(), key=lambda chunk: (chunk.url, chunk.title))
136
  sections = []
137
- size = None
138
  for (url, title), chunks in itertools.groupby(data, lambda chunk: (chunk.url, chunk.title)):
139
  chunks = [Chunk(chunk.content, chunk.n_tokens, chunk.embedding) for chunk in chunks]
140
- _size = max(len(chunk.content) for chunk in chunks)
141
- size = max(_size, size or 0)
142
  content = "".join(chunk.content for chunk in chunks)
143
  sections.append((Section(title, url, content), chunks))
144
 
 
134
  """Write all documents from the dataframe into the db as a new version."""
135
  data = sorted(df.itertuples(), key=lambda chunk: (chunk.url, chunk.title))
136
  sections = []
137
+ size = 0
138
  for (url, title), chunks in itertools.groupby(data, lambda chunk: (chunk.url, chunk.title)):
139
  chunks = [Chunk(chunk.content, chunk.n_tokens, chunk.embedding) for chunk in chunks]
140
+ size = max(size, max(len(chunk.content) for chunk in chunks))
 
141
  content = "".join(chunk.content for chunk in chunks)
142
  sections.append((Section(title, url, content), chunks))
143