Update DiT_Extractor/sentence_extractor.py

DiT_Extractor/sentence_extractor.py

@@ -95,6 +95,11 @@ def sentence_extract(document):
             for sentence in sentences:
                 t += len(sentence)
                 if t <= max_tokens:
+                    # update character indicies from concatenating sentences
+                    if len(word_section) > 0:
+                        last_word_obj = word_section[-1]
+                        _, (_, char_idx_offset), _ = last_word_obj
+                        sentence = [(w, (sc+char_idx_offset+1, ec+char_idx_offset+1), bbox) for w, (sc, ec), bbox in sentence]
                     word_section += sentence
                 else:
                     word_sections.append(word_section)