Spaces:
Runtime error
Runtime error
Update scripts/process.py
Browse files- scripts/process.py +21 -23
scripts/process.py
CHANGED
|
@@ -70,29 +70,27 @@ def load_document(
|
|
| 70 |
Returns a list of type haystack.schema.Document
|
| 71 |
"""
|
| 72 |
with st.spinner("๐ Uploading file"):#+file.name+"..."):
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
return documents
|
| 96 |
|
| 97 |
|
| 98 |
def preprocessing(document):
|
|
|
|
| 70 |
Returns a list of type haystack.schema.Document
|
| 71 |
"""
|
| 72 |
with st.spinner("๐ Uploading file"):#+file.name+"..."):
|
| 73 |
+
if file_name.endswith('.pdf'):
|
| 74 |
+
converter = PDFToTextConverter(remove_numeric_tables=True)
|
| 75 |
+
if file_name.endswith('.txt'):
|
| 76 |
+
converter = TextConverter()
|
| 77 |
+
if file_name.endswith('.docx'):
|
| 78 |
+
converter = DocxToTextConverter()
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
documents = []
|
| 82 |
+
#logger.info("Converting {}".format(file_name))
|
| 83 |
+
# PDFToTextConverter, TextConverter, and DocxToTextConverter
|
| 84 |
+
# return a list containing a single Document
|
| 85 |
+
document = converter.convert(
|
| 86 |
+
file_path=file_path, meta=None,
|
| 87 |
+
encoding=encoding, id_hash_keys=id_hash_keys
|
| 88 |
+
)[0]
|
| 89 |
+
text = document.content
|
| 90 |
+
documents.append(Document(content=text,
|
| 91 |
+
meta={"name": file_name},
|
| 92 |
+
id_hash_keys=id_hash_keys))
|
| 93 |
+
return documents
|
|
|
|
|
|
|
| 94 |
|
| 95 |
|
| 96 |
def preprocessing(document):
|