Spaces:
Runtime error
Runtime error
Update chunk_python_code.py
Browse files- chunk_python_code.py +9 -4
chunk_python_code.py
CHANGED
|
@@ -5,7 +5,12 @@ from langchain.schema import Document
|
|
| 5 |
|
| 6 |
def chunk_python_code_with_metadata(python_code, file_path):
|
| 7 |
"""
|
| 8 |
-
Custom Python Code Splitter
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
Entry point method to process the Python file.
|
| 10 |
It invokes the iterate_ast function.
|
| 11 |
"""
|
|
@@ -61,9 +66,9 @@ def _iterate_ast(python_code, documents, file_path):
|
|
| 61 |
elif isinstance(first_level_node, ast.Assign):
|
| 62 |
documents.extend(
|
| 63 |
_chunk_first_level_assign_node(first_level_node, python_code))
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
|
| 68 |
|
| 69 |
|
|
|
|
| 5 |
|
| 6 |
def chunk_python_code_with_metadata(python_code, file_path):
|
| 7 |
"""
|
| 8 |
+
Custom Python Code Splitter
|
| 9 |
+
chunks python file by length of func/method body
|
| 10 |
+
aims to have one full method/function in a chunk and full body of a class, but cutting of when first method declaration is met
|
| 11 |
+
|
| 12 |
+
able to handles decorators on methods
|
| 13 |
+
|
| 14 |
Entry point method to process the Python file.
|
| 15 |
It invokes the iterate_ast function.
|
| 16 |
"""
|
|
|
|
| 66 |
elif isinstance(first_level_node, ast.Assign):
|
| 67 |
documents.extend(
|
| 68 |
_chunk_first_level_assign_node(first_level_node, python_code))
|
| 69 |
+
# else:
|
| 70 |
+
# documents.extend(
|
| 71 |
+
# _handle_not_defined_case(python_code))
|
| 72 |
|
| 73 |
|
| 74 |
|