Corey Morris
commited on
Commit
·
f228d38
1
Parent(s):
74822dd
moved methods to better match flow
Browse files
details_data_processor.py
CHANGED
|
@@ -16,7 +16,14 @@ class DetailsDataProcessor:
|
|
| 16 |
# self.data = self.process_data()
|
| 17 |
# self.ranked_data = self.rank_data()
|
| 18 |
|
| 19 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
# download a file from a single url and save it to a local directory
|
| 22 |
@staticmethod
|
|
@@ -52,14 +59,7 @@ class DetailsDataProcessor:
|
|
| 52 |
return constructed_url
|
| 53 |
|
| 54 |
|
| 55 |
-
|
| 56 |
-
matching_files = [] # List to hold matching filenames
|
| 57 |
-
for root, dirs, files in os.walk(directory):
|
| 58 |
-
for basename in files:
|
| 59 |
-
if fnmatch.fnmatch(basename, pattern):
|
| 60 |
-
filename = os.path.join(root, basename)
|
| 61 |
-
matching_files.append(filename) # Append the matching filename to the list
|
| 62 |
-
return matching_files # Return the list of matching filenames
|
| 63 |
|
| 64 |
|
| 65 |
def pipeline(self):
|
|
|
|
| 16 |
# self.data = self.process_data()
|
| 17 |
# self.ranked_data = self.rank_data()
|
| 18 |
|
| 19 |
+
def _find_files(self, directory='results', pattern='results*.json'):
|
| 20 |
+
matching_files = [] # List to hold matching filenames
|
| 21 |
+
for root, dirs, files in os.walk(directory):
|
| 22 |
+
for basename in files:
|
| 23 |
+
if fnmatch.fnmatch(basename, pattern):
|
| 24 |
+
filename = os.path.join(root, basename)
|
| 25 |
+
matching_files.append(filename) # Append the matching filename to the list
|
| 26 |
+
return matching_files # Return the list of matching filenames
|
| 27 |
|
| 28 |
# download a file from a single url and save it to a local directory
|
| 29 |
@staticmethod
|
|
|
|
| 59 |
return constructed_url
|
| 60 |
|
| 61 |
|
| 62 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
|
| 65 |
def pipeline(self):
|