om4r932 commited on
Commit
49a2d4c
·
1 Parent(s): f7db7af

Remove SSE for now

Browse files
Files changed (2) hide show
  1. app.py +16 -65
  2. index.html +5 -5
app.py CHANGED
@@ -92,18 +92,8 @@ def index_tdocs_wg_progress(req: IndexTDoc):
92
  folder = meetings_mapping[category][wg_number]
93
  url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}"
94
 
95
- def generate_events():
96
- tdoc_indexer.processed_count = 0 # Reset progress
97
- tdoc_indexer.total_count = 0
98
- tdoc_indexer.process_workgroup(folder, url)
99
-
100
- while tdoc_indexer.processed_count < tdoc_indexer.total_count:
101
- yield f"data: {tdoc_indexer.processed_count}/{tdoc_indexer.total_count}"
102
- import time; time.sleep(0.2)
103
- # Pour afficher la fin de l’indexation
104
- yield f"data: {tdoc_indexer.total_count}/{tdoc_indexer.total_count}"
105
-
106
- return StreamingResponse(generate_events(), media_type="text/event-stream")
107
 
108
  @app.post("/index_tdocs/meeting")
109
  def index_tdocs_meeting_progress(req: IndexTDoc):
@@ -115,67 +105,28 @@ def index_tdocs_meeting_progress(req: IndexTDoc):
115
  category, wg_number = get_folder_name(req.wg)
116
  folder = meetings_mapping[category][wg_number]
117
  url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}/{folder}"
118
-
119
- def generate_events():
120
- tdoc_indexer.processed_count = 0
121
- tdoc_indexer.total_count = len(req.meetings)
122
- for i, meet in enumerate(req.meetings):
123
- tdoc_indexer.process_meeting(meet, url)
124
- yield f"data: {i+1}/{tdoc_indexer.total_count}"
125
- tdoc_indexer.save_indexer()
126
- return StreamingResponse(generate_events(), media_type="text/event-stream")
127
 
128
 
129
  @app.post("/index_tdocs/all")
130
  def index_all_tdocs_progress():
131
- def generate_events():
132
- tdoc_indexer.processed_count = 0
133
- tdoc_indexer.total_count = 0
134
- # On lance l’indexation (la méthode met à jour les compteurs)
135
- tdoc_indexer.index_all_tdocs()
136
- while tdoc_indexer.processed_count < tdoc_indexer.total_count:
137
- yield f"data: {tdoc_indexer.processed_count}/{tdoc_indexer.total_count}"
138
- import time; time.sleep(0.2)
139
- yield f"data: {tdoc_indexer.total_count}/{tdoc_indexer.total_count}"
140
- return StreamingResponse(generate_events(), media_type="text/event-stream")
141
 
142
 
143
  @app.post("/index_specs/3gpp")
144
  def index_3gpp_specs_progress():
145
- def generate_events():
146
- spec_3gpp_indexer.processed_count = 0
147
- spec_3gpp_indexer.total_count = 0
148
- import threading
149
-
150
- def worker():
151
- spec_3gpp_indexer.run()
152
- spec_3gpp_indexer.save()
153
- spec_3gpp_indexer.create_bm25_index()
154
-
155
- t = threading.Thread(target=worker)
156
- t.start()
157
- while t.is_alive() or spec_3gpp_indexer.processed_count < spec_3gpp_indexer.total_count:
158
- yield f"data: {spec_3gpp_indexer.processed_count}/{spec_3gpp_indexer.total_count}"
159
- import time; time.sleep(0.5)
160
- yield f"data: {spec_3gpp_indexer.total_count}/{spec_3gpp_indexer.total_count}"
161
- return StreamingResponse(generate_events(), media_type="text/event-stream")
162
 
163
  @app.post("/index_specs/etsi")
164
  def index_etsi_specs_progress():
165
- def generate_events():
166
- spec_etsi_indexer.processed_count = 0
167
- spec_etsi_indexer.total_count = 0
168
- import threading
169
-
170
- def worker():
171
- spec_etsi_indexer.run()
172
- spec_etsi_indexer.save()
173
- spec_etsi_indexer.create_bm25_index()
174
-
175
- t = threading.Thread(target=worker)
176
- t.start()
177
- while t.is_alive() or spec_etsi_indexer.processed_count < spec_etsi_indexer.total_count:
178
- yield f"data: {spec_etsi_indexer.processed_count}/{spec_etsi_indexer.total_count}"
179
- import time; time.sleep(0.5)
180
- yield f"data: {spec_etsi_indexer.total_count}/{spec_etsi_indexer.total_count}"
181
- return StreamingResponse(generate_events(), media_type="text/event-stream")
 
92
  folder = meetings_mapping[category][wg_number]
93
  url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}"
94
 
95
+ tdoc_indexer.process_workgroup(folder, url)
96
+ return {"status": "OK"}
 
 
 
 
 
 
 
 
 
 
97
 
98
  @app.post("/index_tdocs/meeting")
99
  def index_tdocs_meeting_progress(req: IndexTDoc):
 
105
  category, wg_number = get_folder_name(req.wg)
106
  folder = meetings_mapping[category][wg_number]
107
  url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}/{folder}"
108
+ for i, meet in enumerate(req.meetings):
109
+ tdoc_indexer.process_meeting(meet, url)
110
+ tdoc_indexer.save_indexer()
111
+ return {"status": "OK"}
 
 
 
 
 
112
 
113
 
114
  @app.post("/index_tdocs/all")
115
  def index_all_tdocs_progress():
116
+ tdoc_indexer.index_all_tdocs()
117
+ return {"status": "OK"}
 
 
 
 
 
 
 
 
118
 
119
 
120
  @app.post("/index_specs/3gpp")
121
  def index_3gpp_specs_progress():
122
+ spec_3gpp_indexer.run()
123
+ spec_3gpp_indexer.save()
124
+ spec_3gpp_indexer.create_bm25_index()
125
+ return {"status": "OK"}
 
 
 
 
 
 
 
 
 
 
 
 
 
126
 
127
  @app.post("/index_specs/etsi")
128
  def index_etsi_specs_progress():
129
+ spec_etsi_indexer.run()
130
+ spec_etsi_indexer.save()
131
+ spec_etsi_indexer.create_bm25_index()
132
+ return {"status": "OK"}
 
 
 
 
 
 
 
 
 
 
 
 
 
index.html CHANGED
@@ -319,14 +319,14 @@ function updateDropbtnLabel() {
319
  fetch("/index_tdocs/meeting", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({wg: document.getElementById("tdocs-wg-option").value, meetings: selectedMeetings})})
320
  .then(resp => resp.text())
321
  .then(data => {
322
- logMessage(`${data}`)
323
  enableButtons()
324
  })
325
  } else {
326
  fetch("/index_tdocs/working_group", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({wg: document.getElementById("tdocs-wg-option").value})})
327
  .then(resp => resp.text())
328
  .then(data => {
329
- logMessage(`${data}`)
330
  enableButtons()
331
  })
332
  }
@@ -334,7 +334,7 @@ function updateDropbtnLabel() {
334
  fetch("/index_tdocs/all", {method: "POST", headers: {"Content-Type": "application/json"}})
335
  .then(resp => resp.text())
336
  .then(data => {
337
- logMessage(`${data}`)
338
  enableButtons()
339
  })
340
  }
@@ -346,7 +346,7 @@ function updateDropbtnLabel() {
346
  fetch("/index_specs/3gpp", {method: "POST", headers: {"Content-Type": "application/json"}})
347
  .then(resp => resp.text())
348
  .then(data => {
349
- logMessage(`${data}`)
350
  enableButtons()
351
  })
352
  });
@@ -357,7 +357,7 @@ function updateDropbtnLabel() {
357
  fetch("/index_specs/etsi", {method: "POST", headers: {"Content-Type": "application/json"}})
358
  .then(resp => resp.text())
359
  .then(data => {
360
- logMessage(`${data}`)
361
  enableButtons()
362
  })
363
  });
 
319
  fetch("/index_tdocs/meeting", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({wg: document.getElementById("tdocs-wg-option").value, meetings: selectedMeetings})})
320
  .then(resp => resp.text())
321
  .then(data => {
322
+ logMessage("Index successful")
323
  enableButtons()
324
  })
325
  } else {
326
  fetch("/index_tdocs/working_group", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({wg: document.getElementById("tdocs-wg-option").value})})
327
  .then(resp => resp.text())
328
  .then(data => {
329
+ logMessage("Index successful")
330
  enableButtons()
331
  })
332
  }
 
334
  fetch("/index_tdocs/all", {method: "POST", headers: {"Content-Type": "application/json"}})
335
  .then(resp => resp.text())
336
  .then(data => {
337
+ logMessage("Index successful")
338
  enableButtons()
339
  })
340
  }
 
346
  fetch("/index_specs/3gpp", {method: "POST", headers: {"Content-Type": "application/json"}})
347
  .then(resp => resp.text())
348
  .then(data => {
349
+ logMessage("Index successful")
350
  enableButtons()
351
  })
352
  });
 
357
  fetch("/index_specs/etsi", {method: "POST", headers: {"Content-Type": "application/json"}})
358
  .then(resp => resp.text())
359
  .then(data => {
360
+ logMessage("Index successful")
361
  enableButtons()
362
  })
363
  });