Update app.py

app.py

@@ -47,9 +47,9 @@ except ImportError:
 # ───────────────── Config & Constants ───────────────────────────────────────
 TMP_ROOT = Path(tempfile.gettempdir()) / "rf_datasets"
 TMP_ROOT.mkdir(parents=True, exist_ok=True)
-CPU_COUNT   = int(os.getenv("QC_CPU",        1))   # force single-core by default
-BATCH_SIZE  = int(os.getenv("QC_BATCH",      4))   # small batches
-SAMPLE_LIMIT = int(os.getenv("QC_SAMPLE",   200))
+CPU_COUNT    = int(os.getenv("QC_CPU",      1))   # force single-core by default
+BATCH_SIZE   = int(os.getenv("QC_BATCH",    4))   # small batches
+SAMPLE_LIMIT = int(os.getenv("QC_SAMPLE", 200))
 
 DEFAULT_W = {
     "Integrity":     0.25,
@@ -70,7 +70,7 @@ class QCConfig:
     weights:     str | None
     cpu_count:   int = CPU_COUNT
     batch_size:  int = BATCH_SIZE
-    sample_limit:int = SAMPLE_LIMIT
+    sample_limit: int = SAMPLE_LIMIT
 
 # ─────────── Helpers & Caching ─────────────────────────────────────────────
 def load_yaml(path: Path) -> Dict:
@@ -171,12 +171,12 @@ def qc_class_balance(lbls: List[Path], cfg: QCConfig) -> Dict:
         boxes.append(len(bs))
         counts.update(b[0] for b in bs)
     if not counts:
-        return {"name":"Class balance","score":0,"details":"No labels"}
+        return {"name": "Class balance", "score": 0, "details": "No labels"}
     bal = min(counts.values()) / max(counts.values()) * 100
     return {
-        "name":"Class balance",
-        "score":bal,
-        "details":{
+        "name": "Class balance",
+        "score": bal,
+        "details": {
             "class_counts": dict(counts),
             "boxes_per_image": {
                 "min": min(boxes),
@@ -188,7 +188,7 @@ def qc_class_balance(lbls: List[Path], cfg: QCConfig) -> Dict:
 
 def qc_image_quality(imgs: List[Path], cfg: QCConfig) -> Dict:
     if cv2 is None:
-        return {"name":"Image quality","score":100,"details":"cv2 missing"}
+        return {"name": "Image quality", "score": 100, "details": "cv2 missing"}
     blurry, dark, bright = [], [], []
     sample = imgs[:cfg.sample_limit]
     with ThreadPoolExecutor(max_workers=cfg.cpu_count) as ex:
@@ -200,9 +200,9 @@ def qc_image_quality(imgs: List[Path], cfg: QCConfig) -> Dict:
     bad = len({*blurry, *dark, *bright})
     score = 100 - bad / max(len(sample), 1) * 100
     return {
-        "name":"Image quality",
-        "score":score,
-        "details":{
+        "name": "Image quality",
+        "score": score,
+        "details": {
             "blurry": [str(p) for p in blurry],
             "dark":   [str(p) for p in dark],
             "bright": [str(p) for p in bright]
@@ -214,31 +214,36 @@ def qc_duplicates(imgs: List[Path], cfg: QCConfig) -> Dict:
         try:
             fd = fastdup.create(
                 input_dir=str(Path(imgs[0]).parent.parent),
-                work_dir=str(TMP_ROOT/'fastdup')
+                work_dir=str(TMP_ROOT / "fastdup")
             )
             fd.run()
-            clusters = fd.get_clusters()
-            dup = sum(len(c)-1 for c in clusters)
+            cc = fd.connected_components_grouped(sort_by="comp_size", ascending=False)
+            clusters = cc["files"].tolist()
+            dup = sum(len(c) - 1 for c in clusters)
             return {
-                "name":"Duplicates",
-                "score":100-dup/len(imgs)*100,
-                "details":{"groups":clusters[:50]}
+                "name": "Duplicates",
+                "score": max(0.0, 100 - dup / len(imgs) * 100),
+                "details": {"groups": clusters[:50]}
             }
         except Exception as e:
-            return {"name":"Duplicates","score":100,"details":{"fastdup_error":str(e)}}
-    return {"name":"Duplicates","score":100,"details":{"note":"skipped"}}
+            return {
+                "name": "Duplicates",
+                "score": 100.0,
+                "details": {"fastdup_error": str(e)}
+            }
+    return {"name": "Duplicates", "score": 100.0, "details": {"note": "skipped"}}
 
 def qc_model_qa(imgs: List[Path], lbls: List[Path], cfg: QCConfig) -> Dict:
     model = get_model(cfg.weights)
     if model is None:
-        return {"name":"Model QA","score":100,"details":"skipped"}
+        return {"name": "Model QA", "score": 100, "details": "skipped"}
     ious, mism = [], []
     sample = imgs[:cfg.sample_limit]
     for i in range(0, len(sample), cfg.batch_size):
-        batch = sample[i:i+cfg.batch_size]
+        batch = sample[i:i + cfg.batch_size]
         results = model.predict(batch, verbose=False, half=True, dynamic=True)
         for p, res in zip(batch, results):
-            gt = parse_label_file(Path(p).parent.parent/'labels'/f"{Path(p).stem}.txt")
+            gt = parse_label_file(Path(p).parent.parent / 'labels' / f"{Path(p).stem}.txt")
             for cls, x, y, w, h in gt:
                 best = 0.0
                 for b, c, conf in zip(
@@ -254,52 +259,51 @@ def qc_model_qa(imgs: List[Path], lbls: List[Path], cfg: QCConfig) -> Dict:
                     mism.append(str(p))
     miou = float(np.mean(ious)) if ious else 1.0
     return {
-        "name":"Model QA",
-        "score":miou*100,
-        "details":{"mean_iou":miou, "mismatches":mism[:50]}
+        "name": "Model QA",
+        "score": miou * 100,
+        "details": {"mean_iou": miou, "mismatches": mism[:50]}
     }
 
 def qc_label_issues(imgs: List[Path], lbls: List[Path], cfg: QCConfig) -> Dict:
     if get_noise_indices is None:
-        return {"name":"Label issues","score":100,"details":"skipped"}
+        return {"name": "Label issues", "score": 100, "details": "skipped"}
     labels, idxs = [], []
     sample = imgs[:cfg.sample_limit]
-    model  = get_model(cfg.weights)
     for i, p in enumerate(sample):
         bs = parse_label_file(lbls[i]) if lbls[i] else []
         for cls, *_ in bs:
             labels.append(int(cls))
             idxs.append(i)
     if not labels:
-        return {"name":"Label issues","score":100,"details":"no GT"}
+        return {"name": "Label issues", "score": 100, "details": "no GT"}
     labels_arr = np.array(labels)
     uniq       = sorted(set(labels_arr))
     probs      = np.eye(len(uniq))[np.searchsorted(uniq, labels_arr)]
     noise      = get_noise_indices(labels=labels_arr, probabilities=probs)
     flags      = sorted({idxs[n] for n in noise})
     files      = [str(sample[i]) for i in flags]
-    score      = 100 - len(flags)/len(labels)*100
+    score      = 100 - len(flags) / len(labels) * 100
     return {
-        "name":"Label issues",
-        "score":score,
-        "details":{"files":files[:50]}
+        "name": "Label issues",
+        "score": score,
+        "details": {"files": files[:50]}
     }
 
 def _rel_iou(b1, b2):
     x1, y1, w1, h1 = b1
     x2, y2, w2, h2 = b2
-    xa1, ya1 = x1-w1/2, y1-h1/2
-    xa2, ya2 = x1+w1/2, y1+h1/2
-    xb1, yb1 = x2-w2/2, y2-h2/2
-    xb2, yb2 = x2+w2/2, y2+h2/2
+    xa1, ya1 = x1 - w1/2, y1 - h1/2
+    xa2, ya2 = x1 + w1/2, y1 + h1/2
+    xb1, yb1 = x2 - w2/2, y2 - h2/2
+    xb2, yb2 = x2 + w2/2, y2 + h2/2
     ix1 = max(xa1, xb1); iy1 = max(ya1, yb1)
     ix2 = min(xa2, xb2); iy2 = min(ya2, yb2)
-    inter = max(ix2-ix1, 0) * max(iy2-iy1, 0)
-    union = w1*h1 + w2*h2 - inter
-    return inter/union if union else 0.0
+    inter = max(ix2 - ix1, 0) * max(iy2 - iy1, 0)
+    union = w1 * h1 + w2 * h2 - inter
+    return inter / union if union else 0.0
 
 def aggregate(results: List[Dict]) -> float:
-    return sum(DEFAULT_W[r['name']]*r['score'] for r in results)
+    return sum(DEFAULT_W[r["name"]] * r["score"] for r in results)
 
 RF_RE = re.compile(r"https?://universe\.roboflow\.com/([^/]+)/([^/]+)/dataset/(\d+)")
 
@@ -308,7 +312,7 @@ def download_rf_dataset(url: str, rf_api: Roboflow, dest: Path) -> Path:
     if not m:
         raise ValueError(f"Bad RF URL: {url}")
     ws, proj, ver = m.groups()
-    ds_dir = dest/f"{ws}_{proj}_v{ver}"
+    ds_dir = dest / f"{ws}_{proj}_v{ver}"
     if ds_dir.exists():
         return ds_dir
     pr = rf_api.workspace(ws).project(proj)
@@ -328,9 +332,9 @@ def run_quality(
         qc_integrity(imgs, lbls, cfg),
         qc_class_balance(lbls, cfg),
         qc_image_quality(imgs, cfg),
-        qc_duplicates(imgs, cfg)      if run_dup     else {"name":"Duplicates","score":100,"details":"skipped"},
-        qc_model_qa(imgs, lbls, cfg)  if run_modelqa else {"name":"Model QA","score":100,"details":"skipped"},
-        qc_label_issues(imgs, lbls, cfg) if run_modelqa else {"name":"Label issues","score":100,"details":"skipped"},
+        qc_duplicates(imgs, cfg)      if run_dup     else {"name": "Duplicates", "score": 100, "details": "skipped"},
+        qc_model_qa(imgs, lbls, cfg)  if run_modelqa else {"name": "Model QA",   "score": 100, "details": "skipped"},
+        qc_label_issues(imgs, lbls, cfg) if run_modelqa else {"name": "Label issues", "score": 100, "details": "skipped"},
     ]
     final = aggregate(results)
 
@@ -338,14 +342,14 @@ def run_quality(
     for r in results:
         md.append(f"### {r['name']}  {r['score']:.1f}")
         md.append("<details><summary>details</summary>\n```json")
-        md.append(json.dumps(r['details'], indent=2))
+        md.append(json.dumps(r["details"], indent=2))
         md.append("```\n</details>\n")
 
     df = pd.DataFrame.from_dict(
-        next(r for r in results if r['name']=='Class balance')['details']['class_counts'],
-        orient='index', columns=['count']
+        next(r for r in results if r["name"] == "Class balance")["details"]["class_counts"],
+        orient="index", columns=["count"]
     )
-    df.index.name = 'class'
+    df.index.name = "class"
     return "\n".join(md), df
 
 with gr.Blocks(title="YOLO Dataset Quality Evaluator v3") as demo:
@@ -394,9 +398,9 @@ with gr.Blocks(title="YOLO Dataset Quality Evaluator v3") as demo:
                 if not line.strip():
                     continue
                 try:
-                    ds = download_rf_dataset(line, rf, TMP_ROOT)
-                    md, df = run_quality(
-                        ds, None,
+                    ds, md, df = download_rf_dataset(line, rf, TMP_ROOT), *run_quality(
+                        download_rf_dataset(line, rf, TMP_ROOT),
+                        None,
                         Path(weights.name) if weights else None,
                         cfg, run_dup, run_modelqa
                     )