Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -9,19 +9,30 @@ import pytesseract
|
|
9 |
from werkzeug.utils import secure_filename
|
10 |
from langchain_groq import ChatGroq
|
11 |
from langgraph.prebuilt import create_react_agent
|
12 |
-
from pdf2image import convert_from_path
|
13 |
from concurrent.futures import ThreadPoolExecutor
|
14 |
from pdf2image.exceptions import PDFInfoNotInstalledError
|
15 |
from typing import Dict, TypedDict, Optional, Any
|
16 |
from langgraph.graph import StateGraph, END
|
17 |
import uuid
|
18 |
-
import shutil, time
|
19 |
from langchain_experimental.open_clip.open_clip import OpenCLIPEmbeddings
|
20 |
# from matplotlib.offsetbox import OffsetImage, AnnotationBbox
|
21 |
from io import BytesIO
|
22 |
from pathlib import Path
|
23 |
import os
|
24 |
from utils.block_relation_builder import block_builder, variable_adder_main
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
25 |
global pdf_doc
|
26 |
# ============================== #
|
27 |
# INITIALIZE CLIP EMBEDDER #
|
@@ -64,7 +75,8 @@ pytesseract.pytesseract.tesseract_cmd = (r'/usr/bin/tesseract')
|
|
64 |
|
65 |
# poppler_path = r"C:\poppler\Library\bin"
|
66 |
backdrop_images_path = r"app\blocks\Backdrops"
|
67 |
-
sprite_images_path = r"blocks\sprites"
|
|
|
68 |
|
69 |
count = 0
|
70 |
|
@@ -74,12 +86,12 @@ STATIC_DIR = BASE_DIR / "static"
|
|
74 |
GEN_PROJECT_DIR = BASE_DIR / "generated_projects"
|
75 |
BACKDROP_DIR = BLOCKS_DIR / "Backdrops"
|
76 |
SPRITE_DIR = BLOCKS_DIR / "sprites"
|
77 |
-
|
78 |
# === new: outputs rooted under BASE_DIR ===
|
79 |
OUTPUT_DIR = BASE_DIR / "outputs"
|
80 |
-
DETECTED_IMAGE_DIR = OUTPUT_DIR / "DETECTED_IMAGE"
|
81 |
-
SCANNED_IMAGE_DIR = OUTPUT_DIR / "SCANNED_IMAGE"
|
82 |
-
JSON_DIR = OUTPUT_DIR / "EXTRACTED_JSON"
|
83 |
|
84 |
# make all of them in one go
|
85 |
for d in (
|
@@ -88,10 +100,11 @@ for d in (
|
|
88 |
GEN_PROJECT_DIR,
|
89 |
BACKDROP_DIR,
|
90 |
SPRITE_DIR,
|
|
|
91 |
OUTPUT_DIR,
|
92 |
-
DETECTED_IMAGE_DIR,
|
93 |
-
SCANNED_IMAGE_DIR,
|
94 |
-
JSON_DIR,
|
95 |
):
|
96 |
d.mkdir(parents=True, exist_ok=True)
|
97 |
# def classify_image_type(description_or_name: str) -> str:
|
@@ -124,6 +137,9 @@ class GameState(TypedDict):
|
|
124 |
action_plan: Optional[Dict]
|
125 |
temporary_node: Optional[Dict]
|
126 |
|
|
|
|
|
|
|
127 |
|
128 |
# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables
|
129 |
SYSTEM_PROMPT = """
|
@@ -223,7 +239,7 @@ agent_json_resolver = create_react_agent(
|
|
223 |
prompt=SYSTEM_PROMPT_JSON_CORRECTOR
|
224 |
)
|
225 |
|
226 |
-
# Helper function to load the block catalog from a JSON file
|
227 |
# def _load_block_catalog(file_path: str) -> Dict:
|
228 |
# """Loads the Scratch block catalog from a specified JSON file."""
|
229 |
# try:
|
@@ -287,26 +303,8 @@ def find_block_in_all(opcode: str, all_catalogs: list[dict]) -> dict | None:
|
|
287 |
return None
|
288 |
|
289 |
|
290 |
-
|
291 |
-
# --- Global variable for the block catalog ---
|
292 |
# --- Global variable for the block catalog ---
|
293 |
ALL_SCRATCH_BLOCKS_CATALOG = {}
|
294 |
-
# BLOCK_CATALOG_PATH = r"blocks\blocks.json" # Define the path to your JSON file
|
295 |
-
# HAT_BLOCKS_PATH = r"blocks\hat_blocks.json" # Path to the hat blocks JSON file
|
296 |
-
# STACK_BLOCKS_PATH = r"blocks\stack_blocks.json" # Path to the stack blocks JSON file
|
297 |
-
# REPORTER_BLOCKS_PATH = r"blocks\reporter_blocks.json" # Path to the reporter blocks JSON file
|
298 |
-
# BOOLEAN_BLOCKS_PATH = r"blocks\boolean_blocks.json" # Path to the boolean blocks JSON file
|
299 |
-
# C_BLOCKS_PATH = r"blocks\c_blocks.json" # Path to the C blocks JSON file
|
300 |
-
# CAP_BLOCKS_PATH = r"blocks\cap_blocks.json" # Path to the cap blocks JSON file
|
301 |
-
|
302 |
-
# BLOCK_CATALOG_PATH = r"blocks/blocks.json"
|
303 |
-
# HAT_BLOCKS_PATH = r"blocks/hat_blocks.json"
|
304 |
-
# STACK_BLOCKS_PATH = r"blocks/stack_blocks.json"
|
305 |
-
# REPORTER_BLOCKS_PATH = r"blocks/reporter_blocks.json"
|
306 |
-
# BOOLEAN_BLOCKS_PATH = r"blocks/boolean_blocks.json"
|
307 |
-
# C_BLOCKS_PATH = r"blocks/c_blocks.json"
|
308 |
-
# CAP_BLOCKS_PATH = r"blocks/cap_blocks.json"
|
309 |
-
|
310 |
BLOCK_CATALOG_PATH = "blocks" # Define the path to your JSON file
|
311 |
HAT_BLOCKS_PATH = "hat_blocks" # Path to the hat blocks JSON file
|
312 |
STACK_BLOCKS_PATH = "stack_blocks" # Path to the stack blocks JSON file
|
@@ -315,15 +313,24 @@ BOOLEAN_BLOCKS_PATH = "boolean_blocks" # Path to the boolean blocks JSON file
|
|
315 |
C_BLOCKS_PATH = "c_blocks" # Path to the C blocks JSON file
|
316 |
CAP_BLOCKS_PATH = "cap_blocks" # Path to the cap blocks JSON file
|
317 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
318 |
# Load the block catalogs from their respective JSON files
|
319 |
hat_block_data = _load_block_catalog(HAT_BLOCKS_PATH)
|
320 |
-
|
321 |
-
hat_description = hat_block_data.get("description", "No description available")
|
322 |
# hat_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']} example: standalone use: {block['example_standalone']}" for block in hat_block_data["blocks"]])
|
323 |
hat_opcodes_functionalities = "\n".join([
|
324 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
325 |
for block in hat_block_data.get("blocks", [])
|
326 |
]) if isinstance(hat_block_data.get("blocks"), list) else " No blocks information available."
|
|
|
327 |
print("Hat blocks loaded successfully.", hat_description)
|
328 |
|
329 |
boolean_block_data = _load_block_catalog(BOOLEAN_BLOCKS_PATH)
|
@@ -333,6 +340,7 @@ boolean_opcodes_functionalities = "\n".join([
|
|
333 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
334 |
for block in boolean_block_data.get("blocks", [])
|
335 |
]) if isinstance(boolean_block_data.get("blocks"), list) else " No blocks information available."
|
|
|
336 |
|
337 |
c_block_data = _load_block_catalog(C_BLOCKS_PATH)
|
338 |
c_description = c_block_data["description"]
|
@@ -341,6 +349,7 @@ c_opcodes_functionalities = "\n".join([
|
|
341 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
342 |
for block in c_block_data.get("blocks", [])
|
343 |
]) if isinstance(c_block_data.get("blocks"), list) else " No blocks information available."
|
|
|
344 |
|
345 |
cap_block_data = _load_block_catalog(CAP_BLOCKS_PATH)
|
346 |
cap_description = cap_block_data["description"]
|
@@ -349,6 +358,7 @@ cap_opcodes_functionalities = "\n".join([
|
|
349 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
350 |
for block in cap_block_data.get("blocks", [])
|
351 |
]) if isinstance(cap_block_data.get("blocks"), list) else " No blocks information available."
|
|
|
352 |
|
353 |
reporter_block_data = _load_block_catalog(REPORTER_BLOCKS_PATH)
|
354 |
reporter_description = reporter_block_data["description"]
|
@@ -357,6 +367,7 @@ reporter_opcodes_functionalities = "\n".join([
|
|
357 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
358 |
for block in reporter_block_data.get("blocks", [])
|
359 |
]) if isinstance(reporter_block_data.get("blocks"), list) else " No blocks information available."
|
|
|
360 |
|
361 |
stack_block_data = _load_block_catalog(STACK_BLOCKS_PATH)
|
362 |
stack_description = stack_block_data["description"]
|
@@ -365,6 +376,7 @@ stack_opcodes_functionalities = "\n".join([
|
|
365 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
366 |
for block in stack_block_data.get("blocks", [])
|
367 |
]) if isinstance(stack_block_data.get("blocks"), list) else " No blocks information available."
|
|
|
368 |
|
369 |
# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally
|
370 |
ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH)
|
@@ -435,6 +447,49 @@ def extract_json_from_llm_response(raw_response: str) -> dict:
|
|
435 |
logger.error("Sanitized JSON still invalid:\n%s", json_string)
|
436 |
raise
|
437 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
438 |
# Node 1: Logic updating if any issue here
|
439 |
def pseudo_generator_node(state: GameState):
|
440 |
logger.info("--- Running plan_logic_aligner_node ---")
|
@@ -598,7 +653,8 @@ If you find any "Code-Blocks" then,
|
|
598 |
image_input = {
|
599 |
"type": "image_url",
|
600 |
"image_url": {
|
601 |
-
"url": f"data:image/png;base64,{image}"
|
|
|
602 |
}
|
603 |
}
|
604 |
|
@@ -1458,86 +1514,86 @@ Example output:
|
|
1458 |
return state
|
1459 |
|
1460 |
# Node 10:Function based block builder node
|
1461 |
-
|
1462 |
-
|
1463 |
-
|
1464 |
-
|
1465 |
-
|
1466 |
-
#
|
1467 |
-
|
1468 |
-
|
1469 |
-
|
1470 |
-
|
1471 |
-
|
1472 |
-
|
1473 |
-
|
1474 |
-
|
1475 |
-
|
1476 |
-
|
1477 |
-
|
1478 |
-
#
|
1479 |
-
|
1480 |
-
|
1481 |
-
|
1482 |
-
#
|
1483 |
-
|
1484 |
-
|
1485 |
-
|
1486 |
-
|
1487 |
-
|
1488 |
-
#
|
1489 |
-
|
1490 |
-
|
1491 |
-
|
1492 |
-
|
1493 |
-
|
1494 |
-
|
1495 |
-
|
1496 |
-
|
1497 |
-
|
1498 |
-
|
1499 |
-
|
1500 |
-
|
1501 |
-
|
1502 |
-
#
|
1503 |
-
|
1504 |
-
|
1505 |
-
|
1506 |
-
|
1507 |
-
|
1508 |
-
|
1509 |
-
|
1510 |
-
|
1511 |
-
|
1512 |
-
|
1513 |
-
|
1514 |
-
|
1515 |
-
|
1516 |
-
|
1517 |
-
|
1518 |
-
|
1519 |
-
|
1520 |
-
|
1521 |
-
|
1522 |
-
#
|
1523 |
-
|
1524 |
-
|
1525 |
-
|
1526 |
-
|
1527 |
-
|
1528 |
-
|
1529 |
-
|
1530 |
-
|
1531 |
-
|
1532 |
-
|
1533 |
-
|
1534 |
-
|
1535 |
|
1536 |
-
|
1537 |
-
#
|
1538 |
-
#
|
1539 |
|
1540 |
-
|
1541 |
# Node 10:Function based block builder node
|
1542 |
def overall_block_builder_node_2(state: dict):
|
1543 |
logger.info("--- Running OverallBlockBuilderNode ---")
|
@@ -1641,46 +1697,273 @@ def variable_adder_node(state: GameState):
|
|
1641 |
raise
|
1642 |
|
1643 |
|
1644 |
-
scratch_keywords = [
|
1645 |
-
|
1646 |
-
|
1647 |
-
|
1648 |
-
|
1649 |
-
|
1650 |
-
|
1651 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1652 |
|
1653 |
#def extract_images_from_pdf(pdf_path: Path, json_base_dir: Path, image_base_dir: Path):
|
1654 |
#def extract_images_from_pdf(pdf_path: Path, json_base_dir: Path):
|
1655 |
-
|
|
|
|
|
|
|
|
|
|
|
1656 |
''' Extract images from PDF and generate structured sprite JSON '''
|
1657 |
try:
|
1658 |
-
|
1659 |
-
|
1660 |
-
|
1661 |
-
|
1662 |
-
print("-------------------------------
|
1663 |
-
#
|
1664 |
-
|
1665 |
-
|
1666 |
-
|
1667 |
-
|
1668 |
-
|
1669 |
-
|
1670 |
-
|
1671 |
-
|
1672 |
-
|
1673 |
-
#
|
1674 |
-
|
1675 |
-
|
1676 |
-
|
1677 |
-
|
1678 |
-
|
1679 |
-
|
|
|
|
|
|
|
|
|
1680 |
|
|
|
1681 |
try:
|
1682 |
elements = partition_pdf(
|
1683 |
-
filename=str(pdf_path), # partition_pdf might expect a string
|
|
|
1684 |
strategy="hi_res",
|
1685 |
extract_image_block_types=["Image"],
|
1686 |
hi_res_model_name="yolox",
|
@@ -1691,155 +1974,44 @@ def extract_images_from_pdf(pdf_path: Path):
|
|
1691 |
raise RuntimeError(
|
1692 |
f"❌ Failed to extract images from PDF: {str(e)}")
|
1693 |
|
1694 |
-
|
1695 |
-
|
1696 |
-
|
1697 |
-
|
1698 |
-
|
1699 |
-
|
1700 |
-
|
1701 |
-
|
1702 |
-
|
1703 |
-
|
1704 |
-
|
1705 |
-
|
1706 |
-
|
1707 |
-
|
1708 |
-
#
|
1709 |
-
|
|
|
1710 |
|
1711 |
-
|
1712 |
-
|
1713 |
-
|
1714 |
-
|
|
|
1715 |
|
1716 |
-
|
1717 |
-
|
1718 |
-
|
1719 |
-
|
1720 |
-
|
1721 |
-
|
1722 |
-
|
1723 |
-
|
1724 |
-
|
1725 |
-
prompt=system_prompt
|
1726 |
-
)
|
1727 |
-
|
1728 |
-
# If JSON already exists, load it and find the next available Sprite number
|
1729 |
-
if final_json_path.exists(): # Use Path.exists()
|
1730 |
-
with open(final_json_path, "r") as existing_file:
|
1731 |
-
manipulated = json.load(existing_file)
|
1732 |
-
# Determine the next available index (e.g., Sprite 4 if 1–3 already exist)
|
1733 |
-
existing_keys = [int(k.replace("Sprite ", ""))
|
1734 |
-
for k in manipulated.keys()]
|
1735 |
-
start_count = max(existing_keys, default=0) + 1
|
1736 |
-
else:
|
1737 |
-
start_count = 1
|
1738 |
-
|
1739 |
-
sprite_count = start_count
|
1740 |
-
for i, element in enumerate(file_elements):
|
1741 |
-
if "image_base64" in element["metadata"]:
|
1742 |
-
try:
|
1743 |
-
image_data = base64.b64decode(
|
1744 |
-
element["metadata"]["image_base64"])
|
1745 |
-
print(f"\n ------------------------------image_data: {image_data}")
|
1746 |
-
image = Image.open(BytesIO(image_data)).convert("RGB") # Use BytesIO here
|
1747 |
-
|
1748 |
-
image = upscale_image(image, scale=2)
|
1749 |
-
# image.show(title=f"Extracted Image {i+1}")
|
1750 |
-
|
1751 |
-
# MODIFIED: Store image directly to BytesIO to avoid saving to disk if not needed
|
1752 |
-
# and then converting back to base64.
|
1753 |
-
img_buffer = BytesIO()
|
1754 |
-
image.save(img_buffer, format="PNG")
|
1755 |
-
img_bytes = img_buffer.getvalue()
|
1756 |
-
img_base64 = base64.b64encode(img_bytes).decode("utf-8")
|
1757 |
-
print(f"\n------------------------------------------------Image_Base64: {img_base64}")
|
1758 |
-
# Optionally save image to disk if desired for debugging/permanent storage
|
1759 |
-
image_path = extracted_image_subdir / f"Sprite_{i+1}.png"
|
1760 |
-
image.save(image_path)
|
1761 |
-
|
1762 |
-
prompt_combined = """
|
1763 |
-
Analyze this image and return JSON with keys:# modify prompt for "name", if it detects "code-blocks only then give name as 'scratch-block'"
|
1764 |
-
{
|
1765 |
-
"name": "<short name or 'scratch blocks'>" ,
|
1766 |
-
"description": "<short description>"
|
1767 |
-
}
|
1768 |
-
Guidelines:
|
1769 |
-
- If image contains logical/code blocks from Scratch (e.g., move, turn, repeat, when clicked, etc.), use 'scratch-block' as the name.
|
1770 |
-
- If image is a character, object, or backdrop, give an appropriate descriptive name instead.
|
1771 |
-
- Avoid generic names like 'image1' or 'picture'.
|
1772 |
-
- Keep the response strictly in JSON format.
|
1773 |
-
"""
|
1774 |
-
|
1775 |
-
content = [
|
1776 |
-
{"type": "text", "text": prompt_combined},
|
1777 |
-
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"}}
|
1778 |
-
]
|
1779 |
-
|
1780 |
-
response = agent.invoke({"messages": [{"role": "user", "content": content}]})
|
1781 |
-
|
1782 |
-
# Ensure response is handled correctly, it might be a string that needs json.loads
|
1783 |
-
try:
|
1784 |
-
# Assuming the agent returns a dictionary with 'messages' key,
|
1785 |
-
# and the last message's content is the JSON string.
|
1786 |
-
response_content_str = response.get("messages", [])[-1].content
|
1787 |
-
result_json = json.loads(response_content_str)
|
1788 |
-
except (json.JSONDecodeError, IndexError, AttributeError) as e:
|
1789 |
-
logger.error(f"⚠️ Failed to parse agent response as JSON: {e}. Response was: {response}", exc_info=True)
|
1790 |
-
result_json = {} # Default to empty dict if parsing fails
|
1791 |
-
|
1792 |
-
try:
|
1793 |
-
name = result_json.get("name", "").strip()
|
1794 |
-
description = result_json.get("description", "").strip()
|
1795 |
-
except Exception as e:
|
1796 |
-
logger.error(f"⚠️ Failed to extract name/description from result_json: {str(e)}", exc_info=True)
|
1797 |
-
name = "unknown"
|
1798 |
-
description = "unknown"
|
1799 |
-
|
1800 |
-
manipulated_json[f"Sprite {sprite_count}"] = {
|
1801 |
-
"name": name,
|
1802 |
-
"base64": element["metadata"]["image_base64"],
|
1803 |
-
"file-path": pdf_dir_path,
|
1804 |
-
"description": description
|
1805 |
-
}
|
1806 |
-
print(f"\n ------------------elemente: {element['metadata']['image_base64']}")
|
1807 |
-
print(f"\n ------------------pdf_dir_path: {pdf_dir_path}")
|
1808 |
-
sprite_count += 1
|
1809 |
-
print(f"\n===================manipulated JSON: {manipulated_json}")
|
1810 |
-
except Exception as e:
|
1811 |
-
logger.error(f"⚠️ Error processing Sprite {i+1}: {str(e)}", exc_info=True)
|
1812 |
-
|
1813 |
-
# Save manipulated JSON
|
1814 |
-
with open(final_json_path, "w") as sprite_file:
|
1815 |
-
json.dump(manipulated_json, sprite_file, indent=4)
|
1816 |
-
|
1817 |
-
def is_code_block(name: str) -> bool:
|
1818 |
-
for kw in scratch_keywords:
|
1819 |
-
if kw.lower() in name.lower():
|
1820 |
-
return True
|
1821 |
-
return False
|
1822 |
-
|
1823 |
-
# Filter out code block images
|
1824 |
-
filtered_sprites = {}
|
1825 |
-
for key, value in manipulated_json.items():
|
1826 |
-
sprite_name = value.get("name", "")
|
1827 |
-
if not is_code_block(sprite_name):
|
1828 |
-
filtered_sprites[key] = value
|
1829 |
-
else:
|
1830 |
-
logger.info(f"🛑 Excluded code block-like image: {key}")
|
1831 |
-
|
1832 |
-
# Overwrite with filtered content
|
1833 |
-
with open(final_json_path_2, "w") as sprite_file:
|
1834 |
-
json.dump(filtered_sprites, sprite_file, indent=4)
|
1835 |
-
|
1836 |
-
# MODIFIED RETURN VALUE: Return the Path to the primary extracted_sprites.json file
|
1837 |
-
# and the directory where it's located.
|
1838 |
-
return final_json_path, json_subdir # Return the file path and its parent directory
|
1839 |
except Exception as e:
|
1840 |
raise RuntimeError(f"❌ Error in extract_images_from_pdf: {str(e)}")
|
1841 |
|
1842 |
-
def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
|
1843 |
logger.info("🔍 Running similarity matching…")
|
1844 |
os.makedirs(project_folder, exist_ok=True)
|
1845 |
|
@@ -1847,6 +2019,7 @@ def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
1847 |
# CHANGED: define normalized base-paths so startswith() checks work
|
1848 |
backdrop_base_path = os.path.normpath(str(BACKDROP_DIR))
|
1849 |
sprite_base_path = os.path.normpath(str(SPRITE_DIR))
|
|
|
1850 |
# ----------------------------------------
|
1851 |
|
1852 |
project_json_path = os.path.join(project_folder, "project.json")
|
@@ -1854,15 +2027,23 @@ def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
1854 |
# ==============================
|
1855 |
# READ SPRITE METADATA
|
1856 |
# ==============================
|
1857 |
-
with open(input_json_path, 'r') as f:
|
1858 |
-
|
1859 |
|
1860 |
-
sprite_ids,
|
1861 |
for sid, sprite in sprites_data.items():
|
1862 |
sprite_ids.append(sid)
|
1863 |
-
texts.append("This is " + sprite.get("description", sprite.get("name", "")))
|
1864 |
sprite_base64.append(sprite["base64"])
|
1865 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1866 |
# =========================================
|
1867 |
# Build the list of all candidate images
|
1868 |
# =========================================
|
@@ -1882,6 +2063,13 @@ def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
1882 |
SPRITE_DIR / "Centaur.sprite3" / "2373556e776cad3ba4d6ee04fc34550b.png",
|
1883 |
SPRITE_DIR / "Crab.sprite3" / "bear_element.png",
|
1884 |
SPRITE_DIR / "Soccer Ball.sprite3" / "cat_football.png",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1885 |
]
|
1886 |
folder_image_paths = [os.path.normpath(str(p)) for p in folder_image_paths]
|
1887 |
# =========================================
|
@@ -1891,24 +2079,31 @@ def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
1891 |
# -----------------------------------------
|
1892 |
with open(f"{BLOCKS_DIR}/embeddings.json", "r") as f:
|
1893 |
embedding_json = json.load(f)
|
1894 |
-
img_matrix = np.array([img["embeddings"] for img in embedding_json])
|
1895 |
|
1896 |
# =========================================
|
1897 |
# Decode & embed each sprite image
|
1898 |
# =========================================
|
1899 |
-
sprite_features = []
|
1900 |
-
for b64 in sprite_base64:
|
1901 |
-
|
1902 |
-
|
1903 |
-
|
1904 |
-
|
1905 |
-
|
1906 |
-
|
1907 |
-
|
1908 |
-
|
1909 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1910 |
sprite_matrix = np.vstack(sprite_features)
|
1911 |
-
|
|
|
1912 |
# =========================================
|
1913 |
# Compute similarities & pick best match
|
1914 |
# =========================================
|
@@ -2002,6 +2197,7 @@ def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
2002 |
else:
|
2003 |
logger.warning(f"No project.json in {matched_folder}")
|
2004 |
|
|
|
2005 |
# =========================================
|
2006 |
# Merge into final Scratch project.json
|
2007 |
# =========================================
|
@@ -2179,6 +2375,29 @@ def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
|
2179 |
# # logger.info(f"🎉 Final project saved: {project_json_path}")
|
2180 |
# return project_json_path
|
2181 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2182 |
def delay_for_tpm_node(state: GameState):
|
2183 |
logger.info("--- Running DelayForTPMNode ---")
|
2184 |
time.sleep(60) # Adjust the delay as needed
|
@@ -2229,7 +2448,8 @@ def upscale_image(image: Image.Image, scale: int = 2) -> Image.Image:
|
|
2229 |
except Exception as e:
|
2230 |
logger.error(f"❌ Error during image upscaling: {str(e)}")
|
2231 |
return image
|
2232 |
-
|
|
|
2233 |
def create_sb3_archive(project_folder, project_id):
|
2234 |
"""
|
2235 |
Zips the project folder and renames it to an .sb3 file.
|
@@ -2269,19 +2489,22 @@ def create_sb3_archive(project_folder, project_id):
|
|
2269 |
os.remove(sb3_path)
|
2270 |
return sb3_path
|
2271 |
|
2272 |
-
|
|
|
|
|
2273 |
"""
|
2274 |
-
Copies the PDF at `
|
2275 |
renaming it to <project_id>.pdf.
|
2276 |
|
2277 |
Args:
|
2278 |
-
|
2279 |
project_id (str): Your unique project identifier.
|
2280 |
|
2281 |
Returns:
|
2282 |
str: Path to the copied PDF in the generated directory,
|
2283 |
or None if something went wrong.
|
2284 |
-
"""
|
|
|
2285 |
try:
|
2286 |
# 1) Build the destination directory and base filename
|
2287 |
output_dir = GEN_PROJECT_DIR / project_id
|
@@ -2292,9 +2515,16 @@ def save_pdf_to_generated_dir(pdf_path: str, project_id: str) -> str:
|
|
2292 |
target_pdf = output_dir / f"{project_id}.pdf"
|
2293 |
print(f"\n--------------------------------target_pdf {target_pdf}")
|
2294 |
# 3) Copy the PDF
|
2295 |
-
|
2296 |
-
|
2297 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2298 |
|
2299 |
|
2300 |
return str(target_pdf)
|
@@ -2355,22 +2585,40 @@ def process_pdf():
|
|
2355 |
# Create empty json in project_{random_id} folder #
|
2356 |
# =========================================================================== #
|
2357 |
#os.makedirs(project_folder, exist_ok=True)
|
2358 |
-
|
|
|
2359 |
# Save the uploaded PDF temporarily
|
2360 |
-
filename = secure_filename(pdf_file.filename)
|
2361 |
-
temp_dir = tempfile.mkdtemp()
|
2362 |
-
saved_pdf_path = os.path.join(temp_dir, filename)
|
2363 |
-
pdf_file.save(saved_pdf_path)
|
2364 |
-
pdf_doc = saved_pdf_path
|
2365 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2366 |
# logger.info(f"Created project folder: {project_folder}")
|
2367 |
-
logger.info(f"Saved uploaded PDF to: {saved_pdf_path}")
|
2368 |
logger.info(f"Saved uploaded PDF to: {pdf_file}: {pdf}")
|
2369 |
-
print("--------------------------------pdf_file_path---------------------",pdf_file,
|
|
|
|
|
|
|
|
|
|
|
2370 |
# Extract & process
|
2371 |
-
# output_path, result = extract_images_from_pdf(saved_pdf_path
|
2372 |
-
|
2373 |
-
|
|
|
|
|
|
|
|
|
|
|
2374 |
# Check extracted_sprites.json for "scratch block" in any 'name'
|
2375 |
# extracted_dir = os.path.join(JSON_DIR, os.path.splitext(filename)[0])
|
2376 |
# extracted_sprites_json = os.path.join(extracted_dir, "extracted_sprites.json")
|
@@ -2380,14 +2628,16 @@ def process_pdf():
|
|
2380 |
|
2381 |
# with open(extracted_sprites_json, 'r') as f:
|
2382 |
# sprite_data = json.load(f)
|
2383 |
-
|
2384 |
-
|
2385 |
-
|
|
|
|
|
2386 |
|
2387 |
-
|
2388 |
-
|
2389 |
|
2390 |
-
# images = convert_from_path(
|
2391 |
# print(type)
|
2392 |
# page = images[0]
|
2393 |
# # img_base64 = base64.b64encode(images).decode("utf-8")
|
@@ -2396,31 +2646,39 @@ def process_pdf():
|
|
2396 |
# img_bytes = buf.getvalue()
|
2397 |
# img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
2398 |
#image_paths = await convert_pdf_to_images_async(saved_pdf_path)
|
2399 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2400 |
#updating logic here [Dev Patel]
|
2401 |
-
|
2402 |
-
|
2403 |
-
|
2404 |
-
|
2405 |
-
|
2406 |
-
|
2407 |
-
|
2408 |
-
|
2409 |
-
|
|
|
2410 |
|
2411 |
-
|
2412 |
|
2413 |
-
|
2414 |
# final_project_json = project_skeleton
|
2415 |
|
2416 |
# Save the *final* filled project JSON, overwriting the skeleton
|
2417 |
-
|
2418 |
-
|
2419 |
-
|
2420 |
|
2421 |
# --- Call the new function to create the .sb3 file ---
|
2422 |
-
|
2423 |
-
sb3_file_path = BACKDROP_DIR #create_sb3_archive(project_folder, project_id)
|
2424 |
|
2425 |
if sb3_file_path:
|
2426 |
logger.info(f"Successfully created SB3 file: {sb3_file_path}")
|
@@ -2435,8 +2693,7 @@ def process_pdf():
|
|
2435 |
"output_json": "output_path",
|
2436 |
"sprites": "result",
|
2437 |
"project_output_json": "project_output",
|
2438 |
-
|
2439 |
-
"test_url":r"https://prthm11-scratch-vision-game.hf.space/download_sb3/Event_test"
|
2440 |
})
|
2441 |
else:
|
2442 |
return jsonify(error="Failed to create SB3 archive"), 500
|
|
|
9 |
from werkzeug.utils import secure_filename
|
10 |
from langchain_groq import ChatGroq
|
11 |
from langgraph.prebuilt import create_react_agent
|
12 |
+
from pdf2image import convert_from_path, convert_from_bytes
|
13 |
from concurrent.futures import ThreadPoolExecutor
|
14 |
from pdf2image.exceptions import PDFInfoNotInstalledError
|
15 |
from typing import Dict, TypedDict, Optional, Any
|
16 |
from langgraph.graph import StateGraph, END
|
17 |
import uuid
|
18 |
+
import shutil, time, functools
|
19 |
from langchain_experimental.open_clip.open_clip import OpenCLIPEmbeddings
|
20 |
# from matplotlib.offsetbox import OffsetImage, AnnotationBbox
|
21 |
from io import BytesIO
|
22 |
from pathlib import Path
|
23 |
import os
|
24 |
from utils.block_relation_builder import block_builder, variable_adder_main
|
25 |
+
|
26 |
+
def log_execution_time(func):
|
27 |
+
@functools.wraps(func)
|
28 |
+
def wrapper(*args, **kwargs):
|
29 |
+
start_time = time.time()
|
30 |
+
result = func(*args, **kwargs)
|
31 |
+
end_time = time.time()
|
32 |
+
logger.info(f"⏱ {func.__name__} executed in {end_time - start_time:.2f} seconds")
|
33 |
+
return result
|
34 |
+
return wrapper
|
35 |
+
|
36 |
global pdf_doc
|
37 |
# ============================== #
|
38 |
# INITIALIZE CLIP EMBEDDER #
|
|
|
75 |
|
76 |
# poppler_path = r"C:\poppler\Library\bin"
|
77 |
backdrop_images_path = r"app\blocks\Backdrops"
|
78 |
+
sprite_images_path = r"app\blocks\sprites"
|
79 |
+
code_blocks_image_path = r"app\blocks\code_blocks"
|
80 |
|
81 |
count = 0
|
82 |
|
|
|
86 |
GEN_PROJECT_DIR = BASE_DIR / "generated_projects"
|
87 |
BACKDROP_DIR = BLOCKS_DIR / "Backdrops"
|
88 |
SPRITE_DIR = BLOCKS_DIR / "sprites"
|
89 |
+
CODE_BLOCKS_DIR = BLOCKS_DIR / "code_blocks"
|
90 |
# === new: outputs rooted under BASE_DIR ===
|
91 |
OUTPUT_DIR = BASE_DIR / "outputs"
|
92 |
+
# DETECTED_IMAGE_DIR = OUTPUT_DIR / "DETECTED_IMAGE"
|
93 |
+
# SCANNED_IMAGE_DIR = OUTPUT_DIR / "SCANNED_IMAGE"
|
94 |
+
# JSON_DIR = OUTPUT_DIR / "EXTRACTED_JSON"
|
95 |
|
96 |
# make all of them in one go
|
97 |
for d in (
|
|
|
100 |
GEN_PROJECT_DIR,
|
101 |
BACKDROP_DIR,
|
102 |
SPRITE_DIR,
|
103 |
+
CODE_BLOCKS_DIR,
|
104 |
OUTPUT_DIR,
|
105 |
+
# DETECTED_IMAGE_DIR,
|
106 |
+
# SCANNED_IMAGE_DIR,
|
107 |
+
# JSON_DIR,
|
108 |
):
|
109 |
d.mkdir(parents=True, exist_ok=True)
|
110 |
# def classify_image_type(description_or_name: str) -> str:
|
|
|
137 |
action_plan: Optional[Dict]
|
138 |
temporary_node: Optional[Dict]
|
139 |
|
140 |
+
# class GameState(TypedDict):
|
141 |
+
# image: str
|
142 |
+
# pseudo_node: Optional[Dict]
|
143 |
|
144 |
# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables
|
145 |
SYSTEM_PROMPT = """
|
|
|
239 |
prompt=SYSTEM_PROMPT_JSON_CORRECTOR
|
240 |
)
|
241 |
|
242 |
+
# # Helper function to load the block catalog from a JSON file
|
243 |
# def _load_block_catalog(file_path: str) -> Dict:
|
244 |
# """Loads the Scratch block catalog from a specified JSON file."""
|
245 |
# try:
|
|
|
303 |
return None
|
304 |
|
305 |
|
|
|
|
|
306 |
# --- Global variable for the block catalog ---
|
307 |
ALL_SCRATCH_BLOCKS_CATALOG = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
308 |
BLOCK_CATALOG_PATH = "blocks" # Define the path to your JSON file
|
309 |
HAT_BLOCKS_PATH = "hat_blocks" # Path to the hat blocks JSON file
|
310 |
STACK_BLOCKS_PATH = "stack_blocks" # Path to the stack blocks JSON file
|
|
|
313 |
C_BLOCKS_PATH = "c_blocks" # Path to the C blocks JSON file
|
314 |
CAP_BLOCKS_PATH = "cap_blocks" # Path to the cap blocks JSON file
|
315 |
|
316 |
+
# BLOCK_CATALOG_PATH = BLOCKS_DIR / "blocks.json"
|
317 |
+
# HAT_BLOCKS_PATH = BLOCKS_DIR / "hat_blocks.json"
|
318 |
+
# STACK_BLOCKS_PATH = BLOCKS_DIR / "stack_blocks.json"
|
319 |
+
# REPORTER_BLOCKS_PATH = BLOCKS_DIR / "reporter_blocks.json"
|
320 |
+
# BOOLEAN_BLOCKS_PATH = BLOCKS_DIR / "boolean_blocks.json"
|
321 |
+
# C_BLOCKS_PATH = BLOCKS_DIR / "c_blocks.json"
|
322 |
+
# CAP_BLOCKS_PATH = BLOCKS_DIR / "cap_blocks.json"
|
323 |
+
|
324 |
# Load the block catalogs from their respective JSON files
|
325 |
hat_block_data = _load_block_catalog(HAT_BLOCKS_PATH)
|
326 |
+
hat_description = hat_block_data["description"]
|
327 |
+
#hat_description = hat_block_data.get("description", "No description available")
|
328 |
# hat_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']} example: standalone use: {block['example_standalone']}" for block in hat_block_data["blocks"]])
|
329 |
hat_opcodes_functionalities = "\n".join([
|
330 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
331 |
for block in hat_block_data.get("blocks", [])
|
332 |
]) if isinstance(hat_block_data.get("blocks"), list) else " No blocks information available."
|
333 |
+
hat_opcodes_functionalities = os.path.join(BLOCKS_DIR, "hat_blocks.txt")
|
334 |
print("Hat blocks loaded successfully.", hat_description)
|
335 |
|
336 |
boolean_block_data = _load_block_catalog(BOOLEAN_BLOCKS_PATH)
|
|
|
340 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
341 |
for block in boolean_block_data.get("blocks", [])
|
342 |
]) if isinstance(boolean_block_data.get("blocks"), list) else " No blocks information available."
|
343 |
+
boolean_opcodes_functionalities = os.path.join(BLOCKS_DIR, "boolean_blocks.txt")
|
344 |
|
345 |
c_block_data = _load_block_catalog(C_BLOCKS_PATH)
|
346 |
c_description = c_block_data["description"]
|
|
|
349 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
350 |
for block in c_block_data.get("blocks", [])
|
351 |
]) if isinstance(c_block_data.get("blocks"), list) else " No blocks information available."
|
352 |
+
c_opcodes_functionalities = os.path.join(BLOCKS_DIR, "c_blocks.txt")
|
353 |
|
354 |
cap_block_data = _load_block_catalog(CAP_BLOCKS_PATH)
|
355 |
cap_description = cap_block_data["description"]
|
|
|
358 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
359 |
for block in cap_block_data.get("blocks", [])
|
360 |
]) if isinstance(cap_block_data.get("blocks"), list) else " No blocks information available."
|
361 |
+
cap_opcodes_functionalities = os.path.join(BLOCKS_DIR, "cap_blocks.txt")
|
362 |
|
363 |
reporter_block_data = _load_block_catalog(REPORTER_BLOCKS_PATH)
|
364 |
reporter_description = reporter_block_data["description"]
|
|
|
367 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
368 |
for block in reporter_block_data.get("blocks", [])
|
369 |
]) if isinstance(reporter_block_data.get("blocks"), list) else " No blocks information available."
|
370 |
+
reporter_opcodes_functionalities = os.path.join(BLOCKS_DIR, "reporter_blocks.txt")
|
371 |
|
372 |
stack_block_data = _load_block_catalog(STACK_BLOCKS_PATH)
|
373 |
stack_description = stack_block_data["description"]
|
|
|
376 |
f" - Opcode: {block.get('op_code', 'N/A')}, functionality: {block.get('functionality', 'N/A')}, example: standalone use {block.get('example_standalone', 'N/A')}"
|
377 |
for block in stack_block_data.get("blocks", [])
|
378 |
]) if isinstance(stack_block_data.get("blocks"), list) else " No blocks information available."
|
379 |
+
stack_opcodes_functionalities = os.path.join(BLOCKS_DIR, "stack_blocks.txt")
|
380 |
|
381 |
# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally
|
382 |
ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH)
|
|
|
447 |
logger.error("Sanitized JSON still invalid:\n%s", json_string)
|
448 |
raise
|
449 |
|
450 |
+
def clean_base64_for_model(raw_b64):
|
451 |
+
"""
|
452 |
+
Normalize input into a valid data:image/png;base64,<payload> string.
|
453 |
+
|
454 |
+
Accepts:
|
455 |
+
- a list of base64 strings → picks the first element
|
456 |
+
- a PIL Image instance → encodes to PNG/base64
|
457 |
+
- a raw base64 string → strips whitespace and data URI prefix
|
458 |
+
"""
|
459 |
+
if not raw_b64:
|
460 |
+
return ""
|
461 |
+
|
462 |
+
# 1. If it’s a list, take its first element
|
463 |
+
if isinstance(raw_b64, list):
|
464 |
+
raw_b64 = raw_b64[0] if raw_b64 else ""
|
465 |
+
if not raw_b64:
|
466 |
+
return ""
|
467 |
+
|
468 |
+
# 2. If it’s a PIL Image, convert to base64
|
469 |
+
if isinstance(raw_b64, Image.Image):
|
470 |
+
buf = io.BytesIO()
|
471 |
+
raw_b64.save(buf, format="PNG")
|
472 |
+
raw_b64 = base64.b64encode(buf.getvalue()).decode()
|
473 |
+
|
474 |
+
# 3. At this point it must be a string
|
475 |
+
if not isinstance(raw_b64, str):
|
476 |
+
raise TypeError(f"Expected base64 string or PIL Image, got {type(raw_b64)}")
|
477 |
+
|
478 |
+
# 4. Strip any existing data URI prefix, whitespace, or newlines
|
479 |
+
clean_b64 = re.sub(r"^data:image\/[a-zA-Z]+;base64,", "", raw_b64)
|
480 |
+
clean_b64 = clean_b64.replace("\n", "").replace("\r", "").strip()
|
481 |
+
|
482 |
+
# 5. Validate it’s proper base64
|
483 |
+
try:
|
484 |
+
base64.b64decode(clean_b64)
|
485 |
+
except Exception as e:
|
486 |
+
logger.error(f"Invalid Base64 passed to model: {e}")
|
487 |
+
raise
|
488 |
+
|
489 |
+
# 6. Return with the correct data URI prefix
|
490 |
+
return f"data:image/png;base64,{clean_b64}"
|
491 |
+
|
492 |
+
|
493 |
# Node 1: Logic updating if any issue here
|
494 |
def pseudo_generator_node(state: GameState):
|
495 |
logger.info("--- Running plan_logic_aligner_node ---")
|
|
|
653 |
image_input = {
|
654 |
"type": "image_url",
|
655 |
"image_url": {
|
656 |
+
# "url": f"data:image/png;base64,{image}"
|
657 |
+
"url": clean_base64_for_model(image)
|
658 |
}
|
659 |
}
|
660 |
|
|
|
1514 |
return state
|
1515 |
|
1516 |
# Node 10:Function based block builder node
|
1517 |
+
def overall_block_builder_node_2(state: dict):
|
1518 |
+
logger.info("--- Running OverallBlockBuilderNode ---")
|
1519 |
+
|
1520 |
+
project_json = state["project_json"]
|
1521 |
+
targets = project_json["targets"]
|
1522 |
+
# --- Sprite and Stage Target Mapping ---
|
1523 |
+
sprite_map = {target["name"]: target for target in targets if not target["isStage"]}
|
1524 |
+
stage_target = next((target for target in targets if target["isStage"]), None)
|
1525 |
+
if stage_target:
|
1526 |
+
sprite_map[stage_target["name"]] = stage_target
|
1527 |
+
|
1528 |
+
action_plan = state.get("action_plan", {})
|
1529 |
+
print("[Overall Action Plan received at the block generator]:", json.dumps(action_plan, indent=2))
|
1530 |
+
if not action_plan:
|
1531 |
+
logger.warning("No action plan found in state. Skipping OverallBlockBuilderNode.")
|
1532 |
+
return state
|
1533 |
+
|
1534 |
+
# Initialize offsets for script placement on the Scratch canvas
|
1535 |
+
script_y_offset = {}
|
1536 |
+
script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()}
|
1537 |
+
|
1538 |
+
# This handles potential variations in the action_plan structure.
|
1539 |
+
if action_plan.get("action_overall_flow", {}) == {}:
|
1540 |
+
plan_data = action_plan.items()
|
1541 |
+
else:
|
1542 |
+
plan_data = action_plan.get("action_overall_flow", {}).items()
|
1543 |
+
|
1544 |
+
# --- Extract global project context for LLM ---
|
1545 |
+
all_sprite_names = list(sprite_map.keys())
|
1546 |
+
all_variable_names = {}
|
1547 |
+
all_list_names = {}
|
1548 |
+
all_broadcast_messages = {}
|
1549 |
+
|
1550 |
+
for target in targets:
|
1551 |
+
for var_id, var_info in target.get("variables", {}).items():
|
1552 |
+
all_variable_names[var_info[0]] = var_id # Store name -> ID mapping (e.g., "myVariable": "myVarId123")
|
1553 |
+
for list_id, list_info in target.get("lists", {}).items():
|
1554 |
+
all_list_names[list_info[0]] = list_id # Store name -> ID mapping
|
1555 |
+
for broadcast_id, broadcast_name in target.get("broadcasts", {}).items():
|
1556 |
+
all_broadcast_messages[broadcast_name] = broadcast_id # Store name -> ID mapping
|
1557 |
+
|
1558 |
+
# --- Process each sprite's action plan ---
|
1559 |
+
for sprite_name, sprite_actions_data in plan_data:
|
1560 |
+
if sprite_name in sprite_map:
|
1561 |
+
current_sprite_target = sprite_map[sprite_name]
|
1562 |
+
if "blocks" not in current_sprite_target:
|
1563 |
+
current_sprite_target["blocks"] = {}
|
1564 |
+
|
1565 |
+
if sprite_name not in script_y_offset:
|
1566 |
+
script_y_offset[sprite_name] = 0
|
1567 |
+
|
1568 |
+
for plan_entry in sprite_actions_data.get("plans", []):
|
1569 |
+
logic_sequence = str(plan_entry["logic"])
|
1570 |
+
opcode_counts = plan_entry.get("opcode_counts", {})
|
1571 |
+
|
1572 |
+
try:
|
1573 |
+
generated_blocks=block_builder(opcode_counts,logic_sequence)
|
1574 |
+
if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict):
|
1575 |
+
logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.")
|
1576 |
+
generated_blocks = generated_blocks["blocks"]
|
1577 |
+
|
1578 |
+
# Update block positions for top-level script
|
1579 |
+
for block_id, block_data in generated_blocks.items():
|
1580 |
+
if block_data.get("topLevel"):
|
1581 |
+
block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0)
|
1582 |
+
block_data["y"] = script_y_offset[sprite_name]
|
1583 |
+
script_y_offset[sprite_name] += 150 # Increment for next script
|
1584 |
+
|
1585 |
+
current_sprite_target["blocks"].update(generated_blocks)
|
1586 |
+
print(f"[current_sprite_target block updated]: {current_sprite_target['blocks']}")
|
1587 |
+
state["iteration_count"] = 0
|
1588 |
+
logger.info(f"Action blocks added for sprite '{sprite_name}' by OverallBlockBuilderNode.")
|
1589 |
+
except Exception as e:
|
1590 |
+
logger.error(f"Error generating blocks for sprite '{sprite_name}': {e}")
|
1591 |
|
1592 |
+
state["project_json"] = project_json
|
1593 |
+
# with open("debug_state.json", "w", encoding="utf-8") as f:
|
1594 |
+
# json.dump(state, f, indent=2, ensure_ascii=False)
|
1595 |
|
1596 |
+
return state
|
1597 |
# Node 10:Function based block builder node
|
1598 |
def overall_block_builder_node_2(state: dict):
|
1599 |
logger.info("--- Running OverallBlockBuilderNode ---")
|
|
|
1697 |
raise
|
1698 |
|
1699 |
|
1700 |
+
# scratch_keywords = [
|
1701 |
+
# "move", "turn", "wait", "repeat", "if", "else", "broadcast",
|
1702 |
+
# "glide", "change", "forever", "when", "switch",
|
1703 |
+
# "next costume", "set", "show", "hide", "play sound",
|
1704 |
+
# "go to", "x position", "y position", "think", "say",
|
1705 |
+
# "variable", "stop", "clone",
|
1706 |
+
# "touching", "sensing", "pen", "clear","Scratch","Code","scratch blocks"
|
1707 |
+
# ]
|
1708 |
+
|
1709 |
+
# Node 6: Logic updating if any issue here
|
1710 |
+
# def plan_logic_aligner_node(state: GameState):
|
1711 |
+
# logger.info("--- Running plan_logic_aligner_node ---")
|
1712 |
+
|
1713 |
+
# image = state.get("image", "")
|
1714 |
+
|
1715 |
+
# refinement_prompt = f"""
|
1716 |
+
# You are an expert in Scratch 3.0 game development, specializing in understanding block relationships (stacked, nested).
|
1717 |
+
# "Analyze the Scratch code-block image and generate Pseudo-Code for what this logic appears to be doing."
|
1718 |
+
# From Image, you also have to detect a value of Key given in Text form "Script for: ". Below is the example
|
1719 |
+
# Example: "Script for: Bear", "Script for:" is a key and "Bear" is value.
|
1720 |
+
# --- Scratch 3.0 Block Reference ---
|
1721 |
+
# ### Hat Blocks
|
1722 |
+
# Description: {hat_description}
|
1723 |
+
# Blocks:
|
1724 |
+
# {hat_opcodes_functionalities}
|
1725 |
+
|
1726 |
+
# ### Boolean Blocks
|
1727 |
+
# Description: {boolean_description}
|
1728 |
+
# Blocks:
|
1729 |
+
# {boolean_opcodes_functionalities}
|
1730 |
+
|
1731 |
+
# ### C Blocks
|
1732 |
+
# Description: {c_description}
|
1733 |
+
# Blocks:
|
1734 |
+
# {c_opcodes_functionalities}
|
1735 |
+
|
1736 |
+
# ### Cap Blocks
|
1737 |
+
# Description: {cap_description}
|
1738 |
+
# Blocks:
|
1739 |
+
# {cap_opcodes_functionalities}
|
1740 |
+
|
1741 |
+
# ### Reporter Blocks
|
1742 |
+
# Description: {reporter_description}
|
1743 |
+
# Blocks:
|
1744 |
+
# {reporter_opcodes_functionalities}
|
1745 |
+
|
1746 |
+
# ### Stack Blocks
|
1747 |
+
# Description: {stack_description}
|
1748 |
+
# Blocks:
|
1749 |
+
# {stack_opcodes_functionalities}
|
1750 |
+
# -----------------------------------
|
1751 |
+
|
1752 |
+
# Your task is to:
|
1753 |
+
# If you don't find any "Code-Blocks" then,
|
1754 |
+
# **Don't generate Pseudo Code, and pass the message "No Code-blocks" find...
|
1755 |
+
# If you find any "Code-Blocks" then,
|
1756 |
+
# 1. **Refine the 'logic'**: Make it precise, accurate, and fully aligned with the Game Description. Use Scratch‑consistent verbs and phrasing. **Do NOT** use raw double‑quotes inside the logic string.
|
1757 |
+
|
1758 |
+
# 2. **Structural requirements**:
|
1759 |
+
# - **Numeric values** `(e.g., 0, 5, 0.2, -130)` **must** be in parentheses: `(0)`, `(5)`, `(0.2)`, `(-130)`.
|
1760 |
+
# - **AlphaNumeric values** `(e.g., hello, say 5, 4, hi!)` **must** be in parentheses: `(hello)`, `(say 5)`, `(4)`, `(hi!)`.
|
1761 |
+
# - **Variables** must be in the form `[variable v]` (e.g., `[score v]`), even when used inside expressions two example use `set [score v] to (1)` or `show variable ([speed v])`.
|
1762 |
+
# - **Dropdown options** must be in the form `[option v]` (e.g., `[Game Start v]`, `[blue sky v]`). example use `when [space v] key pressed`.
|
1763 |
+
# - **Reporter blocks** used as inputs must be double‑wrapped: `((x position))`, `((y position))`. example use `if <((y position)) = (-130)> then` or `(((x position)) * (1))`.
|
1764 |
+
# - **Boolean blocks** in conditions must be inside `< >`, including nested ones: `<not <condition>>`, `<<cond1> and <cond2>>`,`<<cond1> or <cond2>>`.
|
1765 |
+
# - **Other Boolean blocks** in conditions must be inside `< >`, including nested ones or values or variables: `<(block/value/variable) * (block/value/variable)>`,`<(block/value/variable) < (block/value/variable)>`, and example of another variable`<[apple v] contains [a v]?>`.
|
1766 |
+
# - **Operator expressions** must use explicit Scratch operator blocks, e.g.:
|
1767 |
+
# ```
|
1768 |
+
# (([ballSpeed v]) * (1.1))
|
1769 |
+
# ```
|
1770 |
+
# - **Every hat block script must end** with a final `end` on its own line.
|
1771 |
+
|
1772 |
+
# 3. **Pseudo‑code formatting**:
|
1773 |
+
# - Represent each block or nested block on its own line.
|
1774 |
+
# - Indent nested blocks by 4 spaces under their parent (`forever`, `if`, etc.).
|
1775 |
+
# - No comments or explanatory text—just the block sequence.
|
1776 |
+
# - a natural language breakdown of each step taken after the event, formatted as a multi-line string representing pseudo-code. Ensure clarity and granularity—each described action should map closely to a Scratch block or tight sequence.
|
1777 |
+
|
1778 |
+
# 4. **Logic content**:
|
1779 |
+
# - Build clear flow for mechanics (movement, jumping, flying, scoring, collisions).
|
1780 |
+
# - Match each action closely to a Scratch block or tight sequence.
|
1781 |
+
# - Do **NOT** include any justification or comments—only the raw logic.
|
1782 |
+
|
1783 |
+
# 5. **Examples for reference**:
|
1784 |
+
# **Correct** pattern for a simple start script:
|
1785 |
+
# ```
|
1786 |
+
# when green flag clicked
|
1787 |
+
# switch backdrop to [blue sky v]
|
1788 |
+
# set [score v] to (0)
|
1789 |
+
# show variable [score v]
|
1790 |
+
# broadcast [Game Start v]
|
1791 |
+
# end
|
1792 |
+
# ```
|
1793 |
+
# **Correct** pattern for updating the high score variable handling:
|
1794 |
+
# ```
|
1795 |
+
# when I receive [Game Over v]
|
1796 |
+
# if <((score)) > (([High Score v]))> then
|
1797 |
+
# set [High Score v] to ([score v])
|
1798 |
+
# end
|
1799 |
+
# switch backdrop to [Game Over v]
|
1800 |
+
# end
|
1801 |
+
# ```
|
1802 |
+
# **Correct** pattern for level up and increase difficulty use:
|
1803 |
+
# ```
|
1804 |
+
# when I receive [Level Up v]
|
1805 |
+
# change [level v] by (1)
|
1806 |
+
# set [ballSpeed v] to ((([ballSpeed v]) * (1.1)))
|
1807 |
+
# end
|
1808 |
+
# ```
|
1809 |
+
# **Correct** pattern for jumping mechanics use:
|
1810 |
+
# ```
|
1811 |
+
# when [space v] key pressed
|
1812 |
+
# if <((y position)) = (-100)> then
|
1813 |
+
# repeat (5)
|
1814 |
+
# change y by (100)
|
1815 |
+
# wait (0.1) seconds
|
1816 |
+
# change y by (-100)
|
1817 |
+
# wait (0.1) seconds
|
1818 |
+
# end
|
1819 |
+
# end
|
1820 |
+
# end
|
1821 |
+
# ```
|
1822 |
+
# **Correct** pattern for continuos moving objects use:
|
1823 |
+
# ```
|
1824 |
+
# when green flag clicked
|
1825 |
+
# go to x: (240) y: (-100)
|
1826 |
+
# set [speed v] to (-5)
|
1827 |
+
# show variable [speed v]
|
1828 |
+
# forever
|
1829 |
+
# change x by ([speed v])
|
1830 |
+
# if <((x position)) < (-240)> then
|
1831 |
+
# go to x: (240) y: (-100)
|
1832 |
+
# end
|
1833 |
+
# end
|
1834 |
+
# end
|
1835 |
+
# ```
|
1836 |
+
# **Correct** pattern for continuos moving objects use:
|
1837 |
+
# ```
|
1838 |
+
# when green flag clicked
|
1839 |
+
# go to x: (240) y: (-100)
|
1840 |
+
# set [speed v] to (-5)
|
1841 |
+
# show variable [speed v]
|
1842 |
+
# forever
|
1843 |
+
# change x by ([speed v])
|
1844 |
+
# if <((x position)) < (-240)> then
|
1845 |
+
# go to x: (240) y: (-100)
|
1846 |
+
# end
|
1847 |
+
# end
|
1848 |
+
# end
|
1849 |
+
# ```
|
1850 |
+
# 6. **Donot** add any explaination of logic or comments to justify or explain just put the logic content in the json.
|
1851 |
+
# 7. **Output**:
|
1852 |
+
# Return **only** a JSON object, using double quotes everywhere:
|
1853 |
+
# ```json
|
1854 |
+
# {{
|
1855 |
+
# "refined_logic":{{
|
1856 |
+
# "name_variable": 'Value of "Sript for: "',
|
1857 |
+
# "pseudocode":"…your fully‑formatted pseudo‑code here…",
|
1858 |
+
# }}
|
1859 |
+
# }}
|
1860 |
+
# ```
|
1861 |
+
# """
|
1862 |
+
# image_input = {
|
1863 |
+
# "type": "image_url",
|
1864 |
+
# "image_url": {
|
1865 |
+
# "url": f"data:image/png;base64,{image}"
|
1866 |
+
# }
|
1867 |
+
# }
|
1868 |
+
|
1869 |
+
# content = [
|
1870 |
+
# {"type": "text", "text": refinement_prompt},
|
1871 |
+
# image_input
|
1872 |
+
# ]
|
1873 |
+
|
1874 |
+
# try:
|
1875 |
+
# # Invoke the main agent for logic refinement and relationship identification
|
1876 |
+
# response = agent.invoke({"messages": [{"role": "user", "content": content}]})
|
1877 |
+
# llm_output_raw = response["messages"][-1].content.strip()
|
1878 |
+
|
1879 |
+
# parsed_llm_output = extract_json_from_llm_response(llm_output_raw)
|
1880 |
+
|
1881 |
+
# # result = parsed_llm_output
|
1882 |
+
# # Extract needed values directly
|
1883 |
+
# logic_data = parsed_llm_output.get("refined_logic", {})
|
1884 |
+
# name_variable = logic_data.get("name_variable", "Unknown")
|
1885 |
+
# pseudocode = logic_data.get("pseudocode", "No logic extracted")
|
1886 |
+
|
1887 |
+
# result = {"pseudo_node": {
|
1888 |
+
# "name_variable": name_variable,
|
1889 |
+
# "pseudocode": pseudocode
|
1890 |
+
# }}
|
1891 |
+
|
1892 |
+
# print(f"result:\n\n {result}")
|
1893 |
+
# return result
|
1894 |
+
# except Exception as e:
|
1895 |
+
# logger.error(f"❌ plan_logic_aligner_node failed: {str(e)}")
|
1896 |
+
# return {"error": str(e)}
|
1897 |
+
# except json.JSONDecodeError as error_json:
|
1898 |
+
# # If JSON parsing fails, use the json resolver agent
|
1899 |
+
# correction_prompt = (
|
1900 |
+
# "Your task is to correct the provided JSON string to ensure it is **syntactically perfect and adheres strictly to JSON rules**.\n"
|
1901 |
+
# "It must be a JSON object with `refined_logic` (string) and `block_relationships` (array of objects).\n"
|
1902 |
+
# f"- **Error Details**: {error_json}\n\n"
|
1903 |
+
# "**Strict Instructions for your response:**\n"
|
1904 |
+
# "1. **ONLY** output the corrected JSON. Do not include any other text or explanations.\n"
|
1905 |
+
# "2. Ensure all keys and string values are enclosed in **double quotes**. Escape internal quotes (`\\`).\n"
|
1906 |
+
# "3. No trailing commas. Correct nesting.\n\n"
|
1907 |
+
# "Here is the problematic JSON string to correct:\n"
|
1908 |
+
# f"```json\n{llm_output_raw}\n```\n"
|
1909 |
+
# "Corrected JSON:\n"
|
1910 |
+
# )
|
1911 |
+
# try:
|
1912 |
+
# correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]})
|
1913 |
+
# corrected_output = extract_json_from_llm_response(correction_response["messages"][-1].content)
|
1914 |
+
|
1915 |
+
# result = {
|
1916 |
+
# #"image_path": image_path,
|
1917 |
+
# "pseudo_code": corrected_output
|
1918 |
+
# }
|
1919 |
+
|
1920 |
+
# return result
|
1921 |
+
|
1922 |
+
# except Exception as e_corr:
|
1923 |
+
# logger.error(f"Failed to correct JSON output for even after retry: {e_corr}")
|
1924 |
|
1925 |
#def extract_images_from_pdf(pdf_path: Path, json_base_dir: Path, image_base_dir: Path):
|
1926 |
#def extract_images_from_pdf(pdf_path: Path, json_base_dir: Path):
|
1927 |
+
|
1928 |
+
# Prepare manipulated sprite JSON structure
|
1929 |
+
manipulated_json = {}
|
1930 |
+
img_elements = []
|
1931 |
+
# { changes: "pdf_stream" in place of "pdf_path"
|
1932 |
+
def extract_images_from_pdf(pdf_stream: io.BytesIO):
|
1933 |
''' Extract images from PDF and generate structured sprite JSON '''
|
1934 |
try:
|
1935 |
+
# {
|
1936 |
+
# pdf_path = Path(pdf_path)
|
1937 |
+
# pdf_filename = pdf_path.stem # e.g., "scratch_crab"
|
1938 |
+
# pdf_dir_path = str(pdf_path.parent).replace("/", "\\")
|
1939 |
+
# print("-------------------------------pdf_filename-------------------------------",pdf_filename)
|
1940 |
+
# print("-------------------------------pdf_dir_path-------------------------------",pdf_dir_path)
|
1941 |
+
|
1942 |
+
if isinstance(pdf_stream, io.BytesIO):
|
1943 |
+
# use a random ID since there's no filename
|
1944 |
+
pdf_id = uuid.uuid4().hex
|
1945 |
+
else:
|
1946 |
+
pdf_id = os.path.splitext(os.path.basename(pdf_stream))[0]
|
1947 |
+
|
1948 |
+
# extracted_image_subdir = DETECTED_IMAGE_DIR / pdf_filename
|
1949 |
+
# json_subdir = JSON_DIR / pdf_filename
|
1950 |
+
# extracted_image_subdir.mkdir(parents=True, exist_ok=True)
|
1951 |
+
# json_subdir.mkdir(parents=True, exist_ok=True)
|
1952 |
+
# print("-------------------------------extracted_image_subdir-------------------------------",extracted_image_subdir)
|
1953 |
+
# print("-------------------------------json_subdir-------------------------------",json_subdir)
|
1954 |
+
# # Output paths (now using Path objects directly)
|
1955 |
+
# output_json_path = json_subdir / "extracted.json"
|
1956 |
+
# final_json_path = json_subdir / "extracted_sprites.json" # Path to extracted_sprites.json
|
1957 |
+
# final_json_path_2 = json_subdir / "extracted_sprites_2.json"
|
1958 |
+
# print("-------------------------------output_json_path-------------------------------",output_json_path)
|
1959 |
+
# print("-------------------------------final_json_path-------------------------------",final_json_path)
|
1960 |
+
# print("-------------------------------final_json_path_2-------------------------------",final_json_path_2)
|
1961 |
|
1962 |
+
# }
|
1963 |
try:
|
1964 |
elements = partition_pdf(
|
1965 |
+
# filename=str(pdf_path), # partition_pdf might expect a string
|
1966 |
+
file=pdf_stream, # 'file=', inplace of 'filename'
|
1967 |
strategy="hi_res",
|
1968 |
extract_image_block_types=["Image"],
|
1969 |
hi_res_model_name="yolox",
|
|
|
1974 |
raise RuntimeError(
|
1975 |
f"❌ Failed to extract images from PDF: {str(e)}")
|
1976 |
|
1977 |
+
file_elements = [element.to_dict() for element in elements]
|
1978 |
+
|
1979 |
+
#{
|
1980 |
+
# try:
|
1981 |
+
# with open(output_json_path, "w") as f:
|
1982 |
+
# json.dump([element.to_dict()
|
1983 |
+
# for element in elements], f, indent=4)
|
1984 |
+
# except Exception as e:
|
1985 |
+
# raise RuntimeError(f"❌ Failed to write extracted.json: {str(e)}")
|
1986 |
+
|
1987 |
+
# try:
|
1988 |
+
# # Display extracted images
|
1989 |
+
# with open(output_json_path, 'r') as file:
|
1990 |
+
# file_elements = json.load(file)
|
1991 |
+
# except Exception as e:
|
1992 |
+
# raise RuntimeError(f"❌ Failed to read extracted.json: {str(e)}")
|
1993 |
+
# }
|
1994 |
|
1995 |
+
sprite_count = 1
|
1996 |
+
for el in file_elements:
|
1997 |
+
img_b64 = el["metadata"].get("image_base64")
|
1998 |
+
if not img_b64:
|
1999 |
+
continue
|
2000 |
|
2001 |
+
manipulated_json[f"Sprite {sprite_count}"] = {
|
2002 |
+
# "id":auto_id,
|
2003 |
+
# "name": name,
|
2004 |
+
"base64": el["metadata"]["image_base64"],
|
2005 |
+
"file-path": pdf_id,
|
2006 |
+
# "description": description
|
2007 |
+
}
|
2008 |
+
sprite_count += 1
|
2009 |
+
return manipulated_json
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010 |
except Exception as e:
|
2011 |
raise RuntimeError(f"❌ Error in extract_images_from_pdf: {str(e)}")
|
2012 |
|
2013 |
+
# def similarity_matching(input_json_path: str, project_folder: str) -> str:
|
2014 |
+
def similarity_matching(sprites_data: str, project_folder: str) -> str:
|
2015 |
logger.info("🔍 Running similarity matching…")
|
2016 |
os.makedirs(project_folder, exist_ok=True)
|
2017 |
|
|
|
2019 |
# CHANGED: define normalized base-paths so startswith() checks work
|
2020 |
backdrop_base_path = os.path.normpath(str(BACKDROP_DIR))
|
2021 |
sprite_base_path = os.path.normpath(str(SPRITE_DIR))
|
2022 |
+
code_blocks_path = os.path.normpath(str(CODE_BLOCKS_DIR))
|
2023 |
# ----------------------------------------
|
2024 |
|
2025 |
project_json_path = os.path.join(project_folder, "project.json")
|
|
|
2027 |
# ==============================
|
2028 |
# READ SPRITE METADATA
|
2029 |
# ==============================
|
2030 |
+
# with open(input_json_path, 'r') as f:
|
2031 |
+
# sprites_data = json.load(f)
|
2032 |
|
2033 |
+
sprite_ids, sprite_base64 = [], []
|
2034 |
for sid, sprite in sprites_data.items():
|
2035 |
sprite_ids.append(sid)
|
2036 |
+
# texts.append("This is " + sprite.get("description", sprite.get("name", "")))
|
2037 |
sprite_base64.append(sprite["base64"])
|
2038 |
|
2039 |
+
sprite_images_bytes = []
|
2040 |
+
for b64 in sprite_base64:
|
2041 |
+
img = Image.open(BytesIO(base64.b64decode(b64.split(",")[-1]))).convert("RGB")
|
2042 |
+
buffer = BytesIO()
|
2043 |
+
img.save(buffer, format="PNG")
|
2044 |
+
buffer.seek(0)
|
2045 |
+
sprite_images_bytes.append(buffer)
|
2046 |
+
|
2047 |
# =========================================
|
2048 |
# Build the list of all candidate images
|
2049 |
# =========================================
|
|
|
2063 |
SPRITE_DIR / "Centaur.sprite3" / "2373556e776cad3ba4d6ee04fc34550b.png",
|
2064 |
SPRITE_DIR / "Crab.sprite3" / "bear_element.png",
|
2065 |
SPRITE_DIR / "Soccer Ball.sprite3" / "cat_football.png",
|
2066 |
+
|
2067 |
+
CODE_BLOCKS_DIR / "script1.jpg",
|
2068 |
+
CODE_BLOCKS_DIR / "script2.jpg",
|
2069 |
+
CODE_BLOCKS_DIR / "script3.jpg",
|
2070 |
+
CODE_BLOCKS_DIR / "script4.jpg",
|
2071 |
+
CODE_BLOCKS_DIR / "script5.jpg",
|
2072 |
+
CODE_BLOCKS_DIR / "script6.jpg"
|
2073 |
]
|
2074 |
folder_image_paths = [os.path.normpath(str(p)) for p in folder_image_paths]
|
2075 |
# =========================================
|
|
|
2079 |
# -----------------------------------------
|
2080 |
with open(f"{BLOCKS_DIR}/embeddings.json", "r") as f:
|
2081 |
embedding_json = json.load(f)
|
|
|
2082 |
|
2083 |
# =========================================
|
2084 |
# Decode & embed each sprite image
|
2085 |
# =========================================
|
2086 |
+
# sprite_features = []
|
2087 |
+
# for b64 in sprite_base64:
|
2088 |
+
# if "," in b64:
|
2089 |
+
# b64 = b64.split(",", 1)[1]
|
2090 |
+
|
2091 |
+
# img_bytes = base64.b64decode(b64)
|
2092 |
+
# pil_img = Image.open(BytesIO(img_bytes)).convert("RGB")
|
2093 |
+
# buf = BytesIO()
|
2094 |
+
# pil_img.save(buf, format="PNG")
|
2095 |
+
# buf.seek(0)
|
2096 |
+
# feats = clip_embd.embed_image([buf])[0]
|
2097 |
+
# sprite_features.append(feats)
|
2098 |
+
|
2099 |
+
# ============================== #
|
2100 |
+
# EMBED SPRITE IMAGES #
|
2101 |
+
# ============================== #
|
2102 |
+
sprite_features = clip_embd.embed_image(sprite_images_bytes)
|
2103 |
+
|
2104 |
sprite_matrix = np.vstack(sprite_features)
|
2105 |
+
img_matrix = np.array([img["embeddings"] for img in embedding_json])
|
2106 |
+
|
2107 |
# =========================================
|
2108 |
# Compute similarities & pick best match
|
2109 |
# =========================================
|
|
|
2197 |
else:
|
2198 |
logger.warning(f"No project.json in {matched_folder}")
|
2199 |
|
2200 |
+
|
2201 |
# =========================================
|
2202 |
# Merge into final Scratch project.json
|
2203 |
# =========================================
|
|
|
2375 |
# # logger.info(f"🎉 Final project saved: {project_json_path}")
|
2376 |
# return project_json_path
|
2377 |
|
2378 |
+
# def convert_bytes_to_image(pdf_bytes: bytes, dpi: int):
|
2379 |
+
# images = convert_from_bytes(pdf_bytes, dpi=dpi, poppler_path=poppler_path)
|
2380 |
+
# # Save each page to an in-memory BytesIO and return a list of BytesIOs
|
2381 |
+
# buffers = []
|
2382 |
+
# for img in images:
|
2383 |
+
# buf = BytesIO()
|
2384 |
+
# img.save(buf, format="PNG")
|
2385 |
+
# buf.seek(0)
|
2386 |
+
# buffers.append(buf)
|
2387 |
+
# return buffers
|
2388 |
+
|
2389 |
+
def convert_pdf_stream_to_images(pdf_stream: io.BytesIO, dpi=300):
|
2390 |
+
# Ensure we are at the start of the stream
|
2391 |
+
pdf_stream.seek(0)
|
2392 |
+
|
2393 |
+
with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_pdf:
|
2394 |
+
tmp_pdf.write(pdf_stream.read())
|
2395 |
+
tmp_pdf_path = tmp_pdf.name
|
2396 |
+
|
2397 |
+
# Now use convert_from_path on the temp file
|
2398 |
+
images = convert_from_path(tmp_pdf_path, dpi=dpi)
|
2399 |
+
return images
|
2400 |
+
|
2401 |
def delay_for_tpm_node(state: GameState):
|
2402 |
logger.info("--- Running DelayForTPMNode ---")
|
2403 |
time.sleep(60) # Adjust the delay as needed
|
|
|
2448 |
except Exception as e:
|
2449 |
logger.error(f"❌ Error during image upscaling: {str(e)}")
|
2450 |
return image
|
2451 |
+
|
2452 |
+
@log_execution_time
|
2453 |
def create_sb3_archive(project_folder, project_id):
|
2454 |
"""
|
2455 |
Zips the project folder and renames it to an .sb3 file.
|
|
|
2489 |
os.remove(sb3_path)
|
2490 |
return sb3_path
|
2491 |
|
2492 |
+
#{ changes -> pdf_stream replacement of pdf_path
|
2493 |
+
# def save_pdf_to_generated_dir(pdf_path: str, project_id: str) -> str:
|
2494 |
+
def save_pdf_to_generated_dir(pdf_stream: io.BytesIO, project_id: str) -> str:
|
2495 |
"""
|
2496 |
+
Copies the PDF at `pdf_stream` into GEN_PROJECT_DIR/project_id/,
|
2497 |
renaming it to <project_id>.pdf.
|
2498 |
|
2499 |
Args:
|
2500 |
+
pdf_stream (io.BytesIO): Any existing stream to a PDF file.
|
2501 |
project_id (str): Your unique project identifier.
|
2502 |
|
2503 |
Returns:
|
2504 |
str: Path to the copied PDF in the generated directory,
|
2505 |
or None if something went wrong.
|
2506 |
+
"""
|
2507 |
+
# }
|
2508 |
try:
|
2509 |
# 1) Build the destination directory and base filename
|
2510 |
output_dir = GEN_PROJECT_DIR / project_id
|
|
|
2515 |
target_pdf = output_dir / f"{project_id}.pdf"
|
2516 |
print(f"\n--------------------------------target_pdf {target_pdf}")
|
2517 |
# 3) Copy the PDF
|
2518 |
+
# {
|
2519 |
+
# shutil.copy2(pdf_path, target_pdf)
|
2520 |
+
if isinstance(pdf_stream, io.BytesIO):
|
2521 |
+
with open(target_pdf, "wb") as f:
|
2522 |
+
f.write(pdf_stream.getbuffer())
|
2523 |
+
else:
|
2524 |
+
shutil.copy2(pdf_stream, target_pdf)
|
2525 |
+
print(f"Copied PDF from {pdf_stream} → {target_pdf}")
|
2526 |
+
logger.info(f"Copied PDF from {pdf_stream} → {target_pdf}")
|
2527 |
+
# }
|
2528 |
|
2529 |
|
2530 |
return str(target_pdf)
|
|
|
2585 |
# Create empty json in project_{random_id} folder #
|
2586 |
# =========================================================================== #
|
2587 |
#os.makedirs(project_folder, exist_ok=True)
|
2588 |
+
|
2589 |
+
# {
|
2590 |
# Save the uploaded PDF temporarily
|
2591 |
+
# filename = secure_filename(pdf_file.filename)
|
2592 |
+
# temp_dir = tempfile.mkdtemp()
|
2593 |
+
# saved_pdf_path = os.path.join(temp_dir, filename)
|
2594 |
+
# pdf_file.save(saved_pdf_path)
|
2595 |
+
# pdf_doc = saved_pdf_path
|
2596 |
+
|
2597 |
+
pdf_bytes = pdf_file.read()
|
2598 |
+
pdf_stream = io.BytesIO(pdf_bytes)
|
2599 |
+
logger.info(f"Saved uploaded PDF to: {pdf_stream}")
|
2600 |
+
|
2601 |
+
# pdf= save_pdf_to_generated_dir(saved_pdf_path, project_id)
|
2602 |
+
start_time = time.time()
|
2603 |
+
pdf= save_pdf_to_generated_dir(pdf_stream, project_id)
|
2604 |
# logger.info(f"Created project folder: {project_folder}")
|
2605 |
+
# logger.info(f"Saved uploaded PDF to: {saved_pdf_path}")
|
2606 |
logger.info(f"Saved uploaded PDF to: {pdf_file}: {pdf}")
|
2607 |
+
print("--------------------------------pdf_file_path---------------------",pdf_file,pdf_stream)
|
2608 |
+
total_time = time.time() - start_time
|
2609 |
+
print(f"-----------------------------Execution Time save_pdf_to_generated_dir() : {total_time}-----------------------------\n")
|
2610 |
+
# }
|
2611 |
+
|
2612 |
+
# {
|
2613 |
# Extract & process
|
2614 |
+
# output_path, result = extract_images_from_pdf(saved_pdf_path)
|
2615 |
+
start_time = time.time()
|
2616 |
+
output_path = extract_images_from_pdf(pdf_stream)
|
2617 |
+
print(" --------------------------------------- zip_path_str ---------------------------------------", output_path)
|
2618 |
+
total_time = time.time() - start_time
|
2619 |
+
print(f"-----------------------------Execution Time extract_images_from_pdf() : {total_time}-----------------------------\n")
|
2620 |
+
# }
|
2621 |
+
|
2622 |
# Check extracted_sprites.json for "scratch block" in any 'name'
|
2623 |
# extracted_dir = os.path.join(JSON_DIR, os.path.splitext(filename)[0])
|
2624 |
# extracted_sprites_json = os.path.join(extracted_dir, "extracted_sprites.json")
|
|
|
2628 |
|
2629 |
# with open(extracted_sprites_json, 'r') as f:
|
2630 |
# sprite_data = json.load(f)
|
2631 |
+
start_time = time.time()
|
2632 |
+
project_output = similarity_matching(output_path, project_folder)
|
2633 |
+
logger.info("Received request to process PDF.")
|
2634 |
+
total_time = time.time() - start_time
|
2635 |
+
print(f"-----------------------------Execution Time similarity_matching() : {total_time}-----------------------------\n")
|
2636 |
|
2637 |
+
with open(project_output, 'r') as f:
|
2638 |
+
project_skeleton = json.load(f)
|
2639 |
|
2640 |
+
# images = convert_from_path(pdf_stream, dpi=300)
|
2641 |
# print(type)
|
2642 |
# page = images[0]
|
2643 |
# # img_base64 = base64.b64encode(images).decode("utf-8")
|
|
|
2646 |
# img_bytes = buf.getvalue()
|
2647 |
# img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
2648 |
#image_paths = await convert_pdf_to_images_async(saved_pdf_path)
|
2649 |
+
|
2650 |
+
# images = convert_bytes_to_image(pdf_stream, dpi=250)
|
2651 |
+
# print("PDF converted to images:", images)
|
2652 |
+
|
2653 |
+
if isinstance(pdf_stream, io.BytesIO):
|
2654 |
+
images = convert_pdf_stream_to_images(pdf_stream, dpi=300)
|
2655 |
+
else:
|
2656 |
+
images = convert_from_path(pdf_stream, dpi=300)
|
2657 |
+
|
2658 |
#updating logic here [Dev Patel]
|
2659 |
+
initial_state_dict = {
|
2660 |
+
"project_json": project_skeleton,
|
2661 |
+
"description": "The pseudo code for the script",
|
2662 |
+
"project_id": project_id,
|
2663 |
+
# "project_image": img_b64,
|
2664 |
+
"project_image": images,
|
2665 |
+
"action_plan": {},
|
2666 |
+
"pseudo_code": {},
|
2667 |
+
"temporary_node": {},
|
2668 |
+
}
|
2669 |
|
2670 |
+
final_state_dict = app_graph.invoke(initial_state_dict) # Pass dictionary
|
2671 |
|
2672 |
+
final_project_json = final_state_dict['project_json'] # Access as dict
|
2673 |
# final_project_json = project_skeleton
|
2674 |
|
2675 |
# Save the *final* filled project JSON, overwriting the skeleton
|
2676 |
+
with open(project_output, "w") as f:
|
2677 |
+
json.dump(final_project_json, f, indent=2)
|
2678 |
+
logger.info(f"Final project JSON saved to {project_output}")
|
2679 |
|
2680 |
# --- Call the new function to create the .sb3 file ---
|
2681 |
+
sb3_file_path = create_sb3_archive(project_folder, project_id)
|
|
|
2682 |
|
2683 |
if sb3_file_path:
|
2684 |
logger.info(f"Successfully created SB3 file: {sb3_file_path}")
|
|
|
2693 |
"output_json": "output_path",
|
2694 |
"sprites": "result",
|
2695 |
"project_output_json": "project_output",
|
2696 |
+
"test_url": download_url
|
|
|
2697 |
})
|
2698 |
else:
|
2699 |
return jsonify(error="Failed to create SB3 archive"), 500
|