Upload 9 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/siteOGImage.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,78 @@
----
-title: 3d Model Playground
-emoji: 🏢
-colorFrom: red
-colorTo: purple
-sdk: static
-pinned: false
-license: mit
-short_description: control 3D models with voice + hand gestures
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 3D Model Playground
+
+Control 3D models using hand gestures and voice commands in real-time.
+
+An interactive web app built with threejs, mediapipe computer vision, web speech API, and rosebud AI.
+
+- Say "drag", "rotate", "scale", or "animate" to change the interaction mode
+- Pinch fingers to control the 3D model
+- Drag/drop a new 3D model onto the page to import it (GLB/GLTF format)
+
+[Video](https://youtu.be/_I1E44Fp1Es?si=lR2otqR_-ZGdIGXT) | [Live Demo](https://collidingscopes.github.io/3d-model-playground/)
+
+## Requirements
+
+- Modern web browser with WebGL support
+- Camera / microphone access
+
+## Technologies
+
+- **Three.js** for 3D rendering
+- **MediaPipe** for hand tracking and gesture recognition
+- **Web Speech API** for speech recognition
+- **HTML5 Canvas** for visual feedback
+- **JavaScript** for real-time interaction
+
+## Setup for Development
+
+```bash
+# Clone this repository
+git clone https://github.com/collidingScopes/3d-model-playground
+
+# Navigate to the project directory
+cd 3d-model-playground
+
+# Serve with your preferred method (example using Python)
+python -m http.server
+```
+
+Then navigate to `http://localhost:8000` in your browser.
+
+## License
+
+MIT License
+
+## Credits
+
+- Three.js - https://threejs.org/
+- MediaPipe - https://mediapipe.dev/
+- Rosebud AI - https://rosebud.ai/
+- Quaternius 3D models - https://quaternius.com/
+
+## Related Projects
+
+I've released several computer vision projects (with code + tutorials) here:
+[Fun With Computer Vision](https://www.funwithcomputervision.com/)
+
+You can purchase lifetime access and receive the full project files and tutorials. I'm adding more content regularly :)
+
+You might also like some of my other open source projects:
+
+- [Particular Drift](https://collidingScopes.github.io/particular-drift) - Turn photos into flowing particle animations
+- [Liquid Logo](https://collidingScopes.github.io/liquid-logo) - Transform logos and icons into liquid metal animations
+- [Video-to-ASCII](https://collidingScopes.github.io/ascii) - Convert videos into ASCII pixel art
+
+## Contact
+
+- Instagram: [@stereo.drift](https://www.instagram.com/stereo.drift/)
+- Twitter/X: [@measure_plan](https://x.com/measure_plan)
+- Email: [[email protected]](mailto:[email protected])
+- GitHub: [collidingScopes](https://github.com/collidingScopes)
+
+## Donations
+
+If you enjoyed this, feel free to buy me a coffee.
+
+My name is Alan, and I enjoy building open source software for computer vision, games, and more. This would be much appreciated during late-night coding sessions!
+
+[![Buy Me A Coffee](https://www.buymeacoffee.com/assets/img/custom_images/yellow_img.png)](https://www.buymeacoffee.com/stereoDrift)

SpeechManager.js

@@ -0,0 +1,361 @@
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) {
+        resolve(value);
+    } else {
+        Promise.resolve(value).then(_next, _throw);
+    }
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _class_call_check(instance, Constructor) {
+    if (!(instance instanceof Constructor)) {
+        throw new TypeError("Cannot call a class as a function");
+    }
+}
+function _defineProperties(target, props) {
+    for(var i = 0; i < props.length; i++){
+        var descriptor = props[i];
+        descriptor.enumerable = descriptor.enumerable || false;
+        descriptor.configurable = true;
+        if ("value" in descriptor) descriptor.writable = true;
+        Object.defineProperty(target, descriptor.key, descriptor);
+    }
+}
+function _create_class(Constructor, protoProps, staticProps) {
+    if (protoProps) _defineProperties(Constructor.prototype, protoProps);
+    if (staticProps) _defineProperties(Constructor, staticProps);
+    return Constructor;
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, g, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    };
+    return g = {
+        next: verb(0),
+        "throw": verb(1),
+        "return": verb(2)
+    }, typeof Symbol === "function" && (g[Symbol.iterator] = function() {
+        return this;
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(_)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+export var SpeechManager = /*#__PURE__*/ function() {
+    "use strict";
+    function SpeechManager(onTranscript, onRecognitionActive, onCommandRecognized) {
+        var _this = this;
+        _class_call_check(this, SpeechManager);
+        this.onTranscript = onTranscript;
+        this.onRecognitionActive = onRecognitionActive; // Callback for recognition state
+        this.onCommandRecognized = onCommandRecognized; // Callback for recognized commands
+        this.recognition = null;
+        this.isRecognizing = false;
+        this.finalTranscript = '';
+        this.interimTranscript = '';
+        var SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
+        if (SpeechRecognition) {
+            this.recognition = new SpeechRecognition();
+            this.recognition.continuous = true; // Keep listening even after a pause
+            this.recognition.interimResults = true; // Get results while speaking
+            this.recognition.onstart = function() {
+                _this.isRecognizing = true;
+                console.log('Speech recognition started.');
+                if (_this.onRecognitionActive) _this.onRecognitionActive(true);
+            };
+            this.recognition.onresult = function(event) {
+                _this.interimTranscript = '';
+                for(var i = event.resultIndex; i < event.results.length; ++i){
+                    if (event.results[i].isFinal) {
+                        // Append to finalTranscript and then clear it for the next utterance
+                        // This way, `finalTranscript` holds the *current complete* utterance.
+                        var currentFinalTranscript = event.results[i][0].transcript.trim().toLowerCase();
+                        _this.finalTranscript += currentFinalTranscript; // Append to potentially longer session transcript if needed, though we process per utterance
+                        if (_this.onTranscript) {
+                            // Display the raw transcript before processing as command
+                            _this.onTranscript(event.results[i][0].transcript, ''); // Send final, clear interim
+                        }
+                        // Check for commands
+                        var commandMap = {
+                            'drag': 'drag',
+                            'rotate': 'rotate',
+                            'rotation': 'rotate',
+                            'scale': 'scale',
+                            'size': 'scale',
+                            'zoom': 'scale',
+                            'animate': 'animate',
+                            'anime': 'animate',
+                            'animation': 'animate' // Alias for animate
+                        };
+                        var spokenCommands = Object.keys(commandMap);
+                        var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                        try {
+                            for(var _iterator = spokenCommands[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                var spokenCmd = _step.value;
+                                if (currentFinalTranscript.includes(spokenCmd)) {
+                                    var actualCommand = commandMap[spokenCmd];
+                                    if (_this.onCommandRecognized) {
+                                        _this.onCommandRecognized(actualCommand);
+                                    }
+                                    break; // Process the first command found (and its alias)
+                                }
+                            }
+                        } catch (err) {
+                            _didIteratorError = true;
+                            _iteratorError = err;
+                        } finally{
+                            try {
+                                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                    _iterator.return();
+                                }
+                            } finally{
+                                if (_didIteratorError) {
+                                    throw _iteratorError;
+                                }
+                            }
+                        }
+                        // Reset finalTranscript for the next full utterance if you are processing utterance by utterance
+                        // If you want to accumulate, then don't reset here.
+                        // For command processing, resetting per utterance is usually best.
+                        _this.finalTranscript = '';
+                    } else {
+                        _this.interimTranscript += event.results[i][0].transcript;
+                        if (_this.onTranscript) {
+                            _this.onTranscript(null, _this.interimTranscript);
+                        }
+                    }
+                }
+                // If only interim results were processed in this event batch, ensure onTranscript is called
+                if (_this.interimTranscript && !event.results[event.results.length - 1].isFinal) {
+                    if (_this.onTranscript) {
+                        _this.onTranscript(null, _this.interimTranscript);
+                    }
+                }
+            };
+            this.recognition.onerror = function(event) {
+                console.error('Speech recognition error:', event.error);
+                var oldIsRecognizing = _this.isRecognizing;
+                _this.isRecognizing = false;
+                _this.finalTranscript = ''; // Clear transcript on error
+                _this.interimTranscript = '';
+                if (_this.onTranscript) _this.onTranscript('', ''); // Clear display
+                if (oldIsRecognizing && _this.onRecognitionActive) _this.onRecognitionActive(false);
+                // Automatically restart if it's an 'aborted' or 'no-speech' error
+                if (event.error === 'aborted' || event.error === 'no-speech') {
+                    console.log('Restarting speech recognition due to inactivity or abort.');
+                // Don't call startRecognition directly, let onend handle it if continuous
+                }
+            };
+            this.recognition.onend = function() {
+                var oldIsRecognizing = _this.isRecognizing;
+                _this.isRecognizing = false;
+                console.log('Speech recognition ended.');
+                _this.finalTranscript = ''; // Clear transcript on end
+                _this.interimTranscript = '';
+                if (_this.onTranscript) _this.onTranscript('', ''); // Clear display
+                if (oldIsRecognizing && _this.onRecognitionActive) _this.onRecognitionActive(false);
+                // If it ended and continuous is true, restart it.
+                // This handles cases where the browser might stop it.
+                if (_this.recognition.continuous) {
+                    console.log('Continuous mode: Restarting speech recognition.');
+                    _this.startRecognition(); // startRecognition already resets transcripts
+                }
+            };
+        } else {
+            console.warn('Web Speech API is not supported in this browser.');
+        }
+    }
+    _create_class(SpeechManager, [
+        {
+            key: "startRecognition",
+            value: function startRecognition() {
+                var _this = this;
+                if (this.recognition && !this.isRecognizing) {
+                    try {
+                        this.finalTranscript = ''; // Reset transcript
+                        this.interimTranscript = '';
+                        this.recognition.start();
+                    } catch (e) {
+                        console.error("Error starting speech recognition:", e);
+                        // This can happen if it's already started or due to permissions
+                        if (e.name === 'InvalidStateError' && this.isRecognizing) {
+                        // Already started, do nothing
+                        } else {
+                            // Attempt to restart if it fails for other reasons (e.g. after an error)
+                            setTimeout(function() {
+                                return _this.startRecognition();
+                            }, 500);
+                        }
+                    }
+                }
+            }
+        },
+        {
+            key: "stopRecognition",
+            value: function stopRecognition() {
+                if (this.recognition && this.isRecognizing) {
+                    this.recognition.stop();
+                }
+            }
+        },
+        {
+            key: "requestPermissionAndStart",
+            value: // Call this on user interaction to request microphone permission
+            function requestPermissionAndStart() {
+                var _this = this;
+                return _async_to_generator(function() {
+                    var err;
+                    return _ts_generator(this, function(_state) {
+                        switch(_state.label){
+                            case 0:
+                                if (!_this.recognition) {
+                                    console.log("Speech recognition not supported.");
+                                    return [
+                                        2
+                                    ];
+                                }
+                                _state.label = 1;
+                            case 1:
+                                _state.trys.push([
+                                    1,
+                                    3,
+                                    ,
+                                    4
+                                ]);
+                                // Attempt to get microphone access (this might prompt the user)
+                                return [
+                                    4,
+                                    navigator.mediaDevices.getUserMedia({
+                                        audio: true
+                                    })
+                                ];
+                            case 2:
+                                _state.sent();
+                                console.log("Microphone permission granted.");
+                                _this.startRecognition();
+                                return [
+                                    3,
+                                    4
+                                ];
+                            case 3:
+                                err = _state.sent();
+                                console.error("Microphone permission denied or error:", err);
+                                if (_this.onTranscript) {
+                                    _this.onTranscript("Microphone access denied. Please allow microphone access in your browser settings.", "");
+                                }
+                                return [
+                                    3,
+                                    4
+                                ];
+                            case 4:
+                                return [
+                                    2
+                                ];
+                        }
+                    });
+                })();
+            }
+        }
+    ]);
+    return SpeechManager;
+}();

audioManager.js

@@ -0,0 +1,85 @@
+// Basic Web Audio API Sound Manager
+function _class_call_check(instance, Constructor) {
+    if (!(instance instanceof Constructor)) {
+        throw new TypeError("Cannot call a class as a function");
+    }
+}
+function _defineProperties(target, props) {
+    for(var i = 0; i < props.length; i++){
+        var descriptor = props[i];
+        descriptor.enumerable = descriptor.enumerable || false;
+        descriptor.configurable = true;
+        if ("value" in descriptor) descriptor.writable = true;
+        Object.defineProperty(target, descriptor.key, descriptor);
+    }
+}
+function _create_class(Constructor, protoProps, staticProps) {
+    if (protoProps) _defineProperties(Constructor.prototype, protoProps);
+    if (staticProps) _defineProperties(Constructor, staticProps);
+    return Constructor;
+}
+export var AudioManager = /*#__PURE__*/ function() {
+    "use strict";
+    function AudioManager() {
+        _class_call_check(this, AudioManager);
+        // Use '||' for broader browser compatibility, though 'webkit' is largely legacy
+        var AudioContext = window.AudioContext || window.webkitAudioContext;
+        this.audioCtx = null;
+        this.isInitialized = false;
+        this.lastClickTime = 0;
+        this.clickInterval = 200; // Milliseconds between clicks for rhythm
+        if (AudioContext) {
+            try {
+                this.audioCtx = new AudioContext();
+                this.isInitialized = true;
+                console.log("AudioContext created successfully.");
+            } catch (e) {
+                console.error("Error creating AudioContext:", e);
+            }
+        } else {
+            console.warn("Web Audio API is not supported in this browser.");
+        }
+    }
+    _create_class(AudioManager, [
+        {
+            // Resume audio context after user interaction (required by many browsers)
+            key: "resumeContext",
+            value: function resumeContext() {
+                if (this.audioCtx && this.audioCtx.state === 'suspended') {
+                    this.audioCtx.resume().then(function() {
+                        console.log("AudioContext resumed successfully.");
+                    }).catch(function(e) {
+                        return console.error("Error resuming AudioContext:", e);
+                    });
+                }
+            }
+        },
+        {
+            key: "playInteractionClickSound",
+            value: function playInteractionClickSound() {
+                if (!this.isInitialized || !this.audioCtx || this.audioCtx.state !== 'running') return;
+                var internalCurrentTime = this.audioCtx.currentTime;
+                // Check if enough time has passed since the last click
+                if (internalCurrentTime - this.lastClickTime < this.clickInterval / 1000) {
+                    return; // Too soon for the next click
+                }
+                this.lastClickTime = internalCurrentTime;
+                var oscillator = this.audioCtx.createOscillator();
+                var gainNode = this.audioCtx.createGain();
+                oscillator.connect(gainNode);
+                gainNode.connect(this.audioCtx.destination);
+                oscillator.type = 'sine'; // Softer waveform for a 'tic'
+                oscillator.frequency.setValueAtTime(1200, this.audioCtx.currentTime); // Lowered base pitch
+                // A very quick pitch drop can make it sound more 'clicky'
+                oscillator.frequency.exponentialRampToValueAtTime(600, this.audioCtx.currentTime + 0.01); // Lowered pitch drop target
+                var clickVolume = 0.08; // Increased volume slightly
+                gainNode.gain.setValueAtTime(0, this.audioCtx.currentTime); // Start silent for a clean attack
+                gainNode.gain.linearRampToValueAtTime(clickVolume, this.audioCtx.currentTime + 0.003); // Very fast attack
+                gainNode.gain.exponentialRampToValueAtTime(0.0001, this.audioCtx.currentTime + 0.005); // Keep decay short for 'tic'
+                oscillator.start(this.audioCtx.currentTime);
+                oscillator.stop(this.audioCtx.currentTime + 0.005); // Match decay duration
+            }
+        }
+    ]);
+    return AudioManager;
+}();

index.html

@@ -1,19 +1,57 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>3D Model Playground</title>
+        <link rel="stylesheet" href="styles.css">
+
+        <!-- Primary Meta Tags -->
+        <meta name="title" content="3D Model Playground">
+        <meta name="description" content="Control 3D models with hand gestures & voice commands">
+
+        <!-- Open Graph / Facebook -->
+        <meta property="og:type" content="website">
+        <meta property="og:url" content="https://collidingscopes.github.io/3d-model-playground/">
+        <meta property="og:title" content="3D Model Playground">
+        <meta property="og:description" content="Control 3D models with hand gestures & voice commands">
+        <meta property="og:image" content="https://raw.githubusercontent.com/collidingScopes/3d-model-playground/main/assets/siteOGImage.jpg">
+
+        <!-- Twitter -->
+        <meta property="twitter:card" content="summary_large_image">
+        <meta property="twitter:url" content="https://collidingscopes.github.io/3d-model-playground/">
+        <meta property="twitter:title" content="3D Model Playground">
+        <meta property="twitter:description" content="Control 3D models with hand gestures & voice commands">
+        <meta property="twitter:image" content="https://raw.githubusercontent.com/collidingScopes/3d-model-playground/main/assets/siteOGImage.jpg">
+
+        <script defer src="https://cloud.umami.is/script.js" data-website-id="eb59c81c-27cb-4e1d-9e8c-bfbe70c48cd9"></script>
+        <script type="importmap">
+            {
+                "imports": {
+                    "three": "https://unpkg.com/[email protected]/build/three.module.js",
+                    "three/examples/": "https://unpkg.com/[email protected]/examples/jsm/",
+                    "three/loaders/": "https://unpkg.com/[email protected]/examples/jsm/loaders/"
+                }
+            }
+        </script>
+    </head>
+    <body style="width: 100%; height: 100%; overflow: hidden; margin: 0;">
+        <div id="renderDiv" style="width: 100%; height: 100%; margin: 0;">
+            <div id="instruction-text" class="text-box"></div>
+            <div id="video-link" class="text-box">
+                <a href="https://youtu.be/_I1E44Fp1Es?si=lR2otqR_-ZGdIGXT" target="_blank">Video Demo</a>
+            </div>
+            <div id="social-links" class="text-box">
+                <a href="https://www.x.com/measure_plan/" target="_blank">Twitter</a><br>
+                <a href="https://www.instagram.com/stereo.drift/" target="_blank">Instagram</a><br>
+                <a href="https://www.youtube.com/@funwithcomputervision" target="_blank">Youtube</a>
+            </div>
+            <div id="coffee-link" class="text-box">
+                <span id="logo">🪬</span><br>
+                <a href="https://www.funwithcomputervision.com/" target="_blank">code & tutorials here</a>
+            </div>
+        </div>
+        <script type="module" src="main.js"></script>
+
+    </body>
 </html>

main.js

@@ -0,0 +1,12 @@
+import { Game } from './game.js';
+// Get the render target div
+var renderDiv = document.getElementById('renderDiv');
+// Check if renderDiv exists
+if (!renderDiv) {
+    console.error('Fatal Error: renderDiv element not found.');
+} else {
+    // Initialize the game with the render target
+    var game = new Game(renderDiv);
+    // Start the game
+    game.start(); // The actual setup happens async within the Game class constructor
+}

styles.css

@@ -0,0 +1,44 @@
+.text-box {
+    padding: 8px 15px;
+    background-color: rgba(255, 255, 255, 0.9);
+    color: black;
+    border-radius: 4px;
+    font-family: "Arial", "Helvetica Neue", Helvetica, sans-serif;
+    border: 2px solid black;
+    box-shadow: 3px 3px 0px black;
+    font-size: clamp(13px, 2vw, 15px);
+    text-align: center;
+    z-index: 200;
+    opacity: 1;
+    transition: opacity 0.3s ease-in-out, bottom 0.3s ease-in-out, box-shadow 0.2s ease;
+}
+
+#instruction-text {
+    position: absolute;
+    bottom: 10px;
+    left: 50%;
+    transform: translateX(-50%);
+    pointer-events: none;
+}
+
+#social-links {
+    position: absolute;
+    bottom: 10px;
+    left: 10px;
+}
+
+#coffee-link {
+    position: absolute;
+    bottom: 10px;
+    right: 10px;
+}
+
+#video-link {
+    position: absolute;
+    top: 10px;
+    left: 10px;
+}
+
+#logo {
+    font-size: 2em;
+}