Spaces:
Build error
Build error
| const ArgumentType = require('../../extension-support/argument-type'); | |
| const BlockType = require('../../extension-support/block-type'); | |
| const Cast = require('../../util/cast'); | |
| const log = require('../../util/log'); | |
| const ml5 = require('ml5'); | |
| /** | |
| * Formatter which is used for translating. | |
| * When it was loaded as a module, 'formatMessage' will be replaced which is used in the runtime. | |
| * @type {Function} | |
| */ | |
| let formatMessage = require('format-message'); | |
| /** | |
| * URL to get this extension as a module. | |
| * When it was loaded as a module, 'extensionURL' will be replaced a URL which is retrieved from. | |
| * @type {string} | |
| */ | |
| let extensionURL = 'https://champierre.github.io/ml2scratch/ml2scratch.mjs'; | |
| const HAT_TIMEOUT = 100; | |
| const blockIconURI = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACgAAAAoCAYAAACM/rhtAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAACXBIWXMAAAsTAAALEwEAmpwYAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpMwidZAAAFX0lEQVRYCe1YTWhcVRQ+8+YvmaRtkibYpBpbBEtSqKkGFy6iGxG36koUXCi40IW7FkQQcemyaotu3PiDuAjiItaoCGKTWmyxTFRobP4aHBKTppPJ/Lz3/L773s17982bISGVdjFnmHn33XvuOd/5zrnnPSbR8917rtzFYt3F2BS0FsC9ZqjFYIvBvTKw1/2pZgaSWEw0U9jjmoP9/DaTWIDctB/Q0omE2OJipGGyp3McvdKFnuO4kWgd70pLG64jNVx5WjkblTqABJcDiDm7LMIvQP4v4gJOwpLOdE7a4a8KJ3EtpQ4gmSO417vul+cGTiAsd0fc7CYIMpW2knLt1oq8uHBB0vCQA7wSrlE6DICsOaaVzBHc2OHju/G7a91Rx5bPCzPy9WZBBlI5pNsGWFMMgETPmlNpZQogZbsqqUQSs94954KaZN0E8+G18HxYnzoOPmkrJVu1irTBNrOkKsk0RVUxAHJCU6x1CS5pmdXBNa3HPbsVy/V2W0ClgOFHlWSMIdNzSCEK4JM/f5I3L30l65VNBc52vAaxCRbeuTwuZ/OTYivWvdBmVhfltelPZRF1RnH9jKixmiH7yBDSrAQOw6z7KrEHx18LotxCmt9fviLvzk3K+YXfPXt+BD8v/yFvzU7Iq1gneM1tfn1JzsyOy+xGQenrjPjG1YUM8iSLU0MqfYNhBYwbMhjW49ZjmU70nwE5t/SbrJU3xYJhsvfx/K+Y75ensZ5Q+fJ2ZlFj0jYgKZzWqChgmMylsvLhyDPyRs+DcrlalD7WY0SaADRjXgWYPhic2FgUska5VLgmn63/LYOpdllgzwyJSpdTUbUVmt4eqnWkvT/XJSf3H1Kdgyc4ymPdIdm2EBmQnAK6vmT2yZn5aelIZ+Xc9Sn0pU6ZQ3s4WWea3qLuwkZxMPChRtnGswQZMSnxdHcOkKYA8KiVkfPlm/LNlS8BLidDVlryYC8eSpzLMEhvHC6N6GqTFJuqCgBSMuvWpALGRtu6ANiWPNJI4PXVw/3c5YG0ocuTbzMLSlzUsTdK8aBgPg5M3JxvILgwwhLbAQr+6sPPyxdHn5CL/+bl7L2PyswjL8gwanAJJ9EyeIR3gNKHhI2Z/TTpp5LN2wH2PNrRD2tz6MjtUgIBUc6bpNgPjzixq4IIn8wekGPdh2UY3+nOXnmo74h6pj6eOygfFP8xKlwxlT4gk8sz6qnB+xqCPIRDMbivV0VfrJXlpavjMlUsyAgy8pdTlawRZMyTJOAtiMVC5BdQZ0fclJSqZbyBZGX0ngeUahVOV22kuVbCIzFIiGrMdklOLfwip65/L5LM4jSsyFjviHz72MuSSXrcDGJ+KpkBAS7KJESKD6QhgxoeI2eapk88q7YQHJ07oJVp4lvJ20NPyWk08w60Ia6xJMYGhuTHzCvqIFOPnwrKoL+jezvtNOj58U6zj8m41AHUwJJ+i1Bg4PT4wfvURgeAmUt+KLxn2r2x54hz3dkOvA0Nq/noj816RlOmBVW3KiiA1c5DGwyAXFfboLlFI5C0nwp1gx8+QUwJ0qKfEBq8qRfcJf2nSwK21lF37IE8MIGlQNcASEhbDAMn6qP5i3KjtK5SQOCExWsjofHoegCUK557nlPepQFqvrQmE+ipvajBFZz4uFaViP55xATyFfwG+1tta9uw4Z7pD3xi7EPzywKbgjl1E/3xwwGTPXhR5R3JieaGuwwGOUElvnoP4onRns3iMOjYuXp7RMdG22uoV4Ije36YhpM6gFwlyFtQL2pmjC2370YD8vmMNRwLUGtqA/r+Tlzj0n4ncDT02QLYkJodLrQY3CFRDdVaDDakZocL/wH/AdPykJ+gGwAAAABJRU5ErkJggg=='; | |
| const Message = { | |
| train_label_1: { | |
| 'ja': 'ラベル1を学習する', | |
| 'ja-Hira': 'ラベル1をがくしゅうする', | |
| 'en': 'train label 1', | |
| 'zh-cn': '学习标签1', | |
| 'zh-tw': '學習標籤1' | |
| }, | |
| train_label_2: { | |
| 'ja': 'ラベル2を学習する', | |
| 'ja-Hira': 'ラベル2をがくしゅうする', | |
| 'en': 'train label 2', | |
| 'zh-cn': '学习标签2', | |
| 'zh-tw': '學習標籤2' | |
| }, | |
| train_label_3: { | |
| 'ja': 'ラベル3を学習する', | |
| 'ja-Hira': 'ラベル3をがくしゅうする', | |
| 'en': 'train label 3', | |
| 'zh-cn': '学习标签3', | |
| 'zh-tw': '學習標籤3' | |
| }, | |
| train: { | |
| 'ja': 'ラベル[LABEL]を学習する', | |
| 'ja-Hira': 'ラベル[LABEL]をがくしゅうする', | |
| 'en': 'train label [LABEL]', | |
| 'zh-cn': '学习标签[LABEL]', | |
| 'zh-tw': '學習標籤[LABEL]' | |
| }, | |
| when_received_block: { | |
| 'ja': 'ラベル[LABEL]を受け取ったとき', | |
| 'ja-Hira': 'ラベル[LABEL]をうけとったとき', | |
| 'en': 'when received label:[LABEL]', | |
| 'zh-cn': '接收到类别[LABEL]时', | |
| 'zh-tw': '接收到類別[LABEL]時' | |
| }, | |
| label_block: { | |
| 'ja': 'ラベル', | |
| 'ja-Hira': 'ラベル', | |
| 'en': 'label', | |
| 'zh-cn': '标签', | |
| 'zh-tw': '標籤' | |
| }, | |
| counts_label_1: { | |
| 'ja': 'ラベル1の枚数', | |
| 'ja-Hira': 'ラベル1のまいすう', | |
| 'en': 'counts of label 1', | |
| 'zh-cn': '标签数量1', | |
| 'zh-tw': '標籤數量1' | |
| }, | |
| counts_label_2: { | |
| 'ja': 'ラベル2の枚数', | |
| 'ja-Hira': 'ラベル2のまいすう', | |
| 'en': 'counts of label 2', | |
| 'zh-cn': '标签数量2', | |
| 'zh-tw': '標籤數量2' | |
| }, | |
| counts_label_3: { | |
| 'ja': 'ラベル3の枚数', | |
| 'ja-Hira': 'ラベル3のまいすう', | |
| 'en': 'counts of label 3', | |
| 'zh-cn': '标签数量3', | |
| 'zh-tw': '標籤數量3' | |
| }, | |
| counts_label_4: { | |
| 'ja': 'ラベル4の枚数', | |
| 'ja-Hira': 'ラベル4のまいすう', | |
| 'en': 'counts of label 4', | |
| 'zh-cn': '标签数量4', | |
| 'zh-tw': '標籤數量4' | |
| }, | |
| counts_label_5: { | |
| 'ja': 'ラベル5の枚数', | |
| 'ja-Hira': 'ラベル5のまいすう', | |
| 'en': 'counts of label 5', | |
| 'zh-cn': '标签数量5', | |
| 'zh-tw': '標籤數量5' | |
| }, | |
| counts_label_6: { | |
| 'ja': 'ラベル6の枚数', | |
| 'ja-Hira': 'ラベル6のまいすう', | |
| 'en': 'counts of label 6', | |
| 'zh-cn': '标签数量6', | |
| 'zh-tw': '標籤數量6' | |
| }, | |
| counts_label_7: { | |
| 'ja': 'ラベル7の枚数', | |
| 'ja-Hira': 'ラベル7のまいすう', | |
| 'en': 'counts of label 7', | |
| 'zh-cn': '标签数量7', | |
| 'zh-tw': '標籤數量7' | |
| }, | |
| counts_label_8: { | |
| 'ja': 'ラベル8の枚数', | |
| 'ja-Hira': 'ラベル8のまいすう', | |
| 'en': 'counts of label 8', | |
| 'zh-cn': '标签数量8', | |
| 'zh-tw': '標籤數量8' | |
| }, | |
| counts_label_9: { | |
| 'ja': 'ラベル9の枚数', | |
| 'ja-Hira': 'ラベル9のまいすう', | |
| 'en': 'counts of label 9', | |
| 'zh-cn': '标签数量9', | |
| 'zh-tw': '標籤數量9' | |
| }, | |
| counts_label_10: { | |
| 'ja': 'ラベル10の枚数', | |
| 'ja-Hira': 'ラベル10のまいすう', | |
| 'en': 'counts of label 10', | |
| 'zh-cn': '标签数量10', | |
| 'zh-tw': '標籤數量10' | |
| }, | |
| counts_label: { | |
| 'ja': 'ラベル[LABEL]の枚数', | |
| 'ja-Hira': 'ラベル[LABEL]のまいすう', | |
| 'en': 'counts of label [LABEL]', | |
| 'zh-cn': '标签数量[LABEL]', | |
| 'zh-tw': '標籤數量[LABEL]' | |
| }, | |
| any: { | |
| 'ja': 'のどれか', | |
| 'ja-Hira': 'のどれか', | |
| 'en': 'any', | |
| 'zh-cn': '任何', | |
| 'zh-tw': '任何' | |
| }, | |
| all: { | |
| 'ja': 'の全て', | |
| 'ja-Hira': 'のすべて', | |
| 'en': 'all', | |
| 'zh-cn': '所有', | |
| 'zh-tw': '所有量' | |
| }, | |
| reset: { | |
| 'ja': 'ラベル[LABEL]の学習をリセット', | |
| 'ja-Hira': 'ラベル[LABEL]のがくしゅうをリセット', | |
| 'en': 'reset label:[LABEL]', | |
| 'zh-cn': '重置[LABEL]', | |
| 'zh-tw': '重置[LABEL]' | |
| }, | |
| download_learning_data: { | |
| 'ja': '学習データをダウンロード', | |
| 'ja-Hira': 'がくしゅうデータをダウンロード', | |
| 'en': 'download learning data', | |
| 'zh-cn': '下载学习数据', | |
| 'zh-tw': '下載學習資料' | |
| }, | |
| upload_learning_data: { | |
| 'ja': '学習データをアップロード', | |
| 'ja-Hira': 'がくしゅうデータをアップロード', | |
| 'en': 'upload learning data', | |
| 'zh-cn': '上传学习数据', | |
| 'zh-tw': '上傳學習資料' | |
| }, | |
| upload: { | |
| 'ja': 'アップロード', | |
| 'ja-Hira': 'アップロード', | |
| 'en': 'upload', | |
| 'zh-cn': '上传', | |
| 'zh-tw': '上傳' | |
| }, | |
| uploaded: { | |
| 'ja': 'アップロードが完了しました。', | |
| 'ja-Hira': 'アップロードがかんりょうしました。', | |
| 'en': 'The upload is complete.', | |
| 'zh-cn': '上传完成。', | |
| 'zh-tw': '上傳完成。' | |
| }, | |
| upload_instruction: { | |
| 'ja': 'ファイルを選び、アップロードボタンをクリックして下さい。', | |
| 'ja-Hira': 'ファイルをえらび、アップロードボタンをクリックしてください。', | |
| 'en': 'Select a file and click the upload button.', | |
| 'zh-cn': '选择一个文件，然后单击上传按钮。', | |
| 'zh-tw': '選擇一個檔案，然後點擊上傳按鈕' | |
| }, | |
| confirm_reset: { | |
| 'ja': '本当にリセットしてもよろしいですか？', | |
| 'ja-Hira': 'ほんとうにリセットしてもよろしいですか？', | |
| 'en': 'Are you sure to reset?', | |
| 'zh-cn': '你确定要重置吗？', | |
| 'zh-tw': '您確定要重置嗎？' | |
| }, | |
| toggle_classification: { | |
| 'ja': 'ラベル付けを[CLASSIFICATION_STATE]にする', | |
| 'ja-Hira': 'ラベルづけを[CLASSIFICATION_STATE]にする', | |
| 'en': 'turn classification [CLASSIFICATION_STATE]', | |
| 'zh-cn': '[CLASSIFICATION_STATE]分类', | |
| 'zh-tw': '[CLASSIFICATION_STATE]分類' | |
| }, | |
| set_classification_interval: { | |
| 'ja': 'ラベル付けを[CLASSIFICATION_INTERVAL]秒間に1回行う', | |
| 'ja-Hira': 'ラベルづけを[CLASSIFICATION_INTERVAL]びょうかんに1かいおこなう', | |
| 'en': 'Label once every [CLASSIFICATION_INTERVAL] seconds', | |
| 'zh-cn': '每隔[CLASSIFICATION_INTERVAL]秒标记一次', | |
| 'zh-tw': '每隔[CLASSIFICATION_INTERVAL]秒標記一次' | |
| }, | |
| video_toggle: { | |
| 'ja': 'ビデオを[VIDEO_STATE]にする', | |
| 'ja-Hira': 'ビデオを[VIDEO_STATE]にする', | |
| 'en': 'turn video [VIDEO_STATE]', | |
| 'zh-cn': '[VIDEO_STATE]摄像头', | |
| 'zh-tw': '視訊設為[VIDEO_STATE]' | |
| }, | |
| set_input: { | |
| 'ja': '[INPUT]の画像を学習/判定する', | |
| 'ja-Hira': '[INPUT]のがぞうをがくしゅう/はんていする', | |
| 'en': 'Learn/Classify [INPUT] image', | |
| 'zh-cn': '学习/分类[INPUT]图像', | |
| 'zh-tw': '學習/分類[INPUT]影像' | |
| }, | |
| on: { | |
| 'ja': '入', | |
| 'ja-Hira': 'いり', | |
| 'en': 'on', | |
| 'zh-cn': '开启', | |
| 'zh-tw': '開啟' | |
| }, | |
| off: { | |
| 'ja': '切', | |
| 'ja-Hira': 'きり', | |
| 'en': 'off', | |
| 'zh-cn': '关闭', | |
| 'zh-tw': '關閉' | |
| }, | |
| video_on_flipped: { | |
| 'ja': '左右反転', | |
| 'ja-Hira': 'さゆうはんてん', | |
| 'en': 'on flipped', | |
| 'zh-cn': '镜像开启', | |
| 'zh-tw': '翻轉' | |
| }, | |
| webcam: { | |
| 'ja': 'カメラ', | |
| 'ja-Hira': 'カメラ', | |
| 'en': 'webcam', | |
| 'zh-cn': '网络摄像头', | |
| 'zh-tw': '網路攝影機' | |
| }, | |
| stage: { | |
| 'ja': 'ステージ', | |
| 'ja-Hira': 'ステージ', | |
| 'en': 'stage', | |
| 'zh-cn': '舞台', | |
| 'zh-tw': '舞台' | |
| }, | |
| first_training_warning: { | |
| 'ja': '最初の学習にはしばらく時間がかかるので、何度もクリックしないで下さい。', | |
| 'ja-Hira': 'さいしょのがくしゅうにはしばらくじかんがかかるので、なんどもクリックしないでください。', | |
| 'en': 'The first training will take a while, so do not click again and again.', | |
| 'zh-cn': '第一项研究需要一段时间，所以不要一次又一次地点击。', | |
| 'zh-tw': '第一次訓練需要一段時間，請稍後，不要一直點擊。' | |
| }, | |
| switch_webcam: { | |
| 'ja': 'カメラを[DEVICE]に切り替える', | |
| 'ja-Hira': 'カメラを[DEVICE]にきりかえる', | |
| 'en': 'switch webcam to [DEVICE]', | |
| 'zh-cn': '网络摄像头切换到[DEVICE]', | |
| 'zh-tw': '網路攝影機切換到[DEVICE]' | |
| } | |
| } | |
| const AvailableLocales = ['en', 'ja', 'ja-Hira', 'zh-cn', 'zh-tw']; | |
| class Scratch3ML2ScratchBlocks { | |
| /** | |
| * @return {string} - the name of this extension. | |
| */ | |
| static get EXTENSION_NAME() { | |
| return 'ML2Scratch'; | |
| } | |
| /** | |
| * @return {string} - the ID of this extension. | |
| */ | |
| static get EXTENSION_ID() { | |
| return 'ml2scratch'; | |
| } | |
| /** | |
| * URL to get this extension. | |
| * @type {string} | |
| */ | |
| static get extensionURL() { | |
| return extensionURL; | |
| } | |
| /** | |
| * Set URL to get this extension. | |
| * extensionURL will be reset when the module is loaded from the web. | |
| * @param {string} url - URL | |
| */ | |
| static set extensionURL(url) { | |
| extensionURL = url; | |
| } | |
| constructor(runtime) { | |
| this.runtime = runtime; | |
| if (runtime.formatMessage) { | |
| // Replace 'formatMessage' to a formatter which is used in the runtime. | |
| formatMessage = runtime.formatMessage; | |
| } | |
| this.when_received = false; | |
| this.when_received_arr = Array(8).fill(false); | |
| this.label = null; | |
| this.locale = this.setLocale(); | |
| this.blockClickedAt = null; | |
| this.counts = null; | |
| this.firstTraining = true; | |
| this.interval = 1000; | |
| this.globalVideoTransparency = 0; | |
| this.setVideoTransparency({ | |
| TRANSPARENCY: this.globalVideoTransparency | |
| }); | |
| this.canvas = document.querySelector('canvas'); | |
| this.runtime.ioDevices.video.enableVideo().then(() => { this.input = this.runtime.ioDevices.video.provider.video }); | |
| this.knnClassifier = ml5.KNNClassifier(); | |
| this.featureExtractor = ml5.featureExtractor('MobileNet', () => { | |
| console.log('[featureExtractor] Model Loaded!'); | |
| this.timer = setInterval(() => { | |
| this.classify(); | |
| }, this.interval); | |
| }); | |
| this.devices = [{ text: 'default', value: '' }]; | |
| const dialog = document.createElement("DIALOG"); | |
| dialog.id = "upload-dialog"; | |
| dialog.innerHTML = ` | |
| <html><body> | |
| <div>${Message.upload_instruction[this.locale]}</p><input type="file" id="upload-files"><input type="button" value="${Message.upload[this.locale]}" id="upload-button"></div> | |
| <div style="margin-top:10px;display:flex;justify-content:flex-end;"><button id="close" aria-label="close" formnovalidate>閉じる</button></div> | |
| </body><body> | |
| `; | |
| this.uploadDialog = dialog; | |
| document.body.appendChild(dialog); | |
| document.getElementById("upload-button").onclick = () =>{ | |
| this.uploadButtonClicked(); | |
| } | |
| document.getElementById("close").onclick = () =>{ | |
| dialog.close(); | |
| } | |
| try { | |
| navigator.mediaDevices.enumerateDevices().then(media => { | |
| for (const device of media) { | |
| if (device.kind === 'videoinput') { | |
| this.devices.push({ | |
| text: device.label, | |
| value: device.deviceId | |
| }); | |
| } | |
| } | |
| }); | |
| } catch (e) { | |
| console.error("failed to load media devices!"); | |
| } | |
| } | |
| getInfo() { | |
| this.locale = this.setLocale(); | |
| return { | |
| id: Scratch3ML2ScratchBlocks.EXTENSION_ID, | |
| name: Scratch3ML2ScratchBlocks.EXTENSION_NAME, | |
| extensionURL: Scratch3ML2ScratchBlocks.extensionURL, | |
| blockIconURI: blockIconURI, | |
| blocks: [ | |
| { | |
| opcode: 'addExample1', | |
| blockType: BlockType.COMMAND, | |
| text: Message.train_label_1[this.locale] | |
| }, | |
| { | |
| opcode: 'addExample2', | |
| blockType: BlockType.COMMAND, | |
| text: Message.train_label_2[this.locale] | |
| }, | |
| { | |
| opcode: 'addExample3', | |
| blockType: BlockType.COMMAND, | |
| text: Message.train_label_3[this.locale] | |
| }, | |
| { | |
| opcode: 'train', | |
| text: Message.train[this.locale], | |
| blockType: BlockType.COMMAND, | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| menu: 'train_menu', | |
| defaultValue: '4' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'trainAny', | |
| text: Message.train[this.locale], | |
| blockType: BlockType.COMMAND, | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| defaultValue: '11' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'getLabel', | |
| text: Message.label_block[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'whenReceived', | |
| text: Message.when_received_block[this.locale], | |
| blockType: BlockType.HAT, | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| menu: 'received_menu', | |
| defaultValue: 'any' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'whenReceivedAny', | |
| text: Message.when_received_block[this.locale], | |
| blockType: BlockType.HAT, | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| defaultValue: '11' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'getCountByLabel1', | |
| text: Message.counts_label_1[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel2', | |
| text: Message.counts_label_2[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel3', | |
| text: Message.counts_label_3[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel4', | |
| text: Message.counts_label_4[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel5', | |
| text: Message.counts_label_5[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel6', | |
| text: Message.counts_label_6[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel7', | |
| text: Message.counts_label_7[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel8', | |
| text: Message.counts_label_8[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel9', | |
| text: Message.counts_label_9[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel10', | |
| text: Message.counts_label_10[this.locale], | |
| blockType: BlockType.REPORTER | |
| }, | |
| { | |
| opcode: 'getCountByLabel', | |
| text: Message.counts_label[this.locale], | |
| blockType: BlockType.REPORTER, | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| defaultValue: '11' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'reset', | |
| blockType: BlockType.COMMAND, | |
| text: Message.reset[this.locale], | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| menu: 'reset_menu', | |
| defaultValue: 'all' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'resetAny', | |
| blockType: BlockType.COMMAND, | |
| text: Message.reset[this.locale], | |
| arguments: { | |
| LABEL: { | |
| type: ArgumentType.STRING, | |
| defaultValue: '11' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'download', | |
| text: Message.download_learning_data[this.locale], | |
| blockType: BlockType.COMMAND | |
| }, | |
| { | |
| opcode: 'upload', | |
| text: Message.upload_learning_data[this.locale], | |
| blockType: BlockType.COMMAND | |
| }, | |
| { | |
| opcode: 'toggleClassification', | |
| text: Message.toggle_classification[this.locale], | |
| blockType: BlockType.COMMAND, | |
| arguments: { | |
| CLASSIFICATION_STATE: { | |
| type: ArgumentType.STRING, | |
| menu: 'classification_menu', | |
| defaultValue: 'off' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'setClassificationInterval', | |
| text: Message.set_classification_interval[this.locale], | |
| blockType: BlockType.COMMAND, | |
| arguments: { | |
| CLASSIFICATION_INTERVAL: { | |
| type: ArgumentType.STRING, | |
| menu: 'classification_interval_menu', | |
| defaultValue: '1' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'videoToggle', | |
| text: Message.video_toggle[this.locale], | |
| blockType: BlockType.COMMAND, | |
| arguments: { | |
| VIDEO_STATE: { | |
| type: ArgumentType.STRING, | |
| menu: 'video_menu', | |
| defaultValue: 'off' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'setVideoTransparency', | |
| text: formatMessage({ | |
| id: 'videoSensing.setVideoTransparency', | |
| default: 'set video transparency to [TRANSPARENCY]', | |
| description: 'Controls transparency of the video preview layer' | |
| }), | |
| arguments: { | |
| TRANSPARENCY: { | |
| type: ArgumentType.NUMBER, | |
| defaultValue: 50 | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'setInput', | |
| text: Message.set_input[this.locale], | |
| blockType: BlockType.COMMAND, | |
| arguments: { | |
| INPUT: { | |
| type: ArgumentType.STRING, | |
| menu: 'input_menu', | |
| defaultValue: 'webcam' | |
| } | |
| } | |
| }, | |
| { | |
| opcode: 'switchCamera', | |
| blockType: BlockType.COMMAND, | |
| text: Message.switch_webcam[this.locale], | |
| arguments: { | |
| DEVICE: { | |
| type: ArgumentType.STRING, | |
| defaultValue: '', | |
| menu: 'mediadevices' | |
| } | |
| } | |
| } | |
| ], | |
| menus: { | |
| received_menu: { | |
| items: this.getMenu('received') | |
| }, | |
| reset_menu: { | |
| items: this.getMenu('reset') | |
| }, | |
| train_menu: { | |
| items: this.getTrainMenu() | |
| }, | |
| count_menu: { | |
| items: this.getTrainMenu() | |
| }, | |
| video_menu: this.getVideoMenu(), | |
| classification_interval_menu: { | |
| acceptReporters: true, | |
| items: this.getClassificationIntervalMenu() | |
| }, | |
| classification_menu: this.getClassificationMenu(), | |
| input_menu: this.getInputMenu(), | |
| mediadevices: { | |
| acceptReporters: true, | |
| items: 'getDevices' | |
| } | |
| } | |
| }; | |
| } | |
| /** | |
| * The transparency setting of the video preview stored in a value | |
| * accessible by any object connected to the virtual machine. | |
| * @type {number} | |
| */ | |
| get globalVideoTransparency() { | |
| const stage = this.runtime.getTargetForStage(); | |
| if (stage) { | |
| return stage.videoTransparency; | |
| } | |
| return 50; | |
| } | |
| set globalVideoTransparency(transparency) { | |
| const stage = this.runtime.getTargetForStage(); | |
| if (stage) { | |
| stage.videoTransparency = transparency; | |
| } | |
| return transparency; | |
| } | |
| addExample1() { | |
| this.firstTrainingWarning(); | |
| let features = this.featureExtractor.infer(this.input); | |
| this.knnClassifier.addExample(features, '1'); | |
| this.updateCounts(); | |
| } | |
| addExample2() { | |
| this.firstTrainingWarning(); | |
| let features = this.featureExtractor.infer(this.input); | |
| this.knnClassifier.addExample(features, '2'); | |
| this.updateCounts(); | |
| } | |
| addExample3() { | |
| this.firstTrainingWarning(); | |
| let features = this.featureExtractor.infer(this.input); | |
| this.knnClassifier.addExample(features, '3'); | |
| this.updateCounts(); | |
| } | |
| train(args) { | |
| this.firstTrainingWarning(); | |
| let features = this.featureExtractor.infer(this.input); | |
| this.knnClassifier.addExample(features, args.LABEL); | |
| this.updateCounts(); | |
| } | |
| trainAny(args) { | |
| this.train(args); | |
| } | |
| getLabel() { | |
| return this.label; | |
| } | |
| whenReceived(args) { | |
| if (args.LABEL === 'any') { | |
| if (this.when_received) { | |
| setTimeout(() => { | |
| this.when_received = false; | |
| }, HAT_TIMEOUT); | |
| return true; | |
| } | |
| return false; | |
| } else { | |
| if (this.when_received_arr[args.LABEL]) { | |
| setTimeout(() => { | |
| this.when_received_arr[args.LABEL] = false; | |
| }, HAT_TIMEOUT); | |
| return true; | |
| } | |
| return false; | |
| } | |
| } | |
| whenReceivedAny(args) { | |
| return this.whenReceived(args); | |
| } | |
| getCountByLabel1() { | |
| if (this.counts) { | |
| return this.counts['1']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel2() { | |
| if (this.counts) { | |
| return this.counts['2']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel3() { | |
| if (this.counts) { | |
| return this.counts['3']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel4() { | |
| if (this.counts) { | |
| return this.counts['4']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel5() { | |
| if (this.counts) { | |
| return this.counts['5']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel6() { | |
| if (this.counts) { | |
| return this.counts['6']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel7() { | |
| if (this.counts) { | |
| return this.counts['7']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel8() { | |
| if (this.counts) { | |
| return this.counts['8']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel9() { | |
| if (this.counts) { | |
| return this.counts['9']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel10() { | |
| if (this.counts) { | |
| return this.counts['10']; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| getCountByLabel(args) { | |
| if (this.counts[args.LABEL]) { | |
| return this.counts[args.LABEL]; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| reset(args) { | |
| if (this.actionRepeated()) { return }; | |
| setTimeout(() => { | |
| let result = confirm(Message.confirm_reset[this.locale]); | |
| if (result) { | |
| if (args.LABEL == 'all') { | |
| this.knnClassifier.clearAllLabels(); | |
| for (let label in this.counts) { | |
| this.counts[label] = 0; | |
| } | |
| } else { | |
| if (this.counts[args.LABEL] > 0) { | |
| this.knnClassifier.clearLabel(args.LABEL); | |
| this.counts[args.LABEL] = 0; | |
| } | |
| } | |
| } | |
| }, 1000); | |
| } | |
| resetAny(args) { | |
| this.reset(args); | |
| } | |
| download() { | |
| if (this.actionRepeated()) { return }; | |
| let fileName = String(Date.now()); | |
| this.knnClassifier.save(fileName); | |
| } | |
| upload() { | |
| if (this.actionRepeated()) { return }; | |
| document.getElementById('upload-dialog').showModal(); | |
| } | |
| toggleClassification(args) { | |
| let state = args.CLASSIFICATION_STATE; | |
| if (this.timer) { | |
| clearTimeout(this.timer); | |
| } | |
| if (state === 'on') { | |
| this.timer = setInterval(() => { | |
| this.classify(); | |
| }, this.interval); | |
| } | |
| } | |
| setClassificationInterval(args) { | |
| if (this.timer) { | |
| clearTimeout(this.timer); | |
| } | |
| this.interval = args.CLASSIFICATION_INTERVAL * 1000; | |
| this.timer = setInterval(() => { | |
| this.classify(); | |
| }, this.interval); | |
| } | |
| videoToggle(args) { | |
| let state = args.VIDEO_STATE; | |
| if (state === 'off') { | |
| this.runtime.ioDevices.video.disableVideo(); | |
| } else { | |
| this.runtime.ioDevices.video.enableVideo().then(() => { this.input = this.runtime.ioDevices.video.provider.video }); | |
| this.runtime.ioDevices.video.mirror = state === "on"; | |
| } | |
| } | |
| /** | |
| * A scratch command block handle that configures the video preview's | |
| * transparency from passed arguments. | |
| * @param {object} args - the block arguments | |
| * @param {number} args.TRANSPARENCY - the transparency to set the video | |
| * preview to | |
| */ | |
| setVideoTransparency(args) { | |
| const transparency = Cast.toNumber(args.TRANSPARENCY); | |
| this.globalVideoTransparency = transparency; | |
| this.runtime.ioDevices.video.setPreviewGhost(transparency); | |
| } | |
| setInput(args) { | |
| let input = args.INPUT; | |
| if (input === 'webcam') { | |
| this.input = this.runtime.ioDevices.video.provider.video; | |
| } else { | |
| this.input = this.canvas; | |
| } | |
| } | |
| uploadButtonClicked() { | |
| let files = document.getElementById('upload-files').files; | |
| if (files.length <= 0) { | |
| alert('Please select JSON file.'); | |
| return false; | |
| } | |
| let fr = new FileReader(); | |
| fr.onload = (e) => { | |
| let data = JSON.parse(e.target.result); | |
| this.knnClassifier.load(data, () => { | |
| console.log('uploaded!'); | |
| this.updateCounts(); | |
| alert(Message.uploaded[this.locale]); | |
| }); | |
| } | |
| fr.onloadend = (e) => { | |
| document.getElementById('upload-files').value = ""; | |
| } | |
| fr.readAsText(files.item(0)); | |
| this.uploadDialog.close(); | |
| } | |
| classify() { | |
| let numLabels = this.knnClassifier.getNumLabels(); | |
| if (numLabels == 0) return; | |
| let features = this.featureExtractor.infer(this.input); | |
| this.knnClassifier.classify(features, (err, result) => { | |
| if (err) { | |
| console.error(err); | |
| } else { | |
| this.label = this.getTopConfidenceLabel(result.confidencesByLabel); | |
| this.when_received = true; | |
| this.when_received_arr[this.label] = true | |
| } | |
| }); | |
| } | |
| getTopConfidenceLabel(confidences) { | |
| let topConfidenceLabel; | |
| let topConfidence = 0; | |
| for (let label in confidences) { | |
| if (confidences[label] > topConfidence) { | |
| topConfidenceLabel = label; | |
| } | |
| } | |
| return topConfidenceLabel; | |
| } | |
| updateCounts() { | |
| this.counts = this.knnClassifier.getCountByLabel(); | |
| console.debug(this.counts); | |
| } | |
| actionRepeated() { | |
| let currentTime = Date.now(); | |
| if (this.blockClickedAt && (this.blockClickedAt + 250) > currentTime) { | |
| console.log('Please do not repeat trigerring this block.'); | |
| this.blockClickedAt = currentTime; | |
| return true; | |
| } else { | |
| this.blockClickedAt = currentTime; | |
| return false; | |
| } | |
| } | |
| getMenu(name) { | |
| let arr = []; | |
| let defaultValue = 'any'; | |
| let text = Message.any[this.locale]; | |
| if (name == 'reset') { | |
| defaultValue = 'all'; | |
| text = Message.all[this.locale]; | |
| } | |
| arr.push({ text: text, value: defaultValue }); | |
| for (let i = 1; i <= 10; i++) { | |
| let obj = {}; | |
| obj.text = i.toString(10); | |
| obj.value = i.toString(10); | |
| arr.push(obj); | |
| }; | |
| return arr; | |
| } | |
| getTrainMenu() { | |
| let arr = []; | |
| for (let i = 4; i <= 10; i++) { | |
| let obj = {}; | |
| obj.text = i.toString(10); | |
| obj.value = i.toString(10); | |
| arr.push(obj); | |
| }; | |
| return arr; | |
| } | |
| getVideoMenu() { | |
| return [ | |
| { | |
| text: Message.off[this.locale], | |
| value: 'off' | |
| }, | |
| { | |
| text: Message.on[this.locale], | |
| value: 'on' | |
| }, | |
| { | |
| text: Message.video_on_flipped[this.locale], | |
| value: 'on-flipped' | |
| } | |
| ] | |
| } | |
| getInputMenu() { | |
| return [ | |
| { | |
| text: Message.webcam[this.locale], | |
| value: 'webcam' | |
| }, | |
| { | |
| text: Message.stage[this.locale], | |
| value: 'stage' | |
| } | |
| ] | |
| } | |
| getClassificationIntervalMenu() { | |
| return [ | |
| { | |
| text: '1', | |
| value: '1' | |
| }, | |
| { | |
| text: '0.5', | |
| value: '0.5' | |
| }, | |
| { | |
| text: '0.2', | |
| value: '0.2' | |
| }, | |
| { | |
| text: '0.1', | |
| value: '0.1' | |
| } | |
| ] | |
| } | |
| getClassificationMenu() { | |
| return [ | |
| { | |
| text: Message.off[this.locale], | |
| value: 'off' | |
| }, | |
| { | |
| text: Message.on[this.locale], | |
| value: 'on' | |
| } | |
| ] | |
| } | |
| firstTrainingWarning() { | |
| if (this.firstTraining) { | |
| alert(Message.first_training_warning[this.locale]); | |
| this.firstTraining = false; | |
| } | |
| } | |
| setLocale() { | |
| let locale = formatMessage.setup().locale; | |
| if (AvailableLocales.includes(locale)) { | |
| return locale; | |
| } else { | |
| return 'en'; | |
| } | |
| } | |
| switchCamera(args) { | |
| if (args.DEVICE !== '') { | |
| if (this.runtime.ioDevices.video.provider._track !== null) { | |
| this.runtime.ioDevices.video.provider._track.stop(); | |
| const deviceId = args.DEVICE; | |
| navigator.mediaDevices.getUserMedia({ audio: false, video: { deviceId } }).then( | |
| stream => { | |
| try { | |
| this.runtime.ioDevices.video.provider._video.srcObject = stream; | |
| } catch (error) { | |
| this.runtime.ioDevices.video.provider._video.src = window.URL.createObjectURL(stream); | |
| } | |
| // Needed for Safari/Firefox, Chrome auto-plays. | |
| this.runtime.ioDevices.video.provider._video.play(); | |
| this.runtime.ioDevices.video.provider._track = stream.getTracks()[0]; | |
| } | |
| ); | |
| } | |
| } | |
| } | |
| getDevices() { | |
| return this.devices; | |
| } | |
| } | |
| exports.blockClass = Scratch3ML2ScratchBlocks; // loadable-extension needs this line. | |
| module.exports = Scratch3ML2ScratchBlocks; | |