soiz1 commited on
Commit
61abc04
·
verified ·
1 Parent(s): 93ff48c

Create scratch3_ic2scratch/index.js

Browse files
local-scratch-vm/src/extensions/scratch3_ic2scratch/index.js ADDED
@@ -0,0 +1,391 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ const ArgumentType = require('../../extension-support/argument-type');
2
+ const BlockType = require('../../extension-support/block-type');
3
+ const Cast = require('../../util/cast');
4
+ const log = require('../../util/log');
5
+ const ml5 = require('ml5');
6
+ const formatMessage = require('format-message');
7
+
8
+ const HAT_TIMEOUT = 100;
9
+
10
+ const blockIconURI = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACgAAAAoCAYAAACM/rhtAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAACXBIWXMAAAsTAAALEwEAmpwYAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpMwidZAAAGD0lEQVRYCe1YTWxUVRT+3sy8melMp5TWASmCLdJGA/6FhQkkmEgIiTEs3LlzYVy4MSFuSUxcaIxsdaExgQ2JcYcsTNREjBqr4g9QQGyAAlNL/9v5n+mM33ffvPExvNd2qAEWnObOe3PvOfd895xzzzlTK/X1B3XcxxS6j7EZaA8ArtVDDyy4VgtG2tnACmBuTQN+fK08AVvdNt0WwArFlzhqqCMEiwPN4YKqmXWQQ7x18xRfhPxhPtulFQFKkZg6qCBmWeZdc9V6HTmqL3OUOKGnKEa+KPnifCYMvwNdhytRpkA+HWK1tCxAqdSpE1SWqZVpkhLNRxtSMSyuhCiuYclGjn2KVF+sVbFQJx+fqHOILPKFY9gUspFtHEbQnWMZDt+PQIASdNVmqgW8mEjjpZ5tGExtxMaOLiSjHUhGYrBDBEbAIYEm1WilOkeF4LKVEnLlAjKFOVxc+AfHZ0YxXJpHN4EuNqDJM9ITBNQKKnWKtU20zGglh8+278fB/l1YrBRxIzuD2eIi5st5THIUlsqo0qq1uuO4EGUiBC3waTuBVCyB3ngKm5M9iIVtHLv0Hd7InMbuSJxWruEaD0K/MDQU27eTwN9GYkzxZNNSShfu69uBGJ/lagWlapmgKigTVJJujhKMAIW5rhHhiHN0EKhcXRRvQy4ZiWL3hkEgP44f8pM4W15EL/l6qCtPGzo+uBWOr4td987L8NygTFCivuR6M27dor1vXXYHjmw7AJtwLuSn8eHsKI1gYwutO8nQsFu28wXoBm+Um+jmWQQpWmq40XtSy/fciiknqtzYUlqyGKcDXWkc2nGgCeO1yct45dwJXKwWCTKCWelrrjoZxPPVedWmDkjnzRVwlXgFXADeOb2L10uKUR1mdH4Cn/x9ChtiXTi45Wk8mx7Ax0P7sffM56gzNGTBxr034r4x6N14pXfB8ButcrrdIl2u9yZ+w6GxU9g+/CnGc7N47uEhPBnvxvVaxeRRr+yaAXo3W+7dDQVdKETX4YXOPqCSx48Tl8xFe4oWBXNnmKf1euWuAXTBK0eClppWEg+FkGOaEiWU+BV/XnScv+sADRoGhbmdngvRxHVr6N4jgAThJmUXWAsu5xz3zIJEpTyoeu7ednU+hlzEDYh33cXKhUrMReXUag79nb0GymSVBY/5ViXWa83ARC0XqGVSG+Um6Mah7ujhJm6VSRSn8Cfr9Dtb9mLPpscxkZ/HieKMAe7NgVIUCFDA1hPgAtulgm7cGincqEZPdPdhZM+bWBdNNMvm0dHvTSv3mN3JcudGp6PQF6D8nuV4RJsyDSyU8oZb4eE1v5lc7UejHVN71pfoZgdUw5npMRy/+jPenTqPrXbSNCeC59XhC1A6l5SvQmTl84/ZMTyT7ucrxZWv2iST+yRDkEfOfYnDmWHekhRjkGawbPRH/7Nc66Vo/W5Uy1Jq1zNybTSFV2/8gpuFBbZTYdP7GaZVfmivKiuELseF2QwO3zyLKCvJo2wMdsZ6MMj9Mzy4ju0Hxm/OqNZCjmOABRyVAt4+84VpWFWqZBE36A1zwIf45C6be4zn5/DWyEl+szDE1irLtb8I/ApjPM5ZHcSPAgFKQF2uTjfIG/fR3GW8/NNRnGZ7JGvozwAVWL9BlSalcI9vbpzD88PHcDI/hUF20iMsdeqiBUwtfxA4LvtaVfOGFLBRjqsEuZNx8lVhBrt+PYqz09fMugEqsH6DikXfZs5j3+/HcIk/uPoJboxW6zTHc6y2HDjJB/4m0aKXSjznVl6Qy2z103Tz6+sHMJh8COlYJ2y6zHQpFKhwvcSkO1nKYiR7E+/PXUGf6gXxqhnVrVwJlFfvqgHKHkVu3UVlU7QC+GOq+ZNS1hICEUE0IbBiRCNJ/j6xTBMaGE9G0P8jMM20skut4kU/1jfTgul4j7l5mtd/Gkx65RdlJtN5c77K+XFaVDK6peJtl1YNUBsLhFpyuWq8XjZR5v5Lw82OS0ShuuMWf5VLWe5OwFHMv9RpIYikSGAU6KJWxVrTcDvo1nXJtENtWdC78UqKV1r37rXc+53E7XL7/e9rDwCu1aT/Ars+OtwtbgbfAAAAAElFTkSuQmCC';
11
+
12
+ const Message = {
13
+ when_received_block: {
14
+ 'ja': '認識の候補を受け取ったとき',
15
+ 'ja-Hira': 'にんしきのこうほをうけとったとき',
16
+ 'en': 'when received classification candidates',
17
+ 'zh-cn': '收到分类结果时'
18
+ },
19
+ result1: {
20
+ 'ja': '候補1',
21
+ 'ja-Hira': 'こうほ1',
22
+ 'en': 'candidate1',
23
+ 'zh-cn': '结果1'
24
+ },
25
+ result2: {
26
+ 'ja': '候補2',
27
+ 'ja-Hira': 'こうほ2',
28
+ 'en': 'candidate2',
29
+ 'zh-cn': '结果2'
30
+ },
31
+ result3: {
32
+ 'ja': '候補3',
33
+ 'ja-Hira': 'こうほ3',
34
+ 'en': 'candidate3',
35
+ 'zh-cn': '结果3'
36
+ },
37
+ confidence1: {
38
+ 'ja': '確信度1',
39
+ 'ja-Hira': 'かくしんど1',
40
+ 'en': 'confidence1',
41
+ 'zh-cn': '置信度1'
42
+ },
43
+ confidence2: {
44
+ 'ja': '確信度2',
45
+ 'ja-Hira': 'かくしんど2',
46
+ 'en': 'confidence2',
47
+ 'zh-cn': '置信度2'
48
+ },
49
+ confidence3: {
50
+ 'ja': '確信度3',
51
+ 'ja-Hira': 'かくしんど3',
52
+ 'en': 'confidence3',
53
+ 'zh-cn': '置信度3'
54
+ },
55
+ toggle_classification: {
56
+ 'ja': '画像認識を[CLASSIFICATION_STATE]にする',
57
+ 'ja-Hira': 'がぞうにんしきを[CLASSIFICATION_STATE]にする',
58
+ 'en': 'turn classification [CLASSIFICATION_STATE]',
59
+ 'zh-cn': '[CLASSIFICATION_STATE]分类'
60
+ },
61
+ set_classification_interval: {
62
+ 'ja': '画像認識を[CLASSIFICATION_INTERVAL]秒間に1回行う',
63
+ 'ja-Hira': 'がぞうにんしきを[CLASSIFICATION_INTERVAL]びょうかんに1かいおこなう',
64
+ 'en': 'Classify once every [CLASSIFICATION_INTERVAL] seconds',
65
+ 'zh-cn': '每隔[CLASSIFICATION_INTERVAL]秒标记一次'
66
+ },
67
+ video_toggle: {
68
+ 'ja': 'ビデオを[VIDEO_STATE]にする',
69
+ 'ja-Hira': 'ビデオを[VIDEO_STATE]にする',
70
+ 'en': 'turn video [VIDEO_STATE]',
71
+ 'zh-cn': '[VIDEO_STATE]摄像头'
72
+ },
73
+ on: {
74
+ 'ja': '入',
75
+ 'ja-Hira': 'いり',
76
+ 'en': 'on',
77
+ 'zh-cn': '开启'
78
+ },
79
+ off: {
80
+ 'ja': '切',
81
+ 'ja-Hira': 'きり',
82
+ 'en': 'off',
83
+ 'zh-cn': '关闭'
84
+ },
85
+ video_on_flipped: {
86
+ 'ja': '左右反転',
87
+ 'ja-Hira': 'さゆうはんてん',
88
+ 'en': 'on flipped',
89
+ 'zh-cn': '镜像开启'
90
+ }
91
+ }
92
+
93
+ const AvailableLocales = ['en', 'ja', 'ja-Hira', 'zh-cn'];
94
+
95
+ class Scratch3ImageClassifierBlocks {
96
+ constructor (runtime) {
97
+ this.runtime = runtime;
98
+ this.when_received = false;
99
+ this.results = [];
100
+ this.locale = this.setLocale();
101
+
102
+ this.blockClickedAt = null;
103
+
104
+ this.interval = 1000;
105
+
106
+ this.detect = () => {
107
+ this.video = this.runtime.ioDevices.video.provider.video;
108
+ this.classifier = ml5.imageClassifier('MobileNet', () => {
109
+ console.log('Model Loaded!');
110
+ this.timer = setInterval(() => {
111
+ this.classify();
112
+ }, this.interval);
113
+ });
114
+ }
115
+
116
+ this.runtime.ioDevices.video.enableVideo().then(this.detect)
117
+ }
118
+
119
+ getInfo() {
120
+ this.locale = this.setLocale();
121
+
122
+ return {
123
+ id: 'ic2scratch',
124
+ name: 'ImageClassifier2Scratch',
125
+ blockIconURI: blockIconURI,
126
+ blocks: [
127
+ {
128
+ opcode: 'getResult1',
129
+ text: Message.result1[this.locale],
130
+ blockType: BlockType.REPORTER
131
+ },
132
+ {
133
+ opcode: 'getResult2',
134
+ text: Message.result2[this.locale],
135
+ blockType: BlockType.REPORTER
136
+ },
137
+ {
138
+ opcode: 'getResult3',
139
+ text: Message.result3[this.locale],
140
+ blockType: BlockType.REPORTER
141
+ },
142
+ {
143
+ opcode: 'getConfidence1',
144
+ text: Message.confidence1[this.locale],
145
+ blockType: BlockType.REPORTER
146
+ },
147
+ {
148
+ opcode: 'getConfidence2',
149
+ text: Message.confidence2[this.locale],
150
+ blockType: BlockType.REPORTER
151
+ },
152
+ {
153
+ opcode: 'getConfidence3',
154
+ text: Message.confidence3[this.locale],
155
+ blockType: BlockType.REPORTER
156
+ },
157
+ {
158
+ opcode: 'whenReceived',
159
+ text: Message.when_received_block[this.locale],
160
+ blockType: BlockType.HAT,
161
+ },
162
+ {
163
+ opcode: 'toggleClassification',
164
+ text: Message.toggle_classification[this.locale],
165
+ blockType: BlockType.COMMAND,
166
+ arguments: {
167
+ CLASSIFICATION_STATE: {
168
+ type: ArgumentType.STRING,
169
+ menu: 'classification_menu',
170
+ defaultValue: 'off'
171
+ }
172
+ }
173
+ },
174
+ {
175
+ opcode: 'setClassificationInterval',
176
+ text: Message.set_classification_interval[this.locale],
177
+ blockType: BlockType.COMMAND,
178
+ arguments: {
179
+ CLASSIFICATION_INTERVAL: {
180
+ type: ArgumentType.STRING,
181
+ menu: 'classification_interval_menu',
182
+ defaultValue: '1'
183
+ }
184
+ }
185
+ },
186
+ {
187
+ opcode: 'videoToggle',
188
+ text: Message.video_toggle[this.locale],
189
+ blockType: BlockType.COMMAND,
190
+ arguments: {
191
+ VIDEO_STATE: {
192
+ type: ArgumentType.STRING,
193
+ menu: 'video_menu',
194
+ defaultValue: 'off'
195
+ }
196
+ }
197
+ },
198
+ {
199
+ opcode: 'setVideoTransparency',
200
+ text: formatMessage({
201
+ id: 'videoSensing.setVideoTransparency',
202
+ default: 'set video transparency to [TRANSPARENCY]',
203
+ description: 'Controls transparency of the video preview layer'
204
+ }),
205
+ arguments: {
206
+ TRANSPARENCY: {
207
+ type: ArgumentType.NUMBER,
208
+ defaultValue: 50
209
+ }
210
+ }
211
+ }
212
+ ],
213
+ menus: {
214
+ video_menu: this.getVideoMenu(),
215
+ classification_interval_menu: this.getClassificationIntervalMenu(),
216
+ classification_menu: this.getClassificationMenu()
217
+ }
218
+ };
219
+ }
220
+
221
+ getResult1() {
222
+ return this.results[0]['label'];
223
+ }
224
+
225
+ getResult2() {
226
+ return this.results[1]['label'];
227
+ }
228
+
229
+ getResult3() {
230
+ return this.results[2]['label'];
231
+ }
232
+
233
+ getConfidence1() {
234
+ return this.results[0]['confidence'];
235
+ }
236
+
237
+ getConfidence2() {
238
+ return this.results[1]['confidence'];
239
+ }
240
+
241
+ getConfidence3() {
242
+ return this.results[2]['confidence'];
243
+ }
244
+
245
+ whenReceived(args) {
246
+ if (this.when_received) {
247
+ setTimeout(() => {
248
+ this.when_received = false;
249
+ }, HAT_TIMEOUT);
250
+ return true;
251
+ }
252
+ return false;
253
+ }
254
+
255
+ toggleClassification (args) {
256
+ if (this.actionRepeated()) { return };
257
+
258
+ let state = args.CLASSIFICATION_STATE;
259
+ if (this.timer) {
260
+ clearTimeout(this.timer);
261
+ }
262
+ if (state === 'on') {
263
+ this.timer = setInterval(() => {
264
+ this.classify();
265
+ }, this.interval);
266
+ }
267
+ }
268
+
269
+ setClassificationInterval (args) {
270
+ if (this.actionRepeated()) { return };
271
+
272
+ if (this.timer) {
273
+ clearTimeout(this.timer);
274
+ }
275
+
276
+ this.interval = args.CLASSIFICATION_INTERVAL * 1000;
277
+ this.timer = setInterval(() => {
278
+ this.classify();
279
+ }, this.interval);
280
+ }
281
+
282
+ videoToggle (args) {
283
+ if (this.actionRepeated()) { return };
284
+
285
+ let state = args.VIDEO_STATE;
286
+ if (state === 'off') {
287
+ this.runtime.ioDevices.video.disableVideo();
288
+ } else {
289
+ this.runtime.ioDevices.video.enableVideo().then(this.detect);
290
+ this.runtime.ioDevices.video.mirror = state === "on";
291
+ }
292
+ }
293
+
294
+ /**
295
+ * A scratch command block handle that configures the video preview's
296
+ * transparency from passed arguments.
297
+ * @param {object} args - the block arguments
298
+ * @param {number} args.TRANSPARENCY - the transparency to set the video
299
+ * preview to
300
+ */
301
+ setVideoTransparency (args) {
302
+ const transparency = Cast.toNumber(args.TRANSPARENCY);
303
+ this.globalVideoTransparency = transparency;
304
+ this.runtime.ioDevices.video.setPreviewGhost(transparency);
305
+ }
306
+
307
+ classify() {
308
+ this.classifier.classify(this.video, (err, results) => {
309
+ if (err) {
310
+ console.error(err);
311
+ } else {
312
+ this.when_received = true;
313
+ this.results = results;
314
+ }
315
+ });
316
+ }
317
+
318
+ actionRepeated() {
319
+ let currentTime = Date.now();
320
+ if (this.blockClickedAt && (this.blockClickedAt + 250) > currentTime) {
321
+ console.log('Please do not repeat trigerring this block.');
322
+ this.blockClickedAt = currentTime;
323
+ return true;
324
+ } else {
325
+ this.blockClickedAt = currentTime;
326
+ return false;
327
+ }
328
+ }
329
+
330
+ getVideoMenu() {
331
+ return [
332
+ {
333
+ text: Message.off[this.locale],
334
+ value: 'off'
335
+ },
336
+ {
337
+ text: Message.on[this.locale],
338
+ value: 'on'
339
+ },
340
+ {
341
+ text: Message.video_on_flipped[this.locale],
342
+ value: 'on-flipped'
343
+ }
344
+ ]
345
+ }
346
+
347
+ getClassificationIntervalMenu() {
348
+ return [
349
+ {
350
+ text: '5',
351
+ value: '5'
352
+ },
353
+ {
354
+ text: '2',
355
+ value: '2'
356
+ },
357
+ {
358
+ text: '1',
359
+ value: '1'
360
+ },
361
+ {
362
+ text: '0.5',
363
+ value: '0.5'
364
+ }
365
+ ]
366
+ }
367
+
368
+ getClassificationMenu() {
369
+ return [
370
+ {
371
+ text: Message.off[this.locale],
372
+ value: 'off'
373
+ },
374
+ {
375
+ text: Message.on[this.locale],
376
+ value: 'on'
377
+ }
378
+ ]
379
+ }
380
+
381
+ setLocale() {
382
+ let locale = formatMessage.setup().locale;
383
+ if (AvailableLocales.includes(locale)) {
384
+ return locale;
385
+ } else {
386
+ return 'en';
387
+ }
388
+ }
389
+ }
390
+
391
+ module.exports = Scratch3ImageClassifierBlocks;