scaffolding/snail-extension.js

class Posenet2Scratch {
  constructor(runtime) {
    this.runtime = runtime;
    this.poses = [];
    this.keypoints = [];
    this._locale = this.setLocale();

    const video = document.createElement("video");
    video.width = 480;
    video.height = 360;
    video.autoplay = true;
    video.style.display = "none";

    navigator.mediaDevices
      .getUserMedia({ video: true, audio: false })
      .then((stream) => {
        video.srcObject = stream;
      });

    video.addEventListener("loadeddata", () => {
      // Load posenet model and process poses
      posenet.load().then((net) => {
        const detectPose = () => {
          net.estimateMultiplePoses(video, {
            flipHorizontal: false,
          }).then((poses) => {
            this.poses = poses;
            this.keypoints = poses[0]?.keypoints || [];
            requestAnimationFrame(detectPose);
          });
        };
        detectPose();
      });
    });

    this.runtime.ioDevices.video.enableVideo();
  }

  setLocale() {
    const supportedLocales = ["en", "ja", "ja-Hira"];
    const locale = navigator.language || "en";
    return supportedLocales.includes(locale) ? locale : "en";
  }

  getX(args) {
    const personIndex = parseInt(args.PERSON_NUMBER, 10) - 1;
    const partIndex = parseInt(args.PART, 10);
    const pose = this.poses[personIndex];

    if (pose && pose.pose.keypoints[partIndex]) {
      const x = pose.pose.keypoints[partIndex].position.x;
      return this.runtime.ioDevices.video.mirror ? 240 - x : x - 240;
    }
    return "";
  }

  getY(args) {
    const personIndex = parseInt(args.PERSON_NUMBER, 10) - 1;
    const partIndex = parseInt(args.PART, 10);
    const pose = this.poses[personIndex];

    if (pose && pose.pose.keypoints[partIndex]) {
      const y = pose.pose.keypoints[partIndex].position.y;
      return 180 - y;
    }
    return "";
  }

  getPeopleCount() {
    return this.poses.length;
  }

  getInfo() {
    return {
      id: "posenet2scratch",
      name: "Posenet2Scratch",
      blocks: [
        {
          opcode: "getX",
          blockType: "reporter",
          text: "[PART] x of person no. [PERSON_NUMBER]",
          arguments: {
            PERSON_NUMBER: { type: "string", menu: "personNumbers", defaultValue: "1" },
            PART: { type: "string", menu: "parts", defaultValue: "0" },
          },
        },
        {
          opcode: "getY",
          blockType: "reporter",
          text: "[PART] y of person no. [PERSON_NUMBER]",
          arguments: {
            PERSON_NUMBER: { type: "string", menu: "personNumbers", defaultValue: "1" },
            PART: { type: "string", menu: "parts", defaultValue: "0" },
          },
        },
        {
          opcode: "getPeopleCount",
          blockType: "reporter",
          text: "people count",
        },
      ],
      menus: {
        personNumbers: {
          acceptReporters: true,
          items: Array.from({ length: 10 }, (_, i) => ({ text: `${i + 1}`, value: `${i + 1}` })),
        },
        parts: {
          acceptReporters: true,
          items: [
            { text: "nose", value: "0" },
            { text: "left eye", value: "1" },
            { text: "right eye", value: "2" },
            { text: "left ear", value: "3" },
            { text: "right ear", value: "4" },
            { text: "left shoulder", value: "5" },
            { text: "right shoulder", value: "6" },
            { text: "left elbow", value: "7" },
            { text: "right elbow", value: "8" },
            { text: "left wrist", value: "9" },
            { text: "right wrist", value: "10" },
            { text: "left hip", value: "11" },
            { text: "right hip", value: "12" },
            { text: "left knee", value: "13" },
            { text: "right knee", value: "14" },
            { text: "left ankle", value: "15" },
            { text: "right ankle", value: "16" },
          ],
        },
      },
    };
  }
}

Scratch.extensions.register(new Posenet2Scratch());