Duplicate from jonigata/PoseMaker2

Co-authored-by: Naoyuki Hirayama <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,31 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y libgl1-mesa-dev
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+RUN mim install mmcv-full==1.7.0
+RUN pip install mmdet mmpose
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,12 @@
+---
+title: Demo Docker Gradio
+emoji: 📈
+colorFrom: indigo
+colorTo: indigo
+sdk: docker
+pinned: false
+license: apache-2.0
+duplicated_from: jonigata/PoseMaker2
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

external/coco.py

@@ -0,0 +1,181 @@
+dataset_info = dict(
+    dataset_name='coco',
+    paper_info=dict(
+        author='Lin, Tsung-Yi and Maire, Michael and '
+        'Belongie, Serge and Hays, James and '
+        'Perona, Pietro and Ramanan, Deva and '
+        r'Doll{\'a}r, Piotr and Zitnick, C Lawrence',
+        title='Microsoft coco: Common objects in context',
+        container='European conference on computer vision',
+        year='2014',
+        homepage='http://cocodataset.org/',
+    ),
+    keypoint_info={
+        0:
+        dict(name='nose', id=0, color=[51, 153, 255], type='upper', swap=''),
+        1:
+        dict(
+            name='left_eye',
+            id=1,
+            color=[51, 153, 255],
+            type='upper',
+            swap='right_eye'),
+        2:
+        dict(
+            name='right_eye',
+            id=2,
+            color=[51, 153, 255],
+            type='upper',
+            swap='left_eye'),
+        3:
+        dict(
+            name='left_ear',
+            id=3,
+            color=[51, 153, 255],
+            type='upper',
+            swap='right_ear'),
+        4:
+        dict(
+            name='right_ear',
+            id=4,
+            color=[51, 153, 255],
+            type='upper',
+            swap='left_ear'),
+        5:
+        dict(
+            name='left_shoulder',
+            id=5,
+            color=[0, 255, 0],
+            type='upper',
+            swap='right_shoulder'),
+        6:
+        dict(
+            name='right_shoulder',
+            id=6,
+            color=[255, 128, 0],
+            type='upper',
+            swap='left_shoulder'),
+        7:
+        dict(
+            name='left_elbow',
+            id=7,
+            color=[0, 255, 0],
+            type='upper',
+            swap='right_elbow'),
+        8:
+        dict(
+            name='right_elbow',
+            id=8,
+            color=[255, 128, 0],
+            type='upper',
+            swap='left_elbow'),
+        9:
+        dict(
+            name='left_wrist',
+            id=9,
+            color=[0, 255, 0],
+            type='upper',
+            swap='right_wrist'),
+        10:
+        dict(
+            name='right_wrist',
+            id=10,
+            color=[255, 128, 0],
+            type='upper',
+            swap='left_wrist'),
+        11:
+        dict(
+            name='left_hip',
+            id=11,
+            color=[0, 255, 0],
+            type='lower',
+            swap='right_hip'),
+        12:
+        dict(
+            name='right_hip',
+            id=12,
+            color=[255, 128, 0],
+            type='lower',
+            swap='left_hip'),
+        13:
+        dict(
+            name='left_knee',
+            id=13,
+            color=[0, 255, 0],
+            type='lower',
+            swap='right_knee'),
+        14:
+        dict(
+            name='right_knee',
+            id=14,
+            color=[255, 128, 0],
+            type='lower',
+            swap='left_knee'),
+        15:
+        dict(
+            name='left_ankle',
+            id=15,
+            color=[0, 255, 0],
+            type='lower',
+            swap='right_ankle'),
+        16:
+        dict(
+            name='right_ankle',
+            id=16,
+            color=[255, 128, 0],
+            type='lower',
+            swap='left_ankle')
+    },
+    skeleton_info={
+        0:
+        dict(link=('left_ankle', 'left_knee'), id=0, color=[0, 255, 0]),
+        1:
+        dict(link=('left_knee', 'left_hip'), id=1, color=[0, 255, 0]),
+        2:
+        dict(link=('right_ankle', 'right_knee'), id=2, color=[255, 128, 0]),
+        3:
+        dict(link=('right_knee', 'right_hip'), id=3, color=[255, 128, 0]),
+        4:
+        dict(link=('left_hip', 'right_hip'), id=4, color=[51, 153, 255]),
+        5:
+        dict(link=('left_shoulder', 'left_hip'), id=5, color=[51, 153, 255]),
+        6:
+        dict(link=('right_shoulder', 'right_hip'), id=6, color=[51, 153, 255]),
+        7:
+        dict(
+            link=('left_shoulder', 'right_shoulder'),
+            id=7,
+            color=[51, 153, 255]),
+        8:
+        dict(link=('left_shoulder', 'left_elbow'), id=8, color=[0, 255, 0]),
+        9:
+        dict(
+            link=('right_shoulder', 'right_elbow'), id=9, color=[255, 128, 0]),
+        10:
+        dict(link=('left_elbow', 'left_wrist'), id=10, color=[0, 255, 0]),
+        11:
+        dict(link=('right_elbow', 'right_wrist'), id=11, color=[255, 128, 0]),
+        12:
+        dict(link=('left_eye', 'right_eye'), id=12, color=[51, 153, 255]),
+        13:
+        dict(link=('nose', 'left_eye'), id=13, color=[51, 153, 255]),
+        14:
+        dict(link=('nose', 'right_eye'), id=14, color=[51, 153, 255]),
+        15:
+        dict(link=('left_eye', 'left_ear'), id=15, color=[51, 153, 255]),
+        16:
+        dict(link=('right_eye', 'right_ear'), id=16, color=[51, 153, 255]),
+        17:
+        dict(link=('left_ear', 'left_shoulder'), id=17, color=[51, 153, 255]),
+        18:
+        dict(
+            link=('right_ear', 'right_shoulder'), id=18, color=[51, 153, 255])
+    },
+    joint_weights=[
+        1., 1., 1., 1., 1., 1., 1., 1.2, 1.2, 1.5, 1.5, 1., 1., 1.2, 1.2, 1.5,
+        1.5
+    ],
+    sigmas=[
+        0.026, 0.025, 0.025, 0.035, 0.035, 0.079, 0.079, 0.072, 0.072, 0.062,
+        0.062, 0.107, 0.107, 0.087, 0.087, 0.089, 0.089
+    ])

external/default_runtime.py

@@ -0,0 +1,20 @@
+checkpoint_config = dict(interval=10)
+
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        # dict(type='TensorboardLoggerHook')
+        # dict(type='PaviLoggerHook') # for internal services
+    ])
+
+log_level = 'INFO'
+load_from = None
+resume_from = None
+dist_params = dict(backend='nccl')
+workflow = [('train', 1)]
+
+# disable opencv multithreading to avoid system being overloaded
+opencv_num_threads = 0
+# set multi-process start method as `fork` to speed up the training
+mp_start_method = 'fork'

external/faster_rcnn_r50_fpn_coco.py

@@ -0,0 +1,182 @@
+checkpoint_config = dict(interval=1)
+# yapf:disable
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        # dict(type='TensorboardLoggerHook')
+    ])
+# yapf:enable
+dist_params = dict(backend='nccl')
+log_level = 'INFO'
+load_from = None
+resume_from = None
+workflow = [('train', 1)]
+# optimizer
+optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='step',
+    warmup='linear',
+    warmup_iters=500,
+    warmup_ratio=0.001,
+    step=[8, 11])
+total_epochs = 12
+
+model = dict(
+    type='FasterRCNN',
+    pretrained='torchvision://resnet50',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch'),
+    neck=dict(
+        type='FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=80,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ))
+
+dataset_type = 'CocoDataset'
+data_root = 'data/coco'
+img_norm_cfg = dict(
+    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='LoadAnnotations', with_bbox=True),
+    dict(type='Resize', img_scale=(1333, 800), keep_ratio=True),
+    dict(type='RandomFlip', flip_ratio=0.5),
+    dict(type='Normalize', **img_norm_cfg),
+    dict(type='Pad', size_divisor=32),
+    dict(type='DefaultFormatBundle'),
+    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']),
+]
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(
+        type='MultiScaleFlipAug',
+        img_scale=(1333, 800),
+        flip=False,
+        transforms=[
+            dict(type='Resize', keep_ratio=True),
+            dict(type='RandomFlip'),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32),
+            dict(type='DefaultFormatBundle'),
+            dict(type='Collect', keys=['img']),
+        ])
+]
+data = dict(
+    samples_per_gpu=2,
+    workers_per_gpu=2,
+    train=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_train2017.json',
+        img_prefix=f'{data_root}/train2017/',
+        pipeline=train_pipeline),
+    val=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        pipeline=test_pipeline),
+    test=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        pipeline=test_pipeline))
+evaluation = dict(interval=1, metric='bbox')

external/hrnet_w48_coco_256x192.py

@@ -0,0 +1,169 @@
+_base_ = [
+    'default_runtime.py',
+    'coco.py'
+]
+evaluation = dict(interval=10, metric='mAP', save_best='AP')
+
+optimizer = dict(
+    type='Adam',
+    lr=5e-4,
+)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='step',
+    warmup='linear',
+    warmup_iters=500,
+    warmup_ratio=0.001,
+    step=[170, 200])
+total_epochs = 210
+channel_cfg = dict(
+    num_output_channels=17,
+    dataset_joints=17,
+    dataset_channel=[
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+    ],
+    inference_channel=[
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+    ])
+
+# model settings
+model = dict(
+    type='TopDown',
+    pretrained='https://download.openmmlab.com/mmpose/'
+    'pretrain_models/hrnet_w48-8ef0771d.pth',
+    backbone=dict(
+        type='HRNet',
+        in_channels=3,
+        extra=dict(
+            stage1=dict(
+                num_modules=1,
+                num_branches=1,
+                block='BOTTLENECK',
+                num_blocks=(4, ),
+                num_channels=(64, )),
+            stage2=dict(
+                num_modules=1,
+                num_branches=2,
+                block='BASIC',
+                num_blocks=(4, 4),
+                num_channels=(48, 96)),
+            stage3=dict(
+                num_modules=4,
+                num_branches=3,
+                block='BASIC',
+                num_blocks=(4, 4, 4),
+                num_channels=(48, 96, 192)),
+            stage4=dict(
+                num_modules=3,
+                num_branches=4,
+                block='BASIC',
+                num_blocks=(4, 4, 4, 4),
+                num_channels=(48, 96, 192, 384))),
+    ),
+    keypoint_head=dict(
+        type='TopdownHeatmapSimpleHead',
+        in_channels=48,
+        out_channels=channel_cfg['num_output_channels'],
+        num_deconv_layers=0,
+        extra=dict(final_conv_kernel=1, ),
+        loss_keypoint=dict(type='JointsMSELoss', use_target_weight=True)),
+    train_cfg=dict(),
+    test_cfg=dict(
+        flip_test=True,
+        post_process='default',
+        shift_heatmap=True,
+        modulate_kernel=11))
+
+data_cfg = dict(
+    image_size=[192, 256],
+    heatmap_size=[48, 64],
+    num_output_channels=channel_cfg['num_output_channels'],
+    num_joints=channel_cfg['dataset_joints'],
+    dataset_channel=channel_cfg['dataset_channel'],
+    inference_channel=channel_cfg['inference_channel'],
+    soft_nms=False,
+    nms_thr=1.0,
+    oks_thr=0.9,
+    vis_thr=0.2,
+    use_gt_bbox=False,
+    det_bbox_thr=0.0,
+    bbox_file='data/coco/person_detection_results/'
+    'COCO_val2017_detections_AP_H_56_person.json',
+)
+
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='TopDownGetBboxCenterScale', padding=1.25),
+    dict(type='TopDownRandomShiftBboxCenter', shift_factor=0.16, prob=0.3),
+    dict(type='TopDownRandomFlip', flip_prob=0.5),
+    dict(
+        type='TopDownHalfBodyTransform',
+        num_joints_half_body=8,
+        prob_half_body=0.3),
+    dict(
+        type='TopDownGetRandomScaleRotation', rot_factor=40, scale_factor=0.5),
+    dict(type='TopDownAffine'),
+    dict(type='ToTensor'),
+    dict(
+        type='NormalizeTensor',
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]),
+    dict(type='TopDownGenerateTarget', sigma=2),
+    dict(
+        type='Collect',
+        keys=['img', 'target', 'target_weight'],
+        meta_keys=[
+            'image_file', 'joints_3d', 'joints_3d_visible', 'center', 'scale',
+            'rotation', 'bbox_score', 'flip_pairs'
+        ]),
+]
+
+val_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='TopDownGetBboxCenterScale', padding=1.25),
+    dict(type='TopDownAffine'),
+    dict(type='ToTensor'),
+    dict(
+        type='NormalizeTensor',
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]),
+    dict(
+        type='Collect',
+        keys=['img'],
+        meta_keys=[
+            'image_file', 'center', 'scale', 'rotation', 'bbox_score',
+            'flip_pairs'
+        ]),
+]
+
+test_pipeline = val_pipeline
+
+data_root = 'data/coco'
+data = dict(
+    samples_per_gpu=32,
+    workers_per_gpu=2,
+    val_dataloader=dict(samples_per_gpu=32),
+    test_dataloader=dict(samples_per_gpu=32),
+    train=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_train2017.json',
+        img_prefix=f'{data_root}/train2017/',
+        data_cfg=data_cfg,
+        pipeline=train_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    val=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        data_cfg=data_cfg,
+        pipeline=val_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    test=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        data_cfg=data_cfg,
+        pipeline=test_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+)

faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:047c8118fc5ca88ba5ae1fab72f2cd6b070501fe3af2f3cba5cfa9a89b44b03e
+size 167287506

fileservice.py

@@ -0,0 +1,35 @@
+from fastapi import FastAPI, Request, Response
+
+filenames = ["js/poseMaker.js"]
+contents = '\n'.join([open(x).read() for x in filenames])
+
+app = FastAPI()
+
+@app.middleware("http")
+async def insert_js(request: Request, call_next):
+    path = request.scope['path']  # get the request route
+    response = await call_next(request)
+    
+    if path == "/":
+        response_body = ""
+        async for chunk in response.body_iterator:
+            response_body += chunk.decode()
+
+        some_javascript = f"""
+    <script type="text/javascript" defer>
+{contents}
+    </script>
+        """
+
+        response_body = response_body.replace("</body>", some_javascript + "</body>")
+
+        del response.headers["content-length"]
+
+        return Response(
+            content=response_body,
+            status_code=response.status_code, 
+            headers=dict(response.headers),
+            media_type=response.media_type
+        )
+
+    return response

hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9e0b3ab0439cb68e166c7543e59d2587cd8d7e9acf5ea62a8378eeb82fb50e5
+size 255011654

js/poseMaker.js

@@ -0,0 +1,813 @@
+console.log("hello from poseEditor.js")
+var canvas = null;
+var ctx = null;
+var canvasBg = null;
+
+const wheelDisplayTime = 500;
+
+const limbSeq = [
+  [1, 2], [2, 3], [3, 4], // 右腕
+  [1, 5], [5, 6], [6, 7], // 左腕
+  [1, 8], [8, 9], [9, 10],  // 右胴→右脚
+  [1, 11], [11, 12], [12, 13], // 左胴→左脚
+  [1, 0], // 首
+  [0, 14], [14, 16], // 右目
+  [0, 15], [15, 17] // 左目
+];
+
+function findParentNodeIndex(nodeIndex) {
+  // limbSeqの各要素の2番目の要素がjointIndexの場合、その要素の1番目の要素を返す
+  // 見つからないばあいは-1を返す
+  limbIndex = limbSeq.findIndex((limb) => limb[1] === nodeIndex);
+  return limbIndex === -1 ? -1 : limbSeq[limbIndex][0];
+}
+
+function cutOffLimb(pose, cutOffIndex) {
+  console.log(`cutOffLimb: ${cutOffIndex}`);
+  // 末端ノードの座標を削除する
+  var newPose = deepCopy(pose);
+  for (let i = 0; i < 18; i++) {
+    if (newPose[i] == null) {continue;}
+    // ルートまで検索し、その間にcuttOffIndexがあれば削除
+    var curr = i;
+    while (curr !== 1) {
+      console.log(`checking: ${i} -> ${curr}`);
+      let parent = findParentNodeIndex(curr);
+      if (parent === cutOffIndex) {
+        console.log(`cutOffLimb: ${i} -> ${cutOffIndex}`);
+        newPose[i] = null;
+        break;
+      }
+      curr = parent;
+    }
+  }
+  return newPose;
+}
+
+function repairPose(sourcePose) {
+  // TODO: ループには対応してないかも
+  var pose = sourcePose;
+  var newPose = new Array(18)
+  for (var k = 0; k < 3; k++) {
+    var processed = 0; // イテレーション用
+    for (let i = 0; i < 18; i++) {
+      if (pose[i] == null) {
+        let parent = findParentNodeIndex(i);
+        if (parent === -1) {continue;} // あり得ない
+        if (pose[parent] == null) {
+          console.log(`repair failed(A): ${i} -> parent loss`);
+          continue;
+        }
+
+        // サンプルデータから引っ張ってくる
+        var v = sampleCandidateSource[i].map((x, j) => x - sampleCandidateSource[parent][j]);
+        newPose[i] = pose[parent].map((x, j) => x + v[j]);
+        console.log(`repaired: ${i} -> ${newPose[newPose.length - 1]}`);
+        processed++;
+      } else {
+        newPose[i] = pose[i].map(x => x);
+      }
+    }
+    if (processed === 0) {break;}
+    pose = newPose;
+  }
+  return newPose;
+}
+
+function deepCopy(arr) {
+  return JSON.parse(JSON.stringify(arr));
+}
+
+function distSq(p0, p1) {
+  return (p0[0] - p1[0]) ** 2 + (p0[1] - p1[1]) ** 2;
+}
+
+// poseDataの形式：[[[x1, y1], [x2, y2], ...],[[x3, y3], [x4, y4], ...], ...]
+// 各要素が人間
+// 人間の各要素が関節
+
+function poseDataToCandidateAndSubset(poseData) {
+  let candidate = [];
+  let subset = [];
+  for (let i = 0; i < poseData.length; i++) {
+    let person = poseData[i];
+    let subsetElement = [];
+    for (let j = 0; j < person.length; j++) {
+      candidate.push(person[j]);
+      subsetElement.push(candidate.length - 1);
+    }
+    subset.push(subsetElement);
+  }
+  return [candidate, subset];
+}
+
+// サンプルデータ  
+const sampleCandidateSource = [[235, 158],[234, 220],[193, 222],[138, 263],[89, 308],[276, 220],[325, 264],[375, 309],[207, 347],[203, 433],[199, 523],[261, 347],[262, 430],[261, 522],[227, 148],[245, 148],[208, 158],[258, 154]].map((p) => [p[0], p[1] - 70]);
+const sampleSubsetElementSource = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17];
+
+// const sampleCandidateSource = [[618.00, 0.00], [618.00, 44.00], [304.00, 81.00], [482.00, 96.00], [66.00, 270.00], [171.00, 280.00], [618.00, 82.00], [307.00, 112.00], [460.00, 143.00], [0.00, 301.00], [65.00, 301.00], [172.00, 303.00], [584.00, 86.00], [275.00, 119.00], [420.00, 139.00], [0.00, 301.00], [41.00, 301.00], [144.00, 303.00], [544.00, 131.00], [348.00, 139.00], [262.00, 160.00], [0.00, 337.00], [52.00, 339.00], [130.00, 348.00], [570.00, 175.00], [283.00, 177.00], [78.00, 338.00], [172.00, 380.00], [651.00, 78.00], [338.00, 111.00], [505.00, 144.00], [92.00, 301.00], [198.00, 305.00], [661.00, 132.00], [349.00, 156.00], [541.00, 179.00], [106.00, 336.00], [203.00, 348.00], [305.00, 159.00], [665.00, 160.00], [563.00, 192.00], [80.00, 343.00], [181.00, 385.00], [614.00, 205.00], [291.00, 220.00], [432.00, 320.00], [152.00, 372.00], [43.00, 380.00], [0.00, 386.00], [623.00, 281.00], [306.00, 290.00], [92.00, 357.00], [509.00, 434.00], [304.00, 357.00], [622.00, 368.00], [47.00, 394.00], [0.00, 395.00], [142.00, 405.00], [535.00, 565.00], [655.00, 200.00], [337.00, 217.00], [467.00, 322.00], [191.00, 372.00], [83.00, 375.00], [344.00, 282.00], [655.00, 282.00], [103.00, 343.00], [237.00, 368.00], [22.00, 377.00], [0.00, 379.00], [460.00, 459.00], [305.00, 352.00], [638.00, 355.00], [0.00, 401.00], [110.00, 412.00], [411.00, 570.00], [608.00, 0.00], [608.00, 40.00], [297.00, 75.00], [469.00, 84.00], [0.00, 261.00], [58.00, 263.00], [165.00, 275.00], [625.00, 0.00], [625.00, 39.00], [309.00, 74.00], [486.00, 83.00], [71.00, 264.00], [180.00, 276.00], [599.00, 0.00], [599.00, 44.00], [284.00, 80.00], [440.00, 93.00], [48.00, 271.00], [0.00, 272.00], [157.00, 277.00], [634.00, 0.00], [633.00, 41.00], [319.00, 77.00], [79.00, 269.00], [190.00, 277.00]];
+// const sampleSubsetElementSource = [1.00,6.00,12.00,18.00,24.00,28.00,33.00,39.00,43.00,49.00,54.00,59.00,65.00,72.00,77.00,84.00,90.00,97.00,32.98,18.00],[5.00,11.00,17.00,23.00,27.00,32.00,37.00,42.00,46.00,-1.00,-1.00,62.00,67.00,-1.00,82.00,88.00,95.00,100.00,25.45,15.00],[4.00,10.00,16.00,22.00,26.00,31.00,36.00,41.00,47.00,51.00,57.00,63.00,66.00,74.00,81.00,87.00,93.00,99.00,26.97,18.00],[3.00,8.00,14.00,19.00,25.00,30.00,35.00,40.00,45.00,52.00,58.00,61.00,70.00,75.00,79.00,86.00,92.00,-1.00,30.45,17.00],[2.00,7.00,13.00,20.00,-1.00,29.00,34.00,38.00,44.00,50.00,53.00,60.00,64.00,71.00,78.00,85.00,91.00,98.00,27.89,17.00],[0.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,-1.00,76.00,83.00,-1.00,96.00,3.33,4.00];
+
+function makePoseFromCandidateAndSubsetElement(candidate, subsetElement) {
+  var pose = [];
+  for (let j = 0 ; j < 18; j++) {
+    let i = subsetElement[j];
+    pose.push(i < 0 || candidate[i] == null ? null : candidate[i].map((x)=>x));
+  }
+  return pose;
+}
+
+function makePoseDataFromCandidateAndSubset(candidate, subset) {
+  return subset.map(subsetElement => makePoseFromCandidateAndSubsetElement(candidate, subsetElement));
+}
+
+function addPerson() {
+  var dx = Math.random() * 100;
+  var dy = Math.random() * 100;
+
+  poseData.push(
+    makePoseFromCandidateAndSubsetElement(
+      sampleCandidateSource.map(point => [point[0] + dx, point[1] + dy]),
+      sampleSubsetElementSource));
+
+  addHistory();    
+  Redraw();
+}
+
+function removePerson(personIndex) {
+  poseData.splice(personIndex, 1);
+  addHistory();
+  Redraw();
+}
+
+function repairPerson(personIndex) {
+  poseData[personIndex] = repairPose(poseData[personIndex]);
+  addHistory();
+  Redraw();
+}
+
+function cutOffPersonLimb(personIndex, limbIndex) {
+  poseData[personIndex] = cutOffLimb(poseData[personIndex], limbIndex);
+  console.log(poseData[personIndex]);
+  console.log(poseData);
+  addHistory();
+  Redraw();
+}
+
+// ドラッグ中の各キーが押されているかどうかのフラグ
+var keyDownFlags = {};
+// マウスカーソル
+var mouseCursor = [-1, -1];
+
+function cross(lhs, rhs) {return lhs[0] * rhs[1] - lhs[1] * rhs[0];}
+function dot(lhs, rhs) {return lhs[0] * rhs[0] + lhs[1] * rhs[1];}
+function directedAngleTo(lhs, rhs) {return Math.atan2(cross(lhs, rhs), dot(lhs, rhs));}
+
+function isMouseOnCanvas() {
+  // mouseCursorがcanvasの範囲内にあるかどうかを判定
+  var rect = canvas.getBoundingClientRect();
+  var f = 0 <= mouseCursor[0] && mouseCursor[0] <= rect.width && 0 <= mouseCursor[1] && mouseCursor[1] <= rect.height;
+  return f;
+}
+
+function clearCanvas() {
+  var w = canvas.width;
+  var h = canvas.height;
+  ctx.fillStyle = 'black';
+  ctx.fillRect(0, 0, w, h);
+}
+
+function drawBackground() {
+  if (canvasBg != null) {
+    ctx.drawImage(canvasBg, 0, 0);
+  }
+}
+
+function resizeCanvas(width, height) {
+  canvas.width = width ? width : canvas.width;
+  canvas.height = height ? height : canvas.height;
+  Redraw();
+}
+
+function calculateCenter(shape) {
+  var center = shape.reduce(function(acc, point) {
+    if (point === null) {
+      acc[0] += point[0];
+      acc[1] += point[1];
+    }
+    return acc;
+  }, [0, 0]);
+  center[0] /= shape.length;
+  center[1] /= shape.length;
+  return center;
+}
+
+// v2d -> v3d
+function rotateX(vector, angle) {
+  var x = vector[0];
+  var y = vector[1];
+  var z = 0;
+
+  // X軸に対して回転する
+  var x1 = x;
+  var y1 = y * Math.cos(angle) - z * Math.sin(angle);
+  var z1 = y * Math.sin(angle) + z * Math.cos(angle);
+
+  return [x1, y1, z1];
+}
+
+// v2d -> v3d
+function rotateY(vector, angle) {
+  var x = vector[0];
+  var y = vector[1];
+  var z = 0;
+
+  // Y軸に対して回転する
+  var x1 = x * Math.cos(angle) + z * Math.sin(angle);
+  var y1 = y;
+  var z1 = -x * Math.sin(angle) + z * Math.cos(angle);
+
+  return [x1, y1, z1];  
+}
+
+// v3d -> v2d
+function perspectiveProjection(vector, cameraDistance) {
+  var x = vector[0];
+  var y = vector[1];
+  var z = vector[2];
+
+  if (z === 0) {
+    return [x, y];
+  }
+
+  var scale = cameraDistance / (cameraDistance - z);
+  var x1 = x * scale;
+  var y1 = y * scale;
+
+  return [x1, y1];
+}  
+
+// v2d -> v3d
+function rotateAndProject(f, p, c, angle) {
+  var v = [p[0] - c[0], p[1] - c[1]];
+  var v1 = f(v, angle);
+  var v2 = perspectiveProjection(v1, 500);
+  return [v2[0] + c[0], v2[1] + c[1]];
+}
+
+function drawBodyPose() {
+  let stickWidth = 4;
+  let imageSize = Math.min(canvas.width, canvas.height);
+  stickWidth *= imageSize / 512;
+
+  const colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0],
+              [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255],
+              [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]];
+
+  ctx.globalAlpha = 0.6;
+
+  // edge
+  for (let i = 0; i < poseData.length; i++) {
+    const pose = poseData[i];
+
+    for (let j = 0; j < 17; j++) {
+      const p = pose[limbSeq[j][0]];
+      const q = pose[limbSeq[j][1]];
+      if (p == null || q == null) continue;
+      const [X0, Y0] = p;
+      const [X1, Y1] = q;
+      let angle = Math.atan2(Y1 - Y0, X1 - X0);
+      let magnitude = ((X0 - X1) ** 2 + (Y0 - Y1) ** 2) ** 0.5
+      let polygon = new Path2D();
+      polygon.ellipse((X0+X1)/2, (Y0+Y1)/2, magnitude / 2, stickWidth, angle, 0, 2 * Math.PI);
+      ctx.fillStyle = `rgb(${colors[j].join(',')})`;
+      ctx.fill(polygon);
+    }
+  }
+
+  ctx.globalAlpha = 1.0;
+
+  // node
+  for (let i = 0; i < poseData.length; i++) {
+    const pose = poseData[i];
+
+    ctx.font = '12px serif';
+    for (let j = 0; j < 18; j++) {
+      const p = pose[j];
+      if (p == null) continue;
+      const [x, y] = p;
+      ctx.beginPath();
+      ctx.arc(x, y, stickWidth, 0, 2 * Math.PI);
+      ctx.fillStyle = `rgb(${colors[j].join(',')})`;
+      ctx.fill();
+      // ctx.fillStyle = 'rgb(255,255,255)'
+      // ctx.fillText(j, x-3, y+4);
+    }
+  }
+}
+
+let lastWheeling = 0;
+
+function drawUI() {
+  if (keyDownFlags['Space'] || keyDownFlags['BracketLeft'] || keyDownFlags['BracketRight'] || 
+      new Date().getTime() - lastWheeling < wheelDisplayTime) {
+    ctx.beginPath();
+    ctx.lineWidth=4;
+    ctx.arc(mouseCursor[0], mouseCursor[1], dragRange, 0, 2 * Math.PI);
+    ctx.strokeStyle = 'rgb(255,255,255)';
+    ctx.stroke();
+  }
+
+  if (isDragging && (dragMode == "rotate" || dragMode == "rotate2")) {
+    ctx.beginPath();
+    ctx.lineWidth=1;
+    ctx.strokeStyle = 'rgb(255,255,255)';
+    ctx.moveTo(dragStart[0], dragStart[1]);
+    ctx.lineTo(dragStart[0]+rotateBaseVector[0], dragStart[1]+rotateBaseVector[1]);
+    ctx.stroke();
+  }
+
+  let operationTextFlags = {
+    "Space": "Range Move",
+    "AltLeft": "Body Move",
+    "AltRight": "Body Move",
+    "ControlLeft": "Scale",
+    "ControlRight": "Scale",
+    "ShiftLeft": "Rotate",
+    "ShiftRight": "Rotate",
+    "KeyQ": "CutOff",
+    "KeyD": "Delete",
+    "KeyX": "X-Axis",
+    "KeyC": "Y-Axis",
+    "KeyR": "Repair",
+  }
+
+  // operationTextFlagsに含まれるものがkeyDownFlagsに含まれるばあい、そのキーの文字列を取得
+  let activeOperations = Object.keys(operationTextFlags).filter(key => keyDownFlags[key]);
+  if (activeOperations.length > 0) {
+    // 左上に表示
+    ctx.font = '20px serif';
+    ctx.fillStyle = 'rgb(255,255,255)';
+    ctx.fillText(operationTextFlags[activeOperations[0]], 10, 30);
+  }
+}
+
+function Redraw() {
+  clearCanvas();
+  drawBackground();
+  drawBodyPose();
+  drawUI();
+}
+
+function getNearestNode(p) {
+    let minDistSq = Infinity;
+    let personIndex = -1;
+    let nodeIndex = -1;
+    for (let i = 0; i < poseData.length; i++) {
+      const pose = poseData[i];
+      for (let j = 0; j < pose.length; j++) {
+        const q = pose[j];
+        if (q == null) continue;
+        const d = distSq(p, q);
+        if (d < minDistSq) {
+          minDistSq = d;
+          personIndex = i;
+          nodeIndex = j;
+        }
+      }
+    }
+    return [personIndex, nodeIndex, Math.sqrt(minDistSq)];
+}
+
+let dragRange = 64;
+let dragRangeDelta = 16;
+
+// ドラッグ中に座標を保持するための変数
+let isDragging = false;
+let dragStart = [0, 0];
+let dragPersonIndex = -1;
+let dragMarks = [];
+let dragMode = "";
+let rotateBaseVector = null;
+let history = [];
+let historyIndex = 0;
+
+function clearHistory() {
+  history = [];
+  historyIndex = 0;  
+}
+
+function addHistory() {
+  history = history.slice(0, historyIndex);
+  history.push(JSON.parse(JSON.stringify(poseData)));
+  historyIndex = history.length;
+}
+
+function undo() {
+  if (1 < historyIndex) {
+    historyIndex--;
+    poseData = deepCopy(history[historyIndex-1]);
+    Redraw();
+  }   
+}
+
+function redo() {
+  if (historyIndex < history.length) {
+    historyIndex++;
+    poseData = deepCopy(history[historyIndex-1]);
+    Redraw();
+  }   
+}
+
+function fetchLatestPoseData() {
+  return history[historyIndex-1];
+}
+
+function getCanvasPosition(event) {
+  const rect = canvas.getBoundingClientRect();
+  const x = event.clientX - rect.left;
+  const y = event.clientY - rect.top;
+  return [x, y];
+}
+
+function forEachMarkedNodes(fn) {
+  for (let i = 0; i < dragMarks.length; i++) {
+    for (let j = 0; j < dragMarks[i].length; j++) {
+      if (dragMarks[i][j]) {
+        fn(i, j, poseData[i][j]);
+      }
+    }
+  }
+}
+
+// Canvas要素上でマウスが押された場合に呼び出される関数
+function handleMouseDown(event) {
+  const p = getCanvasPosition(event);
+  const [personIndex, nodeIndex, minDist] = getNearestNode(p);
+
+  if (keyDownFlags["KeyD"]) {removePerson(personIndex);return;}
+  if (keyDownFlags["KeyR"]) {repairPerson(personIndex);return;}
+
+  if (keyDownFlags["KeyQ"] && minDist < 16) {
+    console.log("pressed KeyQ");
+    cutOffPersonLimb(personIndex, nodeIndex);
+    return;
+  }
+
+  // ドラッグ処理の開始
+  dragStart = p;
+  dragMarks = poseData.map(pose => pose.map(node => false));
+
+  if (event.altKey || event.ctrlKey || event.shiftKey ||
+      keyDownFlags["KeyX"] || keyDownFlags["KeyC"]) {
+    // dragMarksを設定
+    dragMarks[personIndex] = 
+      poseData[personIndex].map((node) => node != null);
+      isDragging = true;
+    if (event.altKey) {
+      dragMode = "move";
+    } else if (event.ctrlKey) {
+      dragMode = "scale";
+    } else if (event.shiftKey) {
+      dragMode = "rotate";
+      rotateBaseVector = [0, 0];
+    } else if (keyDownFlags["KeyX"]) {
+      dragMode = "rotateX";
+    } else if (keyDownFlags["KeyC"]) {
+      dragMode = "rotateY";
+    }
+  } else if (keyDownFlags["Space"]) {
+    dragMarks[personIndex] = 
+      poseData[personIndex].map(
+        (node) => node != null && distSq(p, node) < dragRange ** 2);
+    isDragging = dragMarks[personIndex].some((mark) => mark);
+    dragMode = "move";
+  } else if (minDist < 16) {
+    dragMarks[personIndex][nodeIndex] = true;
+    isDragging = true;
+    dragMode = "move";
+  }
+}
+
+// Canvas要素上でマウスが動いた場合に呼び出される関数
+function handleMouseMove(event) {
+  mouseCursor = getCanvasPosition(event);
+  if (isDragging) {
+    const p = getCanvasPosition(event);
+    const dragOffset = [p[0] - dragStart[0], p[1] - dragStart[1]];
+    const latestPoseData = fetchLatestPoseData();
+
+    if (dragMode == "scale") {
+      // 拡大縮小
+      let xScale = 1 + dragOffset[0] / canvas.width;
+      let yScale = 1 + dragOffset[0] / canvas.height;
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        node[0] = (lp[0] - dragStart[0]) * xScale + dragStart[0];
+        node[1] = (lp[1] - dragStart[1]) * yScale + dragStart[1];
+      });
+    } else if (dragMode == "rotate") {
+      rotateBaseVector = dragOffset;
+      if (!event.shiftKey) {
+        dragMode = "rotate2";
+      }
+    } else if (dragMode == "rotate2") {
+      // 回転
+      let angle = directedAngleTo(rotateBaseVector, dragOffset);
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        let x = lp[0] - dragStart[0];
+        let y = lp[1] - dragStart[1];
+        let sin = Math.sin(angle);
+        let cos = Math.cos(angle);
+        node[0] = x * cos - y * sin + dragStart[0];
+        node[1] = x * sin + y * cos + dragStart[1];
+      });
+    } else if (dragMode == "rotateX") {
+      const center = dragStart;
+      const angle = dragOffset[1] / -40;
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        const np = rotateAndProject(rotateX, lp, center, angle);
+        node[0] = np[0];
+        node[1] = np[1];
+      });
+    } else if (dragMode == "rotateY") {
+      const center = dragStart;
+      const angle = dragOffset[0] / 40;
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        const np = rotateAndProject(rotateY, lp, center, angle);
+        node[0] = np[0];
+        node[1] = np[1];
+      });
+    } else if (dragMode == "move") {
+      // 移動
+      forEachMarkedNodes((i, j, node) => {
+        const lp = latestPoseData[i][j];
+        node[0] = lp[0] + dragOffset[0];
+        node[1] = lp[1] + dragOffset[1];
+      });
+    }
+  }
+
+  Redraw();
+}
+
+function handleMouseUp(event) {
+  isDragging = false; 
+  addHistory();
+  Redraw();
+}
+
+function handleMouseLeave(event) {
+  mouseCursor = [-1,-1];
+  handleMouseUp(event);
+  keyDownFlags = {};
+}
+
+function ModifyDragRange(delta) { dragRange = Math.max(dragRangeDelta, Math.min(512, dragRange + delta)); }
+
+document.addEventListener('wheel', function(event) {
+  if (!isMouseOnCanvas()) {return;}
+  if (!event.altKey && !keyDownFlags['Space']) {return;}
+
+  event.preventDefault();
+  const deltaY = event.deltaY;
+  if (deltaY < 0) {ModifyDragRange(-dragRangeDelta);}
+  if (0 < deltaY) {ModifyDragRange(dragRangeDelta);}
+  lastWheeling = new Date().getTime();
+  Redraw();
+  window.setTimeout(function() { Redraw(); }, wheelDisplayTime+10);
+}, {passive: false});
+
+document.addEventListener("keydown", (event) => { 
+  if (!isMouseOnCanvas()) {return;}
+
+  if (event.code == "BracketLeft") { ModifyDragRange(-dragRangeDelta); }
+  if (event.code == "BracketRight") { ModifyDragRange(dragRangeDelta); }
+  keyDownFlags[event.code] = true; 
+  Redraw();
+  event.preventDefault();
+});
+document.addEventListener("keyup", (event) => { 
+  if (!isMouseOnCanvas()) {return;}
+
+  keyDownFlags[event.code] = false; 
+  if (event.ctrlKey && event.code == "KeyE") {
+    addPerson();
+  } else if (event.ctrlKey && event.code == "KeyZ") {
+    if (event.shiftKey) {
+      redo();
+    } else {
+      undo();
+    }
+  }
+  Redraw();
+  event.preventDefault();
+});
+
+function initializeEditor() {
+  console.log("initializeEditor");
+
+  canvas = document.getElementById('canvas');
+  ctx = canvas.getContext('2d');
+
+  canvas.addEventListener('mousedown', handleMouseDown);
+  canvas.addEventListener('mousemove', handleMouseMove);
+  canvas.addEventListener('mouseup', handleMouseUp);
+  canvas.addEventListener('mouseleave', handleMouseLeave);
+  poseData = [];
+  clearHistory();
+}
+
+function importPose(jsonData) {
+  if (jsonData != null) {
+    newPoseData = makePoseDataFromCandidateAndSubset(jsonData.candidate, jsonData.subset);
+  } else {
+    newPoseData = makePoseDataFromCandidateAndSubset(sampleCandidateSource, [sampleSubsetElementSource]);
+  }
+  poseData = poseData.concat(newPoseData);
+  addHistory();
+  Redraw();
+}
+
+/*
+function savePose() {
+  const canvasUrl = canvas.toDataURL();
+
+  const createEl = document.createElement('a');
+  createEl.href = canvasUrl;
+
+  // This is the name of our downloaded file
+  createEl.download = "pose.png";
+
+  createEl.click();
+  createEl.remove();
+
+  var [candidate, subset] = poseDataToCandidateAndSubset(poseData);
+  return {candidate: candidate, subset: subset};
+}
+*/
+
+// crc32
+// CRC32を初期化
+function initCrc32Table() {
+  const crcTable = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let j = 0; j < 8; j++) {
+      c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+    }
+    crcTable[i] = c;
+  }
+  return crcTable;
+}
+
+// データのCRC32を計算
+function getCrc32(data, crc=0) {
+  const crcTable = initCrc32Table();
+  crc = (crc ^ 0xFFFFFFFF) >>> 0;
+  for (let i = 0; i < data.length; i++) {
+    crc = crcTable[(crc ^ data[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+function stringToUint8Array(str) {
+  var arr = new Uint8Array(str.length);
+  for (var i = 0; i < str.length; i++) {
+    arr[i] = str.charCodeAt(i);
+  }
+  return arr;
+}
+
+function base64ToUint8Array(base64Str) {
+  return stringToUint8Array(atob(base64Str));
+}
+
+function visitPng(png, type) {
+  var dataLength;
+  var chunkType;
+  var nextChunkPos;
+  var Signature = String.fromCharCode(137, 80, 78, 71, 13, 10, 26, 10);
+  var rpos = 0;
+  
+  // シグネチャの確認
+  if (String.fromCharCode.apply(null, png.subarray(rpos, rpos += 8)) !== Signature) {
+      throw new Error('invalid signature');
+  }
+  
+  // チャンクの探索
+  while (rpos < png.length) {
+      dataLength = (
+          (png[rpos++] << 24) |
+          (png[rpos++] << 16) |
+          (png[rpos++] <<  8) |
+          (png[rpos++]      )
+      ) >>> 0;
+  
+      nextChunkPos = rpos + dataLength + 8;
+  
+      chunkType = String.fromCharCode.apply(null, png.subarray(rpos, rpos += 4));
+      
+      if (chunkType === type) {
+        return [rpos - 8, dataLength, nextChunkPos];
+      }
+      
+      rpos = nextChunkPos;
+  }
+}
+
+function createChunk(type, data) {
+  var dataLength = data.length;
+  var chunk = new Uint8Array(4 + 4 + dataLength + 4);
+  var type = stringToUint8Array(type);
+  var pos = 0;
+  
+  // length
+  chunk[pos++] = (dataLength >> 24) & 0xff;
+  chunk[pos++] = (dataLength >> 16) & 0xff;
+  chunk[pos++] = (dataLength >>  8) & 0xff;
+  chunk[pos++] = (dataLength      ) & 0xff;
+  
+  // type
+  chunk[pos++] = type[0];
+  chunk[pos++] = type[1];
+  chunk[pos++] = type[2];
+  chunk[pos++] = type[3];
+  
+  // data
+  for (let i = 0; i < dataLength; ++i) {
+      chunk[pos++] = data[i];
+  }
+  
+  //crc
+  initCrc32Table();
+  let crc = getCrc32(type);
+  crc = getCrc32(data, crc);
+  chunk[pos++] = (crc >> 24) & 0xff;
+  chunk[pos++] = (crc >> 16) & 0xff;
+  chunk[pos++] = (crc >>  8) & 0xff;
+  chunk[pos++] = (crc      ) & 0xff;
+  
+  return chunk;
+}
+
+function insertChunk(destBuffer, sourceBuffer, rpos, chunk) {
+  var pos = 0;
+  
+  // IDAT チャンクの前までコピー
+  destBuffer.set(sourceBuffer.subarray(0, rpos), pos);
+  pos += rpos;
+  
+  // hoGe チャンクをコピー
+  destBuffer.set(chunk, pos);
+  pos += chunk.length;
+  
+  // IDAT チャンク以降をコピー
+  destBuffer.set(sourceBuffer.subarray(rpos), pos);
+}
+
+function mergeCanvasWithPose(keyword, content) {
+  const canvasUrl = canvas.toDataURL();
+
+  var insertion = stringToUint8Array(`${keyword}\0${content}`);
+  var chunk = createChunk("tEXt", insertion);
+  var sourceBuffer = base64ToUint8Array(canvasUrl.split(',')[1]);
+  var destBuffer = new Uint8Array(sourceBuffer.length + insertion.length + 12);
+
+  var [rpos, dataLength, nextChunkPos] = visitPng(sourceBuffer, "IHDR");
+  insertChunk(destBuffer, sourceBuffer, nextChunkPos, chunk);
+
+  var blob = new Blob([destBuffer], {type: "image/png"});
+  var url = URL.createObjectURL(blob);
+  return url;  
+}
+
+function savePose() {
+  var [candidate, subset] = poseDataToCandidateAndSubset(poseData);
+  let jsonData = {candidate: candidate, subset: subset};
+
+  var url = mergeCanvasWithPose("openpose", JSON.stringify(jsonData));
+
+  const createEl = document.createElement('a');
+  createEl.href = url;
+
+  // This is the name of our downloaded file
+  createEl.download = "pose.png";
+
+  createEl.click();
+  createEl.remove();
+
+  return jsonData;
+}
+
+function importBackground(image) {
+  let m = new Image();
+  m.src = image;
+  m.onload = function() {
+    canvasBg = m;
+    Redraw();
+  }
+}

main.py

@@ -0,0 +1,162 @@
+import gradio as gr
+import json as js
+import util
+from fileservice import app
+from pose import infer, draw
+
+
+def image_changed(image):
+  if image == None:
+    return "estimation", {}
+
+  if 'openpose' in image.info:
+    print("pose found")
+    jsonText = image.info['openpose']
+    jsonObj = js.loads(jsonText)
+    subset = jsonObj['subset']
+    return f"""{image.width}px x {image.height}px, {len(subset)} indivisual(s)""", jsonText
+  else:
+    print("pose not found")
+    pose_result, returned_outputs = infer(util.pil2cv(image))
+
+    candidate = []
+    subset = []
+    for d in pose_result:
+        n = len(candidate)
+        if d['bbox'][4] < 0.9: 
+            continue
+        keypoints = d['keypoints'][:, :2].tolist()
+        midpoint = [(keypoints[5][0] + keypoints[6][0]) / 2, (keypoints[5][1] + keypoints[6][1]) / 2]
+        keypoints.append(midpoint)
+        candidate.extend(util.convert_keypoints(keypoints))
+        m = len(candidate)
+        subset.append([j for j in range(n, m)])
+
+    jsonText = "{ \"candidate\": " + util.candidate_to_json_string(candidate) + ", \"subset\": " + util.subset_to_json_string(subset) + " }"
+    return f"""{image.width}px x {image.height}px, {len(subset)} indivisual(s)""", jsonText
+
+html_text = f"""
+    <canvas id="canvas" width="512" height="512"></canvas><img id="canvas-background" style="display:none;"/>
+"""
+
+with gr.Blocks(css="""button { min-width: 80px; }""") as demo:
+  with gr.Row():
+    with gr.Column(scale=1):
+      width = gr.Slider(label="Width", minimum=512, maximum=1024, step=64, value=512, interactive=True)
+      height = gr.Slider(label="Height", minimum=512, maximum=1024, step=64, value=512, interactive=True)
+      with gr.Accordion(label="Pose estimation", open=False):
+        source = gr.Image(type="pil")
+        estimationResult = gr.Markdown("""estimation""")
+        with gr.Row():
+          with gr.Column(min_width=80):
+            applySizeBtn = gr.Button(value="Apply size")
+          with gr.Column(min_width=80):
+            replaceBtn = gr.Button(value="Replace")
+          with gr.Column(min_width=80):
+            importBtn = gr.Button(value="Import")
+          with gr.Column(min_width=80):
+            bgBtn = gr.Button(value="Background")
+      with gr.Accordion(label="Json", open=False):
+        with gr.Row():
+          with gr.Column(min_width=80):
+            replaceWithJsonBtn = gr.Button(value="Replace")
+          with gr.Column(min_width=80):
+            importJsonBtn = gr.Button(value="Import")
+        gr.Markdown("""
+| inout            | how to                                                                               |
+| -----------------| ----------------------------------------------------------------------------------------- |
+| Import | Paste json to "Json source" and click "Read", edit the width/height, then click "Replace" or "Import". |
+| Export | click "Save" and "Copy to clipboard" of "Json" section.                                             |
+""")
+        json = gr.JSON(label="Json")
+        jsonSource = gr.Textbox(label="Json source", lines=10)
+      with gr.Accordion(label="Notes", open=False):
+        gr.Markdown("""
+#### How to bring pose to ControlNet
+1. Press **Save** button
+2. **Drag** the file placed at the bottom left corder of browser
+3. **Drop** the file into ControlNet
+
+#### Reuse pose image
+Pose image generated by this tool has pose data in the image itself. You can reuse pose information by loading it as the image source instead of a regular image.
+
+#### Points to note for pseudo-3D rotation
+When performing pseudo-3D rotation on the X and Y axes, the projection is converted to 2D and Z-axis information is lost when the mouse button is released. This means that if you finish dragging while the shape is collapsed, you may not be able to restore it to its original state. In such a case, please use the "undo" function.
+
+#### Pose estimation
+In this project, MMPose is used for pose estimation.
+""")
+    with gr.Column(scale=2):
+      html = gr.HTML(html_text)
+      with gr.Row():
+        with gr.Column(scale=1, min_width=60):
+          saveBtn = gr.Button(value="Save")
+        with gr.Column(scale=7):
+          gr.Markdown("""
+- "ctrl + drag" to **scale**
+- "alt + drag" to **move**
+- "shift + drag" to **rotate** (move right first, release shift, then up or down)
+- "space + drag" to **range-move**
+- "[", "]" or "Alt + wheel" or "Space + wheel" to shrink or expand **range**
+- "ctrl + Z", "shift + ctrl + Z" to **undo**, **redo**
+- "ctrl + E" **add** new person
+- "D + click" to **delete** person
+- "Q + click" to **cut off** limb
+- "X + drag" to **x-axis** pseudo-3D rotation
+- "C + drag" to **y-axis** pseudo-3D rotation
+- "R + click" to **repair**
+
+When using Q, X, C, R, pressing and dont release until the operation is complete.
+
+[Contact us for feature requests or bug reports (anonymous)](https://t.co/UC3jJOJJtS)
+""")
+
+  width.change(fn=None, inputs=[width], _js="(w) => { resizeCanvas(w,null); }")
+  height.change(fn=None, inputs=[height], _js="(h) => { resizeCanvas(null,h); }")
+
+  source.change(
+    fn = image_changed,
+    inputs = [source],
+    outputs = [estimationResult, json])
+  applySizeBtn.click(
+    fn = lambda x: (x.width, x.height),
+    inputs = [source], 
+    outputs = [width, height])
+  replaceBtn.click(
+    fn = None,
+    inputs = [json],
+    outputs = [],
+    _js="(json) => { initializeEditor(); importPose(json); return []; }")
+  importBtn.click(
+    fn = None,
+    inputs = [json],
+    outputs = [],
+    _js="(json) => { importPose(json); return []; }")
+  bgBtn.click(
+    fn = None,
+    inputs = [source],
+    outputs = [],
+    _js="(image) => { importBackground(image); return []; }"
+  )
+
+  saveBtn.click(
+    fn = None,
+    inputs = [], outputs = [json],
+    _js="() => { return [savePose()]; }")
+  jsonSource.change(
+    fn = lambda x: x,
+    inputs = [jsonSource], outputs = [json])
+  replaceWithJsonBtn.click(
+    fn = None,
+    inputs = [json],
+    outputs = [],
+    _js="(json) => { initializeEditor(); importPose(json); return []; }")
+  importJsonBtn.click(
+    fn = None,
+    inputs = [json],
+    outputs = [],
+    _js="(json) => { importPose(json); return []; }")
+  demo.load(fn=None, inputs=[], outputs=[], _js="() => { initializeEditor(); importPose(); return []; }")
+
+print("mount")
+gr.mount_gradio_app(app, demo, path="/")

pose.py

@@ -0,0 +1,48 @@
+from mmpose.apis import (inference_top_down_pose_model, init_pose_model,
+                         process_mmdet_results, vis_pose_result)
+from mmpose.datasets import DatasetInfo
+from mmdet.apis import inference_detector, init_detector
+
+det_model = init_detector(
+    "./external/faster_rcnn_r50_fpn_coco.py", 
+    "./faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth",
+    device="cpu")
+pose_model = init_pose_model(
+    "./external/hrnet_w48_coco_256x192.py",
+    "./hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth",
+    device="cpu")
+
+dataset = pose_model.cfg.data['test']['type']
+dataset_info = pose_model.cfg.data['test'].get('dataset_info', None)
+
+dataset_info = DatasetInfo(dataset_info)
+
+def infer(image):
+    mmdet_results = inference_detector(det_model, image)
+    person_results = process_mmdet_results(mmdet_results, 1)
+
+    pose_results, returned_outputs = inference_top_down_pose_model(
+        pose_model,
+        image,
+        person_results,
+        bbox_thr=0.3,
+        format='xyxy',
+        dataset=dataset,
+        dataset_info=dataset_info,
+        return_heatmap=False,
+        outputs=None)
+
+    return pose_results, returned_outputs
+
+def draw(image, results):
+    return vis_pose_result(
+        pose_model,
+        image,
+        results,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        kpt_score_thr=0.3,
+        radius=4,
+        thickness=3,
+        show=False,
+        out_file=None)

requirements.txt

@@ -0,0 +1,8 @@
+fastapi==0.92.0
+gradio==3.18.0
+numpy==1.23.5
+opencv_python
+scipy
+torch
+torchvision
+openmim

util.py

@@ -0,0 +1,46 @@
+import numpy as np
+import cv2
+
+def pil2cv(image):
+    ''' PIL型 -> OpenCV型 '''
+    new_image = np.array(image, dtype=np.uint8)
+    if new_image.ndim == 2:  # モノクロ
+        pass
+    elif new_image.shape[2] == 3:  # カラー
+        new_image = cv2.cvtColor(new_image, cv2.COLOR_RGB2BGR)
+    elif new_image.shape[2] == 4:  # 透過
+        new_image = cv2.cvtColor(new_image, cv2.COLOR_RGBA2BGRA)
+    return new_image
+
+def candidate_to_json_string(arr):
+    a = [f'[{x:.2f}, {y:.2f}]' for x, y, *_ in arr]
+    return '[' + ', '.join(a) + ']'
+
+# make subset to json
+def subset_to_json_string(arr):
+    arr_str = ','.join(['[' + ','.join([f'{num:.2f}' for num in row]) + ']' for row in arr])
+    return '[' + arr_str + ']'
+
+keypoint_index_mapping = [
+    0, 
+    17,
+    6,
+    8,
+    10,
+    5,
+    7,
+    9,
+    12,
+    14,
+    16,
+    11,
+    13,
+    15,
+    2,
+    1,
+    4,
+    3,
+]
+
+def convert_keypoints(keypoints):
+    return [keypoints[i] for i in keypoint_index_mapping]