diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..43ae0e2a6c6d8fca34872506ca0f2e64194fec7c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.py[cod] diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fed786619fb07f1d055addd1eba98fb388a990b0 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +--- +title: TEOChat +emoji: 🏢 +colorFrom: green +colorTo: pink +sdk: gradio +sdk_version: 4.44.1 +app_file: videollava/serve/teochat_demo.py +pinned: false +license: apache-2.0 +short_description: A new vision-language assistant for temporal EO data. +--- + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1109d2a73b6a100dff4d7b878abfe9b60e6e430d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,283 @@ +accelerate==0.26.1 +affine==2.4.0 +aiofiles==23.2.1 +aiohttp==3.8.4 +aiosignal==1.3.1 +altair==5.2.0 +annotated-types==0.7.0 +anyio==3.6.2 +appdirs==1.4.4 +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +arrow==1.2.3 +asttokens==2.2.1 +async-timeout==4.0.2 +attrs==23.1.0 +av==11.0.0 +backcall==0.2.0 +beautifulsoup4==4.12.2 +bitsandbytes==0.41.0 +bleach==6.0.0 +blessed==1.20.0 +braceexpand==0.1.7 +Brotli==1.0.9 +certifi==2024.2.2 +cffi==1.15.1 +chardet==4.0.0 +charset-normalizer==3.1.0 +click==8.1.7 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==2.2.1 +cmake==3.26.3 +comm==0.2.1 +contourpy==1.0.7 +coverage==7.4.4 +croniter==1.3.14 +cycler==0.11.0 +cytoolz==0.12.2 +dask==2023.11.0 +dateutils==0.6.12 +debugpy==1.6.7 +decorator==5.1.1 +decord==0.6.0 +deepdiff==6.3.0 +deepspeed==0.14.0 +defusedxml==0.7.1 +distro==1.9.0 +dnspython==2.6.1 +docker-pycreds==0.4.0 +efficientnet-pytorch==0.7.1 +einops==0.6.1 +einops-exts==0.0.4 +email_validator==2.2.0 +exceptiongroup==1.2.0 +executing==1.2.0 +fastapi==0.111.1 +fastapi-cli==0.0.4 +fastjsonschema==2.16.3 +ffmpy==0.3.1 +filelock==3.13.1 +Fiona==1.9.3 +fire==0.5.0 +fonttools==4.39.3 +fqdn==1.5.1 +frozenlist==1.3.3 +fschat==0.2.36 +fsspec==2023.12.2 +ftfy==6.1.3 +fvcore==0.1.5.post20221221 +gdown==5.1.0 +geojson==3.0.1 +gitdb==4.0.10 +GitPython==3.1.31 +gradio==4.39.0 +gradio_client==1.1.1 +h11==0.14.0 +h5py==3.9.0 +hjson==3.1.0 +httpcore==1.0.5 +httptools==0.6.1 +httpx==0.27.0 +huggingface-hub==0.20.3 +idna==3.4 +imagecodecs==2021.8.26 +imageio==2.33.1 +importlib-metadata==7.0.1 +importlib-resources==5.12.0 +iniconfig==2.0.0 +inquirer==3.1.3 +iopath==0.1.10 +ipykernel==6.28.0 +ipython==8.15.0 +ipython-genutils==0.2.0 +ipywidgets==8.1.3 +isoduration==20.11.0 +itsdangerous==2.1.2 +jedi==0.18.2 +Jinja2==3.1.2 +joblib==1.2.0 +jsonpointer==2.3 +jsonschema==4.17.3 +jupyter_client==8.6.0 +jupyter_core==5.5.0 +jupyter-events==0.6.3 +jupyter_server==2.5.0 +jupyter_server_terminals==0.4.4 +jupyterlab-pygments==0.2.2 +jupyterlab_widgets==3.0.11 +kaleido==0.2.1 +kiwisolver==1.4.4 +kornia==0.7.3 +linkify-it-py==2.0.2 +lit==16.0.2 +locket==1.0.0 +markdown-it-py==2.2.0 +markdown2==2.4.12 +MarkupSafe==2.1.2 +matplotlib==3.7.1 +matplotlib-inline==0.1.6 +mdit-py-plugins==0.3.3 +mdurl==0.1.2 +mistune==2.0.5 +mkl-fft==1.3.8 +mkl-random==1.2.4 +mkl-service==2.4.0 +mpmath==1.3.0 +multidict==6.0.4 +munch==2.5.0 +nbclassic==0.5.5 +nbclient==0.7.4 +nbconvert==7.3.1 +nbformat==5.8.0 +nest-asyncio==1.6.0 +networkx==3.1 +nh3==0.2.15 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.5.4 +notebook_shim==0.2.3 +numpy==1.26.4 +open_clip_torch==2.26.1 +openai==0.28.1 +opencv-python==4.7.0.72 +ordered-set==4.1.0 +orjson==3.9.12 +packaging==23.1 +pandas==2.0.1 +pandocfilters==1.5.0 +parameterized==0.9.0 +parso==0.8.3 +partd==1.4.1 +pathtools==0.1.2 +peft==0.7.1 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==10.3.0 +pip==23.0.1 +platformdirs==3.10.0 +plotly==5.22.0 +pluggy==1.4.0 +portalocker==2.8.2 +pprintpp==0.4.0 +pretrainedmodels==0.7.4 +prometheus-client==0.16.0 +prompt-toolkit==3.0.43 +protobuf==4.22.3 +psutil==5.9.5 +ptyprocess==0.7.0 +pure-eval==0.2.2 +py-cpuinfo==9.0.0 +pycountry==23.12.11 +pycountry-convert==0.7.2 +pycparser==2.21 +pydantic==2.8.2 +pydantic_core==2.20.1 +pydub==0.25.1 +Pygments==2.15.1 +PyJWT==2.6.0 +pynvml==11.5.0 +pyparsing==3.0.9 +pyproj==3.5.0 +pyrsistent==0.19.3 +PySocks==1.7.1 +pytest==8.1.1 +pytest-cov==4.1.0 +pytest-mock==3.14.0 +python-dateutil==2.8.2 +python-dotenv==1.0.1 +python-editor==1.0.4 +python-json-logger==2.0.7 +python-multipart==0.0.9 +pytorch-lightning==2.0.2 +pytorchvideo==0.1.3 +pytz==2023.3 +pywavelets==1.5.0 +PyYAML==6.0.1 +pyzmq==25.1.2 +rasterio==1.3.6 +readchar==4.0.5 +regex==2023.12.25 +repoze.lru==0.7 +requests==2.31.0 +reverse-geocoder==1.5.1 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==13.3.4 +Rtree==1.0.1 +ruff==0.5.2 +safetensors==0.4.2 +scikit-image==0.19.3 +scikit-learn==1.2.2 +scipy==1.11.4 +seaborn==0.13.2 +segmentation-models-pytorch==0.3.2 +semantic-version==2.10.0 +Send2Trash==1.8.0 +sentencepiece==0.1.99 +sentry-sdk==1.21.0 +setproctitle==1.3.2 +setuptools==66.0.0 +shapely==2.0.1 +shellingham==1.5.4 +shortuuid==1.0.11 +six==1.16.0 +smmap==5.0.0 +sniffio==1.3.0 +snuggs==1.4.7 +soupsieve==2.4.1 +stack-data==0.6.2 +starlette==0.37.2 +starsessions==1.3.0 +svgwrite==1.4.3 +sympy==1.11.1 +tabulate==0.9.0 +tenacity==8.5.0 +tensorboardX==2.6.2.2 +termcolor==2.3.0 +terminado==0.17.1 +threadpoolctl==3.1.0 +tifffile==2021.7.2 +tiktoken==0.6.0 +timm==0.6.12 +tinycss2==1.2.1 +tokenizers==0.13.3 +tomli==2.0.1 +tomlkit==0.12.0 +toolz==0.12.1 +torchaudio==2.2.1 +torchgeo==0.6.0 +torchinfo==1.7.2 +torchmetrics==0.11.4 +torchvision==0.17.1 +tornado==6.3.3 +tqdm==4.65.0 +traitlets==5.9.0 +transformers==4.31.0 +Tree==0.2.4 +triton==2.2.0 +typer==0.12.3 +typing_extensions==4.9.0 +tzdata==2023.3 +uc-micro-py==1.0.2 +uri-template==1.2.0 +urllib3==2.1.0 +utm==0.7.0 +uvicorn==0.21.1 +uvloop==0.19.0 +wandb==0.15.0 +watchfiles==0.22.0 +wavedrom==2.0.3.post3 +wcwidth==0.2.13 +webcolors==1.13 +webdataset==0.2.86 +webencodings==0.5.1 +websocket-client==1.5.1 +websockets==11.0.2 +wheel==0.38.4 +widgetsnbextension==4.0.11 +yacs==0.1.8 +yarl==1.9.2 +zipp==3.17.0 +--extra-index-url https://download.pytorch.org/whl/cu113 +torch==2.2.1 \ No newline at end of file diff --git a/static/logo.png b/static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..cc5b3618fc73a86e5a0488088f1fdbf2bc143dd3 Binary files /dev/null and b/static/logo.png differ diff --git a/static/logo.svg b/static/logo.svg new file mode 100644 index 0000000000000000000000000000000000000000..0ba343ca3acd294cdc218caddae9bf00e111971f --- /dev/null +++ b/static/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/videollava/__init__.py b/videollava/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4d1f016db1028101d45ba7d68cb3f0bcb558c2bb --- /dev/null +++ b/videollava/__init__.py @@ -0,0 +1 @@ +from .model import LlavaLlamaForCausalLM diff --git a/videollava/constants.py b/videollava/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..956177db3f56f14adb5cf53ff2fdd6ccaa07dac6 --- /dev/null +++ b/videollava/constants.py @@ -0,0 +1,27 @@ +CONTROLLER_HEART_BEAT_EXPIRATION = 30 +WORKER_HEART_BEAT_INTERVAL = 15 + +LOGDIR = "." + +# Model Constants +IGNORE_INDEX = -100 + +IMAGE_TOKEN_INDEX = -200 +DEFAULT_IMAGE_TOKEN = "" +DEFAULT_IMAGE_PATCH_TOKEN = "" +DEFAULT_IM_START_TOKEN = "" +DEFAULT_IM_END_TOKEN = "" +IMAGE_PLACEHOLDER = "" + +# ====================================================================================================== +DEFAULT_VIDEO_TOKEN = "