diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..9430e7d8d45ef1b020f0fc5b49ebc45e6016cbb3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git* +**__pycache__** +docker +Dockerfile* diff --git a/.gitattributes b/.gitattributes index 92ef5e02a525fb8f3779f58199d1d9ed0762f676..17742c0a227912b2e79ae7c16ecbfd821eff4870 100644 --- a/.gitattributes +++ b/.gitattributes @@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text visual-quality-inspection/transfer-learning/workflows/vision_anomaly_detection/assets/Visual_quality_inspection_layered_architecture.JPG filter=lfs diff=lfs merge=lfs -text visual-quality-inspection/transfer-learning/workflows/vision_anomaly_detection/assets/visual_quality_inspection_pipeline.JPG filter=lfs diff=lfs merge=lfs -text +workflows/vision_anomaly_detection/assets/Visual_quality_inspection_layered_architecture.JPG filter=lfs diff=lfs merge=lfs -text +workflows/vision_anomaly_detection/assets/visual_quality_inspection_pipeline.JPG filter=lfs diff=lfs merge=lfs -text diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000000000000000000000000000000000..88388981ffd07907fe3778e6cfd5756bb254ba15 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ + + +**Before requesting a review:** + +- [ ] I have ensured my PR title is accurate +- [ ] I wrote a description of the changes being made, if it's not obvious +- [ ] I have synced by branch with the base (i.e. `develop`) +- [ ] I ran `make lint` on my branch and it passes +- [ ] I ran the pytest tests that could reasonably be affected by my changes and they pass +- [ ] I have performed a self code review of my own code on the "Files changed" tab of the pull request +- [ ] I have commented my code in hard-to-understand areas +- [ ] I have updated the documentation (in docstrings, notebooks, and .rst files) +- [ ] I have added new tests that prove my fix is effective or that my feature works (or provide justification why not) +- [ ] I have applied the appropriate labels to the PR (if your PR is not ready for review use "WIP") diff --git a/.github/workflows/build-container.yaml b/.github/workflows/build-container.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ccd0c5cba0bf8ddfc5b8773cd6f09b52eb77c5e --- /dev/null +++ b/.github/workflows/build-container.yaml @@ -0,0 +1,50 @@ +name: TLT Containers Weekly Builder +on: + workflow_dispatch: # Can be manually executed + schedule: # 1/week Sunday at 11:00PM + - cron: "5 23 * * 0" +jobs: + build: + container: # MLOps Dev container for Compose Automation + image: ${{ vars.GHA_CONTAINER_IMAGE }} + env: # Add ENVS to control compose building + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + credentials: # CAAS Registry Creds + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + runs-on: [aia-devops] # Runner Label + steps: + - uses: actions/checkout@v3 + with: + submodules: true + set-safe-directory: true + - name: Build Container + run: docker compose build + working-directory: ./docker + push: + needs: [build] + strategy: + matrix: + container: ["tlt-devel", "tlt-prod", "tlt-dist-devel", "tlt-dist-prod"] # name of Compose container + container: + image: ${{ vars.GHA_CONTAINER_IMAGE }} + env: # Add ENVS to control compose building + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + credentials: # CAAS Registry Creds + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + runs-on: [aia-devops] + steps: + - uses: docker/login-action@v2 + with: # CAAS Registry Creds + registry: ${{ vars.GHA_REGISTRY }} + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Push Container # tlt-- + run: | + docker tag intel/ai-tools:${{ matrix.container }}-latest ${{ vars.GHA_REGISTRY_REPO }}:ww$(date +"%U")-${{ matrix.container }} + docker push ${{ vars.GHA_REGISTRY_REPO }}:ww$(date +"%U")-${{ matrix.container }} diff --git a/.github/workflows/docs-test.yaml b/.github/workflows/docs-test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..276c10318463794b0e676aff99a775cd71214329 --- /dev/null +++ b/.github/workflows/docs-test.yaml @@ -0,0 +1,34 @@ +name: Build and Test docs +on: + pull_request: + types: [submitted] + # run the workflow if changes pushed to main or release branches + push: + branches: + - '**' + tags: + - '**' + paths: + - '**' + +# installs dependencies, build the docs and push it to `gh-pages` +jobs: + docs-test: + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + volumes: + - /tf_dataset/dataset/transfer_learning:/tmp/data + steps: + - uses: actions/checkout@v3 + # Test the docs + - name: Run documentation tests + run: | + make test_docs diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a249bb351d6f6432b1d27f175e308bb7ce5ece66 --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,41 @@ +name: Integration Test +on: + pull_request_review: + types: [submitted] + # run the workflow if changes pushed to main or release branches + push: + branches: + - develop + - main + - r0.1 + - r0.2 + - r0.3 + - r0.4 + - r0.5 + tags: + - '**' + paths: + - '**' +jobs: + integration-test: + if: github.event.review.state == 'approved' || + github.event.pull_request.merged == true || + github.event_name == 'push' + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + volumes: + - /tf_dataset/dataset/transfer_learning:/tmp/data + steps: + - uses: actions/checkout@v3 + - name: Run Integration Tests + shell: bash + continue-on-error: false + run: make integration diff --git a/.github/workflows/nightly-integration.yaml b/.github/workflows/nightly-integration.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e335f64dd97e97eb6c7af8a998214ba012171934 --- /dev/null +++ b/.github/workflows/nightly-integration.yaml @@ -0,0 +1,25 @@ +name: Nightly Integration Test +on: + workflow_dispatch: # Can be manually executed + schedule: # nightly at 10:00PM + - cron: "0 22 * * *" +jobs: + nightly-test: + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + with: + ref: develop + - name: Run Integration Test + shell: bash + continue-on-error: false + run: make integration diff --git a/.github/workflows/nightly-notebook-test.yaml b/.github/workflows/nightly-notebook-test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55682314479834fffb883185536afc4e4bf582d8 --- /dev/null +++ b/.github/workflows/nightly-notebook-test.yaml @@ -0,0 +1,25 @@ +name: Nightly Notebooks Test +on: + workflow_dispatch: # Can be manually executed + schedule: # nightly at 11:00PM + - cron: "0 23 * * *" +jobs: + notebook-test: + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + DATASET_DIR: /tmp/data + OUTPUT_DIR: /tmp/output + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + with: + ref: develop + - name: Run Notebook Tests + run: make test_notebook_catalog diff --git a/.github/workflows/notebook-test.yaml b/.github/workflows/notebook-test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcc025a0fae594dd544bd73efc9bdb75efe277cd --- /dev/null +++ b/.github/workflows/notebook-test.yaml @@ -0,0 +1,41 @@ +name: Notebooks Test +on: + pull_request_review: + types: [submitted] + # run the workflow if changes pushed to main or release branches + push: + branches: + - develop + - main + - r0.1 + - r0.2 + - r0.3 + - r0.4 + - r0.5 + tags: + - '**' + paths: + - '**' +jobs: + notebook-test: + if: github.event.review.state == 'approved' || + github.event.pull_request.merged == true || + github.event_name == 'push' + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + DATASET_DIR: /tmp/data + OUTPUT_DIR: /tmp/output + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + volumes: + - /tf_dataset/dataset/transfer_learning:/tmp/data + steps: + - uses: actions/checkout@v3 + - name: Run Notebook Tests + run: make test_notebook_custom diff --git a/.github/workflows/style-test.yaml b/.github/workflows/style-test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51a35ed48b9116a49b41bd4cfdcc3ab0282abfcf --- /dev/null +++ b/.github/workflows/style-test.yaml @@ -0,0 +1,30 @@ +name: Style Checks +on: + pull_request: + types: [submitted] + # run the workflow if changes pushed to main or release branches + push: + branches: + - '**' + tags: + - '**' + paths: + - '**' +# installs dependencies and runs the linter +jobs: + style-test: + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Run linter + run: | + make lint diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c135c31088a6ebf9c4bc65a428b29b817f8e9efb --- /dev/null +++ b/.github/workflows/unittest.yaml @@ -0,0 +1,30 @@ +name: Unit Test +on: + pull_request: + types: [submitted] + # run the workflow if changes pushed to main or release branches + push: + branches: + - '**' + tags: + - '**' + paths: + - '**' +jobs: + unit-test: + runs-on: [ aia-devops ] + container: + image: ${{ vars.GHA_IMAGE }} + env: + http_proxy: ${{ secrets.HTTP_PROXY }} + https_proxy: ${{ secrets.HTTPS_PROXY }} + no_proxy: ${{ secrets.NO_PROXY }} + # credentials: + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Run Unit Test + shell: bash + continue-on-error: false + run: make unittest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..c89a2aafc5bdc0ee4aaebedb1c836a3518a6b46d --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +_autosummary +.coverage +.DS_Store +.idea* +.ipynb_checkpoints +.vscode +*.egg-info/ +*.pyc +**.log +**/*.cache +**/**.whl +**/**/models/ +**/**venv +**venv* +build/ +data +dist/ +docs/_build/ +nc_workspace +output diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..0ce3b508a6107b6f54d990ba5e651db7134f5fed --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,59 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @global-owner1 and @global-owner2 will be requested for +# review when someone opens a pull request. +* @ashahba @dmsuehir @etcylfleet @HarshaRamayanam @mhbuehler @okhleif-IL + +# Order is important; the last matching pattern takes the most +# precedence. When someone opens a pull request that only +# modifies JS files, only @js-owner and not the global +# owner(s) will be requested for a review. +# *.js @js-owner #This is an inline comment. + +# You can also use email addresses if you prefer. They'll be +# used to look up users just like we do for commit author +# emails. +# *.go docs@example.com + +# Teams can be specified as code owners as well. Teams should +# be identified in the format @org/team-name. Teams must have +# explicit write access to the repository. In this example, +# the octocats team in the octo-org organization owns all .txt files. +# *.txt @octo-org/octocats + +# In this example, @doctocat owns any files in the build/logs +# directory at the root of the repository and any of its +# subdirectories. +# /build/logs/ @doctocat + +# The `docs/*` pattern will match files like +# `docs/getting-started.md` but not further nested files like +# `docs/build-app/troubleshooting.md`. +# docs/* docs@example.com + +# In this example, @octocat owns any file in an apps directory +# anywhere in your repository. +# apps/ @octocat + +# In this example, @doctocat owns any file in the `/docs` +# directory in the root of your repository and any of its +# subdirectories. +# /docs/ @doctocat + +# In this example, any change inside the `/scripts` directory +# will require approval from @doctocat or @octocat. +# /scripts/ @doctocat @octocat + +# In this example, @octocat owns any file in a `/logs` directory such as +# `/build/logs`, `/scripts/logs`, and `/deeply/nested/logs`. Any changes +# in a `/logs` directory will require approval from @octocat. +# **/logs @octocat + +# In this example, @octocat owns any file in the `/apps` +# directory in the root of your repository except for the `/apps/github` +# subdirectory, as its owners are left empty. +# /apps/ @octocat +# /apps/github diff --git a/DATASETS.md b/DATASETS.md new file mode 100644 index 0000000000000000000000000000000000000000..ed56ed3c471de79a613e7bc539c4d1d1ebb7ae80 --- /dev/null +++ b/DATASETS.md @@ -0,0 +1,30 @@ +# Datasets + +This is a comprehensive list of public datasets used by this repository. + +| Name (Link/Source) | Framework | Use Case | +|--------------------| --------- | -------- | +| [AG News (Hugging Face)](https://huggingface.co/datasets/ag_news) | PyTorch | Text Classification | +| [AG News (TFDS)](https://www.tensorflow.org/datasets/catalog/ag_news_subset) | TensorFlow | Text Classification | +| [Food101 (Torchvision)](https://pytorch.org/vision/stable/generated/torchvision.datasets.Food101.html#torchvision.datasets.Food101) | PyTorch | Image Classification | +| [Food101 (TFDS)](https://www.tensorflow.org/datasets/catalog/food101) | TensorFlow | Image Classification | +| [SMS Spam Collection](https://archive.ics.uci.edu/dataset/228/sms+spam+collection) | PyTorch & TensorFlow | Text Classification | +| [TF Flowers (TFDS)](https://www.tensorflow.org/datasets/catalog/tf_flowers) | PyTorch & TensorFlow | Image Classification | +| [Cats vs. Dogs (TFDS)](https://www.tensorflow.org/datasets/catalog/cats_vs_dogs) | TensorFlow | Image Classification | +| [Country211 (Torchvision)](https://pytorch.org/vision/stable/generated/torchvision.datasets.Country211.html#torchvision.datasets.Country211) | PyTorch | Image Classification | +| [DTD (Torchvision)](https://pytorch.org/vision/stable/generated/torchvision.datasets.DTD.html#torchvision.datasets.DTD) | PyTorch | Image Classification | +| [FGVCAircraft (Torchvision)](https://pytorch.org/vision/stable/generated/torchvision.datasets.FGVCAircraft.html#torchvision.datasets.FGVCAircraft) | PyTorch | Image Classification | +| [RenderedSST2 (Torchvision)](https://pytorch.org/vision/stable/generated/torchvision.datasets.RenderedSST2.html#torchvision.datasets.RenderedSST2) | PyTorch | Image Classification | +| [Rock Paper Scissors (TFDS)](https://www.tensorflow.org/datasets/catalog/rock_paper_scissors) | TensorFlow | Image Classification | +| [Rotten_Tomatoes (Hugging Face)](https://huggingface.co/datasets/rotten_tomatoes) | PyTorch | Text Classification | +| [TweetEval (Hugging Face)](https://huggingface.co/datasets/tweet_eval) | PyTorch | Text Classification | +| [CIFAR10 (Torchvision)](https://pytorch.org/vision/stable/generated/torchvision.datasets.CIFAR10.html#torchvision.datasets.CIFAR10) | PyTorch | Image Classification | +| [IMDB Reviews (Hugging Face)](https://huggingface.co/datasets/imdb) | PyTorch | Text Classification | +| [IMDB Reviews (TFDS)](https://www.tensorflow.org/datasets/catalog/imdb_reviews) | TensorFlow | Text Classification | +| [GLUE/SST2 (TFDS)](https://www.tensorflow.org/datasets/catalog/glue#gluesst2) | TensorFlow | Text Classification | +| [GLUE/COLA (TFDS)](https://www.tensorflow.org/datasets/catalog/glue#gluecola_default_config) | TensorFlow | Text Classification | +| [Colorectal Histology (TFDS)](https://www.tensorflow.org/datasets/catalog/colorectal_histology) | TensorFlow | Image Classification | +| [RESISC45 (TFDS)](https://www.tensorflow.org/datasets/catalog/resisc45) | TensorFlow | Image Classification | +| [CDD-CESM](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=109379611) | PyTorch & TensorFlow | Image & Text Classification | +| [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) | PyTorch & TensorFlow | Text Classification | +| [MVTec](https://www.mvtec.com/company/research/datasets/mvtec-ad) | PyTorch | Anomaly Detection | diff --git a/GetStarted.md b/GetStarted.md new file mode 100644 index 0000000000000000000000000000000000000000..d52f56a86d2871d9057574d59160e767b6411488 --- /dev/null +++ b/GetStarted.md @@ -0,0 +1,263 @@ +# Get Started + +This is a guide for getting started with Intel® Transfer Learning Tool and will +walk you through the steps to check system requirements, install, and then run +the tool with a couple of examples showing no-code CLI and low-code API +approaches. + +

Intel Transfer Learning Tool Get Started Flow

+ +Intel Transfer Learning Tool Get Started Flow + +## ① Check System Requirements + +| Recommended Hardware | Precision | +| ---------------------------- | ---------- | +| Intel® 4th Gen Xeon® Scalable Performance processors | BF16 | +| Intel® 1st, 2nd, 3rd, and 4th Gen Xeon® Scalable Performance processors | FP32 | + +| Resource | Minimum | +| ---------------------------- | ---------- | +| CPU Cores | 8 (16+ recommended) | +| RAM | 16 GB (24-32+ GB recommended) | +| Disk space | 10 GB minimum (can vary based on datasets downloaded) | + +| Required Software | +| ------------------------- | +| Linux\* system (validated on Ubuntu\* 20.04/22.04 LTS) | +| Python (3.8, 3.9, or 3.10) | +| Pip | +| Conda or Python virtualenv | +| git (only required for advanced installation) | + +## ② Install + +1. **Install Dependencies** + + Install required packages using: + + ``` + sudo apt-get install build-essential python3-dev libgl1 libglib2.0-0 + ``` + +2. **Create and activate a Python3 virtual environment** + + We encourage you to use a Python virtual environment (virtualenv or conda) + for consistent package management. There are two ways to do this: + + a. Use `virtualenv`: + + ``` + virtualenv -p python3 tlt_dev_venv + source tlt_dev_venv/bin/activate + ``` + + b. Or use `conda`: + + ``` + conda create --name tlt_dev_venv python=3.9 + conda activate tlt_dev_venv + ``` + +3. **Install Intel Transfer Learning Tool** + + Use the Basic Installation instructions unless you plan on making code changes. + + a. **Basic Installation** + + ``` + pip install intel-transfer-learning-tool + ``` + + b. **Advanced Installation** + + Clone the repo: + + ``` + git clone https://github.com/IntelAI/transfer-learning.git + cd transfer-learning + ``` + + Then either do an editable install to avoid a rebuild and + install after each code change (preferred): + + ``` + pip install --editable . + ``` + + or build and install a wheel: + + ``` + python setup.py bdist_wheel + pip install dist/intel_transfer_learning_tool-0.5.0-py3-none-any.whl + ``` + + +4. **Additional Feature-Specific Steps** + + * For distributed/multinode training, follow these additional + [distributed training instructions](tlt/distributed/README.md). + +5. **Verify Installation** + + Verify that your installation was successful by using the following + command, which displays help information about the Intel Transfer Learning Tool: + + ``` + tlt --help + ``` + +## ③ Run the Intel Transfer Learning Tool + +With the Intel Transfer Learning Tool, you can train AI models with TensorFlow or +PyTorch using either no-code CLI commands at a bash prompt, or low-code API +calls from a Python script. Both approaches provide the same opportunities for +training, evaluation, optimization, and benchmarking. With the CLI, no +programming experience is required, and you'll need basic Python knowledge to +use the API. Choose the approach that works best for you. + + +### Run Using the No-Code CLI + +Let's continue from the previous step where you prepared the dataset, and train +a model using CLI commands. This example uses the CLI to train an image +classifier to identify different types of flowers. You can see a list of all +available image classifier models using the command: + +``` +tlt list models --use-case image_classification +``` + +**Train a Model** + +In this example, we'll use the `tlt train` command to retrain the TensorFlow +ResNet50v1.5 model using a flowers dataset from the +[TensorFlow Datasets catalog](https://www.tensorflow.org/datasets/catalog/tf_flowers). +The `--dataset-dir` and `--output-dir` paths need to point to writable folders on your system. +``` +# Use the follow environment variable setting to reduce the warnings and log output from TensorFlow +export TF_CPP_MIN_LOG_LEVEL="2" + +tlt train -f tensorflow --model-name resnet_v1_50 --dataset-name tf_flowers --dataset-dir "/tmp/data-${USER}" --output-dir "/tmp/output-${USER}" +``` +``` +Model name: resnet_v1_50 +Framework: tensorflow +Dataset name: tf_flowers +Training epochs: 1 +Dataset dir: /tmp/data-user +Output directory: /tmp/output-user +... +Model: "sequential" +_________________________________________________________________ +Layer (type) Output Shape Param # +================================================================= +keras_layer (KerasLayer) (None, 2048) 23561152 +dense (Dense) (None, 5) 10245 +================================================================= +Total params: 23,571,397 +Trainable params: 10,245 +Non-trainable params: 23,561,152 +_________________________________________________________________ +Checkpoint directory: /tmp/output-user/resnet_v1_50_checkpoints +86/86 [==============================] - 24s 248ms/step - loss: 0.4600 - acc: 0.8438 +Saved model directory: /tmp/output-user/resnet_v1_50/1 +``` + +After training completes, the `tlt train` command evaluates the model. The loss and +accuracy values are printed toward the end of the console output. The model is +exported to the output directory you specified in a numbered folder created for +each training run. + +**Next Steps** + +That ends this Get Started CLI example. As a next step, you can also follow the +[Beyond Get Started CLI Example](examples/cli/README.md) for a complete example +that includes evaluation, benchmarking, and quantization in the datasets. + +Read about all the CLI commands in the [CLI reference](/cli.md). +Find more examples in our list of [Examples](examples/README.md). + +### Run Using the Low-Code API + +The following Python code example trains an image classification model with the TensorFlow +flowers dataset using API calls from Python. The model is +benchmarked and quantized to INT8 precision for improved inference performance. + +You can run the API example using a Jupyter notebook. See the [notebook setup +instructions](/notebooks/setup.md) for more details for preparing the Jupyter +notebook environment. + +```python +import os + +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" + +from tlt.datasets import dataset_factory +from tlt.models import model_factory +from tlt.utils.types import FrameworkType, UseCaseType + +username = os.getenv('USER', 'user') + +# Specify a writable directory for the dataset to be downloaded +dataset_dir = '/tmp/data-{}'.format(username) +if not os.path.exists(dataset_dir): + os.makedirs(dataset_dir) + +# Specify a writeable directory for output (such as saved model files) +output_dir = '/tmp/output-{}'.format(username) +if not os.path.exists(output_dir): + os.makedirs(output_dir) + +# Get the model +model = model_factory.get_model(model_name="resnet_v1_50", framework=FrameworkType.TENSORFLOW) + +# Download and preprocess the flowers dataset from the TensorFlow datasets catalog +dataset = dataset_factory.get_dataset(dataset_dir=dataset_dir, + dataset_name='tf_flowers', + use_case=UseCaseType.IMAGE_CLASSIFICATION, + framework=FrameworkType.TENSORFLOW, + dataset_catalog='tf_datasets') +dataset.preprocess(image_size=model.image_size, batch_size=32) +dataset.shuffle_split(train_pct=.75, val_pct=.25) + +# Train the model using the dataset +model.train(dataset, output_dir=output_dir, epochs=1) + +# Evaluate the trained model +metrics = model.evaluate(dataset) +for metric_name, metric_value in zip(model._model.metrics_names, metrics): + print("{}: {}".format(metric_name, metric_value)) + +# Export the model +saved_model_dir = model.export(output_dir=output_dir) + +# Quantize the trained model +quantization_output = os.path.join(output_dir, "quantized_model") +model.quantize(quantization_output, dataset, overwrite_model=True) + +# Benchmark the trained model using the Intel Neural Compressor config file +model.benchmark(dataset, saved_model_dir=quantization_output) + +# Do graph optimization on the trained model +optimization_output = os.path.join(output_dir, "optimized_model") +model.optimize_graph(optimization_output, overwrite_model=True) +``` + +For more information on the API, see the [API Documentation](/api.md). + +## Summary and Next Steps + +The Intel Transfer Learning Tool can be used to develop an AI model and export +an Intel-optimized saved model for deployment. The sample CLI and API commands +we've presented show how to execute end-to-end transfer learning workflows. + +For the no-code CLI, you can follow a +complete example that includes trainng, evaluation, benchmarking, and quantization +in the datasets, as well as some additional models in the [Beyond Get Started +CLI example](examples/cli/README.md) documentation. You can also read about all the +CLI commands in the [CLI reference](/cli.md). + +For the low-code API, read about the API in the [API Documentation](/api.md). + +Find more CLI and API examples in our list of [Examples](examples/README.md). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..01c7ad7facd90ef32a6a409d261ad90689d7e7b1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Legal.md b/Legal.md new file mode 100644 index 0000000000000000000000000000000000000000..09ae9d2a936306484157e53fd0a98c169a86d484 --- /dev/null +++ b/Legal.md @@ -0,0 +1,34 @@ +# Legal Information + +## Disclaimer + +Intel® Transfer Learning Tool scripts are not intended for benchmarking Intel® platforms. For any +performance and/or benchmarking information on specific Intel platforms, visit +https://www.intel.ai/blog. + +Intel is committed to the respect of human rights and avoiding complicity in +human rights abuses, a policy reflected in the Intel Global Human Rights +Principles. Accordingly, by accessing the Intel material on this platform you +agree that you will not use the material in a product or application that causes +or contributes to a violation of an internationally recognized human right. + +## License + +Intel® Transfer Learning Tool, documentation, and example code are all licensed +under Apache License Version 2.0. + +## Datasets + +To the extent that any [public datasets](DATASETS.md) are referenced by Intel or accessed using +tools or code on this site those datasets are provided by the third party +indicated as the data source. Intel does not create the data, or datasets, and +does not warrant their accuracy or quality. By accessing the public dataset(s) +you agree to the terms associated with those datasets and that your use complies +with the applicable license. + +Intel expressly disclaims the accuracy, adequacy, or completeness of any public +datasets, and is not liable for any errors, omissions, or defects in the data, +or for any reliance on the data. Intel is not liable for any liability or +damages relating to your use of public datasets. + +\*Other names and brands may be claimed as the property of others. [Trademarks](http://www.intel.com/content/www/us/en/legal/trademarks.html) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..9afb2db7ce63be3c658894461a6724f06ffa8098 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include tlt * diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..4cf64160173973673a3b31c21ecb714c9fa2838e --- /dev/null +++ b/Makefile @@ -0,0 +1,112 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Note: These are just placeholders for future additions to Makefile. +# You can remove these comments later. +ACTIVATE_TLT_VENV = "tlt_dev_venv/bin/activate" +ACTIVATE_NOTEBOOK_VENV = "tlt_notebook_venv/bin/activate" +ACTIVATE_TEST_VENV = "tlt_test_venv/bin/activate" +ACTIVATE_DOCS_VENV = $(ACTIVATE_TEST_VENV) + +# Customize sample test run commands +# PY_TEST_EXTRA_ARGS="'-vvv -k test_platform_util_with_no_args'" make test +# PY_TEST_EXTRA_ARGS="'--collect-only'" make test +PY_TEST_EXTRA_ARGS ?= "--durations=0" + +tlt_test_venv: $(CURDIR)/tests/requirements-test.txt + @echo "Creating a virtualenv tlt_test_venv..." + @test -d tlt_test_venv || virtualenv -p python3 tlt_test_venv + + @echo "Building the TLT API in tlt_test_venv env..." + @. $(ACTIVATE_TEST_VENV) && pip install --editable . + + @echo "Installing test dependencies..." + @. $(ACTIVATE_TEST_VENV) && pip install -r $(CURDIR)/tests/requirements-test.txt + +tlt_notebook_venv: $(CURDIR)/notebooks/requirements.txt + @echo "Creating a virtualenv tlt_notebook_venv..." + @test -d tlt_notebook_venv || virtualenv -p python3 tlt_notebook_venv + + @echo "Installing TF & PYT notebook dependencies..." + @. $(ACTIVATE_NOTEBOOK_VENV) && pip install -r $(CURDIR)/notebooks/requirements.txt + +test: unittest integration + +unittest: tlt_test_venv + @echo "Testing unit test API..." + @. $(ACTIVATE_TEST_VENV) && PYTHONPATH=$(CURDIR)/tests py.test -vvv -s $(PY_TEST_EXTRA_ARGS) "-k not integration and not skip" + +integration: tlt_test_venv + @echo "Testing integration test API..." + @. $(ACTIVATE_TEST_VENV) && PYTHONPATH=$(CURDIR)/tests py.test -vvv -s $(PY_TEST_EXTRA_ARGS) "-k integration and not skip" + +lint: tlt_test_venv + @echo "Style checks..." + @. $(ACTIVATE_TEST_VENV) && flake8 tlt tests downloader + +clean: + rm -rf tlt_test_venv + +tlt_docs_venv: tlt_test_venv $(CURDIR)/docs/requirements-docs.txt + @echo "Installing docs dependencies..." + @. $(ACTIVATE_DOCS_VENV) && pip install -r $(CURDIR)/docs/requirements-docs.txt + +html: tlt_docs_venv + @echo "Building Sphinx documentation..." + @. $(ACTIVATE_DOCS_VENV) && $(MAKE) -C docs clean html + +test_docs: html + @echo "Testing Sphinx documentation..." + @. $(ACTIVATE_DOCS_VENV) && $(MAKE) -C docs doctest + +tlt_notebook_venv: tlt_test_venv + @echo "Installing notebook dependencies..." + @. $(ACTIVATE_TEST_VENV) && pip install -r $(CURDIR)/notebooks/requirements.txt + +test_notebook_custom: tlt_notebook_venv + @echo "Testing Jupyter notebooks with custom datasets..." + @. $(ACTIVATE_TEST_VENV) && \ + bash run_notebooks.sh $(CURDIR)/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb remove_for_custom_dataset && \ + bash run_notebooks.sh $(CURDIR)/notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb remove_for_custom_dataset && \ + bash run_notebooks.sh $(CURDIR)/notebooks/text_classification/tlt_api_tf_text_classification/TLT_TF_Text_Classification.ipynb remove_for_custom_dataset && \ + bash run_notebooks.sh $(CURDIR)/notebooks/text_classification/tlt_api_pyt_text_classification/TLT_PYT_Text_Classification.ipynb remove_for_custom_dataset + +test_notebook_catalog: tlt_notebook_venv + @echo "Testing Jupyter notebooks with public catalog datasets..." + @. $(ACTIVATE_TEST_VENV) && \ + bash run_notebooks.sh $(CURDIR)/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb remove_for_tf_dataset && \ + bash run_notebooks.sh $(CURDIR)/notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb remove_for_tv_dataset && \ + bash run_notebooks.sh $(CURDIR)/notebooks/text_classification/tlt_api_tf_text_classification/TLT_TF_Text_Classification.ipynb remove_for_tf_dataset && \ + bash run_notebooks.sh $(CURDIR)/notebooks/text_classification/tlt_api_pyt_text_classification/TLT_PYT_Text_Classification.ipynb remove_for_hf_dataset + +test_tf_notebook: tlt_notebook_venv + @. $(ACTIVATE_TEST_VENV) && bash run_notebooks.sh tensorflow + +test_pyt_notebook: tlt_notebook_venv + @. $(ACTIVATE_TEST_VENV) && bash run_notebooks.sh pytorch + +dist: tlt_docs_venv + @echo "Create binary wheel..." + @. $(ACTIVATE_DOCS_VENV) && python setup.py bdist_wheel + +check_dist: dist + @echo "Testing the wheel..." + @. $(ACTIVATE_DOCS_VENV) && \ + pip install twine && \ + python setup.py bdist_wheel && \ + twine check dist/* diff --git a/Models.md b/Models.md new file mode 100644 index 0000000000000000000000000000000000000000..e44dfaeb2ee2d283218c9d2c2e464ccb1a7209e9 --- /dev/null +++ b/Models.md @@ -0,0 +1,178 @@ +# Intel® Transfer Learning Tool Supported Models + +## Image Classification + +| Model name | Framework | Model Hub | +|------------|-----------|-----------| +| alexnet | PyTorch* | Torchvision* | +| convnext_base | PyTorch | Torchvision | +| convnext_large | PyTorch | Torchvision | +| convnext_small | PyTorch | Torchvision | +| convnext_tiny | PyTorch | Torchvision | +| densenet121 | PyTorch | Torchvision | +| densenet161 | PyTorch | Torchvision | +| densenet169 | PyTorch | Torchvision | +| densenet201 | PyTorch | Torchvision | +| efficientnetv2-b0 | TensorFlow* | TensorFlow Hub* | +| efficientnetv2-b1 | TensorFlow | TensorFlow Hub | +| efficientnetv2-b2 | TensorFlow | TensorFlow Hub | +| efficientnetv2-b3 | TensorFlow | TensorFlow Hub | +| efficientnetv2-s | TensorFlow | TensorFlow Hub | +| efficientnet_b0 | TensorFlow | TensorFlow Hub | +| efficientnet_b0 | PyTorch | Torchvision | +| efficientnet_b1 | TensorFlow | TensorFlow Hub | +| efficientnet_b1 | PyTorch | Torchvision | +| efficientnet_b2 | TensorFlow | TensorFlow Hub | +| efficientnet_b2 | PyTorch | Torchvision | +| efficientnet_b3 | TensorFlow | TensorFlow Hub | +| efficientnet_b3 | PyTorch | Torchvision | +| efficientnet_b4 | TensorFlow | TensorFlow Hub | +| efficientnet_b4 | PyTorch | Torchvision | +| efficientnet_b5 | TensorFlow | TensorFlow Hub | +| efficientnet_b5 | PyTorch | Torchvision | +| efficientnet_b6 | TensorFlow | TensorFlow Hub | +| efficientnet_b6 | PyTorch | Torchvision | +| efficientnet_b7 | TensorFlow | TensorFlow Hub | +| efficientnet_b7 | PyTorch | Torchvision | +| googlenet | PyTorch | Torchvision | +| inception_v3 | TensorFlow | TensorFlow Hub | +| mnasnet0_5 | PyTorch | Torchvision | +| mnasnet1_0 | PyTorch | Torchvision | +| mobilenet_v2 | PyTorch | Torchvision | +| mobilenet_v2_100_224 | TensorFlow | TensorFlow Hub | +| mobilenet_v3_large | PyTorch | Torchvision | +| mobilenet_v3_small | PyTorch | Torchvision | +| nasnet_large | TensorFlow | TensorFlow Hub | +| proxyless_cpu | PyTorch | PyTorch Hub* | +| regnet_x_16gf | PyTorch | Torchvision | +| regnet_x_1_6gf | PyTorch | Torchvision | +| regnet_x_32gf | PyTorch | Torchvision | +| regnet_x_3_2gf | PyTorch | Torchvision | +| regnet_x_400mf | PyTorch | Torchvision | +| regnet_x_800mf | PyTorch | Torchvision | +| regnet_x_8gf | PyTorch | Torchvision | +| regnet_y_16gf | PyTorch | Torchvision | +| regnet_y_1_6gf | PyTorch | Torchvision | +| regnet_y_32gf | PyTorch | Torchvision | +| regnet_y_3_2gf | PyTorch | Torchvision | +| regnet_y_400mf | PyTorch | Torchvision | +| regnet_y_800mf | PyTorch | Torchvision | +| regnet_y_8gf | PyTorch | Torchvision | +| resnet101 | PyTorch | Torchvision | +| resnet152 | PyTorch | Torchvision | +| resnet18 | PyTorch | Torchvision | +| resnet18_ssl | PyTorch | PyTorch Hub | +| resnet18_swsl | PyTorch | PyTorch Hub | +| resnet34 | PyTorch | Torchvision | +| resnet50 | PyTorch | Torchvision | +| resnet50_ssl | PyTorch | PyTorch Hub | +| resnet50_swsl | PyTorch | PyTorch Hub | +| resnet_v1_50 | TensorFlow | TensorFlow Hub | +| resnet_v2_101 | TensorFlow | TensorFlow Hub | +| resnet_v2_50 | TensorFlow | TensorFlow Hub | +| resnext101_32x16d_ssl | PyTorch | PyTorch Hub | +| resnext101_32x16d_swsl | PyTorch | PyTorch Hub | +| resnext101_32x16d_wsl | PyTorch | PyTorch Hub | +| resnext101_32x32d_wsl | PyTorch | PyTorch Hub | +| resnext101_32x48d_wsl | PyTorch | PyTorch Hub | +| resnext101_32x4d_ssl | PyTorch | PyTorch Hub | +| resnext101_32x4d_swsl | PyTorch | PyTorch Hub | +| resnext101_32x8d | PyTorch | Torchvision | +| resnext101_32x8d_ssl | PyTorch | PyTorch Hub | +| resnext101_32x8d_swsl | PyTorch | PyTorch Hub | +| resnext101_32x8d_wsl | PyTorch | PyTorch Hub | +| resnext50_32x4d | PyTorch | Torchvision | +| resnext50_32x4d_ssl | PyTorch | PyTorch Hub | +| resnext50_32x4d_swsl | PyTorch | PyTorch Hub | +| shufflenet_v2_x0_5 | PyTorch | Torchvision | +| shufflenet_v2_x1_0 | PyTorch | Torchvision | +| vgg11 | PyTorch | Torchvision | +| vgg11_bn | PyTorch | Torchvision | +| vgg13 | PyTorch | Torchvision | +| vgg13_bn | PyTorch | Torchvision | +| vgg16 | PyTorch | Torchvision | +| vgg16_bn | PyTorch | Torchvision | +| vgg19 | PyTorch | Torchvision | +| vgg19_bn | PyTorch | Torchvision | +| vit_b_16 | PyTorch | Torchvision | +| vit_b_32 | PyTorch | Torchvision | +| vit_l_16 | PyTorch | Torchvision | +| vit_l_32 | PyTorch | Torchvision | +| wide_resnet101_2 | PyTorch | Torchvision | +| wide_resnet50_2 | PyTorch | Torchvision | +| ConvNeXtBase | TensorFlow | Keras* | +| ConvNeXtLarge | TensorFlow | Keras | +| ConvNeXtSmall | TensorFlow | Keras | +| ConvNeXtTiny | TensorFlow | Keras | +| ConvNeXtXLarge | TensorFlow | Keras | +| DenseNet121 | TensorFlow | Keras | +| DenseNet169 | TensorFlow | Keras | +| DenseNet201 | TensorFlow | Keras | +| EfficientNetV2B0 | TensorFlow | Keras | +| EfficientNetV2B1 | TensorFlow | Keras | +| EfficientNetV2B2 | TensorFlow | Keras | +| EfficientNetV2B3 | TensorFlow | Keras | +| EfficientNetV2L | TensorFlow | Keras | +| EfficientNetV2M | TensorFlow | Keras | +| EfficientNetV2S | TensorFlow | Keras | +| InceptionResNetV2 | TensorFlow | Keras | +| InceptionV3 | TensorFlow | Keras | +| MobileNet | TensorFlow | Keras | +| MobileNetV2 | TensorFlow | Keras | +| NASNetLarge | TensorFlow | Keras | +| NASNetMobile | TensorFlow | Keras | +| ResNet101 | TensorFlow | Keras | +| ResNet101V2 | TensorFlow | Keras | +| ResNet152 | TensorFlow | Keras | +| ResNet152V2 | TensorFlow | Keras | +| ResNet50 | TensorFlow | Keras | +| ResNet50V2 | TensorFlow | Keras | +| VGG16 | TensorFlow | Keras | +| VGG19 | TensorFlow | Keras | +| Xception | TensorFlow | Keras | + +## Text Classification + +| Model name | Framework | Model Hub | +|------------|-----------|-----------| +| bert-base-cased | PyTorch | Hugging Face* | +| bert-base-uncased | TensorFlow | Hugging Face | +| bert-large-uncased | TensorFlow | Hugging Face | +| bert-large-uncased | PyTorch | Hugging Face | +| clinical-bert | PyTorch | Hugging Face | +| distilbert-base-uncased | PyTorch | Hugging Face | +| google/bert_uncased_L-10_H-128_A-2 | TensorFlow | Hugging Face | +| google/bert_uncased_L-10_H-256_A-4 | TensorFlow | Hugging Face | +| google/bert_uncased_L-10_H-512_A-8 | TensorFlow | Hugging Face | +| google/bert_uncased_L-10_H-768_A-12 | TensorFlow | Hugging Face | +| google/bert_uncased_L-12_H-128_A-2 | TensorFlow | Hugging Face | +| google/bert_uncased_L-12_H-256_A-4 | TensorFlow | Hugging Face | +| google/bert_uncased_L-12_H-512_A-8 | TensorFlow | Hugging Face | +| google/bert_uncased_L-12_H-768_A-12 | TensorFlow | Hugging Face | +| google/bert_uncased_L-2_H-128_A-2 | TensorFlow | Hugging Face | +| google/bert_uncased_L-2_H-256_A-4 | TensorFlow | Hugging Face | +| google/bert_uncased_L-2_H-512_A-8 | TensorFlow | Hugging Face | +| google/bert_uncased_L-2_H-768_A-12 | TensorFlow | Hugging Face | +| google/bert_uncased_L-4_H-128_A-2 | TensorFlow | Hugging Face | +| google/bert_uncased_L-4_H-256_A-4 | TensorFlow | Hugging Face | +| google/bert_uncased_L-4_H-512_A-8 | TensorFlow | Hugging Face | +| google/bert_uncased_L-4_H-768_A-12 | TensorFlow | Hugging Face | +| google/bert_uncased_L-6_H-128_A-2 | TensorFlow | Hugging Face | +| google/bert_uncased_L-6_H-256_A-4 | TensorFlow | Hugging Face | +| google/bert_uncased_L-6_H-512_A-8 | TensorFlow | Hugging Face | +| google/bert_uncased_L-6_H-768_A-12 | TensorFlow | Hugging Face | +| google/bert_uncased_L-8_H-128_A-2 | TensorFlow | Hugging Face | +| google/bert_uncased_L-8_H-256_A-4 | TensorFlow | Hugging Face | +| google/bert_uncased_L-8_H-512_A-8 | TensorFlow | Hugging Face | +| google/bert_uncased_L-8_H-768_A-12 | TensorFlow | Hugging Face | + +## Image Anomaly Detection + +| Model name | Framework | Model Hub | +|------------|-----------|-----------| +| resnet101 | PyTorch | Torchvision | +| resnet152 | PyTorch | Torchvision | +| resnet18 | PyTorch | Torchvision | +| resnet34 | PyTorch | Torchvision | +| resnet50 | PyTorch | Torchvision | + diff --git a/README.md b/README.md index 884641abca8382d0a85913c3dc2b76afe4102ff6..e158e9ff9133fc8a34c431132fd52e785590c39c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,98 @@ ---- -title: Quality Control Inspector -emoji: 📈 -colorFrom: green -colorTo: green -sdk: gradio -sdk_version: 5.41.1 -app_file: app.py -pinned: false -license: apache-2.0 -short_description: Quality control automation using self-supervised anomaly det ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +*Note: You may find it easier to read about Intel Transfer Learning tool, follow the Get +Started guide, and browse the API material from our published documentation site +https://intelai.github.io/transfer-learning.* + + + +# Intel® Transfer Learning Tool + +Transfer learning workflows use the knowledge learned by a pre-trained model on +a large dataset to improve the performance of a related problem with a smaller +dataset. + +## What is Intel® Transfer Learning Tool + +Intel® Transfer Learning Tool makes it easier and faster for you to +create transfer learning workflows across a variety of AI use cases. Its +open-source Python\* library leverages public pretrained model hubs, +Intel-optimized deep learning frameworks, and your custom dataset to efficiently +generate new models optimized for Intel hardware. + +This project documentation provides information, resource links, and instructions for the Intel +Transfer Learning Tool as well as Jupyter\* notebooks and examples that +demonstrate its usage. + +**Features:** +* Supports PyTorch\* and TensorFlow\* +* Select from over [100 image classification and text classification models](Models.md) from + Torchvision, PyTorch Hub, TensorFlow Hub, Keras, and Hugging Face +* Use your own custom dataset or get started quickly with built-in datasets +* Automatically create a trainable classification layer customized for your dataset +* Pre-process your dataset using scaling, cropping, batching, and splitting +* Use APIs for prediction, evaluation, and benchmarking +* Export your model for deployment or resume training from checkpoints + +**Intel Optimizations:** +* Boost performance with Intel® Optimization for TensorFlow and Intel® Extension for PyTorch +* Quantize to INT8 to reduce model size and speed up inference using Intel® Neural Compressor +* Optimize model for FP32 inference using Intel Neural Compressor +* Reduce training time with auto-mixed precision for select hardware platforms +* Further reduce training time with multinode training for PyTorch + +## How the Intel Transfer Learning Tool Works + +The Intel Transfer Learning Tool lets you train AI models with TensorFlow or +PyTorch using either no-code command line interface (CLI) commands at a bash +prompt, or low-code application programming interface (API) calls from a Python +script. + +Use your own dataset or select an existing image or text classification dataset listed in the +[public datasets](DATASETS.md) documentation. Construct your own CLI or API commands for training, evaluation, +and optimization using the TensorFlow or PyTorch framework, and finally export +your saved model optimized for inference on Intel CPUs. + +An overview of the Intel Transfer Learning Tool flow is shown in this +figure: + +

Intel Transfer Learning Tool Flow

+ +Intel Transfer Learning Tool Flow + +## Get Started + +The [Get Started](GetStarted.md) guide walks you through the steps to check +system requirements, install, and then run the tool with a couple of examples +showing no-code CLI and low-code API approaches. After that, you can check out +these additional CLI and API [Examples](examples/README.md). + + +As described in the [Get Started](GetStarted.md) guide, once you have a Python +3.9 environment set up, you do a basic install of the Intel Transfer Learning +Tool using: + +``` +pip install intel-transfer-learning-tool +``` + +Then you can use the Transfer Learning Tool CLI interface (tlt) to train a +TensorFlow image classification model (resnet_v1_50), download and use an +existing built-in dataset (tf_flowers), and save the trained model to +`/tmp/output` using this one command: + +``` +tlt train --framework tensorflow --model-name resnet_v1_50 --dataset-name tf_flowers \ + --output-dir /tmp/output --dataset-dir /tmp/data +``` + +Use `tlt --help` to see the list of CLI commands. More detailed help for each +command can be found using, for example, `tlt train --help`. + + + +## Support + +The Intel Transfer Learning Tool team tracks bugs and enhancement requests using +[GitHub issues](https://github.com/IntelAI/transfer-learning-tool/issues). Before submitting a +suggestion or bug report, search the existing GitHub issues to see if your issue has already been reported. + +See [Legal Information](Legal.md) for Disclaimers, Trademark, and Licensing information. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000000000000000000000000000000000..556938bdea39190bb47c2cb1e33109301b11e454 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Report a Vulnerability + +Please report security issues or vulnerabilities to the [Intel® Security Center]. + +For more information on how Intel® works to resolve security issues, see +[Vulnerability Handling Guidelines]. + +[Intel® Security Center]:https://www.intel.com/content/www/us/en/security-center/default.html + +[Vulnerability Handling Guidelines]:https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000000000000000000000000000000000000..fff4ab923ce55f11569939421ffa39cd3566f5d9 --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-minimal diff --git a/api.md b/api.md new file mode 100644 index 0000000000000000000000000000000000000000..a936a4da26b02ef4e908928c25c8144a2618c03e --- /dev/null +++ b/api.md @@ -0,0 +1,4 @@ +# API Reference + +Low-code Python\* API documentation is automatically generated from the code and +appears in the Transfer Learning Tool documentation website's [API](https://intelai.github.io/transfer-learning/main/api.html) page. diff --git a/bandit.yaml b/bandit.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7a1dd00122650038aa096642b9c7178c464f9c1 --- /dev/null +++ b/bandit.yaml @@ -0,0 +1,11 @@ +# FILE: bandit.yaml +exclude_dirs: [ '.venv', '.tox', 'tlt_test_venv', 'tlt_notebook_venv', 'tests' ] +skips: [ 'B301', 'B311', 'B403', 'B404' ] +# B301 - dill usage scoope is different from what's described in CWE-502 +# dill is mostly used used for dumping/saving models to disk(serialization) +# When loading previously saved models from disk(descerialization), +# either Keras model loader or PyTorch loader used first to verify the model, +# and then create a copy to be passed to dill for loading. +# B311 - random usage scope is different from what's described in CWE-330 +# B403 - this one is reported everytime 'dill' is imported, so it's actually covered by B301 justification +# B404 - this one is reported everytime 'subprocess' is imported but this modules is not used as described in CWE-78 diff --git a/cli.md b/cli.md new file mode 100644 index 0000000000000000000000000000000000000000..8b41a5919fd49c86ce9fbd2c22f8a98244f0161b --- /dev/null +++ b/cli.md @@ -0,0 +1,4 @@ +# CLI Reference + +No-code bash CLI documentation is automatically generated from the code and +appears in the Transfer Learning Tool documentation website's [CLI](https://intelai.github.io/transfer-learning/main/cli.html) page. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c093d932c114e42e6182d098ce3225091b02c20a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,143 @@ +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +ARG IMAGE_NAME=ubuntu +ARG IMAGE_TAG=22.04 +FROM ${IMAGE_NAME}:${IMAGE_TAG} as base + +# TLT base target +FROM base as tlt-base + +ARG PYTHON=python3 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + libgl1 \ + libglib2.0-0 \ + ${PYTHON} \ + python3-pip && \ + apt-get clean autoclean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -sf "$(which ${PYTHON})" /usr/bin/python + +# TLT target for GitHub actions +FROM tlt-base as tlt-ci + +ENV DEBIAN_FRONTEND=noninteractive + +ENV LANG C.UTF-8 +ARG PYTHON=python3 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + ${PYTHON}-dev \ + ${PYTHON}-distutils \ + build-essential \ + ca-certificates \ + make \ + pandoc && \ + apt-get clean autoclean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ${PYTHON} -m pip install virtualenv + +# TLT target for development +FROM tlt-ci as tlt-devel + +COPY . /tmp/intel-transfer-learning + +WORKDIR /tmp/intel-transfer-learning + +RUN ${PYTHON} setup.py bdist_wheel && \ + pip install --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html dist/*.whl + +# TLT target for deployment +FROM tlt-base as tlt-prod + +COPY --from=tlt-devel /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages +COPY --from=tlt-devel /usr/local/bin /usr/local/bin + +ENV DATASET_DIR=/tmp/data +ENV OUTPUT_DIR=/tmp/output + +# TLT target for running with MPI +FROM tlt-prod as tlt-mpi + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + libopenmpi-dev \ + openmpi-bin \ + openmpi-common \ + openssh-client \ + openssh-server && \ + apt-get clean autoclean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +# Create a wrapper for OpenMPI to allow running as root by default +RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \ + echo '#!/bin/bash' > /usr/bin/mpirun && \ + echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \ + chmod a+x /usr/bin/mpirun + +# Configure OpenMPI to run good defaults: +RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf + +# Install OpenSSH for MPI to communicate between containers and allow OpenSSH to +# talk to containers without asking for confirmation +RUN mkdir -p /var/run/sshd && \ + cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \ + echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \ + mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config + +# TLT target for with MPI, Horovod and all development tools +FROM tlt-mpi as tlt-dist-devel + +ARG HOROVOD_WITH_PYTORCH=1 +ARG HOROVOD_WITHOUT_MXNET=1 +ARG HOROVOD_WITH_TENSORFLOW=1 +ARG HOROVOD_VERSION + +ARG PYTHON=python3 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + cmake \ + g++ \ + gcc \ + git \ + ${PYTHON}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir horovod==${HOROVOD_VERSION} + +ARG ONECCL_VERSION +ARG ONECCL_URL=https://developer.intel.com/ipex-whl-stable-cpu + +RUN python -m pip install --no-cache-dir oneccl_bind_pt==${ONECCL_VERSION} -f ${ONECCL_URL} + +COPY . /tmp/intel-transfer-learning + +WORKDIR /tmp/intel-transfer-learning + +FROM tlt-mpi as tlt-dist-prod + +COPY --from=tlt-dist-devel /usr/local/lib/${PYTHON}/dist-packages /usr/local/lib/python3.10/dist-packages +COPY --from=tlt-dist-devel /usr/local/bin /usr/local/bin diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cdd8121774c788d3798dbdc3d4ab5b5a3a577a10 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,73 @@ +# Docker +Follow these instructions to set up and run our provided Docker image. + +## Set Up Docker Engine and Docker Compose +You'll need to install Docker Engine on your development system. Note that while **Docker Engine** is free to use, **Docker Desktop** may require you to purchase a license. See the [Docker Engine Server installation instructions](https://docs.docker.com/engine/install/#server) for details. + +To build and run this workload inside a Docker Container, ensure you have Docker Compose installed on your machine. If you don't have this tool installed, consult the official [Docker Compose installation documentation](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually). + +```bash +DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} +mkdir -p $DOCKER_CONFIG/cli-plugins +curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose +chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose +docker compose version +``` + +## Set Up Docker Image +Build or Pull the provided docker images. + +```bash +cd docker +docker compose build +``` +OR +```bash +docker pull intel/ai-tools:tlt-0.5.0 +docker pull intel/ai-tools:tlt-devel-0.5.0 +docker pull intel/ai-tools:tlt-dist-0.5.0 +docker pull intel/ai-tools:tlt-dist-devel-0.5.0 +``` + +## Use Docker Image +Utilize the TLT CLI without installation by using the provided docker image and docker compose. + +```bash +docker compose run tlt-prod +# OR +docker compose run tlt-prod tlt --help +``` + +## Kubernetes +### 1. Install Helm +- Install [Helm](https://helm.sh/docs/intro/install/) +```bash +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ +chmod 700 get_helm.sh && \ +./get_helm.sh +``` +### 2. Setting up Training Operator +Install the standalone operator from GitHub or use a pre-existing Kubeflow configuration. +```bash +kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone" +``` +OR +```bash +helm repo add cowboysysop https://cowboysysop.github.io/charts/ +helm install cowboysysop/training-operator +``` +### 3. Deploy TLT Distributed Job +For more customization information, see the chart [README](./docker/chart/README.md) +```bash +export NAMESPACE=kubeflow +helm install --namespace ${NAMESPACE} --set ... tlt-distributed ./docker/chart +``` +### 4. View +To view your workflow progress +```bash +kubectl get -o yaml mpijob tf-tlt-distributed -n ${NAMESPACE} +``` +OR +```bash +kubectl logs tf-tlt-distributed-launcher -n ${NAMESPACE} +``` diff --git a/docker/chart/.helmignore b/docker/chart/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778 --- /dev/null +++ b/docker/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/docker/chart/Chart.yaml b/docker/chart/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24e0665278b03ec08ee6017557057f663e2c42e6 --- /dev/null +++ b/docker/chart/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: TLT TF Distributed Training +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/docker/chart/README.md b/docker/chart/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c2ca7a49409fcac9867154c1bf47a5d8bd00174b --- /dev/null +++ b/docker/chart/README.md @@ -0,0 +1,31 @@ +# TLT TF Distributed Training + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| batchDenom | int | `1` | Batch denominator to be used to divide global batch size | +| batchSize | int | `128` | Global batch size to distributed data | +| datasetName | string | `"cifar10"` | Dataset name to load from tfds | +| epochs | int | `1` | Total epochs to train the model | +| imageName | string | `"intel/ai-tools"` | | +| imageTag | string | `"0.5.0-dist-devel"` | | +| metadata.name | string | `"tlt-distributed"` | | +| metadata.namespace | string | `"kubeflow"` | | +| modelName | string | `"https://tfhub.dev/google/efficientnet/b1/feature-vector/1"` | TF Hub or HuggingFace model URL | +| pvcName | string | `"tlt"` | | +| pvcResources.data | string | `"2Gi"` | Amount of Storage for Dataset | +| pvcResources.output | string | `"1Gi"` | Amount of Storage for Output Directory | +| pvcScn | string | `"nil"` | PVC `StorageClassName` | +| resources.cpu | int | `2` | Number of Compute for Launcher | +| resources.memory | string | `"4Gi"` | Amount of Memory for Launcher | +| scaling | string | `"strong"` | For `weak` scaling, `lr` is scaled by a factor of `sqrt(batch_size/batch_denom)` and uses global batch size for all the processes. For `strong` scaling, lr is scaled by world size and divides global batch size by world size | +| slotsPerWorker | int | `1` | Number of Processes Per Worker | +| useCase | string | `"image_classification"` | Use case (`image_classification`|`text_classification`) | +| workerResources.cpu | int | `4` | Number of Compute per Worker | +| workerResources.memory | string | `"8Gi"` | Amount of Memory per Worker | +| workers | int | `4` | Number of Workers | diff --git a/docker/chart/templates/mpijob.yaml b/docker/chart/templates/mpijob.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e54854b497bc52ba8ae09c21938d23a946d3df9 --- /dev/null +++ b/docker/chart/templates/mpijob.yaml @@ -0,0 +1,92 @@ +apiVersion: kubeflow.org/v1 +kind: MPIJob +metadata: + name: tf-{{ .Values.metadata.name }} + namespace: {{ .Values.metadata.namespace }} +spec: + slotsPerWorker: {{ .Values.slotsPerWorker }} + runPolicy: + cleanPodPolicy: None + mpiReplicaSpecs: + Launcher: + replicas: 1 + template: + spec: + hostIPC: true + containers: + - image: "{{ .Values.imageName }}:{{ .Values.imageTag }}" + name: mpi + command: + - horovodrun + args: + - --verbose + - -np + - {{ .Values.workers }} + - --hostfile + - /etc/mpi/hostfile + - python + - /tmp/intel-transfer-learning/tlt/distributed/tensorflow/run_train_tf.py + - --batch_denom + - "{{ .Values.batchDenom }}" + - --batch_size + - "{{ .Values.batchSize }}" + - --dataset-dir + - /tmp/data + - --dataset-name + - {{ .Values.datasetName }} + - --epochs + - "{{ .Values.epochs }}" + - --model-name + - {{ .Values.modelName }} + - --output-dir + - /tmp/output + - --scaling + - "{{ .Values.scaling }}" + - --shuffle + - --use-case + - {{ .Values.useCase }} + resources: + limits: + cpu: {{ .Values.resources.cpu }} + memory: {{ .Values.resources.memory }} + volumeMounts: + - name: dataset-dir + mountPath: /tmp/data + - name: output-dir + mountPath: /tmp/output + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: dataset-dir + persistentVolumeClaim: + claimName: "{{ .Values.pvcName }}-data" + - name: output-dir + persistentVolumeClaim: + claimName: "{{ .Values.pvcName }}-output" + Worker: + replicas: {{ .Values.workers }} + template: + spec: + containers: + - image: "{{ .Values.imageName }}:{{ .Values.imageTag }}" + name: mpi + resources: + limits: + cpu: {{ .Values.workerResources.cpu }} + memory: {{ .Values.workerResources.memory }} + volumeMounts: + - name: dataset-dir + mountPath: /tmp/data + - name: output-dir + mountPath: /tmp/output + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: dataset-dir + persistentVolumeClaim: + claimName: "{{ .Values.pvcName }}-data" + - name: output-dir + persistentVolumeClaim: + claimName: "{{ .Values.pvcName }}-output" diff --git a/docker/chart/templates/pvc.yaml b/docker/chart/templates/pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcafcf6817374812ef72a7abbbcec9a0a81da9c6 --- /dev/null +++ b/docker/chart/templates/pvc.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcName }}-data + namespace: {{ .Values.metadata.namespace }} +spec: + storageClassName: {{ .Values.pvcScn }} + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: {{ .Values.pvcResources.data }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pvcName }}-output + namespace: {{ .Values.metadata.namespace }} +spec: + storageClassName: {{ .Values.pvcScn }} + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: {{ .Values.pvcResources.output }} diff --git a/docker/chart/values.yaml b/docker/chart/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45fe27ae6639b5d25dd2c8261f0a8cedd55cf838 --- /dev/null +++ b/docker/chart/values.yaml @@ -0,0 +1,28 @@ +metadata: + name: tlt-distributed + namespace: kubeflow + +imageName: intel/ai-tools +imageTag: 0.5.0-dist-devel + +batchDenom: 1 +batchSize: 128 +datasetName: cifar10 +epochs: 1 +modelName: https://tfhub.dev/google/efficientnet/b1/feature-vector/1 +scaling: strong +slotsPerWorker: 1 +useCase: image_classification +workers: 4 + +pvcName: tlt +pvcScn: nil +pvcResources: + data: 2Gi + output: 1Gi +resources: + cpu: 2 + memory: 4Gi +workerResources: + cpu: 4 + memory: 8Gi diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..4c1cbc805c5fca455f91a42b809ff6dbb75a0329 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,54 @@ +version: "3" +services: + tlt-devel: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: "" + IMAGE_NAME: ubuntu + IMAGE_TAG: 22.04 + PYTHON: python3.10 # Version must be specified for prod + context: ../ + dockerfile: ./docker/Dockerfile + target: tlt-devel + image: intel/ai-tools:tlt-devel-latest + pull_policy: always + tlt-prod: + extends: + service: tlt-devel + build: + args: + DATASET_DIR: /tmp/data + OUTPUT_DIR: /tmp/output + target: tlt-prod + image: intel/ai-tools:tlt-prod-latest + volumes: + - /${DATASET_DIR:-$PWD/../data}:/tmp/data + - /${OUTPUT_DIR:-$PWD/../output}:/tmp/output + tlt-dist-devel: + extends: + service: tlt-prod + build: + args: + HOROVOD_VERSION: 0.28.0 + ONECCL_VERSION: 2.0.0 + ONECCL_URL: https://developer.intel.com/ipex-whl-stable-cpu + target: tlt-dist-devel + image: intel/ai-tools:tlt-dist-devel-latest + tlt-dist-prod: + extends: + service: tlt-dist-devel + build: + target: tlt-dist-prod + command: | + tlt train -f tensorflow + --dataset-name cifar10 + --model-name resnet_v1_50 + --dataset-dir /tmp/data + --output-dir /tmp/output + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + image: intel/ai-tools:tlt-dist-prod-latest diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0918c9768895c23af5ecf35282391d7b12ed301a --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +markdown diff --git a/docs/DATASETS.rst b/docs/DATASETS.rst new file mode 100644 index 0000000000000000000000000000000000000000..fa73e6f958343ccf1184ac8ad4e2585b1f38c08e --- /dev/null +++ b/docs/DATASETS.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: ../DATASETS.md + :parser: myst_parser.sphinx_ diff --git a/docs/GetStarted.rst b/docs/GetStarted.rst new file mode 100644 index 0000000000000000000000000000000000000000..a30052eb3269c69d8db37ca27f2b931ac15d0419 --- /dev/null +++ b/docs/GetStarted.rst @@ -0,0 +1,2 @@ +.. include:: ../GetStarted.md + :parser: myst_parser.sphinx_ diff --git a/docs/Legal.rst b/docs/Legal.rst new file mode 100644 index 0000000000000000000000000000000000000000..0bdc11ee5a7620a867da960c92dbf5add75e38a6 --- /dev/null +++ b/docs/Legal.rst @@ -0,0 +1,2 @@ +.. include:: ../Legal.md + :parser: myst_parser.sphinx_ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..88499e06e60ee5449598e8a558b2388459051ad2 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,43 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +LISTEN_IP ?= 127.0.0.1 +LISTEN_PORT ?= 9999 +serve: + @python -m http.server --directory ./_build/html ${LISTEN_PORT} --bind ${LISTEN_IP} diff --git a/docs/Models.rst b/docs/Models.rst new file mode 100644 index 0000000000000000000000000000000000000000..0a55d827ec140a4fe6282719bc9b6b50f749e251 --- /dev/null +++ b/docs/Models.rst @@ -0,0 +1,2 @@ +.. include:: ../Models.md + :parser: myst_parser.sphinx_ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88fe2cdb5e81816cbd796fb0a80a084378d343e7 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,58 @@ +# Building Documentation + +## Sphinx Documentation + +Install `tlt` and its dependencies for developers as described the [Get Started](/GetStarted) guide. +```bash +# Run these commands from root of the project +python3 -m virtualenv tlt_dev_venv +source tlt_dev_venv/bin/activate +python -m pip install --editable . +``` + +Install Pandoc, Sphinx and a few other tools required to build docs +```bash +sudo apt-get install pandoc +pip install -r docs/requirements-docs.txt +``` + +Navigate to the `docs` directory and run the doctests to ensure all tests pass: +```bash +# run this command from within docs directory +make doctest +``` + +This should produce output similiar to: +```bash +Doctest summary +=============== + 6 tests + 0 failures in tests + 0 failures in setup code + 0 failures in cleanup code +build succeeded. +``` + +Finally generate the html docs (from within `docs` directory): +```bash +make clean html +``` + +The output HTML files will be located in `transfer-learning/docs/_build/html`. + +To start a local HTTP server and view the docs locally, try: +```bash +make serve +Serving HTTP on 127.0.1.1 port 9999 (http://127.0.1.1:9999/) ... +``` + +If you need to view the docs from another machine, please try either port forwarding or +provide appropriate values for `LISTEN_IP/LISTEN_PORT` arguments. +For example: +```bash +LISTEN_IP=0.0.0.0 make serve +Serving HTTP on 0.0.0.0 port 9999 (http://0.0.0.0:9999/) ... +``` + +runs the docs server on the host while listening to all hosts. +Now you can navigate to `HOSTNAME:9999` to view the docs. diff --git a/docs/_static/tlt-custom.css b/docs/_static/tlt-custom.css new file mode 100644 index 0000000000000000000000000000000000000000..f0e1c0f33567cde2cdfc83606bc93653b20b2a23 --- /dev/null +++ b/docs/_static/tlt-custom.css @@ -0,0 +1,43 @@ +/* allow the page to use more of the window width */ +.wy-nav-content { + max-width: 1100px; + } + +/* allow text wrapping in a table to avoid table horizontal scrolling */ +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal !important; + } + +/* center all images */ +.rst-content img { + margin-left: auto; + margin-right: auto; + display: block; +} + +/* add an underline to title headings and wrap long API headings + * Note: we use JavaScript to add a ​ after the dot + * in really long H1 titles created automatically from the code */ +.rst-content h1,h2,h3,h4,h5 { + text-decoration: underline; + word-wrap: break-word; +} + +/* add link color to module xref generated by autodoc */ +.rst-content a.internal code.xref span.pre { + color: #2980b9; +} + +/* change red text color to dark gray in code literals */ +.rst-content code.literal, .rst-content tt.literal { + color: #404040; +} + +/* change background color of search area/site title to increase contrast */ +.wy-side-nav-search { + background-color: #2f71b2; +} +/* change href link color to increase contrast */ +a { + color: #2f71b2; +} diff --git a/docs/_static/tlt-custom.js b/docs/_static/tlt-custom.js new file mode 100644 index 0000000000000000000000000000000000000000..70536ffcf42ec180326244f9ca47648e65cdbc69 --- /dev/null +++ b/docs/_static/tlt-custom.js @@ -0,0 +1,19 @@ +/* Extra tlt-specific javascript */ + +$(document).ready(function(){ + + /* open external links in a new tab */ + $('a[class*=external]').attr({target: '_blank', rel: 'noopener'}); + + /* add word break points (zero-width space) after a period in really long titles */ + $('h1').html(function(index, html){ + return html.replace(/\./g, '.\u200B'); + }); + + /* copy image alt tags as title so hover text tool tip by browser + * (Looks like the myst-parser isn't passing the title tag through to Sphinx, + * but is passing the alt tag) */ + $("img[alt]").each(function(){ + $(this).attr('title', $(this).attr('alt')); + }); +}); diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 0000000000000000000000000000000000000000..c159a811f98dd8201f2ed2045c4556a7089fe697 --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,5 @@ +{% extends '!footer.html' %} +{% block extrafooter %} +*Other names and brands may be claimed as the property of others. +Trademarks +{% endblock %} diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000000000000000000000000000000000000..97694e119ea1a132a1bbc3f6035fdba2e762886a --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,132 @@ +API Reference +============= + +Datasets +-------- + +.. currentmodule:: tlt.datasets + +The simplest way to create datasets is with the dataset factory methods :meth:`load_dataset`, for using a +custom dataset, and :meth:`get_dataset`, for downloading and using a third-party dataset from a catalog such as TensorFlow +Datasets or Torchvision. + +Factory Methods +*************** + +.. automodule:: tlt.datasets.dataset_factory + :members: load_dataset, get_dataset + +Class Reference +*************** + +Image Classification +^^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: tlt.datasets.image_classification + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + tfds_image_classification_dataset.TFDSImageClassificationDataset + torchvision_image_classification_dataset.TorchvisionImageClassificationDataset + tf_custom_image_classification_dataset.TFCustomImageClassificationDataset + pytorch_custom_image_classification_dataset.PyTorchCustomImageClassificationDataset + image_classification_dataset.ImageClassificationDataset + +Text Classification +^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: tlt.datasets.text_classification + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + tfds_text_classification_dataset.TFDSTextClassificationDataset + hf_text_classification_dataset.HFTextClassificationDataset + tf_custom_text_classification_dataset.TFCustomTextClassificationDataset + hf_custom_text_classification_dataset.HFCustomTextClassificationDataset + text_classification_dataset.TextClassificationDataset + +Base Classes +^^^^^^^^^^^^ + +.. note:: Users should rarely need to interact directly with these. + +.. currentmodule:: tlt.datasets + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + pytorch_dataset.PyTorchDataset + tf_dataset.TFDataset + hf_dataset.HFDataset + dataset.BaseDataset + +Models +------ + +.. currentmodule:: tlt.models + +Discover and work with available models by using model factory methods. The :meth:`get_model` function will download +third-party models, while the :meth:`load_model` function will load a custom model, from either a path location or a +model object in memory. The model discovery and inspection methods are :meth:`get_supported_models` and +:meth:`print_supported_models`. + +Factory Methods +*************** + +.. automodule:: tlt.models.model_factory + :members: get_model, load_model, get_supported_models, print_supported_models + +Class Reference +*************** + +Image Classification +^^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: tlt.models.image_classification + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + tfhub_image_classification_model.TFHubImageClassificationModel + tf_image_classification_model.TFImageClassificationModel + keras_image_classification_model.KerasImageClassificationModel + torchvision_image_classification_model.TorchvisionImageClassificationModel + pytorch_image_classification_model.PyTorchImageClassificationModel + pytorch_hub_image_classification_model.PyTorchHubImageClassificationModel + image_classification_model.ImageClassificationModel + +Text Classification +^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: tlt.models.text_classification + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + tf_text_classification_model.TFTextClassificationModel + pytorch_hf_text_classification_model.PyTorchHFTextClassificationModel + tf_hf_text_classification_model.TFHFTextClassificationModel + text_classification_model.TextClassificationModel + +Base Classes +^^^^^^^^^^^^ + +.. note:: Users should rarely need to interact directly with these. + +.. currentmodule:: tlt.models + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + pytorch_model.PyTorchModel + tf_model.TFModel + hf_model.HFModel + model.BaseModel diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000000000000000000000000000000000000..28bf6f03c819283b41b9ed45e75260423ffa1fd5 --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,7 @@ +CLI Reference +============= + +.. click:: tlt.tools.cli.main:cli_group + :prog: tlt + :nested: full + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..6bae572f39e2a843b0945d7333017d4480c5e040 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +import shutil +import glob +sys.path.insert(0, os.path.abspath('../..')) +sys.setrecursionlimit(1500) +import sphinx_rtd_theme +from datetime import datetime + +# -- Project information ----------------------------------------------------- + +project = 'Intel® Transfer Learning Tool' +author = 'Intel Corporation' +copyright = '2022-' + str(datetime.now().year) + u', ' + author + +# The full version, including alpha/beta/rc tags +release = '0.2.0' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'myst_parser', + 'nbsphinx', + 'nbsphinx_link', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.napoleon', + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', + 'sphinx_click', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '.venv3', 'README.md'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +html_last_updated_fmt = '%b %d, %Y' +html_show_sphinx = False +html_favicon = 'images/favicon-intel-32x32.png' + +html_static_path = ['_static'] +templates_path = ['_templates'] + +def setup(app): + app.add_css_file("tlt-custom.css") + app.add_js_file("tlt-custom.js") + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +autodoc_member_order = 'bysource' +nbsphinx_execute = 'never' +nbsphinx_prolog = """ +:orphan: + +""" +myst_heading_anchors = 2 +suppress_warnings = ["myst.xref_missing", "myst.header"] + +# ask the myst parser to process tags so Sphinx can handle the properly +myst_enable_extensions = [ "html_image" ] diff --git a/docs/distributed.rst b/docs/distributed.rst new file mode 100644 index 0000000000000000000000000000000000000000..5a4e5389e4b37d1be3a43f1cdc3d35c578c1e532 --- /dev/null +++ b/docs/distributed.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: ../tlt/distributed/README.md + :parser: myst_parser.sphinx_ diff --git a/docs/docbuild.rst b/docs/docbuild.rst new file mode 100644 index 0000000000000000000000000000000000000000..ee5e314b1ed114aa0a25f3c6041713625e80cced --- /dev/null +++ b/docs/docbuild.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: README.md + :parser: myst_parser.sphinx_ diff --git a/docs/examples/README.rst b/docs/examples/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..8d2a38cb8c00e6bf2508262549d2195aca99c36f --- /dev/null +++ b/docs/examples/README.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: ../../examples/README.md + :parser: myst_parser.sphinx_ diff --git a/docs/examples/cli/README.rst b/docs/examples/cli/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..d2a98679730db87110ccaee80e4a670232a5cffc --- /dev/null +++ b/docs/examples/cli/README.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: ../../../examples/cli/README.md + :parser: myst_parser.sphinx_ diff --git a/docs/examples/cli/image_classification.rst b/docs/examples/cli/image_classification.rst new file mode 100644 index 0000000000000000000000000000000000000000..601104ca62b44a68cfd0e0e8e104e97275a544f0 --- /dev/null +++ b/docs/examples/cli/image_classification.rst @@ -0,0 +1,2 @@ +.. include:: ../../../examples/cli/image_classification.md + :parser: myst_parser.sphinx_ diff --git a/docs/examples/cli/text_classification.rst b/docs/examples/cli/text_classification.rst new file mode 100644 index 0000000000000000000000000000000000000000..fbb183aa596c9019033322ec858f88e8f401ee13 --- /dev/null +++ b/docs/examples/cli/text_classification.rst @@ -0,0 +1,2 @@ +.. include:: ../../../examples/cli/text_classification.md + :parser: myst_parser.sphinx_ diff --git a/docs/examples/examples.rst b/docs/examples/examples.rst new file mode 100644 index 0000000000000000000000000000000000000000..65b30408f03f6739631782a45278fc04e50a2ee4 --- /dev/null +++ b/docs/examples/examples.rst @@ -0,0 +1,16 @@ +Examples +######## + +.. include:: ../../examples/README.md + :parser: myst_parser.sphinx_ + :start-after: # Examples + +.. toctree:: + :maxdepth: 1 + :hidden: + + cli/README + Image Classification CLI Example + Text Classification CLI Example + Jupyter Notebook API Examples <../notebooks/README> + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..c892c319c3d47025f10bbba15dfb947d2189164c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,23 @@ +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + :end-before: + +.. include:: ../README.md + :parser: myst_parser.sphinx_ + :start-after: + +.. toctree:: + :maxdepth: 1 + :hidden: + + Documentation Home + GetStarted + examples/examples + cli + api + Supported Models + Legal + genindex + GitHub Repository + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000000000000000000000000000000000000..32bb24529f92346af26219baed295b7488b77534 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/methods.rst b/docs/methods.rst new file mode 100644 index 0000000000000000000000000000000000000000..151c82716d45273a1c4f67c239fb2c8bfc3cdfd3 --- /dev/null +++ b/docs/methods.rst @@ -0,0 +1,68 @@ +:orphan: + +Image Classification Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. currentmodule:: tlt.models.image_classification + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + tfhub_image_classification_model.TFHubImageClassificationModel.train + tfhub_image_classification_model.TFHubImageClassificationModel.quantize + tfhub_image_classification_model.TFHubImageClassificationModel.optimize_graph + tfhub_image_classification_model.TFHubImageClassificationModel.benchmark + + tf_image_classification_model.TFImageClassificationModel.train + tf_image_classification_model.TFImageClassificationModel.quantize + tf_image_classification_model.TFImageClassificationModel.optimize_graph + tf_image_classification_model.TFImageClassificationModel.benchmark + + keras_image_classification_model.KerasImageClassificationModel.train + keras_image_classification_model.KerasImageClassificationModel.quantize + keras_image_classification_model.KerasImageClassificationModel.optimize_graph + keras_image_classification_model.KerasImageClassificationModel.benchmark + + torchvision_image_classification_model.TorchvisionImageClassificationModel.train + torchvision_image_classification_model.TorchvisionImageClassificationModel.quantize + torchvision_image_classification_model.TorchvisionImageClassificationModel.benchmark + + pytorch_image_classification_model.PyTorchImageClassificationModel.train + pytorch_image_classification_model.PyTorchImageClassificationModel.quantize + pytorch_image_classification_model.PyTorchImageClassificationModel.benchmark + + pytorch_hub_image_classification_model.PyTorchHubImageClassificationModel.train + pytorch_hub_image_classification_model.PyTorchHubImageClassificationModel.quantize + pytorch_hub_image_classification_model.PyTorchHubImageClassificationModel.benchmark + + image_classification_model.ImageClassificationModel.train + image_classification_model.ImageClassificationModel.quantize + image_classification_model.ImageClassificationModel.benchmark + +Text Classification Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: tlt.models.text_classification + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + :recursive: + + tf_text_classification_model.TFTextClassificationModel.train + tf_text_classification_model.TFTextClassificationModel.quantize + tf_text_classification_model.TFTextClassificationModel.optimize_graph + tf_text_classification_model.TFTextClassificationModel.benchmark + + pytorch_hf_text_classification_model.PyTorchHFTextClassificationModel.train + pytorch_hf_text_classification_model.PyTorchHFTextClassificationModel.quantize + pytorch_hf_text_classification_model.PyTorchHFTextClassificationModel.benchmark + + tf_hf_text_classification_model.TFHFTextClassificationModel.train + tf_hf_text_classification_model.TFHFTextClassificationModel.quantize + tf_hf_text_classification_model.TFHFTextClassificationModel.optimize_graph + tf_hf_text_classification_model.TFHFTextClassificationModel.benchmark + + text_classification_model.TextClassificationModel.train + text_classification_model.TextClassificationModel.quantize + text_classification_model.TextClassificationModel.benchmark diff --git a/docs/notebooks/Medical_Imaging_Classification.nblink b/docs/notebooks/Medical_Imaging_Classification.nblink new file mode 100644 index 0000000000000000000000000000000000000000..cfcc35b0d445825a4b4e2821e55c3af1f6ed8292 --- /dev/null +++ b/docs/notebooks/Medical_Imaging_Classification.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/e2e_workflows/Medical_Imaging_Classification.ipynb" +} diff --git a/docs/notebooks/README.rst b/docs/notebooks/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..0bf4ee834d47f58a4a017dc408aaedda9dfba726 --- /dev/null +++ b/docs/notebooks/README.rst @@ -0,0 +1,96 @@ +Intel® Transfer Learning Tool API Notebook Examples +=================================================== + +.. toctree:: + :maxdepth: 1 + :hidden: + + setup + +API examples are demonstrated using Jupyter notebooks. + +Prerequisites +************* + +Before running these Jupyter notebook examples, use these :doc:`notebook setup +instructions` to install required dependencies. + + +Intel Transfer Learning Tool API Tutorial Notebooks +*************************************************** + +.. |imageClassPyTorch| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _imageClassPyTorch: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb + +.. |imageClassTensorFlow| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _imageClassTensorflow: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb + +.. |textClassPyTorch| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _textClassPyTorch: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/text_classification/tlt_api_pyt_text_classification/TLT_PYT_Text_Classification.ipynb + +.. |textClassTensorFlow| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _textClassTensorflow: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/text_classification/tlt_api_tf_text_classification/TLT_TF_Text_Classification.ipynb + +.. |imageAnomalyPyTorch| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _imageAnomalyPyTorch: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/image_anomaly_detection/tlt_api_pyt_anomaly_detection/Anomaly_Detection.ipynb + +.. csv-table:: + :header: "Notebook Title", ".ipynb Link", "Use Case", "Framework" + :widths: 30, 10, 20, 20 + + :doc:`Image Classification with PyTorch `, |imageClassPyTorch|_ , Image Classification, PyTorch & Intel Transfer Learning Tool + :doc:`Image Classification with TensorFlow `, |imageClassTensorFlow|_ , Image Classification, TensorFlow & Intel Transfer Learning Tool + :doc:`Text Classification with PyTorch `, |textClassPyTorch|_ , Text Classification, PyTorch & Intel Transfer Learning Tool + :doc:`Text Classification with TensorFlow `, |textClassTensorflow|_ , Text Classification, TensorFlow & Intel Transfer Learning Tool + :doc:`Anomaly Detection using PyTorch `, |imageAnomalyPyTorch|_, Image Anomaly Detection, PyTorch & Intel Transfer Learning Tool + +Intel Transfer Learning Tool API End-to-End Pipelines +***************************************************** + +.. |imageClassMedical| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _imageClassMedical: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/e2e_workflows/Medical_Imaging_Classification.ipynb + +.. |imageClassRemote| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _imageClassRemote: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/e2e_workflows/Remote_Sensing_Image_Scene_Classification.ipynb + + +.. csv-table:: + :header: "Notebook Title", ".ipynb Link", "Use Case", "Framework" + :widths: 30, 10, 20, 20 + + :doc:`Medical Imaging Classification (Colorectal histology) using TensorFlow `, |imageClassMedical|_ , Image Classification, TensorFlow & Intel Transfer Learning Tool + :doc:`Remote Sensing Image Scene Classification (Resisc) using TensorFlow `, |imageClassRemote|_ , Image Classification, TensorFlow & Intel Transfer Learning Tool + +Intel Transfer Learning Tool Performance Comparison +***************************************************** + +.. |imageClassTFPerf| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _imageClassTFPerf: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/performance/tf_image_classification_performance.ipynb + +.. |textClassHFPerf| image:: /images/Jupyter_logo.svg + :alt: Jupyter notebook .ipynb file + :height: 35 +.. _textClassHFPerf: https://github.com/IntelAI/transfer-learning/blob/main/notebooks/performance/hf_text_classification_performance.ipynb + +.. csv-table:: + :header: "Notebook Title", ".ipynb Link", "Use Case", "Framework" + :widths: 30, 10, 20, 20 + + :doc:`Performance Comparison: Image Classification with TensorFlow `, |imageClassTFPerf|_ , Image Classification, TensorFlow & Intel Transfer Learning Tool + :doc:`Performance Comparison: Text Classification with Hugging Face `, |textClassHFPerf|_ , Text Classification, "Hugging Face, PyTorch & Intel Transfer Learning Tool" diff --git a/docs/notebooks/Remote_Sensing_Image_Scene_Classification.nblink b/docs/notebooks/Remote_Sensing_Image_Scene_Classification.nblink new file mode 100644 index 0000000000000000000000000000000000000000..049a7432c5a4bec934bca5b158f04f36a2c49b17 --- /dev/null +++ b/docs/notebooks/Remote_Sensing_Image_Scene_Classification.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/e2e_workflows/Remote_Sensing_Image_Scene_Classification.ipynb" +} diff --git a/docs/notebooks/TLT_HF_Text_Classification_Performance.nblink b/docs/notebooks/TLT_HF_Text_Classification_Performance.nblink new file mode 100644 index 0000000000000000000000000000000000000000..16675ab76dfdfbc554343cd9e26484e6f347374c --- /dev/null +++ b/docs/notebooks/TLT_HF_Text_Classification_Performance.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/performance/hf_text_classification_performance.ipynb" +} diff --git a/docs/notebooks/TLT_PyTorch_Anomly_Detection.nblink b/docs/notebooks/TLT_PyTorch_Anomly_Detection.nblink new file mode 100644 index 0000000000000000000000000000000000000000..991c14663d63c2ce9324e1a4400ae61413c5815d --- /dev/null +++ b/docs/notebooks/TLT_PyTorch_Anomly_Detection.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/image_anomaly_detection/tlt_api_pyt_anomaly_detection/Anomaly_Detection.ipynb" +} diff --git a/docs/notebooks/TLT_PyTorch_Image_Classification_Transfer_Learning.nblink b/docs/notebooks/TLT_PyTorch_Image_Classification_Transfer_Learning.nblink new file mode 100644 index 0000000000000000000000000000000000000000..5c4860d1590528aa461ea97df59b148bccac173c --- /dev/null +++ b/docs/notebooks/TLT_PyTorch_Image_Classification_Transfer_Learning.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb" +} diff --git a/docs/notebooks/TLT_PyTorch_Text_Classification_Transfer_Learning.nblink b/docs/notebooks/TLT_PyTorch_Text_Classification_Transfer_Learning.nblink new file mode 100644 index 0000000000000000000000000000000000000000..35799bb73ee5e94fbc2a0628471972451aef80c7 --- /dev/null +++ b/docs/notebooks/TLT_PyTorch_Text_Classification_Transfer_Learning.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/text_classification/tlt_api_pyt_text_classification/TLT_PYT_Text_Classification.ipynb" +} diff --git a/docs/notebooks/TLT_TF_Image_Classification_Performance.nblink b/docs/notebooks/TLT_TF_Image_Classification_Performance.nblink new file mode 100644 index 0000000000000000000000000000000000000000..20e7f8f8ff391aaf9f096c952d1bb26ba0f309e1 --- /dev/null +++ b/docs/notebooks/TLT_TF_Image_Classification_Performance.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/performance/tf_image_classification_performance.ipynb" +} diff --git a/docs/notebooks/TLT_TF_Image_Classification_Transfer_Learning.nblink b/docs/notebooks/TLT_TF_Image_Classification_Transfer_Learning.nblink new file mode 100644 index 0000000000000000000000000000000000000000..7cef52ca9f36d4fe330639f0fb6456611abef3fd --- /dev/null +++ b/docs/notebooks/TLT_TF_Image_Classification_Transfer_Learning.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb" +} diff --git a/docs/notebooks/TLT_TF_Text_Classification_Transfer_Learning.nblink b/docs/notebooks/TLT_TF_Text_Classification_Transfer_Learning.nblink new file mode 100644 index 0000000000000000000000000000000000000000..bf4a8a38a7dcccb986d047ebd01b24d2855a1a3a --- /dev/null +++ b/docs/notebooks/TLT_TF_Text_Classification_Transfer_Learning.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../notebooks/text_classification/tlt_api_tf_text_classification/TLT_TF_Text_Classification.ipynb" +} diff --git a/docs/notebooks/setup.rst b/docs/notebooks/setup.rst new file mode 100644 index 0000000000000000000000000000000000000000..a0e63106ebf3367f8014cd4f41c6b6fe7786dc09 --- /dev/null +++ b/docs/notebooks/setup.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: ../../notebooks/setup.md + :parser: myst_parser.sphinx_ diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 0000000000000000000000000000000000000000..f3de818d0c8f1ccd68e97836531445220595e9f9 --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,8 @@ +docutils~=0.17.1 +ipykernel~=6.23.0 +myst-parser~=0.18.1 +nbsphinx~=0.9.1 +nbsphinx-link~=1.3.0 +sphinx~=5.2.0 +sphinx_click~=4.4.0 +sphinx_rtd_theme~=1.2.0 diff --git a/docs/tlt/distributed/README.rst b/docs/tlt/distributed/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..147065b960eb20ae04045c2f3569c3b93630242f --- /dev/null +++ b/docs/tlt/distributed/README.rst @@ -0,0 +1,4 @@ +:orphan: + +.. include:: ../../../tlt/distributed/README.md + :parser: myst_parser.sphinx_ diff --git a/downloader/README.md b/downloader/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1a47f58169aa44a94b5b258af5e6f78553d42e69 --- /dev/null +++ b/downloader/README.md @@ -0,0 +1,71 @@ +# Downloader + +An easy-to-use, unified tool for downloading and managing AI datasets and models. + +## Datasets + +### Supported Catalogs & File Types + +| Source | Info | +|----------|-----------| +| TensorFlow Datasets | [https://www.tensorflow.org/datasets](https://www.tensorflow.org/datasets) | +| Torchvision | [https://pytorch.org/vision/stable/datasets.html](https://pytorch.org/vision/stable/datasets.html) | +| Hugging Face | [https://huggingface.co/docs/datasets/index](https://huggingface.co/docs/datasets/index) | +| Generic Web URL | Publicly downloadable files: `.zip`, `.gz`, `.bz2`, `.txt`, `.csv`, `.png`, `.jpg`, etc. | + +### Usage + +Dataset catalog example: +``` +from downloader.datasets import DataDownloader + +downloader = DataDownloader('tf_flowers', dataset_dir='/home/user/datasets', catalog='tensorflow_datasets') +downloader.download(split='train') +``` + +URL example: +``` +from downloader.datasets import DataDownloader + +downloader = DataDownloader('my_dataset', dataset_dir='/home/user/datasets', url='http:///.zip') +downloader.download() +``` + +## Models + +### Supported Model Hubs + +| Source | Info | +|----------|-----------| +| TensorFlow Hub | [https://www.tensorflow.org/hub](https://www.tensorflow.org/hub) | +| Torchvision | [https://pytorch.org/vision/stable/models.html](https://pytorch.org/vision/stable/models.html) | +| Hugging Face | [https://huggingface.co/models](https://huggingface.co/models) (AutoModelForSequenceClassification or TFBertModel types) | + +### Usage + +Example: +``` +from downloader.models import ModelDownloader + +# Hugging Face +downloader = ModelDownloader('bert-large-uncased', hub='hugging_face', num_labels=2) +downloader.download() + +# Torchvision +downloader = ModelDownloader('resnet34', hub='torchvision') +downloader.download() +``` + +## Build and Install + +To install the downloader, follow [The setup instructions for Intel Transfer Learning Tool](/README.md#build-and-install). The downloader is currently +packaged alongside the Intel Transfer Learning Tool and uses its requirements.txt files, but the tools can be separated at some future time. The +downloader's dependencies are tracked in [requirements.txt](requirements.txt). + +## Testing +With an activated environment that has the dependencies for the downloader and `pytest` in it, run this command from +the root repository directory: + +``` +py.test -s downloader/tests +``` diff --git a/downloader/__init__.py b/downloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c754cd09acd3651e1431bb4997d7a37b861c32d6 --- /dev/null +++ b/downloader/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os + +BASE_DIR = os.path.dirname(__file__) diff --git a/downloader/datasets.py b/downloader/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..16bfb8d62cdab75f9363e01aacd6dea32686c37c --- /dev/null +++ b/downloader/datasets.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +from pydoc import locate +import tarfile +import zipfile +import inspect + +from downloader.types import DatasetType +from downloader import utils + + +class DataDownloader(): + """ + A unified dataset downloader class. + + Can download from TensorFlow Datasets, Torchvision, Hugging Face, and generic web URLs. If initialized for a + dataset catalog, the download method will return a dataset object of type tensorflow.data.Dataset, + torch.utils.data.Dataset, or datasets.arrow_dataset.Dataset. If initialized for a web URL that is a zipfile or a + tarfile, the file will be extracted and the path, or list of paths, to the extracted contents will be returned. + """ + def __init__(self, dataset_name, dataset_dir, catalog=None, url=None, **kwargs): + """ + Class constructor for a DataDownloader. + + Args: + dataset_name (str): Name of the dataset + dataset_dir (str): Local destination directory of dataset + catalog (str, optional): The catalog to download the dataset from; options are 'tensorflow_datasets', + 'torchvision', 'hugging_face', and None which will result in a GENERIC type dataset which expects + an accompanying url input + url (str, optional): If downloading from the web, provide the URL location + kwargs (optional): Some catalogs accept additional keyword arguments when downloading + + raises: + ValueError if both catalog and url are omitted or if both are provided + + """ + if catalog is None and url is None: + raise ValueError("Must provide either a catalog or url as the source.") + if catalog is not None and url is not None: + raise ValueError("Only one of catalog or url should be provided. Found {} and {}.".format(catalog, url)) + + if not os.path.isdir(dataset_dir): + os.makedirs(dataset_dir) + + self._dataset_name = dataset_name + self._dataset_dir = dataset_dir + self._type = DatasetType.from_str(catalog) + self._url = url + self._args = kwargs + + def download(self, split='train'): + """ + Download the dataset + + Args: + split (str): desired split, optional + + Returns: + tensorflow.data.Dataset, torch.utils.data.Dataset, datasets.arrow_dataset.Dataset, str, or list[str] + + """ + if self._type == DatasetType.TENSORFLOW_DATASETS: + import tensorflow_datasets as tfds + if isinstance(split, str): + split = [split] + os.environ['NO_GCE_CHECK'] = 'true' + return tfds.load(self._dataset_name, + data_dir=self._dataset_dir, + split=split, + **self._args) + + elif self._type == DatasetType.TORCHVISION: + from torchvision.datasets import __all__ as torchvision_datasets + dataset_class = locate('torchvision.datasets.{}'.format(self._dataset_name)) + if dataset_class: + params = inspect.signature(dataset_class).parameters + kwargs = dict(download=True, split=split, train=split == 'train') + kwargs = dict([(k, v) for k, v in kwargs.items() if k in params]) + return dataset_class(self._dataset_dir, **kwargs) + else: + raise ValueError("Torchvision dataset {} not found in following: {}" + .format(self._dataset_name, torchvision_datasets)) + + elif self._type == DatasetType.HUGGING_FACE: + from datasets import load_dataset + if 'subset' in self._args: + return load_dataset(self._dataset_name, self._args['subset'], split=split, cache_dir=self._dataset_dir) + else: + return load_dataset(self._dataset_name, split=split, cache_dir=self._dataset_dir) + + elif self._type == DatasetType.GENERIC: + file_path = utils.download_file(self._url, self._dataset_dir) + if os.path.isfile(file_path): + if tarfile.is_tarfile(file_path): + contents = utils.extract_tar_file(file_path, self._dataset_dir) + elif zipfile.is_zipfile(file_path): + contents = utils.extract_zip_file(file_path, self._dataset_dir) + else: + return file_path + + # Contents are a list of top-level extracted members + # Convert to absolute paths and return a single string if length is 1 + if len(contents) > 1: + return [os.path.join(self._dataset_dir, i) for i in contents] + else: + return os.path.join(self._dataset_dir, contents[0]) + + else: + raise FileNotFoundError("Unable to find the downloaded file at:", file_path) diff --git a/downloader/models.py b/downloader/models.py new file mode 100644 index 0000000000000000000000000000000000000000..d2c93f3c1f19d478c992ab4e324d570383b7bc68 --- /dev/null +++ b/downloader/models.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +from pydoc import locate + +from downloader.types import ModelType + + +class ModelDownloader(): + """ + A unified model downloader class. + + Can download models from TF Hub, Torchvision, and Hugging Face. + """ + def __init__(self, model_name, hub, model_dir=None, **kwargs): + """ + Class constructor for a ModelDownloader. + + Args: + model_name (str): Name of the model + hub (str, optional): The catalog to download the model from; options are 'tf_hub', + 'torchvision', 'pytorch_hub', 'hugging_face', and 'keras' + model_dir (str): Local destination directory of the model, if None the model hub's default cache + directory will be used + kwargs (optional): Some model hubs accept additional keyword arguments when downloading + + """ + if model_dir is not None and not os.path.isdir(model_dir): + os.makedirs(model_dir) + + self._model_name = model_name + self._model_dir = model_dir + self._type = ModelType.from_str(hub) + self._args = kwargs + + def download(self): + """ + Download the model + + Returns: + A torch.nn.Module, keras.engine.functional.Functional, or tensorflow_hub.keras_layer.KerasLayer object + + """ + if self._type == ModelType.TF_HUB: + from tensorflow_hub import KerasLayer + if self._model_dir is not None: + os.environ['TFHUB_CACHE_DIR'] = self._model_dir + + return KerasLayer(self._model_name, **self._args) + + elif self._type == ModelType.TORCHVISION: + if self._model_dir is not None: + os.environ['TORCH_HOME'] = self._model_dir + pretrained_model_class = locate('torchvision.models.{}'.format(self._model_name)) + + return pretrained_model_class(**self._args) + + elif self._type == ModelType.PYTORCH_HUB: + from tlt.utils.file_utils import read_json_file + from tlt import TLT_BASE_DIR + import torch + + if self._model_dir is not None: + os.environ['TORCH_HOME'] = self._model_dir + + config_file = os.path.join(TLT_BASE_DIR, "models/configs/pytorch_hub_image_classification_models.json") + pytorch_hub_model_map = read_json_file(config_file) + self._repo = pytorch_hub_model_map[self._model_name]["repo"] + + # Some models have pretrained=True by default, which error out if passed in load() + if pytorch_hub_model_map[self._model_name]["pretrained_default"] == "True": + return torch.hub.load(self._repo, self._model_name) + else: + return torch.hub.load(self._repo, self._model_name, pretrained=True) + + elif self._type == ModelType.HUGGING_FACE: + if self._model_dir is not None: + os.environ['TRANSFORMERS_CACHE'] = self._model_dir + # AutoModelForSequenceClassification is currently the only supported model type + from transformers import AutoModelForSequenceClassification + + return AutoModelForSequenceClassification.from_pretrained(self._model_name, **self._args) + + elif self._type == ModelType.KERAS_APPLICATIONS: + if self._model_dir is not None: + os.environ['KERAS_HOME'] = self._model_dir + try: + pretrained_model_class = locate('keras.applications.{}'.format(self._model_name)) + except TypeError: + pretrained_model_class = locate('keras.applications.{}.{}'.format(self._model_name.lower(), + self._model_name)) + + return pretrained_model_class(**self._args) + + elif self._type == ModelType.TF_BERT_HUGGINGFACE: + if self._model_dir is not None: + os.environ['TRANSFORMERS_CACHE'] = self._model_dir + from transformers import BertConfig, TFBertModel + + config = BertConfig.from_pretrained(self._model_name, output_hidden_states=True) + return TFBertModel.from_pretrained(self._model_name, config=config, from_pt=True, **self._args) diff --git a/downloader/tests/__init__.py b/downloader/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..55899ddaa2b193dbc3b2d314ec2712c6378dd375 --- /dev/null +++ b/downloader/tests/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# diff --git a/downloader/tests/test_dataset_download.py b/downloader/tests/test_dataset_download.py new file mode 100644 index 0000000000000000000000000000000000000000..c24d1596160ca962faf066f9367310abba8ef7cb --- /dev/null +++ b/downloader/tests/test_dataset_download.py @@ -0,0 +1,116 @@ +import os +import pytest +import shutil +import tempfile + +try: + from datasets.arrow_dataset import Dataset as HF_Dataset +except ModuleNotFoundError: + print("WARNING: datasets may not be installed") + +try: + from torch.utils.data import Dataset as TV_Dataset +except ModuleNotFoundError: + print("WARNING: torch may not be installed") + +try: + from tensorflow.data import Dataset as TF_Dataset +except ModuleNotFoundError: + print("WARNING: tensorflow may not be installed") + +from downloader import datasets +from downloader.types import DatasetType + + +@pytest.mark.parametrize('dataset_name,catalog,url', + [['foo', 'tfds', 'https:...'], + ['bar', 'bar', None], + ['baz', None, None]]) +def test_bad_download(dataset_name, catalog, url): + """ + Tests downloader throws ValueError for bad inputs + """ + with pytest.raises(ValueError): + datasets.DataDownloader(dataset_name, dataset_dir='/tmp/data', catalog=catalog, url=url) + + +class TestDatasetDownload: + """ + Tests the dataset downloader with a temp download directory that is initialized and cleaned up + """ + URLS = {'sms_spam_collection': + 'https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip', + 'flowers': + 'https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz', + 'imagenet_labels': + 'https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt', + 'peacock': + 'https://c8.staticflickr.com/8/7095/7210797228_c7fe51c3cb_z.jpg', + 'pennfudan': + 'https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip'} + + @classmethod + def setup_class(cls): + cls._dataset_dir = tempfile.mkdtemp() + + @classmethod + def teardown_class(cls): + if os.path.exists(cls._dataset_dir): + print("Deleting test directory:", cls._dataset_dir) + shutil.rmtree(cls._dataset_dir) + + @pytest.mark.integration + @pytest.mark.parametrize('dataset_name,catalog,split,kwargs,size', + [['tf_flowers', 'tfds', 'train', {}, 3670], + ['CIFAR10', 'torchvision', 'train', {}, 50000], + ['CIFAR10', 'torchvision', 'val', {}, 10000], + ['imdb', 'huggingface', 'train', {}, 25000], + ['glue', 'huggingface', 'test', {'subset': 'sst2'}, 1821]]) + def test_catalog_download(self, dataset_name, catalog, split, kwargs, size): + """ + Tests downloader for different dataset catalog types and splits + """ + downloader = datasets.DataDownloader(dataset_name, dataset_dir=self._dataset_dir, catalog=catalog, **kwargs) + data = downloader.download(split=split) + + # Check the type of the downloader and returned object + if catalog == 'tfds': + data = data[0] # TFDS returns a list with the dataset in it + assert downloader._type == DatasetType.TENSORFLOW_DATASETS + assert isinstance(data, TF_Dataset) + elif catalog == 'torchvision': + assert downloader._type == DatasetType.TORCHVISION + assert isinstance(data, TV_Dataset) + elif catalog == 'huggingface': + assert downloader._type == DatasetType.HUGGING_FACE + assert isinstance(data, HF_Dataset) + + # Verify the split size + assert len(data) == size + + # Check that the directory is not empty + assert os.listdir(self._dataset_dir) is not None + + @pytest.mark.parametrize('dataset_name,url,num_contents', + [['sms_spam_collection', URLS['sms_spam_collection'], 2], + ['flowers', URLS['flowers'], 1], + ['imagenet_labels', URLS['imagenet_labels'], 1], + ['peacock', URLS['peacock'], 1], + ['pennfudan', URLS['pennfudan'], 1]]) + def test_generic_download(self, dataset_name, url, num_contents): + """ + Tests downloader for different web URLs and file types + """ + downloader = datasets.DataDownloader(dataset_name, dataset_dir=self._dataset_dir, url=url) + data_path = downloader.download() + + assert downloader._type == DatasetType.GENERIC + + # Test that the returned object is the expected type and length + if num_contents == 1: + assert isinstance(data_path, str) + assert os.path.exists(data_path) + else: + assert isinstance(data_path, list) + for path in data_path: + assert os.path.exists(path) diff --git a/downloader/tests/test_model_download.py b/downloader/tests/test_model_download.py new file mode 100644 index 0000000000000000000000000000000000000000..d79068c11f3ba16b9db56e3ac60748f13e5bf3f7 --- /dev/null +++ b/downloader/tests/test_model_download.py @@ -0,0 +1,91 @@ +import os +import pytest +import shutil +import tempfile + +try: + from torch.nn import Module +except ModuleNotFoundError: + print("WARNING: Unable to import torch. Torch may not be installed") + +try: + from tensorflow_hub.keras_layer import KerasLayer +except ModuleNotFoundError: + print("WARNING: Unable to import KerasLayer. Tensorflow Hub may not be installed") + +try: + from tensorflow.keras import Model +except ModuleNotFoundError: + print("WARNING: Unable to import Keras Model. Tensorflow may not be installed") + +from downloader import models +from downloader.types import ModelType + + +@pytest.mark.parametrize('hub', + [['foo'], + ['bar'], + ['baz']]) +def test_bad_hub(hub): + """ + Tests downloader throws ValueError for bad inputs + """ + model_name = 'model' + with pytest.raises(ValueError): + models.ModelDownloader(model_name, hub) + + +class TestModelDownload: + """ + Tests the model downloader with a temp download directory that is initialized and cleaned up + """ + @classmethod + def setup_class(cls): + cls._model_dir = tempfile.mkdtemp() + + @classmethod + def teardown_class(cls): + if os.path.exists(cls._model_dir): + print("Deleting test directory:", cls._model_dir) + shutil.rmtree(cls._model_dir) + + # Has previously been skipped due to HTTP Error 403: rate limit exceeded') + @pytest.mark.parametrize('model_name,hub,kwargs', + [['https://tfhub.dev/google/efficientnet/b0/feature-vector/1', 'tf_hub', {}], + ['https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3', 'tfhub', + {'name': 'encoder', 'trainable': True}], + ['resnet34', 'torchvision', {}], + ['mobilenet_v2', 'torchvision', {}], + ['resnet18_ssl', 'pytorch_hub', {}], + ['resnet50_swsl', 'pytorch_hub', {}], + ['distilbert-base-uncased', 'huggingface', {}], + ['bert-base-cased', 'hugging_face', {}], + ['Xception', 'keras_applications', {}], + ['ResNet50', 'keras', {'weights': 'imagenet', 'include_top': False}], + ['google/bert_uncased_L-2_H-128_A-2', 'tf_bert_huggingface', {}], + ['bert-base-uncased', 'tf_bert_hugging_face', {}]]) + def test_hub_download(self, model_name, hub, kwargs): + """ + Tests downloader for different model hubs + """ + downloader = models.ModelDownloader(model_name, hub, model_dir=self._model_dir, **kwargs) + model = downloader.download() + + # Check the type of the downloader and returned object + if downloader._type == ModelType.TF_HUB: + assert isinstance(model, KerasLayer) + elif downloader._type == ModelType.TORCHVISION: + assert isinstance(model, Module) + elif downloader._type == ModelType.PYTORCH_HUB: + assert isinstance(model, Module) + elif downloader._type == ModelType.HUGGING_FACE: + assert isinstance(model, Module) + elif downloader._type == ModelType.KERAS_APPLICATIONS: + assert isinstance(model, Model) + elif downloader._type == ModelType.TF_BERT_HUGGINGFACE: + assert isinstance(model, Model) + else: + assert False + + # Check that the directory is not empty + assert os.listdir(self._model_dir) is not None diff --git a/downloader/types.py b/downloader/types.py new file mode 100644 index 0000000000000000000000000000000000000000..a935fef8288e617e58d5b9b36ac5e393d987bdea --- /dev/null +++ b/downloader/types.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from enum import Enum, auto + + +class DatasetType(Enum): + TENSORFLOW_DATASETS = auto() + TORCHVISION = auto() + HUGGING_FACE = auto() + GENERIC = auto() + + def __str__(self): + return self.name.lower() + + @staticmethod + def from_str(dataset_str): + if dataset_str is None: + return DatasetType.GENERIC + + dataset_str = dataset_str.lower() + + if dataset_str in ["tfds", "tensorflow", "tensorflow_datasets", "tensorflow datasets", "tensorflow_dataset", + "tensorflow dataset"]: + return DatasetType.TENSORFLOW_DATASETS + elif dataset_str in ["torchvision"]: + return DatasetType.TORCHVISION + elif dataset_str in ["huggingface", "hugging_face", "hugging face"]: + return DatasetType.HUGGING_FACE + elif dataset_str in ["generic"]: + return DatasetType.GENERIC + else: + options = [e.name for e in DatasetType] + raise ValueError("Unsupported dataset type: {} (Select from: {})".format( + dataset_str, options)) + + +class ModelType(Enum): + TF_HUB = auto() + TORCHVISION = auto() + PYTORCH_HUB = auto() + HUGGING_FACE = auto() + KERAS_APPLICATIONS = auto() + TF_BERT_HUGGINGFACE = auto() + GENERIC = auto() + + def __str__(self): + return self.name.lower() + + @staticmethod + def from_str(model_str): + if model_str is None: + return ModelType.GENERIC + + model_str = model_str.lower() + + if model_str in ["tfhub", "tf_hub", "tf hub", "tensorflow_hub", "tensorflow hub"]: + return ModelType.TF_HUB + elif model_str in ["torchvision"]: + return ModelType.TORCHVISION + elif model_str in ["pytorch_hub", "pyt_hub", "torch_hub", "torch hub", "pytorch hub"]: + return ModelType.PYTORCH_HUB + elif model_str in ["huggingface", "hugging_face", "hugging face"]: + return ModelType.HUGGING_FACE + elif model_str in ["keras", "keras_applications", "keras applications"]: + return ModelType.KERAS_APPLICATIONS + elif model_str in ["tf_bert_huggingface", "tf bert huggingface", "tf_bert_hugging_face", + "tf bert hugging face"]: + return ModelType.TF_BERT_HUGGINGFACE + elif model_str in ["generic"]: + return ModelType.GENERIC + else: + options = [e.name for e in ModelType] + raise ValueError("Unsupported model type: {} (Select from: {})".format(model_str, options)) diff --git a/downloader/utils.py b/downloader/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0ae94b4d6a29926326fbe67827bc70f2ce1f8486 --- /dev/null +++ b/downloader/utils.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import tarfile +import requests +import shutil +import zipfile + + +def download_file(download_url, destination_directory): + """ + Downloads a file using the specified url to the destination directory. Returns the + path to the downloaded file. + """ + if not os.path.isdir(destination_directory): + os.makedirs(destination_directory) + + destination_file_path = os.path.join(destination_directory, os.path.basename(download_url)) + + print("Downloading {} to {}".format(download_url, destination_directory)) + response = requests.get(download_url, stream=True, timeout=30) + with open(destination_file_path, 'wb') as out_file: + shutil.copyfileobj(response.raw, out_file) + + return destination_file_path + + +def extract_tar_file(tar_file_path, destination_directory): + """ + Extracts a tar file on the local file system to the destination directory. Returns a list + of top-level contents (files and folders) of the extracted archive. + """ + if not os.path.isdir(destination_directory): + os.makedirs(destination_directory) + + print("Extracting {} to {}".format(tar_file_path, destination_directory)) + with tarfile.open(tar_file_path) as t: + t.extractall(path=destination_directory) + contents = {i.split('/')[0] for i in t.getnames()} + return list(contents) + + +def extract_zip_file(zip_file_path, destination_directory): + """ + Extracts a zip file on the local file system to the destination directory. Returns a list + of top-level contents (files and folders) of the extracted archive. + """ + if not os.path.isdir(destination_directory): + os.makedirs(destination_directory) + + print("Extracting {} to {}".format(zip_file_path, destination_directory)) + with zipfile.ZipFile(zip_file_path, "r") as z: + z.extractall(path=destination_directory) + contents = {i.split('/')[0] for i in z.namelist()} + return list(contents) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000000000000000000000000000000000..951bf606e5acefae844735dc1024abcf6160bd08 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,11 @@ +# Examples + +Here are some examples using no-code bash commands: + +* [Beyond Get Started CLI Example](cli/README.md) +* [Text Classification Intel® Transfer Learning Tool CLI Example](cli/text_classification.md) +* [Image Classification Intel® Transfer Learning Tool CLI Example](cli/image_classification.md) + +Here are Jupyter notebook examples using low-code Python\* API calls: + +* [Intel® Transfer Learning Tool API Examples](../notebooks/README.md) diff --git a/examples/cli/README.md b/examples/cli/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f0dc6d3c2217fd95972d177a7e0a7741e87ce0bb --- /dev/null +++ b/examples/cli/README.md @@ -0,0 +1,227 @@ +# Beyond Get Started CLI Example + +If you already walked through the [Get Started Guide](/GetStarted.md), this +example begins with the same training steps and then continues with more examples of +using CLI commands to work with that trained model. + +The following example walks through a full workflow using the Intel Transfer Learning +Tool CLI to train a model, and then benchmark, quantize, and optimize the +trained model. It uses a TensorFlow image classification model, but the +same commands and concepts can be applied when working with other frameworks +and use cases. + +Use `tlt --help` to see the list of CLI commands. More detailed information on +each command can be found using `tlt --help` (such as `tlt train --help`). + +## List the Available Models + +Use the `tlt list` command to see a list of available models for each framework. +Use the `--use-case` flag to limit the list to models for a particular use case. +``` +tlt list models --use-case image_classification +``` +``` +------------------------------ +IMAGE CLASSIFICATION +------------------------------ +alexnet (pytorch) +convnext_base (pytorch) +convnext_large (pytorch) +convnext_small (pytorch) +convnext_tiny (pytorch) +densenet121 (pytorch) +densenet161 (pytorch) +densenet169 (pytorch) +densenet201 (pytorch) +efficientnet_b0 (pytorch) +efficientnet_b0 (tensorflow) +efficientnet_b1 (pytorch) +efficientnet_b1 (tensorflow) +efficientnet_b2 (pytorch) +efficientnet_b2 (tensorflow) +efficientnet_b3 (pytorch) +efficientnet_b3 (tensorflow) +efficientnet_b4 (pytorch) +efficientnet_b4 (tensorflow) +... +``` + +## Train a Model + +For this example, we use the TensorFlow flowers dataset. Download and extract +the dataset by following this example: +``` +# Create a directory for the dataset to be downloaded +DATASET_DIR=/tmp/dataset +mkdir -p ${DATASET_DIR} + +# Download and extract the dataset +wget -P ${DATASET_DIR} https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz +tar -xzf ${DATASET_DIR}/flower_photos.tgz -C ${DATASET_DIR} + +# Set the DATASET_DIR to the extracted images folder +DATASET_DIR=${DATASET_DIR}/flower_photos + +# Supress debug information from TensorFlow 2.12 +TF_CPP_MIN_LOG_LEVEL=2 +``` + +After the dataset directory is ready, use the `tlt train` command to train one of the models from +`tlt list`. In this example, we use the TensorFlow ResNet50v1.5 model. Make sure to specify +your own file path for the `output-dir`. The `dataset-dir` should point to the extracted dataset folder. +``` +tlt train -f tensorflow --model-name resnet_v1_50 --dataset-dir ${DATASET_DIR} --output-dir /tmp/output +``` +``` +Model name: resnet_v1_50 +Framework: tensorflow +Training epochs: 1 +Dataset dir: /tmp/dataset/flower_photos +Output directory: /tmp/output +Found 3670 files belonging to 5 classes. +... +Model: "sequential" +_________________________________________________________________ +Layer (type) Output Shape Param # +================================================================= +keras_layer (KerasLayer) (None, 2048) 23561152 +dense (Dense) (None, 5) 10245 +================================================================= +Total params: 23,571,397 +Trainable params: 10,245 +Non-trainable params: 23,561,152 +_________________________________________________________________ +Checkpoint directory: /tmp/output/resnet_v1_50_checkpoints +86/86 [==============================] - 24s 248ms/step - loss: 0.4600 - acc: 0.8438 +Saved model directory: /tmp/output/resnet_v1_50/1 +``` + +The `tlt train` command evaluates the model after training completes. The loss and +accuracy values are printed toward the end of the console output, along with the +location where the trained model has been saved (including a new numbered folder for +each run). + +A trained model can also be evaluated using the `tlt eval` command: +``` +tlt eval --model-dir /tmp/output/resnet_v1_50/1 --dataset-dir ${DATASET_DIR} +``` + +## Benchmark the Trained Model + +Benchmark the performance of the trained model using `tlt benchmark`. Make sure +to specify your own file paths for `model-dir` (including the numbered folder). +The `dataset-dir` should point to the extracted dataset folder. +``` +tlt benchmark --model-dir /tmp/output/resnet_v1_50/1 --dataset-dir ${DATASET_DIR} --batch-size 512 +``` +``` +Model directory: /tmp/output/resnet_v1_50/1 +Dataset directory: /tmp/dataset/flower_photos +Batch size: 512 +Model name: resnet_v1_50 +Framework: tensorflow +... +performance mode benchmark result: +2022-06-28 10:22:10 [INFO] Batch size = 512 +2022-06-28 10:22:10 [INFO] Latency: 3.031 ms +2022-06-28 10:22:10 [INFO] Throughput: 329.878 images/sec + ``` + +## Quantize the Model + +Perform post-training quantization using the [Intel® Neural +Compressor](https://intel.github.io/neural-compressor) using the `tlt quantize` +command. Make sure to specify your own file paths for `model-dir` (including the +numbered folder), `dataset-dir`, and `output-dir`. The quantized model will be +saved to the output directory. +``` +tlt quantize --model-dir /tmp/output/resnet_v1_50/1 --dataset-dir ${DATASET_DIR} --batch-size 512 \ +--accuracy-criterion 0.01 --output-dir /tmp/output +``` +``` +Model directory: /tmp/output/resnet_v1_50/1 +Dataset directory: /tmp/dataset/flower_photos +Accuracy criterion: 0.01 +Exit policy timeout: 0 +Exit policy max trials: 50 +Batch size: 512 +Output directory: /tmp/output +... +2022-06-28 10:25:58 [INFO] |******Mixed Precision Statistics*****| +2022-06-28 10:25:58 [INFO] +-----------------+----------+--------+ +2022-06-28 10:25:58 [INFO] | Op Type | Total | INT8 | +2022-06-28 10:25:58 [INFO] +-----------------+----------+--------+ +2022-06-28 10:25:58 [INFO] | Conv2D | 53 | 53 | +2022-06-28 10:25:58 [INFO] | MatMul | 1 | 1 | +2022-06-28 10:25:58 [INFO] | MaxPool | 4 | 4 | +2022-06-28 10:25:58 [INFO] | QuantizeV2 | 5 | 5 | +2022-06-28 10:25:58 [INFO] | Dequantize | 4 | 4 | +2022-06-28 10:25:58 [INFO] +-----------------+----------+--------+ +2022-06-28 10:25:58 [INFO] Pass quantize model elapsed time: 32164.27 ms +2022-06-28 10:25:58 [INFO] Start to evaluate the TensorFlow model. +2022-06-28 10:26:12 [INFO] Model inference elapsed time: 13921.64 ms +2022-06-28 10:26:12 [INFO] Tune 1 result is: [Accuracy (int8|fp32): 0.9008|0.9022, Duration (seconds) (int8|fp32): 13.9226|17.3321], Best tune result is: [Accuracy: 0.9008, Duration (seconds): 13.9226] +2022-06-28 10:26:12 [INFO] |**********************Tune Result Statistics**********************| +2022-06-28 10:26:12 [INFO] +--------------------+----------+---------------+------------------+ +2022-06-28 10:26:12 [INFO] | Info Type | Baseline | Tune 1 result | Best tune result | +2022-06-28 10:26:12 [INFO] +--------------------+----------+---------------+------------------+ +2022-06-28 10:26:12 [INFO] | Accuracy | 0.9022 | 0.9008 | 0.9008 | +2022-06-28 10:26:12 [INFO] | Duration (seconds) | 17.3321 | 13.9226 | 13.9226 | +2022-06-28 10:26:12 [INFO] +--------------------+----------+---------------+------------------+ +2022-06-28 10:26:12 [INFO] Save tuning history to /tmp/output/nc_workspace/./history.snapshot. +2022-06-28 10:26:12 [INFO] Specified timeout or max trials is reached! Found a quantized model which meet accuracy goal. Exit. +.. +INFO:tensorflow:SavedModel written to: /tmp/output/quantized/resnet_v1_50/1/saved_model.pb +2022-06-28 10:26:13 [INFO] SavedModel written to: /tmp/output/quantized/resnet_v1_50/1/saved_model.pb +2022-06-28 10:26:13 [INFO] Save quantized model to /tmp/output/quantized/resnet_v1_50/1 +``` + +## Benchmark the Quantized Model + +The `tlt benchmark` command is used again, but this time the `model-dir` should point +to the quantized model directory. +Make sure to specify your own file paths for `model-dir` and `dataset-dir`. You can then compare +the performance of the full precision model to the quantized model. +``` +tlt benchmark --model-dir /tmp/output/quantized/resnet_v1_50/1 --dataset-dir ${DATASET_DIR} --batch-size 512 +``` +``` +Model directory: /tmp/output/quantized/resnet_v1_50/1 +Dataset directory: /tmp/dataset/flower_photos +Benchmarking mode: performance +Batch size: 512 +Model name: resnet_v1_50 +Framework: tensorflow +... +performance mode benchmark result: +2022-06-28 10:28:33 [INFO] Batch size = 512 +2022-06-28 10:28:33 [INFO] Latency: 0.946 ms +2022-06-28 10:28:33 [INFO] Throughput: 1056.940 images/sec +``` + +## Perform Graph Optimization on the Trained Model + +Alternatively, the [Intel Neural Compressor](https://intel.github.io/neural-compressor) can be used to optimize +the full precision graph. Make sure to specify your own file paths for `model-dir` and `output-dir`. +Note that graph optimization is also done as part of the quantization flow, so there is no need to call +`tlt optimize` on a quantized model. +``` +tlt optimize --model-dir /tmp/output/resnet_v1_50/1 --output-dir /tmp/output +``` +``` +Model directory: /tmp/output/resnet_v1_50/1 +Model name: resnet_v1_50 +Output directory: /tmp/output +Framework: tensorflow +Starting graph optimization +... +2022-06-28 13:50:01 [INFO] Graph optimization is done. +... +2022-06-28 13:51:21 [INFO] SavedModel written to: /tmp/output/optimized/resnet_v1_50/1/saved_model.pb +``` + +## Addititional CLI Examples + +More CLI examples can be found here: +* [Image classification examples](/examples/cli/image_classification.md) +* [Text classification examples](/examples/cli/text_classification.md) diff --git a/examples/cli/image_classification.md b/examples/cli/image_classification.md new file mode 100644 index 0000000000000000000000000000000000000000..86d6cbb1e8888725bb9f647cfc4f30e4f4bfaaa2 --- /dev/null +++ b/examples/cli/image_classification.md @@ -0,0 +1,84 @@ +# Image Classification Intel® Transfer Learning Tool CLI Example + +## Transfer Learning Using your Own Dataset + +The example below shows how the Intel Transfer Learning Tool CLI can be used for image classification transfer learning +using your own dataset. The dataset is expected to be organized with subfolders for each image +class. Each subfolder should contain .jpg images for the class. The name of the subfolder will +be used as the class label. + +This example downloads a flower photos dataset from TensorFlow, which has images of +flowers belonging to 5 classes: daisy, dandelion, roses, sunflowers, and tulips. The extracted +dataset is already formatted in the expected format with subfolders for each class. +```bash +# Create dataset and output directories +DATASET_DIR=/tmp/data +OUTPUT_DIR=/tmp/output +mkdir -p ${DATASET_DIR} +mkdir -p ${OUTPUT_DIR} + +# Download and extract the dataset +wget -P ${DATASET_DIR} https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz +tar -xzf ${DATASET_DIR}/flower_photos.tgz -C ${DATASET_DIR} + +# Set the DATASET_DIR to the extracted images folder +DATASET_DIR=${DATASET_DIR}/flower_photos + +# Train resnet_v1_50 using the flower photos directory +tlt train \ + -f tensorflow \ + --model-name resnet_v1_50 \ + --dataset-dir ${DATASET_DIR} \ + --output-dir ${OUTPUT_DIR} \ + --epochs 2 + +# Evaluate the model exported after training +# Note that your --model-dir path may vary, since each training run creates a new directory +tlt eval \ + --model-dir /tmp/output/resnet_v1_50/1 \ + --dataset-dir ${DATASET_DIR} +``` + +## Transfer Learning Using a Dataset from the TFDS Catalog + +This example shows the Intel Transfer Learning Tool CLI being used for image classification transfer learning +using the `tf_flowers` dataset from the +[TensorFlow Datasets (TFDS) catalog](https://www.tensorflow.org/datasets/catalog/overview). + +```bash +# Create dataset and output directories +DATASET_DIR=/tmp/data +OUTPUT_DIR=/tmp/output +mkdir -p ${DATASET_DIR} +mkdir -p ${OUTPUT_DIR} + +# Name of the dataset to use +DATASET_NAME=tf_flowers + +# Train resnet_v1_50 using the TFDS catalog dataset +tlt train \ + -f tensorflow \ + --model-name resnet_v1_50 \ + --dataset-name ${DATASET_NAME} \ + --dataset-dir ${DATASET_DIR} \ + --output-dir ${OUTPUT_DIR} \ + --epochs 2 + +# Evaluate the model exported after training +# Note that your --model-dir path may vary, since each training run creates a new directory +tlt eval \ + --model-dir ${OUTPUT_DIR}/resnet_v1_50/1 \ + --dataset-name ${DATASET_NAME} \ + --dataset-dir ${DATASET_DIR} +``` + +## Citations + +``` +@ONLINE {tfflowers, +author = "The TensorFlow Team", +title = "Flowers", +month = "jan", +year = "2019", +url = "http://download.tensorflow.org/example_images/flower_photos.tgz" } +``` diff --git a/examples/cli/text_classification.md b/examples/cli/text_classification.md new file mode 100644 index 0000000000000000000000000000000000000000..cdd417b4e21e72c104e8d79220377c383c125b70 --- /dev/null +++ b/examples/cli/text_classification.md @@ -0,0 +1,183 @@ +# Text Classification Intel® Transfer Learning Tool CLI Example + +## Fine Tuning Using Your Own Dataset + +The example below shows how to fine tune a TensorFlow text classification model using your own +dataset in the .csv format. The .csv file is expected to have 2 columns: a numerical class label +and the text/sentence to classify. Note that although the TLT API is more flexible and allows for +providing map functions to translate string class names to numerical values and filtering which +columns are being used, the CLI only allows using .csv files in the expected format. + +The `--dataset-dir` argument is the path to the directory where your dataset is located, and the +`--dataset-file` is the name of the .csv file to load from that directory. Use the `--class-names` +argument to specify a list of the classes and the `--delimiter` to specify the character that +separates the two columns. If no `--delimiter` is specified, the CLI will default to use a comma (`,`). + +This example is downloading the [SMS Spam Collection](https://archive.ics.uci.edu/dataset/228/sms+spam+collection) +dataset, which has a tab separated value file in the .zip file. This dataset has labeled SMS text +messages that are either being classified as `ham` or `spam`. The first column in the data file has +the label (`ham` or `spam`) and the second column is the text of the SMS message. The string class +labels are replaced with numerical values before training. +```bash +# Create dataset and output directories +DATASET_DIR=/tmp/data +OUTPUT_DIR=/tmp/output +mkdir -p ${DATASET_DIR} +mkdir -p ${OUTPUT_DIR} + +# Download and extract the dataset +wget -P ${DATASET_DIR} https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip +unzip ${DATASET_DIR}/sms+spam+collection.zip + +# Make a copy of the .csv file with 'numerical' in the file name +DATASET_FILE=SMSSpamCollection_numerical.csv +cp ${DATASET_DIR}/SMSSpamCollection ${DATASET_DIR}/${DATASET_FILE} + +# Replace string class labels with numerical values in the .csv file\ +# The list numerical class labels passed as the --class-names during training and evaluation +sed -i 's/ham/0/g' ${DATASET_DIR}/${DATASET_FILE} +sed -i 's/spam/1/g' ${DATASET_DIR}/${DATASET_FILE} + +# Train google/bert_uncased_L-10_H-256_A-4 using our dataset file, which has tab delimiters +tlt train \ + -f tensorflow \ + --model-name google/bert_uncased_L-10_H-256_A-4 \ + --output-dir ${OUTPUT_DIR} \ + --dataset-dir ${DATASET_DIR} \ + --dataset-file ${DATASET_FILE} \ + --epochs 2 \ + --class-names 0,1 \ + --delimiter $'\t' + +# Evaluate the model exported after training +# Note that your --model-dir path may vary, since each training run creates a new directory +tlt eval \ + --model-dir ${OUTPUT_DIR}/google_bert_uncased_L-10_H-256_A-4/1 \ + --model-name google/bert_uncased_L-10_H-256_A-4 \ + --dataset-dir ${DATASET_DIR} \ + --dataset-file ${DATASET_FILE} \ + --class-names 0,1 \ + --delimiter $'\t' +``` + +## Fine Tuning Using a Dataset from the TFDS Catalog + +This example demonstrates using the Intel Transfer Learning Tool CLI to fine tune a text classification model using a +dataset from the [TensorFlow Datasets (TFDS) catalog](https://www.tensorflow.org/datasets/catalog/overview). +Intel Transfer Learning Tool supports the following text classification datasets from TFDS: +[imdb_reviews](https://www.tensorflow.org/datasets/catalog/imdb_reviews), +[glue/sst2](https://www.tensorflow.org/datasets/catalog/imdb_reviews), +and [glue/cola](https://www.tensorflow.org/datasets/catalog/glue#gluecola_default_config). + +```bash +# Create dataset and output directories +DATASET_DIR=/tmp/data +OUTPUT_DIR=/tmp/output +mkdir -p ${DATASET_DIR} +mkdir -p ${OUTPUT_DIR} + +# Name of the dataset to use +DATASET_NAME=imdb_reviews + +# Train google/bert_uncased_L-10_H-256_A-4 using the TFDS dataset +tlt train \ + -f tensorflow \ + --model-name google/bert_uncased_L-10_H-256_A-4 \ + --output-dir ${OUTPUT_DIR} \ + --dataset-dir ${DATASET_DIR} \ + --dataset-name ${DATASET_NAME} \ + --epochs 2 + +# Evaluate the model exported after training +# Note that your --model-dir path may vary, since each training run creates a new directory +tlt eval \ + --model-dir ${OUTPUT_DIR}/google_bert_uncased_L-10_H-256_A-4/2 \ + --model-name google/bert_uncased_L-10_H-256_A-4 \ + --dataset-dir ${DATASET_DIR} \ + --dataset-name ${DATASET_NAME} +``` + +## Distributed Transfer Learning Using a Dataset from Hugging Face +This example runs a distributed PyTorch training job using the TLT CLI. It fine tunes a text classification model +for document-level sentiment analysis using a dataset from the [Hugging Face catalog](https://huggingface.co/datasets). +Intel Transfer Learning Tool supports the following text classification datasets from Hugging Face: +* [imdb](https://huggingface.co/datasets/imdb) +* [tweet_eval](https://huggingface.co/datasets/tweet_eval) +* [rotten_tomatoes](https://huggingface.co/datasets/rotten_tomatoes) +* [ag_news](https://huggingface.co/datasets/ag_news) +* [sst2](https://huggingface.co/datasets/sst2) + +Follow [these instructions](/tlt/distributed/README.md) to set up your machines for distributed training with PyTorch. This will +ensure your environment has the right prerequisites, package dependencies, and hostfile configuration. When +you have successfully run the sanity check, the following commands will fine-tune `bert-large-uncased` with sst2 for +one epoch using 2 nodes and 2 processes per node. + +```bash +# Create dataset and output directories +DATASET_DIR=/tmp/data +OUTPUT_DIR=/tmp/output +mkdir -p ${DATASET_DIR} +mkdir -p ${OUTPUT_DIR} + +# Name of the dataset to use +DATASET_NAME=sst2 + +# Train bert-large-uncased using the Hugging Face dataset sst2 +tlt train \ + -f pytorch \ + --model_name bert-large-uncased \ + --dataset_name sst2 \ + --output_dir $OUTPUT_DIR \ + --dataset_dir $DATASET_DIR \ + --distributed \ + --hostfile hostfile \ + --nnodes 2 \ + --nproc_per_node 2 +``` + +## Citations +``` +@InProceedings{maas-EtAl:2011:ACL-HLT2011, + author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher}, + title = {Learning Word Vectors for Sentiment Analysis}, + booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies}, + month = {June}, + year = {2011}, + address = {Portland, Oregon, USA}, + publisher = {Association for Computational Linguistics}, + pages = {142--150}, + url = {http://www.aclweb.org/anthology/P11-1015} +} + +@inproceedings{wang2019glue, + title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, + author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, + note={In the Proceedings of ICLR.}, + year={2019} +} + +@misc{misc_sms_spam_collection_228, + author = {Almeida, Tiago}, + title = {{SMS Spam Collection}}, + year = {2012}, + howpublished = {UCI Machine Learning Repository} +} + +@inproceedings{socher-etal-2013-recursive, + title = "Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank", + author = "Socher, Richard and + Perelygin, Alex and + Wu, Jean and + Chuang, Jason and + Manning, Christopher D. and + Ng, Andrew and + Potts, Christopher", + booktitle = "Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing", + month = oct, + year = "2013", + address = "Seattle, Washington, USA", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/D13-1170", + pages = "1631--1642", +} +``` diff --git a/images/Jupyter_logo.svg b/images/Jupyter_logo.svg new file mode 100644 index 0000000000000000000000000000000000000000..ab255087431725b5ae37c0fe279a6572f1064252 --- /dev/null +++ b/images/Jupyter_logo.svg @@ -0,0 +1,90 @@ + +Group.svg +Created using Figma 0.90 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/TLT-GSG_flow.svg b/images/TLT-GSG_flow.svg new file mode 100644 index 0000000000000000000000000000000000000000..070d7d7596ab20e75ade5b678b28ffdc1d739ef8 --- /dev/null +++ b/images/TLT-GSG_flow.svg @@ -0,0 +1,59 @@ + + + + + + + + + + Check system + requirements + + + + Install the Intel + Transfer Learning + Tool + + + + Choose how to run + the Intel Transfer + Learning Tool + + + + a) Run using No- + Code CLI + + + + b) Run using Low- + Code API + + 1 + + 2 + + 3 + + + diff --git a/images/TLT-tool_flow.svg b/images/TLT-tool_flow.svg new file mode 100644 index 0000000000000000000000000000000000000000..7f67c1da2b77cddfc2c26ef454dd0f0ab0e8bbb1 --- /dev/null +++ b/images/TLT-tool_flow.svg @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + Select Framework, Use Case, Dataset & Model + Framework : TensorFlow, PyTorch + Use Case : Image Classification, Text Classification + Dataset : Public catalog (TF Hub, Torchvision, Hugging face), or Custom + Model : EfficientNet, InceptionV3, MobileNetV2, ResNet, BERT… + + + Pre-Process & Split Dataset + Resize, Batch, Split Train/Val, Augmentation + + + Train + Num Epochs, Initial Checkpoint, Distributed… + + + Evaluate & Predict + Evaluation, Prediction, Benchmarking + + + Optimize + Quantization or FP32 Graph Optimization + + + Export Model + + + + + + + + Optimizations for Intel + + + + + + + + A + + Retraining not possible once optimized + + + + A + + + diff --git a/images/favicon-intel-32x32.png b/images/favicon-intel-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..5a6b7816762e4d5a558c24986fe539c6a6c3e3b8 Binary files /dev/null and b/images/favicon-intel-32x32.png differ diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dcb95e2203e270b2cb07883a0ddd15ec60c2e929 --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,48 @@ +# Transfer Learning Notebooks + +## Environment setup and running the notebooks + +Use the [setup instructions](setup.md) to install the dependencies required to run the notebooks. + +This directory has Jupyter notebooks that demonstrate transfer learning with +and without Intel® Transfer Learning Tool. All of the notebooks use models from public model repositories +and leverage optimized libraries [Intel-optimized TensorFlow](https://pypi.org/project/intel-tensorflow/) +and [Intel Extension for PyTorch](https://github.com/intel/intel-extension-for-pytorch). + +## Intel® Transfer Learning Tool Tutorial Notebooks + +| Notebook | Domain: Use Case | Framework| Description | +| ---------| ---------|----------|-------------| +| [BERT Text Classification with TensorFlow using the Intel® Transfer Learning Tool](/notebooks/text_classification/tlt_api_tf_text_classification) | NLP: Text Classification | TensorFlow and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to fine tune a BERT model from Hugging Face using text classification datasets. | +| [BERT Text Classification with PyTorch using the Intel® Transfer Learning Tool](/notebooks/text_classification/tlt_api_pyt_text_classification) | NLP: Text Classification | PyTorch and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to fine tune a BERT model from Hugging Face using text classification datasets. | +| [Image Classification with TensorFlow using Intel® Transfer Learning Tool](/notebooks/image_classification/tlt_api_tf_image_classification) | CV: Image Classification | TensorFlow and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for image classification using a TensorFlow model. | +| [Image Classification with TensorFlow using Graph Optimization and Intel® Transfer Learning Tool](/notebooks/image_classification/tlt_api_tf_image_classification) | CV: Image Classification | TensorFlow and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning with graph optimization that increases throughput for image classification using a TensorFlow model. | +| [Image Classification with PyTorch using Intel® Transfer Learning Tool](/notebooks/image_classification/tlt_api_pyt_image_classification) | CV: Image Classification | PyTorch and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for image classification using a PyTorch model. | +| [Image Anomaly Detection with PyTorch using Intel® Transfer Learning Tool](/notebooks/image_anomaly_detection/tlt_api_pyt_anomaly_detection) | CV: Image Anomaly Detection| PyTorch and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do feature extraction and pca analysis using a single function for image anomaly detection using a Torchvision model. | + +## Native Framework Transfer Learning Notebooks + +| Notebook | Domain: Use Case | Framework| Description | +| ---------| ---------|----------|-------------| +| [BERT SQuAD fine tuning with TF Hub](/notebooks/question_answering/tfhub_question_answering) | NLP: Question Answering | TensorFlow | Demonstrates BERT fine tuning using scripts from the [TensorFlow Model Garden](https://github.com/tensorflow/models) and the [SQuAD dataset](https://rajpurkar.github.io/SQuAD-explorer/). The notebook allows for selecting a BERT large or BERT base model from [TF Hub](https://tfhub.dev). The fine tuned model is evaluated and exported as a saved model. | +| [BERT Text Classification with TF Hub](/notebooks/text_classification/tfhub_text_classification) | NLP: Text Classification | TensorFlow | Demonstrates BERT binary text classification fine tuning using the [IMDb movie review dataset](https://www.tensorflow.org/datasets/catalog/imdb_reviews) and multiclass text classification fine tuning using the [AG News datasets](https://www.tensorflow.org/datasets/catalog/ag_news_subset) from [TensorFlow Datasets](https://www.tensorflow.org/datasets) or a custom dataset (for binary classification). The notebook allows for selecting a BERT encoder (BERT large, BERT base, or small BERT) to use along with a preprocessor from [TF Hub](https://tfhub.dev). The fine tuned model is evaluated and exported as a saved model. | +| [Text Classifier fine tuning with PyTorch & Hugging Face](/notebooks/text_classification/pytorch_text_classification) | NLP: Text Classification | PyTorch |Demonstrates fine tuning [Hugging Face models](https://huggingface.co/models) to do sentiment analysis using the [IMDb movie review dataset from Hugging Face Datasets](https://huggingface.co/datasets/imdb) or a custom dataset with [Intel® Extension for PyTorch*](https://github.com/intel/intel-extension-for-pytorch) | +| [Image Classification with TF Hub](/notebooks/image_classification/tf_image_classification) | CV: Image Classification | TensorFlow | Demonstrates transfer learning with multiple [TF Hub](https://tfhub.dev) image classifiers, TF datasets, and custom image datasets | +| [Image Classification with PyTorch & Torchvision](/notebooks/image_classification/pytorch_image_classification) | CV: Image Classification | PyTorch | Demonstrates transfer learning with multiple [Torchvision](https://pytorch.org/vision/stable/index.html) image classification models, Torchvision datasets, and custom datasets | + +## Transfer Learning Tool End-to-End Pipelines + +| Notebook | Domain: Use Case | Framework| Description | +| ---------| ---------|----------|-------------| +| [Document-Level Sentiment Analysis (SST2) using PyTorch and the Intel® Transfer Learning Tool API](/notebooks/e2e_workflows/Document_Level_Sentiment_Analysis.ipynb) | NLP: Text Classification | PyTorch and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for text classification using a PyTorch model from Hugging Face for a document-level sentiment analysis workflow. | +| [Medical Imaging Classification (Colorectal histology) using TensorFlow and the Intel® Transfer Learning Tool API](/notebooks/e2e_workflows/Medical_Imaging_Classification.ipynb) | CV: Image Classification | TensorFlow and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for image classification using a TensorFlow model for a medical imaging classification application. | +| [Remote Sensing Image Scene Classification (Resisc) using TensorFlow and the Intel® Transfer Learning Tool API](/notebooks/e2e_workflows/Remote_Sensing_Image_Scene_Classification.ipynb) | CV: Image Classification | TensorFlow and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for image classification using a TensorFlow model for a remote sensing image scene classification application. | +| [Multimodal Cancer Detection using TensorFlow, PyTorch, and the Intel® Transfer Learning Tool API](/notebooks/e2e_workflows/Multimodal_Cancer_Detection.ipynb) | CV: Image Classification
NLP: Text Classification | TensorFlow, PyTorch, and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for a late fusion multimodal ensemble application using both NLP and computer vision models from PyTorch and Tensorflow, respectively. | +| [Anomaly Detection with PyTorch using Intel® Transfer Learning Tool](/notebooks/e2e_workflows/Anomaly_Detection_MVTec.ipynb) | CV: Image Anomaly Detection | PyTorch and the Intel Transfer Learning Tool API | Demonstrates how to use the Intel Transfer Learning Tool API to do feature extraction and pca analysis using dedicated function calls for image anomaly detection using a Torchvision model. | + +## Performance Comparison Notebooks + +| Notebook | Domain: Use Case | Framework| Description | +| ---------| -----------------|----------|-------------| +| [Performance Comparison: Image Classification Transfer Learning with TensorFlow and the Intel Transfer Learning Tool](/notebooks/performance/tf_image_classification_performance.ipynb) | CV: Image Classification | TensorFlow and the Intel Transfer Learning Tool API | Compares training and evaluation metrics and performance for image classification transfer learning using TensorFlow libraries and the Intel Transfer Learning Tool. | +| [Performance Comparison: Text Classification Transfer Learning with Hugging Face and the Intel Transfer Learning Tool](/notebooks/performance/hf_text_classification_performance.ipynb) | NLP: Text Classification | Hugging Face, PyTorch, and the Intel Transfer Learning Tool API | Compares training and evaluation metrics for text classification transfer learning using the Hugging Face Trainer and the Intel Transfer Learning Tool. | diff --git a/notebooks/e2e_workflows/Anomaly_Detection_MVTec.ipynb b/notebooks/e2e_workflows/Anomaly_Detection_MVTec.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..186def4bc2204297b7e8735a15dfb8ca1a038356 --- /dev/null +++ b/notebooks/e2e_workflows/Anomaly_Detection_MVTec.ipynb @@ -0,0 +1,603 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3405d28d", + "metadata": {}, + "source": [ + "# Image Anomaly Detection with PyTorch using
Intel® Transfer Learning Tool\n", + "\n", + "This notebook demonstrates anomaly detection using the Intel Transfer Learning Toolkit. It performs defect analysis with the MVTec dataset using PyTorch. The workflow uses a pretrained ResNet50 v1.5 model from torchvision." + ] + }, + { + "cell_type": "markdown", + "id": "1d61b7ac", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions to setup a PyTorch environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0bf9fd0", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import PIL.Image as Image\n", + "import torch, torchvision\n", + "from torchvision.transforms.functional import InterpolationMode\n", + "import requests\n", + "from io import BytesIO\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_and_extract_tar_file, download_file\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "8f1fc78c", + "metadata": {}, + "source": [ + "## 2. Get or load the model\n", + "\n", + "In this step, we use the model factory to get the desired model. The `get_model` function returns a pretrained model object from a public model hub, while the `load_model` function loads a pretrained model from a checkpoint on your local disk or in memory.\n", + "\n", + "Here we are getting the pretrained `resnet50` model from Torchvision:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad4aeafd", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name=\"resnet50\", framework=\"pytorch\", use_case='anomaly_detection')" + ] + }, + { + "cell_type": "markdown", + "id": "9d087ee7", + "metadata": {}, + "source": [ + "To load a previously trained model from a file, use this:\n", + "```\n", + "model = model_factory.load_model(model_name=\"resnet50\", model=, framework=\"pytorch\", \n", + " use_case='anomaly_detection')\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "dabd4183", + "metadata": {}, + "source": [ + "## 3. Get the dataset" + ] + }, + { + "cell_type": "markdown", + "id": "2d314ba0", + "metadata": {}, + "source": [ + "To use [MVTec](https://www.mvtec.com/company/research/datasets/mvtec-ad) or your own image dataset for anomaly detection, your image files (`.jpg` or `.png`) should be arranged in one of two ways. \n", + "\n", + "### Method 1: Category Folders\n", + "\n", + "Arrange them in folders in the root dataset directory like this:\n", + "\n", + "```\n", + "hazelnut\n", + " └── crack\n", + " └── cut\n", + " └── good\n", + " └── hole\n", + " └── print\n", + "```\n", + "\n", + "IMPORTANT: There must be a subfolder named `good` and at least one other folder of defective examples. It does not matter what the names of the other folders are or how many there, as long as there is at least one. This would also be an acceptable Method 1 layout:\n", + "\n", + "```\n", + "toothbrush\n", + " └── defective\n", + " └── good\n", + "```\n", + "\n", + "TLT will encode all of the non-good images as \"bad\" and use the \"good\" images in the training set and a mix of good and bad images in the validation set.\n", + "\n", + "### Method 2: Train & Test Folders with Category Subfolders\n", + "\n", + "Arrange them in folders in the root dataset directory like this:\n", + "\n", + "```\n", + "hazelnut\n", + " └── train\n", + " └── good\n", + " └── test\n", + " └── crack\n", + " └── cut\n", + " └── good\n", + " └── hole\n", + " └── print\n", + "```\n", + "\n", + "When using this layout, TLT will use the exact defined split for train and validation subsets unless you use the `shuffle_split` method to re-shuffle and split up the \"good\" images with certain percentages. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64b24c5b-9b48-4041-a6a2-7c438ca3a0c5", + "metadata": {}, + "outputs": [], + "source": [ + "img_dir = os.path.join(dataset_dir, 'hazelnut')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "357f3dfd", + "metadata": {}, + "outputs": [], + "source": [ + "# Select the subdirectory in dataset_dir to use\n", + "dataset = dataset_factory.load_dataset(img_dir,\n", + " use_case='image_anomaly_detection', \n", + " framework=\"pytorch\")\n", + "\n", + "print(dataset._dataset)\n", + "print(\"Class names:\", str(dataset.class_names))\n", + "print(\"Defect names:\", dataset.defect_names)" + ] + }, + { + "cell_type": "markdown", + "id": "2200ef4e", + "metadata": {}, + "source": [ + "Note: The defects argument can be used to filter the validation set to use only a subset of defect types. For example:\n", + "```\n", + "dataset = dataset_factory.load_dataset(img_dir, \n", + " use_case='image_anomaly_detection', \n", + " framework=\"pytorch\",\n", + " defects=['crack', 'hole'])\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "99f23249", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "Once you have your dataset, use the following cells to split and preprocess the data. We split them into training and test subsets, then resize the images to match the selected model, and then batch the images. Pass in optional arguments to customize the [Resize](https://pytorch.org/vision/main/generated/torchvision.transforms.Resize.html) or [Normalize](https://pytorch.org/vision/main/generated/torchvision.transforms.Normalize.html) transforms.\n", + "Data augmentation can be applied to the training set by specifying the augmentations to be applied in the `add_aug` parameter. Supported augmentations are given below:\n", + "1. hflip - RandomHorizontalFlip\n", + "2. rotate - RandomRotate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd91fbcf", + "metadata": {}, + "outputs": [], + "source": [ + "# If using Method 1 layout, split the dataset into training and test subsets.\n", + "if dataset._validation_type is None:\n", + " dataset.shuffle_split(train_pct=.75, val_pct=0.0, test_pct=0.25)" + ] + }, + { + "cell_type": "markdown", + "id": "4fbe27a3-1b1e-4add-9725-28bceb62c474", + "metadata": {}, + "source": [ + "For __cutpaste__ feature extractor, cutpaste_type can be specified in the dataset.preprocess() method as follows. The option available are - _normal_, _scar_, _3way_ and _union_. Default is _normal_.\n", + "```\n", + "dataset.preprocess(224, batch_size=batch_size, interpolation=InterpolationMode.LANCZOS, cutpaste_type='normal')\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7c95a70", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess with an image size that matches the model, batch size 32, and the desired interpolation method\n", + "batch_size = 64\n", + "cutpaste_type = 'normal'\n", + "dataset.preprocess(224, batch_size=batch_size, interpolation=InterpolationMode.LANCZOS, cutpaste_type=cutpaste_type)" + ] + }, + { + "cell_type": "markdown", + "id": "3704772b", + "metadata": {}, + "source": [ + "## 5. Visualize samples from the dataset\n", + "\n", + "We get a single batch from our training and test subsets and visualize the images as a sanity check." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd6782b0", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_images(images, labels, sup_title, predictions=None):\n", + " plt.figure(figsize=(18,14))\n", + " plt.subplots_adjust(hspace=0.5)\n", + " for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " inp = images[n]\n", + " inp = inp.numpy().transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " inp = std * inp + mean\n", + " inp = np.clip(inp, 0, 1)\n", + " plt.imshow(inp)\n", + " if predictions:\n", + " correct_prediction = labels[n] == predictions[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predictions[n] if correct_prediction else \"{}\".format(predictions[n])\n", + " else:\n", + " good_sample = labels[n] == 'good'\n", + " color = \"darkgreen\" if labels[n] == 'good' else (\"crimson\" if labels[n] == 'bad' else \"black\")\n", + " title = labels[n]\n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + " _ = plt.suptitle(sup_title, fontsize=20)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffcd2071", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot some images from the training set\n", + "images, labels = dataset.get_batch()\n", + "labels = [dataset.class_names[id] for id in labels]\n", + "plot_images(images, labels, 'Training Samples')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d37b808f", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot some images from the test set\n", + "test_images, test_labels = dataset.get_batch(subset='test')\n", + "test_labels = [dataset.class_names[id] for id in test_labels]\n", + "plot_images(test_images, test_labels, 'Test Samples')" + ] + }, + { + "cell_type": "markdown", + "id": "a49ec7b7", + "metadata": {}, + "source": [ + "## 6. Training and Evaluation\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the torchvision feature extractor for the user's desired layer and extract features from the training set. The extracted features are used to perform a [principal component analysis](https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html). The model's evaluate function returns the AUROC metric ([area under](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.auc.html) the [roc curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_curve.html)) calculated from the dataset's test subset." + ] + }, + { + "cell_type": "markdown", + "id": "ab510f51", + "metadata": {}, + "source": [ + "### Train Arguments\n", + "\n", + "#### Required\n", + "- **dataset** (ImageAnomalyDetectionDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory\n", + "\n", + "#### Optional\n", + "- **generate_checkpoints** (bool): Whether to save/preserve the best weights during SimSiam training (default: True)\n", + "- **initial_checkpoints** (str): The path to a starting weights file\n", + "- **layer_name** (str): The layer name whose output is desired for the extracted features\n", + "- **pooling** (str): Pooling to be applied on the extracted layer ('avg' or 'max') (default: 'avg')\n", + "- **kernel_size** (int): Kernel size in the pooling layer (default: 2)\n", + "- **pca_threshold** (float): Threshold to apply to PCA model (default: 0.99)\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd9420d", + "metadata": {}, + "outputs": [], + "source": [ + "# Examine the model's layers and decide which to use for feature extraction\n", + "model.list_layers(verbose=False)\n", + "layer = 'layer3'" + ] + }, + { + "cell_type": "markdown", + "id": "b19be956-e3c6-4d9d-847d-779c1c35da38", + "metadata": {}, + "source": [ + "## Feature Extraction\n", + "There are three feature extractor options available within the `model.train()` function.\n", + "1. __No fine-tuning__ - To use a pretrained ResNet50/ResNet18 model for feature extraction, simply call `model.load_pretrained_model()`\n", + "2. [__SimSiam__](https://arxiv.org/abs/2011.10566) - A self-supervised neural network based on Siamese networks. It learns a meaningful representation of dataset without using any labels. If selected, SimSiam generates quality features that can help differentiate between regular and anomaly images in a given context. SimSiam produces two different augmented images from one underlying image. The end goal is to train the network to produce the same features for both images. It takes a ResNet model as the backbone and fine-tunes the model on the augmented dataset to get a better feature embedding. To use this feature extractor, download the SimSiam weights based on ResNet50 - https://dl.fbaipublicfiles.com/simsiam/models/100ep-256bs/pretrain/checkpoint_0099.pth.tar - set `initial_checkpoints` to the path of the downloaded checkpoints in the `model.train_simsiam()` function.\n", + "3. [__Cut-paste__](https://arxiv.org/abs/2104.04015#) - A self-supervised method for Anomaly Detection and Localization that takes ResNet50/ ResNet18 model as backbone and fine-tune the model on custom dataset to get better feature embedding. data augmentation strategy that cuts an image patch and pastes at a random location of a large image. To use this feature extractor, call `model.train_cutpaste()` function.\n", + "\n", + "\n", + "### Optional: The SimSiam TwoCropTransform\n", + "To train a Simsiam model, it is required to apply a TwoCropTransform augmentation technique on the dataset used for training. You can preview this augmentation on a sample batch after preprocessing by using `get_batch(simsiam=True)` and then use them for simsiam training in `model.train_simsiam()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b49522f", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a batch of training data with the simsiam transform applied to it\n", + "simsiam_images, _ = dataset.get_batch(simsiam=True)\n", + "\n", + "# Plot the \"A\" samples showing the first set of augmented images\n", + "plot_images(simsiam_images[0], ['{}A'.format(i) for i in range(batch_size)], 'SimSiam \"A\" Samples')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5da06df", + "metadata": {}, + "outputs": [], + "source": [ + "# Now plot the \"B\" samples showing the second set of augmented images based on the same underlying originals\n", + "plot_images(simsiam_images[1], ['{}B'.format(i) for i in range(batch_size)], 'SimSiam \"B\" Samples')" + ] + }, + { + "cell_type": "markdown", + "id": "ace7d296-74d9-47c1-aeaf-386433bac411", + "metadata": {}, + "source": [ + "### Optional: The Cut-paste Transforms\n", + "To train a model with Cut-paste , it is required to apply one of the four augmentations - __CutPasteNormal, CutPasteScar, CutPaste3Way, CutPasteUnion__ on the dataset used for training. You can preview this augmentation on a sample batch after preprocessing by using `get_batch(cutpaste=True)` and then use them for cutpaste training in `model.train_cutpaste()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21cbadd5-8387-4130-b5b4-e016d4ea4e5e", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a batch of training data with the cutpaste transform applied to it\n", + "cutpaste_images, _ = dataset.get_batch(cutpaste=True)\n", + "\n", + "# Plot the \"A\" samples showing the first set of augmented images\n", + "plot_images(cutpaste_images[1], ['{}A'.format(i) for i in range(batch_size)], 'CutPaste \"A\" Samples')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "750bc599-80e4-4e70-8aaf-5f63082b9198", + "metadata": {}, + "outputs": [], + "source": [ + "if cutpaste_type == '3way':\n", + " # Now plot the \"B\" samples showing the second set of augmented images based on the same underlying originals\n", + " plot_images(cutpaste_images[2], ['{}B'.format(i) for i in range(batch_size)], 'CutPaste \"B\" Samples')" + ] + }, + { + "cell_type": "markdown", + "id": "73ecdb31-4105-40fa-a1b1-89c1a9b08108", + "metadata": {}, + "source": [ + "To use a ResNet50 model for feature extraction, run the below command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5663a407-44d7-447d-aa75-a05ae8355716", + "metadata": {}, + "outputs": [], + "source": [ + "extract_model = model.load_pretrained_model()" + ] + }, + { + "cell_type": "markdown", + "id": "275002cd-4708-4b19-a0bc-3e4a74af9ee2", + "metadata": {}, + "source": [ + "There is no fine-tuning being demonstrated here, but you can use `simsiam` or `cutpaste` if desired.\n", + "\n", + "To use simsiam, pass the checkpoint file in `initial_checkpoints` to `model.train_simsiam()` as follows\n", + "```\n", + "components = model.train_simsiam(dataset, output_dir, epochs=2, feature_dim=1000,\n", + " pred_dim=250, initial_checkpoints=,\n", + " generate_checkpoints=False, precision='float32')\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "392f7546-0ae5-4b6b-bba7-fbcf6220d0f0", + "metadata": {}, + "outputs": [], + "source": [ + "extract_model = model.train_simsiam(dataset, output_dir, epochs=2, feature_dim=1000, \n", + " pred_dim=250, initial_checkpoints=None,\n", + " generate_checkpoints=False, precision='float32')" + ] + }, + { + "cell_type": "markdown", + "id": "bd5c4fb9-0f81-4753-b990-c9a721dc95e0", + "metadata": {}, + "source": [ + "To use cutpaste, run `model.train_cutpaste` as given below. Optionally, to load a pretrained checkpoint pass the checkpoint file in `initial_checkpoints` to `model.train_cutpaste()` as follows.\n", + "```\n", + "components = model.train_cutpaste(dataset, output_dir, optim='sgd', epochs=2, freeze_resnet=20,\n", + " head_layer=2, cutpaste_type='normal', initial_checkpoints=,\n", + " generate_checkpoints=False, precision='float32')\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b34e029-5235-4593-ab0b-12d27e672461", + "metadata": {}, + "outputs": [], + "source": [ + "extract_model = model.train_cutpaste(dataset, output_dir, optim='sgd', epochs=2, freeze_resnet=20, head_layer=2, cutpaste_type='normal',\n", + " generate_checkpoints=False, precision='float32')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30fc18dd-7d78-4b77-b29b-8895e249cd34", + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "from tlt.models.image_anomaly_detection.pytorch_image_anomaly_detection_model import extract_features, pca, get_feature_extraction_model\n", + "\n", + "layer_name = layer\n", + "pool = 'avg'\n", + "kernel_size = 2\n", + "dataset._dataset.transform = dataset._train_transform\n", + "images, labels = dataset.get_batch()\n", + "extract_model = get_feature_extraction_model(extract_model, layer_name)\n", + "outputs_inner = extract_features(extract_model, images.to('cpu'), layer_name,\n", + " pooling=[pool, kernel_size])\n", + "data_mats_orig = torch.empty((outputs_inner.shape[1], len(dataset.train_subset))).to('cpu')\n", + "\n", + "# Feature extraction\n", + "with torch.no_grad():\n", + " data_idx = 0\n", + " num_ims = 0\n", + " for images, labels in tqdm(dataset._train_loader):\n", + " images, labels = images.to('cpu'), labels.to('cpu')\n", + " num_samples = len(labels)\n", + " outputs = extract_features(extract_model, images, layer_name, pooling=[pool, kernel_size])\n", + " oi = torch.squeeze(outputs)\n", + " data_mats_orig[:, data_idx:data_idx + num_samples] = oi.transpose(1, 0)\n", + " num_ims += 1\n", + " data_idx += num_samples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "030f5e0e-b7db-41e8-9a13-bcd16d9457c2", + "metadata": {}, + "outputs": [], + "source": [ + "# PCA\n", + "pca_threshold = 0.99\n", + "_pca_mats = pca(data_mats_orig, pca_threshold)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f60192d", + "metadata": {}, + "outputs": [], + "source": [ + "threshold, auroc = model.evaluate(dataset, _pca_mats, use_test_set=True)" + ] + }, + { + "cell_type": "markdown", + "id": "0a877f33", + "metadata": {}, + "source": [ + "## 7. Export" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abc054ff", + "metadata": {}, + "outputs": [], + "source": [ + "model.export(os.path.join(output_dir, 'anomaly'))" + ] + }, + { + "cell_type": "markdown", + "id": "0947915a", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "Paul Bergmann, Kilian Batzner, Michael Fauser, David Sattlegger, Carsten Steger: The MVTec Anomaly Detection Dataset: A Comprehensive Real-World Dataset for Unsupervised Anomaly Detection; in: International Journal of Computer Vision 129(4):1038-1059, 2021, DOI: 10.1007/s11263-020-01400-4.\n", + "\n", + "Paul Bergmann, Michael Fauser, David Sattlegger, Carsten Steger: MVTec AD — A Comprehensive Real-World Dataset for Unsupervised Anomaly Detection; in: IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 9584-9592, 2019, DOI: 10.1109/CVPR.2019.00982." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/e2e_workflows/Document_Level_Sentiment_Analysis.ipynb b/notebooks/e2e_workflows/Document_Level_Sentiment_Analysis.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..d3753e8bfa7e886e8724b6346c2c9e011f63a522 --- /dev/null +++ b/notebooks/e2e_workflows/Document_Level_Sentiment_Analysis.ipynb @@ -0,0 +1,362 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "ac1059eb", + "metadata": {}, + "source": [ + "# Document-Level Sentiment Analysis using
PyTorch and the Intel® Transfer Learning Tool API\n", + "\n", + "This notebook uses the Intel® Transfer Learning Tool to fine-tune a HuggingFace pretrained BERT model for text classification. While this notebook runs on a single node, this workload can also be run in a multinode setting using the TLT CLI. Consult the project documentation and examples to run it using PyTorch distributed training." + ] + }, + { + "cell_type": "markdown", + "id": "0bb70464", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions to setup a Pytorch environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20ab9972", + "metadata": {}, + "outputs": [], + "source": [ + "import intel_extension_for_pytorch as ipex\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_and_extract_zip_file\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "47787deb", + "metadata": {}, + "source": [ + "## 2. Get the model\n", + "\n", + "In this step, we call the Intel Transfer Learning Tool model factory to list supported Huggingface text classification models. This is a list of pretrained models from Huggingface that we tested with our API. Optionally, the `verbose=True` argument can be added to the `print_supported_models()` function call to get more information about each model (such as the links to Huggingface, the original dataset, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52a4af60", + "metadata": {}, + "outputs": [], + "source": [ + "# See a list of available text classification models\n", + "model_factory.print_supported_models(use_case='text_classification', framework='pytorch')" + ] + }, + { + "cell_type": "markdown", + "id": "7293733f", + "metadata": {}, + "source": [ + "Use the TLT model factory to get one of the models listed in the previous cell. The `get_model` function returns a model object that will later be used for training. For this example, we will use bert-large-uncased." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "050d7b0a", + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"bert-large-uncased\"\n", + "framework = \"pytorch\"\n", + "\n", + "model = model_factory.get_model(model_name, framework, num_classes=2)\n", + "\n", + "print(\"Model name:\", model.model_name)\n", + "print(\"Framework:\", model.framework)\n", + "print(\"Use case:\", model.use_case)" + ] + }, + { + "cell_type": "markdown", + "id": "37bf5a93", + "metadata": {}, + "source": [ + "## 3. Get the dataset" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c833ce65", + "metadata": {}, + "source": [ + "### Option A: Use the Hugging Face catalog\n", + "\n", + "Here we are using the dataset in the [Hugging Face datasets catalog](https://huggingface.co/datasets)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf29cc7e", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_name = \"sst2\"\n", + "dataset = dataset_factory.get_dataset(dataset_dir, model.use_case, model.framework, dataset_name,\n", + " dataset_catalog=\"huggingface\", shuffle_files=True, \n", + " split=['train', 'validation'])\n", + "\n", + "print(dataset.info)\n", + "print(\"\\nClass names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "28504679", + "metadata": {}, + "source": [ + "Skip to the next step [4. Preprocess the dataset](#4.-Preprocess-the-dataset) to continue using your own dataset." + ] + }, + { + "cell_type": "markdown", + "id": "6867f79e", + "metadata": {}, + "source": [ + "### Option B: Download the SST2 dataset\n", + "Option B explicitly downloads the `SST-2.zip` file and extracts a `.tsv` file of training data that is tab separated. The dataset factory expects custom text classification input files to have at least two columns where one is the label and the second column is the text/sentence to classify.\n", + "\n", + "For example, the header and first three rows of the file should look similar to this:\n", + "```\n", + "sentence\tlabel\n", + "hide new secretions from the parental units \t0\n", + "contains no wit , only labored gags \t0\n", + "that loves its characters and communicates something rather beautiful about human nature \t1\n", + "```\n", + "\n", + "When using your own dataset, update the path to your dataset directory, as well the other variables with properties about the dataset like the .csv (or .tsv) file name, class names, delimiter, header, and the map function (if string labels need to be translated into numerical values)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41edc8fc", + "metadata": {}, + "outputs": [], + "source": [ + "# Modify the variables below to use a different dataset or a csv file on your local system.\n", + "dataset_url = \"https://dl.fbaipublicfiles.com/glue/data/SST-2.zip\"\n", + "csv_name = \"train.tsv\"\n", + "delimiter = \"\\t\"\n", + "dataset_subdir = os.path.join(dataset_dir, 'SST-2')\n", + "# If we don't already have the csv file, download and extract the zip file to get it.\n", + "if not os.path.exists(os.path.join(dataset_subdir, csv_name)):\n", + " download_and_extract_zip_file(dataset_url, dataset_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94e348c2", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset_factory.load_dataset(dataset_dir=dataset_subdir, \n", + " use_case=\"text_classification\",\n", + " framework=\"pytorch\", csv_file_name=csv_name,\n", + " column_names=[\"sentence\", \"label\"], \n", + " delimiter=delimiter, header=True, label_col=1)\n", + "\n", + "print(dataset.info)\n", + "print(\"\\nClass names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e0771e4", + "metadata": {}, + "outputs": [], + "source": [ + "# Create splits for training and validation\n", + "dataset.shuffle_split(train_pct=0.75, val_pct=0.25)" + ] + }, + { + "cell_type": "markdown", + "id": "539d53b7", + "metadata": {}, + "source": [ + "## 4. Preprocess the dataset\n", + "\n", + "Once you have your dataset from Option A or Option B above, use the following cell to preprocess the dataset. The dataset subsets are tokenized and then batched." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "587d1d9e", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "dataset.preprocess(model_name, batch_size=32, max_length=55)" + ] + }, + { + "cell_type": "markdown", + "id": "352eda54", + "metadata": {}, + "source": [ + "## 5. Fine tuning\n", + "\n", + "The TLT model's train function is called with the dataset that was just prepared, along with an output directory for checkpoints, and the number of training epochs.\n", + "\n", + "With the do_eval paramter set to True by default, this step will also show how the model can be evaluated. The model's evaluate function returns a list of metrics calculated from the dataset's validation subset." + ] + }, + { + "cell_type": "markdown", + "id": "492ee811", + "metadata": {}, + "source": [ + "### Arguments\n", + "\n", + "#### Required\n", + "- **dataset** (ImageClassificationDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory for checkpoint files\n", + "- **epochs** (int): Number of epochs to train the model (default: 1)\n", + "\n", + "#### Optional\n", + "- **initial_checkpoints** (str): Path to checkpoint weights to load. If the path provided is a directory, the latest checkpoint will be used.\n", + "- **ipex_optimize** (bool): Optimize the model using Intel® Extension for PyTorch (default: True)\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "955a4a7e", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.train(dataset, output_dir, epochs=1, ipex_optimize=True)" + ] + }, + { + "cell_type": "markdown", + "id": "7e08a1c9", + "metadata": {}, + "source": [ + "## 6. Predict\n", + "\n", + "The model's predict function can be called with a sentence." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f3cbd35", + "metadata": {}, + "outputs": [], + "source": [ + "result = model.predict(\"Terrible movie\")\n", + "\n", + "print(\"Predicted score:\", float(result))\n", + "print(\"Predicted label:\", dataset.get_str_label(float(result)))" + ] + }, + { + "cell_type": "markdown", + "id": "64ada826", + "metadata": {}, + "source": [ + "## 7. Export the saved model\n", + "\n", + "Lastly, we can call the model export function to generate a saved_model.pb. Each time the model is exported, a new numbered directory is created, which allows serving to pick up the latest model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3981b2f5", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "3d0ed367", + "metadata": {}, + "source": [ + "## Citation\n", + "\n", + "```\n", + "@inproceedings{socher-etal-2013-recursive,\n", + " title = \"Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank\",\n", + " author = \"Socher, Richard and\n", + " Perelygin, Alex and\n", + " Wu, Jean and\n", + " Chuang, Jason and\n", + " Manning, Christopher D. and\n", + " Ng, Andrew and\n", + " Potts, Christopher\",\n", + " booktitle = \"Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing\",\n", + " month = oct,\n", + " year = \"2013\",\n", + " address = \"Seattle, Washington, USA\",\n", + " publisher = \"Association for Computational Linguistics\",\n", + " url = \"https://www.aclweb.org/anthology/D13-1170\",\n", + " pages = \"1631--1642\",\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/e2e_workflows/Medical_Imaging_Classification.ipynb b/notebooks/e2e_workflows/Medical_Imaging_Classification.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3045a212f73cb059f98dd25ca264985708981a03 --- /dev/null +++ b/notebooks/e2e_workflows/Medical_Imaging_Classification.ipynb @@ -0,0 +1,413 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fcfb563d-3f9c-4731-be1f-9c9c4b2c9dfd", + "metadata": { + "tags": [] + }, + "source": [ + "# Medical Imaging Classification (Colorectal histology) using TensorFlow and the Intel® Transfer Learning Tool API" + ] + }, + { + "cell_type": "markdown", + "id": "353f2782-043f-4afd-9858-cc773275e6c5", + "metadata": {}, + "source": [ + "This notebook facilitates implementation of medical imaging classification using Transfer Learning Toolkit. It performs Multi-class texture analysis in colorectal cancer histology dataset. The workflow uses pretrained SOTA models ( RESNET V1.5) from TF hub and transfers the knowledge from a pretrained domain to a different custom domain achieving required accuracy." + ] + }, + { + "cell_type": "markdown", + "id": "8325300b-4a75-42fb-aa14-2b089b4edea5", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04e3f113-4887-49c9-aa8b-a3c2058157a4", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import os\n", + "import pickle\n", + "import tensorflow as tf\n", + "from sklearn.metrics import classification_report\n", + "\n", + "#tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.types import FrameworkType, UseCaseType\n", + "\n", + "from notebooks.plot_utils import plot_curves\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "a47ea534-0661-4cbe-97ed-ca288b6203e5", + "metadata": {}, + "source": [ + "## 2. Get the model" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c9548167-cdfd-44ad-be60-cc3d131850a3", + "metadata": {}, + "source": [ + "In this step, we call the Intel Transfer Learning Tool model factory to list supported TensorFlow image classification models. This is a list of pretrained models from TFHub that we tested with our API. Optionally, the verbose=True argument can be added to the print_supported_models function call to get more information about each model (such as the link to TFHub, image size, the original dataset, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bae74045-f024-498f-9ea7-3ee8ac2ea5d3", + "metadata": {}, + "outputs": [], + "source": [ + "# See a list of available models\n", + "model_factory.print_supported_models(use_case='image_classification', framework='tensorflow')" + ] + }, + { + "cell_type": "markdown", + "id": "3c06b359-1054-45c5-9223-47d4e1b7772d", + "metadata": {}, + "source": [ + "#### Option A: Load a model\n", + "\n", + "Next, use the model factory to get one of the models listed in the previous cell. The `get_model` function returns a model object that will later be used for training. By default, resnet_v1_50 is used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf842217-d992-4b6f-99bf-aef475fa13dd", + "metadata": {}, + "outputs": [], + "source": [ + "# Get the model\n", + "model = model_factory.get_model(model_name=\"resnet_v1_50\", framework=\"tensorflow\")" + ] + }, + { + "cell_type": "markdown", + "id": "afa83899-0a99-4c9e-9e3d-00b040fe3e1f", + "metadata": {}, + "source": [ + "#### Option B: Load a pretrained checkpoint\n", + "\n", + "Optionally, to continue training using a pretrained checkpoint, the user can specify the path to folder containing __saved_model.pb__. The user can specify the path in __model__ parameter.\n", + "\n", + "_Note: The path is same as saved_model_dir_" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b49423b-e57d-478c-87b0-7daec6401b76", + "metadata": {}, + "outputs": [], + "source": [ + "#Load a pretrained checkpoint\n", + "model = model_factory.load_model(model_name='resnet_v1_50', \n", + " model='/home/intel/output/resnet_v1_50/1', \n", + " framework='tensorflow', use_case='image_classification')" + ] + }, + { + "cell_type": "markdown", + "id": "ba2e4d10-458b-4479-8d1f-8cc67304d5c2", + "metadata": {}, + "source": [ + "## 3. Get the dataset\n", + "Use dataset __colorectal_histology__ from the TensorFlow Datasets catalog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cddcfdcf-ba16-4faa-a708-61f6aa01b3b0", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset_factory.get_dataset(dataset_dir=dataset_dir,\n", + " use_case='image_classification', \n", + " framework='tensorflow',\n", + " dataset_name='colorectal_histology',\n", + " dataset_catalog='tf_datasets')\n", + "\n", + "print(dataset.info)\n", + "\n", + "print(\"\\nClass names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "8ae3d375-554c-487b-8907-e895396d20e7", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "\n", + "Once you have your dataset from Option A or Option B above, use the following cells to preprocess the dataset. We resize the images to match the selected models and batch the images, then split them into training and validation subsets. Data augmentation can be applied by specifying the augmentations to be applied in __add_aug__ parameter. Supported augmentations are \n", + "1. hvflip - RandomHorizontalandVerticalFlip\n", + "2. hflip - RandomHorizontalFlip\n", + "3. vflip - RandomVerticalFlip\n", + "4. rotate - RandomRotate\n", + "5. zoom - RandomZoom" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00d529b1-fe40-482a-b3f0-3875b13ba534", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess the dataset with an image size that matches the model and a batch size of 32\n", + "batch_size = 32\n", + "dataset.preprocess(model.image_size, batch_size=batch_size, add_aug=['hvflip', 'rotate'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c4af8c6-c418-45d1-b0bc-dd8c7e2625c9", + "metadata": {}, + "outputs": [], + "source": [ + "# Split the dataset into training, validation and test subsets\n", + "dataset.shuffle_split(train_pct=.80, val_pct=.10, test_pct=0.10)" + ] + }, + { + "cell_type": "markdown", + "id": "ab02c7d0-5b63-4be3-b6a6-cdfc03e48995", + "metadata": {}, + "source": [ + "## 5. Transfer Learning" + ] + }, + { + "cell_type": "markdown", + "id": "feeb23c9-66a0-4670-8fd0-f7950dd2832e", + "metadata": {}, + "source": [ + "This step calls the model's train function with the dataset that was just prepared. The training function will get the TFHub feature vector and add on a dense layer based on the number of classes in the dataset. The model is then compiled and trained based on the number of epochs specified in the argument. With the do_eval paramter set to True by default, this step will also show how the model can be evaluated and will return a list of metrics calculated from the dataset's validation subset.\n", + "### Arguments\n", + "#### Required\n", + "- **dataset** (ImageClassificationDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory for checkpoint files\n", + "- **epochs** (int): Number of epochs to train the model (default: 1)\n", + "#### Optional\n", + "- **initial_checkpoints** (str): Path to checkpoint weights to load. If the path provided is a directory, the latest checkpoint will be used.\n", + "- **early_stopping** (bool): Enable early stopping if convergence is reached while training at the end of each epoch. (default: False)\n", + "- **lr_decay** (bool): If lr_decay is True and do_eval is True, learning rate decay on the validation loss is applied at the end of each epoch.\n", + "- **enable_auto_mixed_precision** (bool or None): Enable auto mixed precision for training. Mixed precision uses both 16-bit and 32-bit floating point types to make training run faster and use less memory. It is recommended to enable auto mixed precision training when running on platforms that support bfloat16 (Intel third or fourth generation Xeon processors). If it is enabled on a platform that does not support bfloat16, it can be detrimental to the training performance. If enable_auto_mixed_precision is set to None, auto mixed precision will be automatically enabled when running with Intel fourth generation Xeon processors, and disabled for other platforms.\n", + "- **extra_layers** (list[int]): Optionally insert additional dense layers between the base model and output layer. This can help increase accuracy when fine-tuning a TFHub model. The input should be a list of integers representing the number and size of the layers, for example [1024, 512] will insert two dense layers, the first with 1024 neurons and the second with 512 neurons.\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa477dbb-58a9-4e06-9933-047699c35797", + "metadata": {}, + "outputs": [], + "source": [ + "# Mixed precision uses both 16-bit and 32-bit floating point types to make training run faster and use less memory.\n", + "# It is recommended to enable auto mixed precision training when running on platforms that support\n", + "# bfloat16 (Intel third or fourth generation Xeon processors). If it is enabled on a platform that\n", + "# does not support bfloat16, it can be detrimental to the training performance.\n", + "# If enable_auto_mixed_precision is set to None, auto mixed precision will be automatically enabled when\n", + "# running with Intel fourth generation Xeon processors, and disabled for other platforms.\n", + "enable_auto_mixed_precision = None\n", + "\n", + "# Train the model using the dataset\n", + "history = model.train(dataset, output_dir=output_dir, epochs=50, \n", + " enable_auto_mixed_precision=None, early_stopping=True)" + ] + }, + { + "cell_type": "markdown", + "id": "40e11437-5666-49c0-9547-22e7696c0ea0", + "metadata": {}, + "source": [ + "## 6. Evaluate" + ] + }, + { + "cell_type": "markdown", + "id": "1d413b5d-ed24-4756-928d-dbb8e7bf1af2", + "metadata": {}, + "source": [ + "The next step shows how the model can be evaluated. The model's evaluate function returns a list of metrics calculated from the dataset's validation subset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bfc8e8-f8c6-4b56-bfc8-51b88288fcdd", + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate model on validation subset\n", + "val_loss, val_acc = model.evaluate(dataset)\n", + "print(\"Validation Accuracy :\", val_acc)\n", + "print(\"Validation Loss :\", val_loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c478191f-2c0c-49d6-a295-0a3acfd183a2", + "metadata": {}, + "outputs": [], + "source": [ + "plot_curves(history, os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name)))\n", + "pickle.dump(history, open(os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name), 'hist.pkl'), 'wb'))" + ] + }, + { + "cell_type": "markdown", + "id": "3c6fd051-97af-4142-b039-482cb742d988", + "metadata": {}, + "source": [ + "## 7. Export" + ] + }, + { + "cell_type": "markdown", + "id": "9225921d-5340-4865-ad4b-a2c19a5210ed", + "metadata": {}, + "source": [ + "Next, we can call the model export function to generate a saved_model.pb. The model is saved in a format that is ready to use with TensorFlow Serving. Each time the model is exported, a new numbered directory is created, which allows serving to pick up the latest model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c4fc5f8-cb72-4ab5-989b-43d7ad315e8b", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "5d79f06c-39cf-4c05-93bf-b35377ec06b1", + "metadata": {}, + "source": [ + "## 8. Inference\n", + "To perform only Inference using a saved model, follow the steps below\n", + "1. Execute Step 2(b) to load a pretrained checkpoint with the appropriate model name.\n", + "2. Execute Steps 3 and 4 to load and prepare the dataset.\n", + "3. Continue with the steps below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abb2182c-6302-474b-abab-7cf16c6b0966", + "metadata": {}, + "outputs": [], + "source": [ + "history = pickle.load(open(os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name), 'hist.pkl'), 'rb'))\n", + "plot_curves(history, os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5da7be1-76d9-4554-ace6-7bf6ef0f8c21", + "metadata": {}, + "outputs": [], + "source": [ + "loss, accuracy = model.evaluate(dataset, use_test_set=True)\n", + "print('Test accuracy :', accuracy)" + ] + }, + { + "cell_type": "markdown", + "id": "eb585ceb-d120-4ded-a3b2-e78a5bd3dd7b", + "metadata": {}, + "source": [ + "We get the test subset from our dataset, and use that to call predict on our model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e1e0450-f6b7-4abd-990c-b65b8d77e599", + "metadata": {}, + "outputs": [], + "source": [ + "actual_labels = np.concatenate([y for x, y in dataset._test_subset], axis=0)\n", + "predicted_labels = model.predict(dataset._test_subset)\n", + "report = classification_report(actual_labels, predicted_labels)\n", + "print(\"Classification report\")\n", + "print(report)" + ] + }, + { + "cell_type": "markdown", + "id": "7863e336-7996-4518-9ada-2b7fdd9672b5", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "@article{kather2016multi,
\n", + " title={Multi-class texture analysis in colorectal cancer histology},
\n", + " author={Kather, Jakob Nikolas and Weis, Cleo-Aron and Bianconi, Francesco and Melchers, Susanne M and Schad, Lothar R and Gaiser, Timo and Marx, Alexander and Z{\"o}llner, Frank Gerrit},
\n", + " journal={Scientific reports},
\n", + " volume={6},
\n", + " pages={27988},
\n", + " year={2016},
\n", + " publisher={Nature Publishing Group}
\n", + " }\n", + " \n", + "Kather, J. N., Zöllner, F. G., Bianconi, F., Melchers, S. M., Schad, L. R., Gaiser, T., Marx, A., & Weis, C.-A. (2016). Collection of textures in colorectal cancer histology [Data set]. Zenodo. https://doi.org/10.5281/zenodo.53169" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/e2e_workflows/Multimodal_Cancer_Detection.ipynb b/notebooks/e2e_workflows/Multimodal_Cancer_Detection.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f07eee98b5e3540917ccf914416460778ee40b5e --- /dev/null +++ b/notebooks/e2e_workflows/Multimodal_Cancer_Detection.ipynb @@ -0,0 +1,511 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2e3e807d", + "metadata": {}, + "source": [ + "# Multimodal Cancer Detection using the Intel® Transfer Learning Tool API\n", + "\n", + "This application is a multimodal solution for predicting cancer diagnosis using categorized contrast enhanced mammography data and radiology notes. It trains two models - one for image classification and the other for text classification - which can be combined into an ensemble classifier.\n", + "\n", + "## Import Dependencies and Setup Directories" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a722a7", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "import torch\n", + "\n", + "from transformers import EvalPrediction, TrainingArguments\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "\n", + "# Specify the root directory where the images and annotations are located\n", + "dataset_dir = os.path.join(os.environ[\"DATASET_DIR\"]) if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "bb53162b", + "metadata": {}, + "source": [ + "## Dataset\n", + "\n", + "Download the images and radiology annotations from https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=109379611\n", + "\n", + "Image files should have the .jpg extension and be arranged in subfolders for each class. The annotation file should be a .csv. The data directory should look something like this:\n", + "\n", + "```\n", + "brca\n", + " ├── annotation\n", + " │ └── annotation.csv\n", + " └── vision_images\n", + " ├── Benign\n", + " │ ├── P100_L_CM_CC.jpg\n", + " │ ├── P100_L_CM_MLO.jpg\n", + " │ └── ...\n", + " ├── Malignant\n", + " │ ├── P102_R_CM_CC.jpg\n", + " │ ├── P102_R_CM_MLO.jpg\n", + " │ └── ...\n", + " └── Normal\n", + " ├── P100_R_CM_CC.jpg\n", + " ├── P100_R_CM_MLO.jpg\n", + " └── ...\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd9c3ef2", + "metadata": {}, + "outputs": [], + "source": [ + "# User input needed - supply the path to the images in the dataset_dir according to your system\n", + "source_image_path = os.path.join(dataset_dir, 'brca', 'vision_images')\n", + "image_path = source_image_path\n", + "\n", + "# User input needed - supply the path and name of the annotation file in the dataset_dir\n", + "source_annotation_path = os.path.join(dataset_dir, 'brca', 'annotation', 'annotation.csv')\n", + "annotation_path = source_annotation_path\n", + "label_col = 3 # Index of the label column in the data file" + ] + }, + { + "cell_type": "markdown", + "id": "245df47c", + "metadata": {}, + "source": [ + "### Optional: Group Data by Patient ID\n", + "\n", + "This section is not required to run the workload, but it is helpful to assign all of a subject's records to be entirely in the train set or test set. This section will do a random stratification based on patient ID and save new copies of the grouped data files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44dbd990", + "metadata": {}, + "outputs": [], + "source": [ + "from data_utils import split_images, split_annotation\n", + "\n", + "grouped_image_path = '{}_grouped'.format(source_image_path)\n", + "\n", + "if os.path.isdir(grouped_image_path):\n", + " print(\"Grouped directory already exists and will be used: {}\".format(grouped_image_path))\n", + "else:\n", + " split_images(source_image_path, grouped_image_path)\n", + "\n", + "image_path = grouped_image_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d21bdff", + "metadata": {}, + "outputs": [], + "source": [ + "file_dir, file_name = os.path.split(source_annotation_path)\n", + "grouped_annotation_path = os.path.join(file_dir, '{}_grouped.csv'.format(os.path.splitext(file_name)[0]))\n", + "\n", + "if os.path.isfile(grouped_annotation_path):\n", + " print(\"Grouped annotation already exists and will be used: {}\".format(grouped_annotation_path))\n", + "else:\n", + " train_dataset = split_annotation(file_dir, file_name, image_path)\n", + " train_dataset.to_csv(grouped_annotation_path)\n", + " print('Grouped annotation saved to: {}'.format(grouped_annotation_path))\n", + "\n", + "annotation_path = grouped_annotation_path\n", + "label_col = 1 # Index of the label column in the grouped data file" + ] + }, + { + "cell_type": "markdown", + "id": "01e9e5cf", + "metadata": {}, + "source": [ + "## Model 1: Image Classification with TensorFlow\n", + "\n", + "### Get the Model and Dataset\n", + "Call the model factory to get a pretrained model from TensorFlow Hub and the dataset factory to load the images from their location. The `get_model` function returns a model object that will later be used for training. We will use resnet_v1_50 by default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9c93b18", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name=\"resnet_v1_50\", framework='tensorflow')\n", + "\n", + "# Load the dataset from the custom dataset path\n", + "dataset = dataset_factory.load_dataset(dataset_dir=image_path,\n", + " use_case='image_classification',\n", + " framework='tensorflow')\n", + "\n", + "print(\"Class names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "6472bedd", + "metadata": {}, + "source": [ + "### Data Preparation\n", + "Once you have your dataset loaded, use the following cell to preprocess the dataset. We split the images into training and validation subsets, resize them to match the model, and then batch the images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98dcf057", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 16\n", + "if 'grouped' not in image_path:\n", + " # Split if not pre-defined\n", + " dataset.shuffle_split(train_pct=.80, val_pct=0.0, test_pct=0.2)\n", + "dataset.preprocess(model.image_size, batch_size=batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "f2f49c77", + "metadata": {}, + "source": [ + "### Transfer Learning\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the TFHub feature vector and add on a dense layer based on the number of classes in the dataset. The model is then compiled and trained based on the number of epochs specified in the argument. We also add two more dense layers using the `extra_layers` parameter.\n", + "\n", + "To optionally insert additional dense layers between the base model and output layer, `extra_layers=[1024, 512]` will insert two dense layers, the first with 1024 neurons and the second with 512 neurons." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21a92e4e", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.train(dataset, output_dir=output_dir, epochs=5, seed=10, extra_layers=[1024, 512], do_eval=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45289d48", + "metadata": {}, + "outputs": [], + "source": [ + "metrics = model.evaluate(dataset, use_test_set=True)\n", + "for metric_name, metric_value in zip(model._model.metrics_names, metrics):\n", + " print(\"{}: {}\".format(metric_name, metric_value))" + ] + }, + { + "cell_type": "markdown", + "id": "bce6bafe", + "metadata": {}, + "source": [ + "### Save the Computer Vision Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "093905b2", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "5621b571", + "metadata": {}, + "source": [ + "## Model 2: Text Classification with PyTorch\n", + "\n", + "### Get the Model and Dataset\n", + "Now we will call the model factory to get a pretrained model from Hugging Face and load the annotation file using the dataset factory. We will use clinical-bert for this part." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d18cebff", + "metadata": {}, + "outputs": [], + "source": [ + "# Set up NLP parameters\n", + "model_name = 'clinical-bert'\n", + "seq_length = 64\n", + "batch_size = 5\n", + "quantization_criterion = 0.05\n", + "quantization_max_trial = 50\n", + "epochs = 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d939924f", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name=model_name, framework='pytorch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e9dff00", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a label map function and reverse label map for the dataset\n", + "def label_map_func(label):\n", + " if label == 'Benign':\n", + " return 0\n", + " elif label == 'Malignant':\n", + " return 1\n", + " elif label == 'Normal':\n", + " return 2\n", + " \n", + "reverse_label_map = {0: 'Benign', 1: 'Malignant', 2: 'Normal'}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "879bad74", + "metadata": {}, + "outputs": [], + "source": [ + "file_dir, file_name = os.path.split(annotation_path)\n", + "dataset = dataset_factory.load_dataset(dataset_dir=file_dir,\n", + " use_case='text_classification',\n", + " framework='pytorch',\n", + " dataset_name='brca',\n", + " csv_file_name=file_name,\n", + " label_map_func=label_map_func,\n", + " class_names=['Benign', 'Malignant', 'Normal'],\n", + " header=True,\n", + " label_col=label_col,\n", + " shuffle_files=True)" + ] + }, + { + "cell_type": "markdown", + "id": "e2b9ddba", + "metadata": {}, + "source": [ + "### Data Preparation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b166b757", + "metadata": {}, + "outputs": [], + "source": [ + "dataset.preprocess(model.hub_name, batch_size=batch_size, max_length=seq_length)\n", + "dataset.shuffle_split(train_pct=0.67, val_pct=0.33)" + ] + }, + { + "cell_type": "markdown", + "id": "020303ee", + "metadata": {}, + "source": [ + "### Transfer Learning\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the pretrained model from HuggingFace and add on a dense layer based on the number of classes in the dataset. The model is then trained using an instance of Hugging Face Trainer for the number of epochs specified. If desired, a native PyTorch loop can be invoked instead of Trainer by setting `use_trainer=False`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41fb0612", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.train(dataset, output_dir, epochs=epochs, use_trainer=True)" + ] + }, + { + "cell_type": "markdown", + "id": "de70a029", + "metadata": {}, + "source": [ + "### Save the NLP Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba08847d", + "metadata": {}, + "outputs": [], + "source": [ + "model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "45752dd6", + "metadata": {}, + "source": [ + "### Int8 Quantization\n", + "\n", + "We can use the [Intel® Extension for Transformers](https://github.com/intel/intel-extension-for-transformers) to quantize the trained model for faster inference. If you want to run this part of the notebook, make sure you have `intel-extension-for-transformers` installed in your environment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44036b44", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install intel-extension-for-transformers==1.0.1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce0687ce", + "metadata": {}, + "outputs": [], + "source": [ + "from intel_extension_for_transformers.optimization.trainer import NLPTrainer\n", + "from intel_extension_for_transformers.optimization import metrics, objectives, OptimizedModel, QuantizationConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9557a68", + "metadata": {}, + "outputs": [], + "source": [ + "# Set up quantization config\n", + "tune_metric = metrics.Metric(\n", + " name=\"eval_accuracy\",\n", + " greater_is_better=True,\n", + " is_relative=True,\n", + " criterion=quantization_criterion,\n", + " weight_ratio=None,\n", + ")\n", + "\n", + "objective = objectives.Objective(\n", + " name=\"performance\", greater_is_better=True, weight_ratio=None\n", + ")\n", + "\n", + "quantization_config = QuantizationConfig(\n", + " approach=\"PostTrainingDynamic\",\n", + " max_trials=quantization_max_trial,\n", + " metrics=[tune_metric],\n", + " objectives=[objective],\n", + ")\n", + "\n", + "# Set up metrics computation\n", + "def compute_metrics(p: EvalPrediction):\n", + " preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions\n", + " preds = np.argmax(preds, axis=1)\n", + " return {\"accuracy\": (preds == p.label_ids).astype(np.float32).mean().item()}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f406d6db", + "metadata": {}, + "outputs": [], + "source": [ + "quantizer = NLPTrainer(model=model._model,\n", + " train_dataset=dataset.train_subset,\n", + " eval_dataset=dataset.validation_subset,\n", + " compute_metrics=compute_metrics,\n", + " tokenizer=dataset._tokenizer)\n", + "quantized_model = quantizer.quantize(quant_config=quantization_config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56e5f2f5", + "metadata": {}, + "outputs": [], + "source": [ + "results = quantizer.evaluate()\n", + "eval_acc = results.get(\"eval_accuracy\")\n", + "print(\"Final Eval Accuracy: {:.5f}\".format(eval_acc))" + ] + }, + { + "cell_type": "markdown", + "id": "b69df1a0", + "metadata": {}, + "source": [ + "## Citations\n", + "\n", + "### Data Citation\n", + "Khaled R., Helal M., Alfarghaly O., Mokhtar O., Elkorany A., El Kassas H., Fahmy A. Categorized Digital Database for Low energy and Subtracted Contrast Enhanced Spectral Mammography images [Dataset]. (2021) The Cancer Imaging Archive. DOI: [10.7937/29kw-ae92](https://doi.org/10.7937/29kw-ae92)\n", + "\n", + "### Publication Citation\n", + "Khaled, R., Helal, M., Alfarghaly, O., Mokhtar, O., Elkorany, A., El Kassas, H., & Fahmy, A. Categorized contrast enhanced mammography dataset for diagnostic and artificial intelligence research. (2022) Scientific Data, Volume 9, Issue 1. DOI: [10.1038/s41597-022-01238-0](https://doi.org/10.1038/s41597-022-01238-0)\n", + "\n", + "### TCIA Citation\n", + "Clark K, Vendt B, Smith K, Freymann J, Kirby J, Koppel P, Moore S, Phillips S, Maffitt D, Pringle M, Tarbox L, Prior F. The Cancer Imaging Archive (TCIA): Maintaining and Operating a Public Information Repository, Journal of Digital Imaging, Volume 26, Number 6, December, 2013, pp 1045-1057. DOI: [10.1007/s10278-013-9622-7](https://doi.org/10.1007/s10278-013-9622-7)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/e2e_workflows/Remote_Sensing_Image_Scene_Classification.ipynb b/notebooks/e2e_workflows/Remote_Sensing_Image_Scene_Classification.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7cf0478d3b33209c0a69c4d5c8f109d6a9138e54 --- /dev/null +++ b/notebooks/e2e_workflows/Remote_Sensing_Image_Scene_Classification.ipynb @@ -0,0 +1,427 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fcfb563d-3f9c-4731-be1f-9c9c4b2c9dfd", + "metadata": { + "tags": [] + }, + "source": [ + "# Remote Sensing Image Scene Classification (Resisc) using TensorFlow and the Intel® Transfer Learning Tool API" + ] + }, + { + "cell_type": "markdown", + "id": "353f2782-043f-4afd-9858-cc773275e6c5", + "metadata": {}, + "source": [ + "This notebook facilitates implementation of remote sensing image scene classification using Transfer Learning Toolkit. It performs Multi-class scene classification on RESISC45 dataset. The workflow uses pretrained SOTA models ( RESNET V1.5) from TF hub and transfers the knowledge from a pretrained domain to a different custom domain achieving required accuracy." + ] + }, + { + "cell_type": "markdown", + "id": "8325300b-4a75-42fb-aa14-2b089b4edea5", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04e3f113-4887-49c9-aa8b-a3c2058157a4", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import os\n", + "import pickle\n", + "import tensorflow as tf\n", + "from sklearn.metrics import classification_report\n", + "\n", + "#tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.types import FrameworkType, UseCaseType\n", + "\n", + "from notebooks.plot_utils import plot_curves\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "a47ea534-0661-4cbe-97ed-ca288b6203e5", + "metadata": {}, + "source": [ + "## 2. Get the model" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c9548167-cdfd-44ad-be60-cc3d131850a3", + "metadata": {}, + "source": [ + "In this step, we call the Intel Transfer Learning Tool model factory to list supported TensorFlow image classification models. This is a list of pretrained models from TFHub that we tested with our API. Optionally, the verbose=True argument can be added to the print_supported_models function call to get more information about each model (such as the link to TFHub, image size, the original dataset, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df3c14ab-c4d3-4aa4-bd24-5265ccd7b534", + "metadata": {}, + "outputs": [], + "source": [ + "# See a list of available models\n", + "model_factory.print_supported_models(use_case='image_classification', framework='tensorflow')" + ] + }, + { + "cell_type": "markdown", + "id": "3c06b359-1054-45c5-9223-47d4e1b7772d", + "metadata": {}, + "source": [ + "#### Option A: Load a model\n", + "\n", + "Next, use the model factory to get one of the models listed in the previous cell. The `get_model` function returns a model object that will later be used for training. By default, resnet_v1_50 is used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf842217-d992-4b6f-99bf-aef475fa13dd", + "metadata": {}, + "outputs": [], + "source": [ + "# Get the model\n", + "model = model_factory.get_model(model_name=\"resnet_v1_50\", framework=\"tensorflow\")" + ] + }, + { + "cell_type": "markdown", + "id": "afa83899-0a99-4c9e-9e3d-00b040fe3e1f", + "metadata": {}, + "source": [ + "#### Option B: Load a pretrained checkpoint\n", + "\n", + "Optionally, to continue training using a pretrained checkpoint, the user can specify the path to folder containing __saved_model.pb__. The user can specify the path in __model__ parameter.\n", + "\n", + "_Note: The path is same as saved_model_dir_" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b49423b-e57d-478c-87b0-7daec6401b76", + "metadata": {}, + "outputs": [], + "source": [ + "#Load a pretrained checkpoint\n", + "model = model_factory.load_model(model_name='resnet_v1_50', \n", + " model='/home/intel/output/resnet_v1_50/1', \n", + " framework='tensorflow', use_case='image_classification')" + ] + }, + { + "cell_type": "markdown", + "id": "ba2e4d10-458b-4479-8d1f-8cc67304d5c2", + "metadata": {}, + "source": [ + "## 3. Get the dataset" + ] + }, + { + "cell_type": "markdown", + "id": "0bff208b-e899-49cc-a4d5-5abab6baa4c9", + "metadata": {}, + "source": [ + "The dataset used for remote sensing domain is resisc45. More details are at the location : https://www.tensorflow.org/datasets/catalog/resisc45. \n", + "To download dataset follow the steps:\n", + "1. Download the rar file from https://onedrive.live.com/?authkey=%21AHHNaHIlzp%5FIXjs&cid=5C5E061130630A68&id=5C5E061130630A68%21107&parId=5C5E061130630A68%21112&action=locate \n", + "2. Unzip the folder\n", + "3. Set custom_dataset_path to point to resisc folder" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1840f1a-2a32-4ef7-89b2-d32ee231463b", + "metadata": {}, + "outputs": [], + "source": [ + "# Set the custom_dataset_path to point to your dataset's directory.\n", + "custom_dataset_path = os.path.join(dataset_dir, \"resisc/\")\n", + "\n", + "# Load the dataset from the custom dataset path\n", + "dataset = dataset_factory.load_dataset(dataset_dir=custom_dataset_path,\n", + " use_case='image_classification', \n", + " framework='tensorflow')\n", + "\n", + "print(\"Class names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "a27d6271-5afa-46b2-aed9-f543c62e1443", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "\n", + "Once you have your dataset from Option A or Option B above, use the following cells to preprocess the dataset. We resize the images to match the selected models and batch the images, then split them into training and validation subsets. Data augmentation can be applied by specifying the augmentations to be applied in __add_aug__ parameter. Supported augmentations are \n", + "1. hvflip - RandomHorizontalandVerticalFlip\n", + "2. hflip - RandomHorizontalFlip\n", + "3. vflip - RandomVerticalFlip\n", + "4. rotate - RandomRotate\n", + "5. zoom - RandomZoom" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00d529b1-fe40-482a-b3f0-3875b13ba534", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess the dataset with an image size that matches the model and a batch size of 256\n", + "batch_size = 256\n", + "dataset.preprocess(model.image_size, batch_size=batch_size, add_aug=['hvflip', 'rotate', 'zoom'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c4af8c6-c418-45d1-b0bc-dd8c7e2625c9", + "metadata": {}, + "outputs": [], + "source": [ + "# Split the dataset into training, validation and test subsets\n", + "dataset.shuffle_split(train_pct=.80, val_pct=.10, test_pct=0.10)" + ] + }, + { + "cell_type": "markdown", + "id": "ab02c7d0-5b63-4be3-b6a6-cdfc03e48995", + "metadata": {}, + "source": [ + "## 5. Transfer Learning" + ] + }, + { + "cell_type": "markdown", + "id": "feeb23c9-66a0-4670-8fd0-f7950dd2832e", + "metadata": {}, + "source": [ + "This step calls the model's train function with the dataset that was just prepared. The training function will get the TFHub feature vector and add on a dense layer based on the number of classes in the dataset. The model is then compiled and trained based on the number of epochs specified in the argument. With the do_eval paramter set to True by default, this step will also show how the model can be evaluated and will return a list of metrics calculated from the dataset's validation subset.\n", + "### Arguments\n", + "#### Required\n", + "- **dataset** (ImageClassificationDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory for checkpoint files\n", + "- **epochs** (int): Number of epochs to train the model (default: 1)\n", + "#### Optional\n", + "- **initial_checkpoints** (str): Path to checkpoint weights to load. If the path provided is a directory, the latest checkpoint will be used.\n", + "- **early_stopping** (bool): Enable early stopping if convergence is reached while training at the end of each epoch. (default: False)\n", + "- **lr_decay** (bool): If lr_decay is True and do_eval is True, learning rate decay on the validation loss is applied at the end of each epoch.\n", + "- **enable_auto_mixed_precision** (bool or None): Enable auto mixed precision for training. Mixed precision uses both 16-bit and 32-bit floating point types to make training run faster and use less memory. It is recommended to enable auto mixed precision training when running on platforms that support bfloat16 (Intel third or fourth generation Xeon processors). If it is enabled on a platform that does not support bfloat16, it can be detrimental to the training performance. If enable_auto_mixed_precision is set to None, auto mixed precision will be automatically enabled when running with Intel fourth generation Xeon processors, and disabled for other platforms.\n", + "- **extra_layers** (list[int]): Optionally insert additional dense layers between the base model and output layer. This can help increase accuracy when fine-tuning a TFHub model. The input should be a list of integers representing the number and size of the layers, for example [1024, 512] will insert two dense layers, the first with 1024 neurons and the second with 512 neurons.\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa477dbb-58a9-4e06-9933-047699c35797", + "metadata": {}, + "outputs": [], + "source": [ + "# Mixed precision uses both 16-bit and 32-bit floating point types to make training run faster and use less memory.\n", + "# It is recommended to enable auto mixed precision training when running on platforms that support\n", + "# bfloat16 (Intel third or fourth generation Xeon processors). If it is enabled on a platform that\n", + "# does not support bfloat16, it can be detrimental to the training performance.\n", + "# If enable_auto_mixed_precision is set to None, auto mixed precision will be automatically enabled when\n", + "# running with Intel fourth generation Xeon processors, and disabled for other platforms.\n", + "enable_auto_mixed_precision = None\n", + "\n", + "# Train the model using the dataset\n", + "history = model.train(dataset, output_dir=output_dir, epochs=50, \n", + " enable_auto_mixed_precision=None, extra_layers=[1024,512], early_stopping=True)" + ] + }, + { + "cell_type": "markdown", + "id": "40e11437-5666-49c0-9547-22e7696c0ea0", + "metadata": {}, + "source": [ + "## 6. Evaluate" + ] + }, + { + "cell_type": "markdown", + "id": "1d413b5d-ed24-4756-928d-dbb8e7bf1af2", + "metadata": {}, + "source": [ + "The next step shows how the model can be evaluated. The model's evaluate function returns a list of metrics calculated from the dataset's validation subset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bfc8e8-f8c6-4b56-bfc8-51b88288fcdd", + "metadata": {}, + "outputs": [], + "source": [ + "# Evaluate model on validation subset\n", + "val_loss, val_acc = model.evaluate(dataset)\n", + "print(\"Validation Accuracy :\", val_acc)\n", + "print(\"Validation Loss :\", val_loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "685f31eb-a32a-4198-afba-782e31ed506a", + "metadata": {}, + "outputs": [], + "source": [ + "plot_curves(history, os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name)))\n", + "pickle.dump(history, open(os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name), 'resisc45_hist.pkl'), 'wb'))" + ] + }, + { + "cell_type": "markdown", + "id": "3c6fd051-97af-4142-b039-482cb742d988", + "metadata": {}, + "source": [ + "## 7. Export" + ] + }, + { + "cell_type": "markdown", + "id": "9225921d-5340-4865-ad4b-a2c19a5210ed", + "metadata": {}, + "source": [ + "Next, we can call the model export function to generate a saved_model.pb. The model is saved in a format that is ready to use with TensorFlow Serving. Each time the model is exported, a new numbered directory is created, which allows serving to pick up the latest model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c4fc5f8-cb72-4ab5-989b-43d7ad315e8b", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "bb29d2b5-31e5-4136-a9d5-9f97d556d4a8", + "metadata": {}, + "source": [ + "## 8. Inference\n", + "To perform only Inference using a saved model, follow the steps below\n", + "1. Execute Step 2(b) to load a pretrained checkpoint with the appropriate model name.\n", + "2. Execute Steps 3 and 4 to load and prepare the dataset.\n", + "3. Continue with the steps below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef0819f7-c9fa-4eae-be48-f83a74777002", + "metadata": {}, + "outputs": [], + "source": [ + "history = pickle.load(open(os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name), 'resisc45_hist.pkl'), 'rb'))\n", + "plot_curves(history, os.path.join(output_dir, \"{}_checkpoints\".format(model.model_name)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39a5b54a-6781-4496-b29f-55a1ce18d8e4", + "metadata": {}, + "outputs": [], + "source": [ + "loss, accuracy = model.evaluate(dataset, use_test_set=True)\n", + "print('Test accuracy :', accuracy)" + ] + }, + { + "cell_type": "markdown", + "id": "f517d0f1-011c-4704-85fa-fd7d57f8aaa4", + "metadata": {}, + "source": [ + "We get the test subset from our dataset, and use that to call predict on our model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "295ac0ab-accd-4433-8143-a087e7b218b5", + "metadata": {}, + "outputs": [], + "source": [ + "actual_labels = np.concatenate([y for x, y in dataset._test_subset], axis=0)\n", + "predicted_labels = model.predict(dataset._test_subset)\n", + "report = classification_report(actual_labels, predicted_labels)\n", + "print(\"Classification report\")\n", + "print(report)" + ] + }, + { + "cell_type": "markdown", + "id": "27daa00f-4e83-419c-ba75-38c03a0d2475", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "@article{Cheng_2017,
\n", + " title={Remote Sensing Image Scene Classification: Benchmark and State of the Art},
\n", + " volume={105},
\n", + " ISSN={1558-2256},
\n", + " url={http://dx.doi.org/10.1109/JPROC.2017.2675998},
\n", + " DOI={10.1109/jproc.2017.2675998},
\n", + " number={10},
\n", + " journal={Proceedings of the IEEE},
\n", + " publisher={Institute of Electrical and Electronics Engineers (IEEE)},
\n", + " author={Cheng, Gong and Han, Junwei and Lu, Xiaoqiang},
\n", + " year={2017},
\n", + " month={Oct},
\n", + " pages={1865-1883}
\n", + "}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/e2e_workflows/data_utils.py b/notebooks/e2e_workflows/data_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0975dc57bcc2b18bc8fc31f1fbe92449650c91a0 --- /dev/null +++ b/notebooks/e2e_workflows/data_utils.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import shutil +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split +from collections import defaultdict + + +def copy_files_src_to_tgt(samples, fns_dict, src_folder, tgt_folder): + for sample in samples: + files_to_copy = fns_dict.get(sample) + for _file in files_to_copy: + src_fn = os.path.join(src_folder, _file) + tgt_fn = os.path.join(tgt_folder, _file) + shutil.copy2(src_fn, tgt_fn) + + +def split_images(src_folder, tgt_folder): + labels = os.listdir(src_folder) + print("Number of labels = ", len(labels)) + print("Labels are: \n", labels) + for label in labels: + fns = os.listdir(os.path.join(src_folder, label)) + fns.sort() + fns_root = ['_'.join(x.split('_')[:2]) for x in fns] + # Convert list of tuples to dictionary value lists + print("\nCreating default dict for stratifying the subject in {}.".format(label)) + fns_dict = defaultdict(list) + for i, j in zip(fns_root, fns): + fns_dict[i].append(j) + train_samples, test_samples = train_test_split(list(fns_dict.keys()), test_size=0.2, random_state=100) + + src_dir = os.path.join(src_folder, label) + tgt_dir = os.path.join(tgt_folder, 'train', label) + os.makedirs(tgt_dir, exist_ok=True) + copy_files_src_to_tgt(train_samples, fns_dict, src_dir, tgt_dir) + + tgt_dir = os.path.join(tgt_folder, 'test', label) + os.makedirs(tgt_dir, exist_ok=True) + copy_files_src_to_tgt(test_samples, fns_dict, src_dir, tgt_dir) + + print("Done splitting the files for label = {}\n".format(label)) + print("Done splitting the data. Output data is here: ", tgt_folder) + + +def get_subject_id(image_name): + image_name = image_name.split("/")[-1] + patient_id = "".join(image_name.split("_")[:2])[1:] + return patient_id + + +def create_patient_id_list(image_data_folder, folder): + folder_pth = os.path.join(folder, image_data_folder) + patient_id_list = [] + for fldr in os.listdir(folder_pth): + for f in os.listdir(os.path.join(folder_pth, fldr)): + patient_id_list.append(get_subject_id(f)) + + return np.unique(patient_id_list) + + +def read_annotation_file( + folder, + file_name, + label_column, + data_column, + patient_id, + patient_id_list, + image_data_folder +): + df_path = os.path.join(folder, file_name) + df = pd.read_csv(df_path) + label_map, reverse_label_map = label2map(df, label_column) + + if patient_id_list is not None: + df = df[df[patient_id].isin(patient_id_list)] + else: + image_name_list = [] + for label in os.listdir(image_data_folder): + image_name_list.extend(os.listdir(os.path.join(image_data_folder, label))) + df = df[df[patient_id].isin(np.unique([get_subject_id(i) for i in image_name_list]))] + + df_new = pd.DataFrame(columns=[label_column, data_column, patient_id]) + for i in df[patient_id].unique(): + annotation = " ".join(df[df[patient_id].isin([i])][data_column].to_list()) + temp_labels = df[df[patient_id] == i][label_column].unique() + if len(temp_labels) == 1: + df_new.loc[len(df_new)] = [temp_labels[0], annotation, i] + else: + if patient_id_list is not None: + # this is the case only shows for inference + # label assigne as a place holder + df_new.loc[len(df_new)] = ["Normal", annotation, i] + else: + Warning("Conflict in labelling ....") + + return df_new, label_map, reverse_label_map + + +def label2map(df, label_column): + label_map, reverse_label_map = {}, {} + for i, v in enumerate(df[label_column].unique().tolist()): + label_map[v] = i + reverse_label_map[i] = v + + return label_map, reverse_label_map + + +def create_train_test_set(df, patient_id, patient_id_list): + train_label, test_label = train_test_split( + patient_id_list, test_size=0.33, random_state=42 + ) + + df_test = df[df[patient_id].isin(test_label)] + df_train = df[df[patient_id].isin(train_label)] + + return df_train, df_test + + +def split_annotation(folder, file_name, image_data_folder): + label_column = "label" + data_column = "symptoms" + patient_id = "Patient_ID" + patient_id_list = None + + df, label_map, reverse_label_map = read_annotation_file( + folder, + file_name, + label_column, + data_column, + patient_id, + patient_id_list, + image_data_folder + ) + + patient_id_list = create_patient_id_list(image_data_folder, folder) + df_train, df_test = create_train_test_set(df, patient_id, patient_id_list) + + return df_train diff --git a/notebooks/image_anomaly_detection/tlt_api_pyt_anomaly_detection/Anomaly_Detection.ipynb b/notebooks/image_anomaly_detection/tlt_api_pyt_anomaly_detection/Anomaly_Detection.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f8e386ca307e05187a63cca15088285bbffd3be0 --- /dev/null +++ b/notebooks/image_anomaly_detection/tlt_api_pyt_anomaly_detection/Anomaly_Detection.ipynb @@ -0,0 +1,558 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3405d28d", + "metadata": {}, + "source": [ + "# Image Anomaly Detection with PyTorch using
Intel® Transfer Learning Tool\n", + "\n", + "This notebook demonstrates anomaly detection using the Intel Transfer Learning Toolkit. It performs defect analysis with the MVTec dataset using PyTorch. The workflow uses a pretrained ResNet50 v1.5 model from torchvision." + ] + }, + { + "cell_type": "markdown", + "id": "1d61b7ac", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions to setup a PyTorch environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0bf9fd0", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import PIL.Image as Image\n", + "import torch, torchvision\n", + "from torchvision.transforms.functional import InterpolationMode\n", + "import requests\n", + "from io import BytesIO\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_and_extract_tar_file, download_file\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "8f1fc78c", + "metadata": {}, + "source": [ + "## 2. Get or load the model\n", + "\n", + "In this step, we use the model factory to get the desired model. The `get_model` function returns a pretrained model object from a public model hub, while the `load_model` function loads a pretrained model from a checkpoint on your local disk or in memory.\n", + "\n", + "Here we are getting the pretrained `resnet50` model from Torchvision:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad4aeafd", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name=\"resnet50\", framework=\"pytorch\", use_case='anomaly_detection')" + ] + }, + { + "cell_type": "markdown", + "id": "9d087ee7", + "metadata": {}, + "source": [ + "To load a previously trained model from a file, use this:\n", + "```\n", + "model = model_factory.load_model(model_name=\"resnet50\", model=, framework=\"pytorch\", \n", + " use_case='anomaly_detection')\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "dabd4183", + "metadata": {}, + "source": [ + "## 3. Get the dataset" + ] + }, + { + "cell_type": "markdown", + "id": "2d314ba0", + "metadata": {}, + "source": [ + "To use [MVTec](https://www.mvtec.com/company/research/datasets/mvtec-ad) or your own image dataset for anomaly detection, your image files (`.jpg` or `.png`) should be arranged in one of two ways. \n", + "\n", + "### Method 1: Category Folders\n", + "\n", + "Arrange them in folders in the root dataset directory like this:\n", + "\n", + "```\n", + "hazelnut\n", + " └── crack\n", + " └── cut\n", + " └── good\n", + " └── hole\n", + " └── print\n", + "```\n", + "\n", + "IMPORTANT: There must be a subfolder named `good` and at least one other folder of defective examples. It does not matter what the names of the other folders are or how many there, as long as there is at least one. This would also be an acceptable Method 1 layout:\n", + "\n", + "```\n", + "toothbrush\n", + " └── defective\n", + " └── good\n", + "```\n", + "\n", + "TLT will encode all of the non-good images as \"bad\" and use the \"good\" images in the training set and a mix of good and bad images in the validation set.\n", + "\n", + "### Method 2: Train & Test Folders with Category Subfolders\n", + "\n", + "Arrange them in folders in the root dataset directory like this:\n", + "\n", + "```\n", + "hazelnut\n", + " └── train\n", + " └── good\n", + " └── test\n", + " └── crack\n", + " └── cut\n", + " └── good\n", + " └── hole\n", + " └── print\n", + "```\n", + "\n", + "When using this layout, TLT will use the exact defined split for train and validation subsets unless you use the `shuffle_split` method to re-shuffle and split up the \"good\" images with certain percentages. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64b24c5b-9b48-4041-a6a2-7c438ca3a0c5", + "metadata": {}, + "outputs": [], + "source": [ + "img_dir = os.path.join(dataset_dir, 'hazelnut')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "357f3dfd", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset_factory.load_dataset(img_dir, \n", + " use_case='image_anomaly_detection', \n", + " framework=\"pytorch\")\n", + "\n", + "print(dataset._dataset)\n", + "print(\"Class names:\", str(dataset.class_names))\n", + "print(\"Defect names:\", dataset.defect_names)" + ] + }, + { + "cell_type": "markdown", + "id": "2200ef4e", + "metadata": {}, + "source": [ + "Note: The defects argument can be used to filter the validation set to use only a subset of defect types. For example:\n", + "```\n", + "dataset = dataset_factory.load_dataset(img_dir, \n", + " use_case='image_anomaly_detection', \n", + " framework=\"pytorch\",\n", + " defects=['crack', 'hole'])\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "99f23249", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "Once you have your dataset, use the following cells to split and preprocess the data. We split them into training and test subsets, then resize the images to match the selected model, and then batch the images. Pass in optional arguments to customize the [Resize](https://pytorch.org/vision/main/generated/torchvision.transforms.Resize.html) or [Normalize](https://pytorch.org/vision/main/generated/torchvision.transforms.Normalize.html) transforms.\n", + "Data augmentation can be applied to the training set by specifying the augmentations to be applied in the `add_aug` parameter. Supported augmentations are given below:\n", + "1. hflip - RandomHorizontalFlip\n", + "2. rotate - RandomRotate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd91fbcf", + "metadata": {}, + "outputs": [], + "source": [ + "# If using Method 1 layout, split the dataset into training and test subsets.\n", + "if dataset._validation_type is None:\n", + " dataset.shuffle_split(train_pct=.75, val_pct=0.0, test_pct=0.25)" + ] + }, + { + "cell_type": "markdown", + "id": "4fbe27a3-1b1e-4add-9725-28bceb62c474", + "metadata": {}, + "source": [ + "For __cutpaste__ feature extractor, cutpaste_type can be specified in the dataset.preprocess() method as follows. The option available are - _normal_, _scar_, _3way_ and _union_. Default is _normal_.\n", + "```\n", + "dataset.preprocess(224, batch_size=batch_size, interpolation=InterpolationMode.LANCZOS, cutpaste_type='normal')\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7c95a70", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess with an image size that matches the model, batch size 32, and the desired interpolation method\n", + "batch_size = 64\n", + "cutpaste_type = 'normal'\n", + "dataset.preprocess(image_size=224, batch_size=batch_size, interpolation=InterpolationMode.LANCZOS, cutpaste_type=cutpaste_type)" + ] + }, + { + "cell_type": "markdown", + "id": "3704772b", + "metadata": {}, + "source": [ + "## 5. Visualize samples from the dataset\n", + "\n", + "We get a single batch from our training and test subsets and visualize the images as a sanity check." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd6782b0", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_images(images, labels, sup_title, predictions=None):\n", + " plt.figure(figsize=(18,14))\n", + " plt.subplots_adjust(hspace=0.5)\n", + " for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " inp = images[n]\n", + " inp = inp.numpy().transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " inp = std * inp + mean\n", + " inp = np.clip(inp, 0, 1)\n", + " plt.imshow(inp)\n", + " if predictions:\n", + " correct_prediction = labels[n] == predictions[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predictions[n] if correct_prediction else \"{}\".format(predictions[n])\n", + " else:\n", + " good_sample = labels[n] == 'good'\n", + " color = \"darkgreen\" if labels[n] == 'good' else (\"crimson\" if labels[n] == 'bad' else \"black\")\n", + " title = labels[n]\n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + " _ = plt.suptitle(sup_title, fontsize=20)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffcd2071", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot some images from the training set\n", + "images, labels = dataset.get_batch()\n", + "labels = [dataset.class_names[id] for id in labels]\n", + "plot_images(images, labels, 'Training Samples')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d37b808f", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot some images from the test set\n", + "images, labels = dataset.get_batch(subset='test')\n", + "labels = [dataset.class_names[id] for id in labels]\n", + "plot_images(images, labels, 'Test Samples')" + ] + }, + { + "cell_type": "markdown", + "id": "a49ec7b7", + "metadata": {}, + "source": [ + "## 6. Training and Evaluation\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the torchvision feature extractor for the user's desired layer and extract features from the training set. The extracted features are used to perform a [principal component analysis](https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html). The model's evaluate function returns the AUROC metric ([area under](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.auc.html) the [roc curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_curve.html)) calculated from the dataset's test subset." + ] + }, + { + "cell_type": "markdown", + "id": "ab510f51", + "metadata": {}, + "source": [ + "### Train Arguments\n", + "\n", + "#### Required\n", + "- **dataset** (ImageAnomalyDetectionDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory\n", + "\n", + "#### Optional\n", + "- **generate_checkpoints** (bool): Whether to save/preserve the best weights during SimSiam training (default: True)\n", + "- **initial_checkpoints** (str): The path to a starting weights file\n", + "- **layer_name** (str): The layer name whose output is desired for the extracted features\n", + "- **pooling** (str): Pooling to be applied on the extracted layer ('avg' or 'max') (default: 'avg')\n", + "- **kernel_size** (int): Kernel size in the pooling layer (default: 2)\n", + "- **pca_threshold** (float): Threshold to apply to PCA model (default: 0.99)\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd9420d", + "metadata": {}, + "outputs": [], + "source": [ + "# Examine the model's layers and decide which to use for feature extraction\n", + "model.list_layers(verbose=False)\n", + "layer = 'layer3'" + ] + }, + { + "cell_type": "markdown", + "id": "b19be956-e3c6-4d9d-847d-779c1c35da38", + "metadata": {}, + "source": [ + "## Feature Extraction\n", + "There are three feature extractor options available within the `model.train()` function.\n", + "1. __No fine-tuning__ - To use a pretrained ResNet50/ResNet18 model for feature extraction, simply do not change the default `simsiam=False` input argument.\n", + "2. [__SimSiam__](https://arxiv.org/abs/2011.10566) - A self-supervised neural network based on Siamese networks. It learns a meaningful representation of dataset without using any labels. If selected, SimSiam generates quality features that can help differentiate between regular and anomaly images in a given context. SimSiam produces two different augmented images from one underlying image. The end goal is to train the network to produce the same features for both images. It takes a ResNet model as the backbone and fine-tunes the model on the augmented dataset to get a better feature embedding. To use this feature extractor, download the SimSiam weights based on ResNet50 - https://dl.fbaipublicfiles.com/simsiam/models/100ep-256bs/pretrain/checkpoint_0099.pth.tar - set `simsiam=True`, and set `initial_checkpoints` to the path of the downloaded checkpoints in the `model.train()` function.\n", + "3. [__Cut-paste__](https://arxiv.org/abs/2104.04015#) - A self-supervised method for Anomaly Detection and Localization that takes ResNet50/ ResNet18 model as backbone and fine-tune the model on custom dataset to get better feature embedding. data augmentation strategy that cuts an image patch and pastes at a random location of a large image. To use this feature extractor, set `cutpaste=True` in the `model.train()` function.\n", + "\n", + "\n", + "### Optional: The SimSiam TwoCropTransform\n", + "To train a Simsiam model, it is required to apply a TwoCropTransform augmentation technique on the dataset used for training. You can preview this augmentation on a sample batch after preprocessing by using `get_batch(simsiam=True)` and then use them for simsiam training by using `simsiam=True` in `model.train()` also." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b49522f", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a batch of training data with the simsiam transform applied to it\n", + "simsiam_images, _ = dataset.get_batch(simsiam=True)\n", + "\n", + "# Plot the \"A\" samples showing the first set of augmented images\n", + "plot_images(simsiam_images[0], ['{}A'.format(i) for i in range(batch_size)], 'SimSiam \"A\" Samples')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5da06df", + "metadata": {}, + "outputs": [], + "source": [ + "# Now plot the \"B\" samples showing the second set of augmented images based on the same underlying originals\n", + "plot_images(simsiam_images[1], ['{}B'.format(i) for i in range(batch_size)], 'SimSiam \"B\" Samples')" + ] + }, + { + "cell_type": "markdown", + "id": "ace7d296-74d9-47c1-aeaf-386433bac411", + "metadata": {}, + "source": [ + "### Optional: The Cut-paste Transforms\n", + "To train a model with Cut-paste , it is required to apply one of the four augmentations - __CutPasteNormal, CutPasteScar, CutPaste3Way, CutPasteUnion__ on the dataset used for training. You can preview this augmentation on a sample batch after preprocessing by using `get_batch(cutpaste=True)` and then use them for cutpaste training by using `cutpaste=True` in `model.train()` also." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21cbadd5-8387-4130-b5b4-e016d4ea4e5e", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a batch of training data with the cutpaste transform applied to it\n", + "cutpaste_images, _ = dataset.get_batch(cutpaste=True)\n", + "\n", + "# Plot the \"A\" samples showing the first set of augmented images\n", + "plot_images(cutpaste_images[1], ['{}A'.format(i) for i in range(batch_size)], 'CutPaste \"A\" Samples')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "750bc599-80e4-4e70-8aaf-5f63082b9198", + "metadata": {}, + "outputs": [], + "source": [ + "if cutpaste_type == '3way':\n", + " # Now plot the \"B\" samples showing the third set of augmented images based on the same underlying originals\n", + " plot_images(cutpaste_images[2], ['{}B'.format(i) for i in range(batch_size)], 'CutPaste \"B\" Samples')" + ] + }, + { + "cell_type": "markdown", + "id": "a2ba878d-1b03-4f7c-8f5a-6507ee1494a9", + "metadata": {}, + "source": [ + "There is no fine-tuning being demonstrated here, but you can use `simsiam` or `cutpaste` if desired.\n", + "\n", + "To use simsiam, set `simsiam=True` and pass the checkpoint file to `model.train()` as follows\n", + "```\n", + "pca_components, trained_model = model.train(dataset, output_dir, epochs=2, feature_dim=1000,\n", + " pred_dim=250, initial_checkpoints=,\n", + " pooling='avg', kernel_size=2, pca_threshold=0.99, simsiam=True,\n", + " generate_checkpoints=False, precision='float32')\n", + "```\n", + "\n", + "To use cutpaste, set `cutpaste=True`. Optionally, to load a pretrained checkpoint pass the checkpoint file to `model.train()` as follows.\n", + "```\n", + "pca_components, trained_model = model.train(dataset, output_dir, optim='sgd', epochs=2, freeze_resnet=20,\n", + " head_layer=2, cutpaste_type='normal', initial_checkpoints=,\n", + " pooling='avg', kernel_size=2, pca_threshold=0.99, cutpaste=True,\n", + " generate_checkpoints=False, precision='float32')\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b601fc", + "metadata": {}, + "outputs": [], + "source": [ + "pca_components, trained_model = model.train(dataset, output_dir, layer_name=layer, epochs=2,\n", + " seed=None, pooling='avg', kernel_size=2, pca_threshold=0.99,\n", + " precision='float32')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f60192d", + "metadata": {}, + "outputs": [], + "source": [ + "threshold, auroc = model.evaluate(dataset, pca_components, use_test_set=True)" + ] + }, + { + "cell_type": "markdown", + "id": "3cb8fc62", + "metadata": {}, + "source": [ + "## 7. Predict\n", + "\n", + "Using the same batch of test samples from above, get and view the model's predictions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f7ffe59", + "metadata": {}, + "outputs": [], + "source": [ + "predictions = model.predict(images, pca_components, return_type='class', threshold=threshold)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a40d0670", + "metadata": {}, + "outputs": [], + "source": [ + "plot_images(images, labels, 'Predictions', predictions=predictions)\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red\")\n", + "\n", + "accuracy = sum([1 if p==labels[i] else 0 for i, p in enumerate(predictions)])/len(predictions)\n", + "print(\"Accuracy: {}\".format(accuracy))" + ] + }, + { + "cell_type": "markdown", + "id": "0a877f33", + "metadata": {}, + "source": [ + "## 8. Export" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abc054ff", + "metadata": {}, + "outputs": [], + "source": [ + "model.export(os.path.join(output_dir, 'anomaly'))" + ] + }, + { + "cell_type": "markdown", + "id": "0947915a", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "Paul Bergmann, Kilian Batzner, Michael Fauser, David Sattlegger, Carsten Steger: The MVTec Anomaly Detection Dataset: A Comprehensive Real-World Dataset for Unsupervised Anomaly Detection; in: International Journal of Computer Vision 129(4):1038-1059, 2021, DOI: 10.1007/s11263-020-01400-4.\n", + "\n", + "Paul Bergmann, Michael Fauser, David Sattlegger, Carsten Steger: MVTec AD — A Comprehensive Real-World Dataset for Unsupervised Anomaly Detection; in: IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 9584-9592, 2019, DOI: 10.1109/CVPR.2019.00982." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/image_classification/pytorch_image_classification/PyTorch_Image_Classification_Transfer_Learning.ipynb b/notebooks/image_classification/pytorch_image_classification/PyTorch_Image_Classification_Transfer_Learning.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..e99ec5a6b2799fe0f81bdde5bf402154882e8132 --- /dev/null +++ b/notebooks/image_classification/pytorch_image_classification/PyTorch_Image_Classification_Transfer_Learning.ipynb @@ -0,0 +1,640 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "67b1e912", + "metadata": {}, + "source": [ + "# Transfer Learning for Image Classification\n", + "\n", + "This notebook uses image classification models from [Torchvision](https://pytorch.org/vision/stable/index.html) that were originally trained using [ImageNet](https://image-net.org/) and does transfer learning with a Torchvision dataset or your own raw images.\n", + "\n", + "The notebook performs the following steps:\n", + "1. [Import dependencies and setup parameters](#1.-Import-dependencies-and-setup-parameters)\n", + "2. [Prepare the dataset](#2.-Prepare-the-dataset)\n", + "3. [Predict using the original model](#3.-Predict-using-the-original-model)\n", + "4. [Transfer learning](#4.-Transfer-learning)\n", + "5. [Visualize the model output](#5.-Visualize-the-model-output)\n", + "6. [Export the saved model](#6.-Export-the-saved-model)" + ] + }, + { + "cell_type": "markdown", + "id": "939a7f98", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [README.md](/notebooks/README.md) to setup a PyTorch environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25f98c12", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "import math\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch\n", + "import torchvision\n", + "from torchvision import datasets, models, transforms\n", + "from PIL import Image\n", + "from pydoc import locate\n", + "import warnings\n", + "\n", + "import intel_extension_for_pytorch as ipex\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from tlt.utils.file_utils import download_and_extract_tar_file, download_file\n", + "from model_utils import torchvision_model_map, get_retrainable_model\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "print('Supported models:')\n", + "print('\\n'.join(torchvision_model_map.keys()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e42b6e79", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify a model from the list above\n", + "model_name = \"efficientnet_b0\"\n", + "\n", + "# Specify the the parent directory for the custom or Torchvision dataset\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "# Batch size\n", + "batch_size = 32\n", + "\n", + "print(\"Dataset directory:\", dataset_directory)\n", + "print(\"Output directory:\", output_directory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00f85b73", + "metadata": {}, + "outputs": [], + "source": [ + "if model_name not in torchvision_model_map.keys():\n", + " raise ValueError(\"The specified model_name ({}) is invalid. Please select from: {}\".\n", + " format(model_name, torchvision_model_map.keys()))\n", + " \n", + "print(\"Pretrained Image Classification Model:\", model_name) " + ] + }, + { + "cell_type": "markdown", + "id": "1aaecdff", + "metadata": {}, + "source": [ + "## 2. Prepare the dataset" + ] + }, + { + "cell_type": "markdown", + "id": "98db4b34", + "metadata": {}, + "source": [ + "Define transforms for data resizing and augmentation. The normalization means and standard deviations `[0.485, 0.456, 0.406], [0.229, 0.224, 0.225]` are specific to torchvision image classification models and are explained in the [documentation](https://pytorch.org/vision/stable/models.html#classification)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "429a90ea", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocessing transforms\n", + "import torchvision.transforms as T\n", + "\n", + "def get_transform(train):\n", + " transforms = []\n", + " transforms.append(T.Resize([256, 256]))\n", + " if train:\n", + " transforms.append(T.RandomHorizontalFlip())\n", + " transforms.append(T.ToTensor())\n", + " transforms.append(T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]))\n", + " \n", + " return T.Compose(transforms)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "7cd4df40", + "metadata": {}, + "source": [ + "### Option A: Use a Torchvision dataset" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "f0b814cb", + "metadata": {}, + "source": [ + "To use a Torchvision dataset, load from the Torchvision.datasets library, applying transforms for image augmentation, normalization, and resizing. This example uses the Food101 dataset from the [Torchvision datasets for image classification](https://pytorch.org/vision/stable/datasets.html#image-classification), but you can choose from a variety of options. If the dataset is not found in the dataset directory it is downloaded. Subsequent runs will reuse the already downloaded dataset.\n", + "\n", + "Note: Some Torchvision datasets use a `train=True/False` argument and others have a `split=\"train\"/\"test\"` convention. See the Torchvision documentation to see how to specify the subset you want to use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "537d83e1", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = torchvision.datasets.Food101(dataset_directory, split='train',\n", + " transform=get_transform(True), download=True)\n", + "dataset_test = torchvision.datasets.Food101(dataset_directory, split='test',\n", + " transform=get_transform(False), download=True) \n", + "class_names = dataset.classes\n", + "\n", + "print('Training data size: {}'.format(len(dataset)))\n", + "print('Validation data size: {}'.format(len(dataset_test)))" + ] + }, + { + "cell_type": "markdown", + "id": "93bb716b", + "metadata": {}, + "source": [ + "Now skip ahead to the [Predict using the original model](#3.-Predict-using-the-original-model) section." + ] + }, + { + "cell_type": "markdown", + "id": "43f9656f", + "metadata": {}, + "source": [ + "### Option B: Use a downloaded or custom dataset" + ] + }, + { + "cell_type": "markdown", + "id": "f235fa0f", + "metadata": {}, + "source": [ + "To use your own image dataset for transfer learning with the rest of this notebook, format your images as `.jpg` files and save them in folders named after the classes that you want the model to predict. To provide a working example using the correct layout, we will download and extract a flower species dataset. After downloading and extracting, you will have the following subdirectories in your dataset directory. Each species subfolder will contain numerous `.jpg` files:\n", + "\n", + "```\n", + "dataset_directory\n", + "└── flower_photos\n", + " └── daisy\n", + " └── dandelion\n", + " └── roses\n", + " └── sunflowers\n", + " └── tulips\n", + "```\n", + "\n", + "Use this as an example to organize your own image files accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29105a9e", + "metadata": {}, + "outputs": [], + "source": [ + "# When you have your own properly organized subdirectory of images, adjust this variable\n", + "dataset_subdir = os.path.join(dataset_directory, \"flower_photos\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13e96b9e", + "metadata": {}, + "outputs": [], + "source": [ + "# Only run this if you want to use the example flowers dataset\n", + "if not os.path.exists(dataset_subdir):\n", + " os.makedirs(dataset_subdir)\n", + " dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + " \n", + " # Download and extract the tar\n", + " download_and_extract_tar_file(dataset_url, dataset_directory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5482b8a", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = datasets.ImageFolder(dataset_subdir, get_transform(True))\n", + "dataset_test = datasets.ImageFolder(dataset_subdir, get_transform(False))\n", + "class_names = dataset.classes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e791a208", + "metadata": {}, + "outputs": [], + "source": [ + "# Use 25% for validation and 75% for training\n", + "indices = torch.randperm(len(dataset)).tolist()\n", + "num_training_samples = math.floor(len(dataset)*.75)\n", + "\n", + "dataset_test = torch.utils.data.Subset(dataset, indices[-num_training_samples:])\n", + "dataset = torch.utils.data.Subset(dataset, indices[:num_training_samples]) " + ] + }, + { + "cell_type": "markdown", + "id": "dbc2c043", + "metadata": {}, + "source": [ + "## 3. Predict using the original model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a89f78d4", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a data loader just for visualization\n", + "data_loader = torch.utils.data.DataLoader(dataset, batch_size=30,\n", + " shuffle=True, num_workers=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be4fa7fe", + "metadata": {}, + "outputs": [], + "source": [ + "# Get the ImageNet labels for displaying with the predictions\n", + "imagenet_classes = []\n", + "labels_file_url = 'https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt'\n", + "labels_file_path = os.path.join(dataset_directory, os.path.basename(labels_file_url))\n", + "if not os.path.exists(labels_file_url):\n", + " download_file(labels_file_url, dataset_directory)\n", + "\n", + "with open(labels_file_path) as f:\n", + " imagenet_labels = f.readlines()\n", + " imagenet_classes = [l.strip() for l in imagenet_labels]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7444f838", + "metadata": {}, + "outputs": [], + "source": [ + "# Get the pretrained torchvision model\n", + "pretrained_model_class = locate('torchvision.models.{}'.format(model_name))\n", + "model = pretrained_model_class(pretrained=True)\n", + "\n", + "# Get a batch of training data\n", + "inputs, classes = next(iter(data_loader))\n", + "\n", + "# Get predictions from the pretrained model\n", + "model.eval()\n", + "outputs = model(inputs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e893dcf", + "metadata": {}, + "outputs": [], + "source": [ + "# List of the actual labels for this batch\n", + "actual_label_batch = [class_names[int(id)] for id in classes]\n", + "\n", + "# List of the predicted labels for this batch\n", + "_, predicted_id = torch.max(outputs, 1)\n", + "predicted_label_batch = [imagenet_classes[id] for id in predicted_id]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfb66dc7", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a results table to list out the ImageNet class prediction vs the actual dataset label\n", + "results_table = []\n", + "for prediction, actual in zip(predicted_label_batch, actual_label_batch):\n", + " results_table.append([prediction, actual])\n", + "\n", + "pd.DataFrame(results_table, columns=[\"ImageNet Prediction\", \"Actual Label\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d9409af", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(10,9))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(30):\n", + " plt.subplot(6,5,n+1)\n", + " inp = inputs[n]\n", + " inp = inp.numpy().transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " inp = std * inp + mean\n", + " inp = np.clip(inp, 0, 1)\n", + " plt.imshow(inp)\n", + " plt.title(predicted_label_batch[n].title(), fontsize=9)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"ImageNet predictions\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "eba7d039", + "metadata": {}, + "source": [ + "## 4. Transfer learning" + ] + }, + { + "cell_type": "markdown", + "id": "cc029ef3", + "metadata": {}, + "source": [ + "Replace the pretrained head of the network with a new layer based on the number of classes in our dataset. Train the model using the new dataset for the specified number of epochs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02f0eee5", + "metadata": {}, + "outputs": [], + "source": [ + "# Number of training epochs\n", + "num_epochs = 1\n", + "\n", + "# To reduce training time, the feature extractor layer can remain frozen (do_fine_tuning=False).\n", + "# Fine-tuning can be enabled to potentially get better accuracy. Note that enabling fine-tuning\n", + "# will increase training time.\n", + "do_fine_tuning = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41c2ee68", + "metadata": {}, + "outputs": [], + "source": [ + "def main(model, criterion, optimizer, dataset, dataset_test, num_epochs=10):\n", + " since = time.time()\n", + " \n", + " device = torch.device(\"cpu\")\n", + " model = model.to(device)\n", + " best_acc = 0.0\n", + "\n", + " # Create data loaders for training and validation\n", + " data_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,\n", + " shuffle=True, num_workers=4)\n", + " data_loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size,\n", + " shuffle=False, num_workers=4)\n", + " \n", + " for epoch in range(num_epochs):\n", + " print(f'Epoch {epoch}/{num_epochs - 1}')\n", + " print('-' * 10)\n", + "\n", + " # Training phase\n", + " model.train()\n", + " running_loss = 0.0\n", + " running_corrects = 0\n", + "\n", + " # Iterate over data.\n", + " for inputs, labels in data_loader:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device)\n", + "\n", + " # Zero the parameter gradients\n", + " optimizer.zero_grad()\n", + "\n", + " # Forward and backward pass\n", + " with torch.set_grad_enabled(True):\n", + " outputs = model(inputs)\n", + " _, preds = torch.max(outputs, 1)\n", + " loss = criterion(outputs, labels)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # Statistics\n", + " running_loss += loss.item() * inputs.size(0)\n", + " running_corrects += torch.sum(preds == labels.data)\n", + "\n", + " epoch_loss = running_loss / len(dataset)\n", + " epoch_acc = running_corrects.double() / len(dataset)\n", + "\n", + " print(f'Training Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n", + "\n", + " # Evaluation phase\n", + " model.eval()\n", + " running_loss = 0.0\n", + " running_corrects = 0\n", + " \n", + " # Iterate over data.\n", + " for inputs, labels in data_loader_test:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device)\n", + "\n", + " # Zero the parameter gradients\n", + " optimizer.zero_grad()\n", + "\n", + " # Forward pass\n", + " with torch.set_grad_enabled(False):\n", + " outputs = model(inputs)\n", + " _, preds = torch.max(outputs, 1)\n", + " loss = criterion(outputs, labels)\n", + " \n", + " # Statistics\n", + " running_loss += loss.item() * inputs.size(0)\n", + " running_corrects += torch.sum(preds == labels.data)\n", + " \n", + " epoch_loss = running_loss / len(dataset_test)\n", + " epoch_acc = running_corrects.double() / len(dataset_test)\n", + "\n", + " if epoch_acc > best_acc:\n", + " best_acc = epoch_acc\n", + " \n", + " print(f'Validation Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n", + " print()\n", + " \n", + "\n", + " time_elapsed = time.time() - since\n", + " print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n", + " print(f'Best Validation Accuracy: {best_acc:4f}')\n", + "\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8b754b4", + "metadata": {}, + "outputs": [], + "source": [ + "model = get_retrainable_model(model_name, len(class_names), do_fine_tuning)\n", + "criterion = torch.nn.CrossEntropyLoss()\n", + "\n", + "# Adam optimizer\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=0.005)\n", + "\n", + "print('Trainable parameters: {}'.format(sum(p.numel() for p in model.parameters() if p.requires_grad)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a2c8e8b", + "metadata": {}, + "outputs": [], + "source": [ + "model, optimizer = ipex.optimize(model, optimizer=optimizer)\n", + "model = main(model, criterion, optimizer, dataset, dataset_test, num_epochs)" + ] + }, + { + "cell_type": "markdown", + "id": "f22b2493", + "metadata": {}, + "source": [ + "## 5. Visualize the model output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc700939", + "metadata": {}, + "outputs": [], + "source": [ + "model.eval()\n", + "outputs = model(inputs)\n", + "_, predicted_id = torch.max(outputs, 1)\n", + "predicted_label_batch = [class_names[id] for id in predicted_id]\n", + "\n", + "# Display the results\n", + "plt.figure(figsize=(10,9))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(30):\n", + " plt.subplot(6,5,n+1)\n", + " inp = inputs[n]\n", + " inp = inp.numpy().transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " inp = std * inp + mean\n", + " inp = np.clip(inp, 0, 1)\n", + " plt.imshow(inp)\n", + " correct_prediction = actual_label_batch[n] == predicted_label_batch[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predicted_label_batch[n].title() if correct_prediction else \"{}\\n({})\".format(predicted_label_batch[n], actual_label_batch[n]) \n", + " plt.title(title, fontsize=9, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"Model predictions\")\n", + "plt.show()\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "549d3602", + "metadata": {}, + "source": [ + "## 6. Export the saved model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a21d129", + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists(output_directory):\n", + " os.makedirs(output_directory)\n", + "file_path = \"{}/image_classification.pt\".format(output_directory)\n", + "torch.save(model.state_dict(), file_path)\n", + "print(\"Saved to {}\".format(file_path))" + ] + }, + { + "cell_type": "markdown", + "id": "edc0250c", + "metadata": {}, + "source": [ + "## Dataset citations\n", + "```\n", + "@inproceedings{bossard14,\n", + " title = {Food-101 -- Mining Discriminative Components with Random Forests},\n", + " author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc},\n", + " booktitle = {European Conference on Computer Vision},\n", + " year = {2014}\n", + "}\n", + "\n", + "@ONLINE {tfflowers,\n", + "author = \"The TensorFlow Team\",\n", + "title = \"Flowers\",\n", + "month = \"jan\",\n", + "year = \"2019\",\n", + "url = \"http://download.tensorflow.org/example_images/flower_photos.tgz\" }\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/image_classification/pytorch_image_classification/README.md b/notebooks/image_classification/pytorch_image_classification/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5efc318364791afb4258788fa0c206d441ac1d8b --- /dev/null +++ b/notebooks/image_classification/pytorch_image_classification/README.md @@ -0,0 +1,37 @@ +# Transfer Learning for Image Classification using PyTorch + +This notebook uses image classification models from Torchvision that were originally trained +using ImageNet and does transfer learning with the Food101 dataset, a flowers dataset, or +a custom image dataset. + +The notebook performs the following steps: + +1. Import dependencies and setup parameters +2. Prepare the dataset +3. Predict using the original model +4. Transfer learning +5. Visualize the model output +6. Export the saved model + +## Running the notebook + +To run the notebook, follow the instructions to setup the [PyTorch notebook environment](/notebooks#pytorch-environment). + +## References + +Dataset citations: +``` +@inproceedings{bossard14, + title = {Food-101 -- Mining Discriminative Components with Random Forests}, + author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc}, + booktitle = {European Conference on Computer Vision}, + year = {2014} +} + +@ONLINE {tfflowers, +author = "The TensorFlow Team", +title = "Flowers", +month = "jan", +year = "2019", +url = "http://download.tensorflow.org/example_images/flower_photos.tgz" } +``` diff --git a/notebooks/image_classification/pytorch_image_classification/model_utils.py b/notebooks/image_classification/pytorch_image_classification/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ab981d5e759a6bc7def8d7ee3aa59cbac3296f62 --- /dev/null +++ b/notebooks/image_classification/pytorch_image_classification/model_utils.py @@ -0,0 +1,79 @@ +# +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torchvision +from pydoc import locate + + +# Dictionary of Torchvision image classification models +torchvision_model_map = { + "resnet18": { + "classifier": "fc" + }, + "resnet50": { + "classifier": "fc" + }, + "efficientnet_b0": { + "classifier": ["classifier", 1] + }, + "efficientnet_b1": { + "classifier": ["classifier", 1] + }, + "efficientnet_b2": { + "classifier": ["classifier", 1] + }, + "densenet121": { + "classifier": "classifier" + }, + "densenet161": { + "classifier": "classifier" + }, + "googlenet": { + "classifier": "fc" + }, + "shufflenet_v2_x1_0": { + "classifier": "fc" + }, + "mobilenet_v2": { + "classifier": ["classifier", 1] + } +} + +def get_retrainable_model(model_name, num_classes, do_fine_tuning=False): + # Load an image classification model pretrained on ImageNet + pretrained_model_class = locate('torchvision.models.{}'.format(model_name)) + classifier_layer = torchvision_model_map[model_name]['classifier'] + + model = pretrained_model_class(pretrained=True) + + if not do_fine_tuning: + for param in model.parameters(): + param.requires_grad = False + + if type(classifier_layer) == list: + classifier = getattr(model, classifier_layer[0])[classifier_layer[1]] + num_features = classifier.in_features + model.classifier[classifier_layer[1]] = torch.nn.Linear(num_features, num_classes) + else: + classifier = getattr(model, classifier_layer) + num_features = classifier.in_features + setattr(model, classifier_layer, torch.nn.Linear(num_features, num_classes)) + + return model + diff --git a/notebooks/image_classification/tf_image_classification/Image_Classification_Transfer_Learning.ipynb b/notebooks/image_classification/tf_image_classification/Image_Classification_Transfer_Learning.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..032513d4d9cb6aae7f16c4de53f1a9fa4fdfa888 --- /dev/null +++ b/notebooks/image_classification/tf_image_classification/Image_Classification_Transfer_Learning.ipynb @@ -0,0 +1,498 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Transfer Learning for Image Classification\n", + "\n", + "This notebook uses a classifier model that was originally trained using [ImageNet](https://image-net.org) and does transfer learning with either a TF dataset or your own raw images.\n", + "The notebook performs the following steps:\n", + "1. [Import dependencies and setup parameters](#1.-Import-dependencies-and-setup-parameters)\n", + "2. [Prepare the dataset](#2.-Prepare-the-dataset) using either a TF dataset or your own images\n", + "3. [Predict using the original model](#3.-Predict-using-the-original-model)\n", + "4. [Transfer learning](#4.-Transfer-Learning)\n", + "5. [Evaluate the model](#5.-Evaluate-the-model)\n", + "6. [Export the saved model](#6.-Export-the-saved-model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [README.md](/notebooks/README.md) to setup a TensorFlow environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import tensorflow_hub as hub\n", + "import tensorflow_datasets as tfds\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "\n", + "from model_util import tfhub_model_map\n", + "from tlt.utils.file_utils import download_and_extract_tar_file\n", + "\n", + "print('Supported models:')\n", + "print('\\n'.join(tfhub_model_map.keys()))\n", + "\n", + "# Specify the the parent directory for the custom or tf dataset\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "# Batch size\n", + "batch_size = 32\n", + "\n", + "print(\"\\nDataset directory:\", dataset_directory)\n", + "print(\"Output directory:\", output_directory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Specify a model from the list above\n", + "model_name = \"efficientnet_b0\"\n", + "\n", + "if model_name not in tfhub_model_map.keys():\n", + " raise ValueError(\"The specified model_name ({}) is invalid. Please select from: {}\".\n", + " format(model_name, tfhub_model_map.keys()))\n", + " \n", + "# Get the info for the specified model from the map\n", + "model_map_values = tfhub_model_map[model_name]\n", + "model_handle = tfhub_model_map[model_name][\"imagenet_model\"]\n", + "feature_vector_handle = tfhub_model_map[model_name][\"feature_vector\"]\n", + "image_size = tfhub_model_map[model_name][\"image_size\"]\n", + "print(\"Model:\", model_name)\n", + "print(\"Classifier model:\", model_handle)\n", + "print(\"Feature vector:\", feature_vector_handle)\n", + "print(\"Image size:\", image_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Prepare the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option A: Use your own image dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use your own image dataset for transfer learning with the rest of this notebook, format your images as `.jpg` files and save them in folders named after the classes that you want the model to predict. To provide a working example using the correct layout, we will download and extract a flower species dataset. This is different from using the TF dataset called `tf_flowers`, although they are the same images, because the download contains image files, not tf_records, and we are not using the TF datasets API. After downloading and extracting, you will have the following subdirectories in your dataset directory. Each species subfolder will contain numerous `.jpg` files:\n", + "\n", + "```\n", + "dataset_directory\n", + "└── flower_photos\n", + " └── daisy\n", + " └── dandelion\n", + " └── roses\n", + " └── sunflowers\n", + " └── tulips\n", + "```\n", + "\n", + "Use this as an example to organize your own image files accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# When you have your own properly organized subdirectory of images, adjust this variable\n", + "dataset_subdir = os.path.join(dataset_directory, \"flower_photos\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Only run this if you want to use the example flowers dataset\n", + "if not os.path.exists(dataset_subdir):\n", + " os.makedirs(dataset_subdir)\n", + " dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + " download_and_extract_tar_file(dataset_url, dataset_directory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pathlib\n", + "data_subdir_path = pathlib.Path(dataset_subdir)\n", + "image_count = len(list(data_subdir_path.glob('*/*.jpg')))\n", + "print('Images:', image_count)\n", + "print('Image Size:', image_size)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create an image data generator and partition the data into train and test sets\n", + "img_gen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255, validation_split=0.25)\n", + "train_ds = img_gen.flow_from_directory(data_subdir_path, batch_size=batch_size, \n", + " target_size=(image_size, image_size),\n", + " class_mode='sparse', subset='training')\n", + "test_ds = img_gen.flow_from_directory(data_subdir_path, batch_size=batch_size, \n", + " target_size=(image_size, image_size),\n", + " class_mode='sparse', subset='validation')\n", + "\n", + "# Get class names for the dataset\n", + "class_names = [k for k in train_ds.class_indices.keys()]\n", + "print(\"Number of classes:\", len(class_names))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Skip to the next step [3. Predict using the original model](#3.-Predict-using-the-original-model) to continue using your own image dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option B: Use a TF dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use a TF Dataset, specify the name of the dataset to load from the TF Datasets catalog, preprocess the images to convert them to float32, and resize the images. This example uses the [Food-101 dataset using the TensorFlow datasets API](https://www.tensorflow.org/datasets/catalog/food101) dataset, but you can choose from a wide variety of [options](https://www.tensorflow.org/datasets/catalog/overview) (click on the \"Image classification\" section). If the dataset is not found in the dataset directory it is downloaded. Subsequent runs will reuse the already downloaded dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Options include: \"food101\", cats_vs_dogs\", \"rock_paper_scissors\", and \"tf_flowers\"\n", + "tf_dataset = \"food101\"\n", + "\n", + "# Load the dataset using the TensorFlow datasets API\n", + "[train_ds, test_ds], info = tfds.load(tf_dataset,\n", + " data_dir=dataset_directory,\n", + " split=[\"train[:75%]\", \"train[75%:]\"],\n", + " as_supervised=True,\n", + " shuffle_files=True,\n", + " with_info=True)\n", + "\n", + "# Preprocess the images to convert them to float32 and resize the images to match our model\n", + "def preprocess_image(image, label):\n", + " image = tf.image.convert_image_dtype(image, tf.float32)\n", + " image = tf.image.resize_with_pad(image, image_size, image_size)\n", + " return (image, label)\n", + "\n", + "train_ds = train_ds.map(preprocess_image)\n", + "test_ds = test_ds.map(preprocess_image)\n", + "\n", + "print(\"Dataset directory: \", dataset_directory)\n", + "print(\"Training dataset size:\", len(train_ds))\n", + "print(\"Validation dataset size:\", len(test_ds))\n", + "\n", + "# Training data is shuffled for randomness\n", + "# https://www.tensorflow.org/datasets/keras_example#build_a_training_pipeline\n", + "train_ds = train_ds.cache()\n", + "train_ds = train_ds.shuffle(info.splits['train'].num_examples)\n", + "train_ds = train_ds.batch(batch_size)\n", + "train_ds = train_ds.prefetch(tf.data.AUTOTUNE)\n", + "\n", + "# Test data does not need to be shuffled, and caching is done after batching\n", + "# https://www.tensorflow.org/datasets/keras_example#build_an_evaluation_pipeline\n", + "test_ds = test_ds.batch(batch_size)\n", + "test_ds = test_ds.cache()\n", + "test_ds = test_ds.prefetch(tf.data.AUTOTUNE)\n", + "\n", + "# Get class names for the dataset\n", + "class_names = info.features[\"label\"].names\n", + "print(\"Number of classes:\", len(class_names))\n", + "print('After processing and batching: ', train_ds.element_spec)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Predict using the original model\n", + "\n", + "Use the classifier model that was trained using ImageNet to do predictions with the dataset and view the results for a single batch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get a batch of the dataset to use for testing\n", + "batch = next(iter(test_ds))\n", + "image_batch, label_batch = batch\n", + "\n", + "# List of the actual labels for this batch\n", + "actual_label_batch = [class_names[int(id)] for id in label_batch]\n", + "\n", + "# Download the ImageNet labels and load them into a list\n", + "labels_file = \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n", + "downloaded_file = tf.keras.utils.get_file(\"labels.txt\", origin=labels_file)\n", + "imagenet_classes = []\n", + "\n", + "with open(downloaded_file) as f:\n", + " imagenet_labels = f.readlines()\n", + " imagenet_classes = [l.strip() for l in imagenet_labels]\n", + "\n", + "# Predict using the TF Hub classifier that was trained using ImageNet\n", + "classifier = tf.keras.Sequential([\n", + " hub.KerasLayer(model_handle, input_shape=(image_size, image_size)+(3,))\n", + "])\n", + "predicted_batch = classifier.predict(image_batch)\n", + "predicted_id = np.argmax(predicted_batch, axis=-1)\n", + "predicted_label_batch = [imagenet_classes[id] for id in predicted_id]\n", + "\n", + "# Visualize the results\n", + "plt.figure(figsize=(16,16))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " plt.imshow(image_batch[n])\n", + " correct_prediction = actual_label_batch[n] == predicted_label_batch[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predicted_label_batch[n].title() if correct_prediction else \"{}\\n({})\".format(predicted_label_batch[n], actual_label_batch[n]) \n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"ImageNet predictions\", fontsize=16)\n", + "plt.show()\n", + "\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Transfer Learning\n", + "\n", + "Get the feature vector from TF Hub and add on a dense layer based on the number of classes in our dataset. Train the model using the training dataset for the specified number of epochs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Number of training epochs\n", + "training_epochs = 1\n", + "\n", + "# To reduce training time, the feature extractor layer can remain frozen (do_fine_tuning=False).\n", + "# Fine-tuning can be enabled to potentially get better accuracy. Note that enabling fine-tuning\n", + "# will increase training time.\n", + "do_fine_tuning = False\n", + "\n", + "# Optionally add a dropout layer (set to a float between 0 and 1, or None).\n", + "# If set to None, no dropout layer will be added.\n", + "dropout_layer_rate = None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "feature_extractor_layer = hub.KerasLayer(feature_vector_handle,\n", + " input_shape=(image_size, image_size, 3),\n", + " trainable=do_fine_tuning)\n", + "\n", + "if dropout_layer_rate == None:\n", + " model = tf.keras.Sequential([\n", + " feature_extractor_layer,\n", + " tf.keras.layers.Dense(len(class_names))\n", + " ])\n", + "else:\n", + " model = tf.keras.Sequential([\n", + " feature_extractor_layer,\n", + " tf.keras.layers.Dropout(dropout_layer_rate),\n", + " tf.keras.layers.Dense(len(class_names))\n", + " ])\n", + "\n", + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "model.compile(\n", + " optimizer=tf.keras.optimizers.Adam(),\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['acc'])\n", + "\n", + "class CollectBatchStats(tf.keras.callbacks.Callback):\n", + " def __init__(self):\n", + " self.batch_losses = []\n", + " self.batch_acc = []\n", + "\n", + " def on_train_batch_end(self, batch, logs=None):\n", + " self.batch_losses.append(logs['loss'])\n", + " self.batch_acc.append(logs['acc'])\n", + " self.model.reset_metrics()\n", + "\n", + "batch_stats_callback = CollectBatchStats()\n", + "\n", + "history = model.fit(train_ds, epochs=training_epochs, shuffle=True, callbacks=[batch_stats_callback])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Evaluate the model\n", + "\n", + "After the training completes, evaluate the model's accuracy using the validation dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "model.evaluate(test_ds, batch_size=batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Also, predict using the same sample batch that we used earlier with the ImageNet trained classier to visualize the results after training the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Predict using the sample batch\n", + "predicted_batch = model.predict(image_batch)\n", + "predicted_id = np.argmax(predicted_batch, axis=-1)\n", + "predicted_label_batch = [class_names[id] for id in predicted_id]\n", + "\n", + "# Display the results\n", + "plt.figure(figsize=(16,16))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " plt.imshow(image_batch[n])\n", + " correct_prediction = actual_label_batch[n] == predicted_label_batch[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predicted_label_batch[n].title() if correct_prediction else \"{}\\n({})\".format(predicted_label_batch[n], actual_label_batch[n]) \n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"Model predictions\", fontsize=16)\n", + "plt.show()\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Export the saved model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = os.path.join(output_directory, \"{}_saved_model\".format(model_name))\n", + "model.save(saved_model_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataset citations\n", + "```\n", + "@inproceedings{bossard14,\n", + " title = {Food-101 -- Mining Discriminative Components with Random Forests},\n", + " author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc},\n", + " booktitle = {European Conference on Computer Vision},\n", + " year = {2014}\n", + "}\n", + "\n", + "@ONLINE {tfflowers,\n", + "author = \"The TensorFlow Team\",\n", + "title = \"Flowers\",\n", + "month = \"jan\",\n", + "year = \"2019\",\n", + "url = \"http://download.tensorflow.org/example_images/flower_photos.tgz\" }\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/image_classification/tf_image_classification/README.md b/notebooks/image_classification/tf_image_classification/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3803ad1316ed4aa500064f0d42fd135ac9be2b09 --- /dev/null +++ b/notebooks/image_classification/tf_image_classification/README.md @@ -0,0 +1,35 @@ +# Transfer Learning for Image Classification with TF Hub + +This notebook uses transfer learning with multiple [TF Hub](https://tfhub.dev) image classifiers, +[TF datasets](https://www.tensorflow.org/datasets/), and custom image datasets. + +The notebook performs the following steps: +1. Import dependencies and setup parameters +1. Prepare the dataset +1. Predict using the original model +1. Transfer Learning +1. Evaluate the model +1. Export the saved model + +## Running the notebook + +To run the notebook, follow the instructions to setup the [TensorFlow notebook environment](/notebooks/setup.md). + +## References + +Dataset citations +``` +@inproceedings{bossard14, + title = {Food-101 -- Mining Discriminative Components with Random Forests}, + author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc}, + booktitle = {European Conference on Computer Vision}, + year = {2014} +} + +@ONLINE {tfflowers, +author = "The TensorFlow Team", +title = "Flowers", +month = "jan", +year = "2019", +url = "http://download.tensorflow.org/example_images/flower_photos.tgz" } +``` diff --git a/notebooks/image_classification/tf_image_classification/model_util.py b/notebooks/image_classification/tf_image_classification/model_util.py new file mode 100644 index 0000000000000000000000000000000000000000..9ef82a0061cd6446efbce338ad19f2705253c417 --- /dev/null +++ b/notebooks/image_classification/tf_image_classification/model_util.py @@ -0,0 +1,72 @@ +# +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Dictionary of TFHub models +tfhub_model_map = { + "resnet_v1_50": { + "imagenet_model": "https://tfhub.dev/google/imagenet/resnet_v1_50/classification/5", + "feature_vector": "https://tfhub.dev/google/imagenet/resnet_v1_50/feature_vector/5", + "image_size": 224 + }, + "resnet_v2_50": { + "imagenet_model": "https://tfhub.dev/google/imagenet/resnet_v2_50/classification/5", + "feature_vector": "https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/5", + "image_size": 224 + }, + "resnet_v2_101": { + "imagenet_model": "https://tfhub.dev/google/imagenet/resnet_v2_101/classification/5", + "feature_vector": "https://tfhub.dev/google/imagenet/resnet_v2_101/feature_vector/5", + "image_size": 224 + }, + "mobilenet_v2_100_224": { + "imagenet_model": "https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/classification/5", + "feature_vector": "https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/feature_vector/4", + "image_size": 224 + }, + "efficientnetv2-s": { + "imagenet_model": "https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_s/classification/2", + "feature_vector": "https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_s/feature_vector/2", + "image_size": 384 + }, + "efficientnet_b0": { + "imagenet_model": "https://tfhub.dev/google/efficientnet/b0/classification/1", + "feature_vector": "https://tfhub.dev/google/efficientnet/b0/feature-vector/1", + "image_size": 224 + }, + "efficientnet_b1": { + "imagenet_model": "https://tfhub.dev/google/efficientnet/b1/classification/1", + "feature_vector": "https://tfhub.dev/google/efficientnet/b1/feature-vector/1", + "image_size": 240 + }, + "efficientnet_b2": { + "imagenet_model": "https://tfhub.dev/google/efficientnet/b2/classification/1", + "feature_vector": "https://tfhub.dev/google/efficientnet/b2/feature-vector/1", + "image_size": 260 + }, + "inception_v3": { + "imagenet_model": "https://tfhub.dev/google/imagenet/inception_v3/classification/5", + "feature_vector": "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/5", + "image_size": 299 + }, + "nasnet_large": { + "imagenet_model": "https://tfhub.dev/google/imagenet/nasnet_large/classification/5", + "feature_vector": "https://tfhub.dev/google/imagenet/nasnet_large/feature_vector/5", + "image_size": 331 + } +} + diff --git a/notebooks/image_classification/tlt_api_pyt_image_classification/README.md b/notebooks/image_classification/tlt_api_pyt_image_classification/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5b77288c399810e058fd8fe472bc621bcf076f5f --- /dev/null +++ b/notebooks/image_classification/tlt_api_pyt_image_classification/README.md @@ -0,0 +1,57 @@ +# Transfer Learning for PyTorch Image Classification using the Intel® Transfer Learning Tool API + +This notebook demonstrates how to use the Intel Transfer Learning Tool API to do transfer learning for +image classification using PyTorch. + +The notebook performs the following steps: +1. Import dependencies and setup parameters +1. Get the model +1. Get the dataset +1. Prepare the dataset +1. Predict using the original model +1. Transfer learning +1. Predict +1. Export + +## Running the notebook + +To run the notebook, follow the instructions to setup the [PyTorch notebook environment](/notebooks/setup.md). + +## References + +Dataset citations +``` +@ONLINE {tfflowers, +author = "The TensorFlow Team", +title = "Flowers", +month = "jan", +year = "2019", +url = "http://download.tensorflow.org/example_images/flower_photos.tgz" } + +@ONLINE {CIFAR10, +author = "Alex Krizhevsky", +title = "CIFAR-10", +year = "2009", +url = "http://www.cs.toronto.edu/~kriz/cifar.html" } + +@article{openimages, + title={OpenImages: A public dataset for large-scale multi-label and multi-class image classification.}, + author={Krasin, Ivan and Duerig, Tom and Alldrin, Neil and Veit, Andreas and Abu-El-Haija, Sami + and Belongie, Serge and Cai, David and Feng, Zheyun and Ferrari, Vittorio and Gomes, Victor + and Gupta, Abhinav and Narayanan, Dhyanesh and Sun, Chen and Chechik, Gal and Murphy, Kevin}, + journal={Dataset available from https://github.com/openimages}, + year={2016} +} +``` +Model citations +``` +@misc{yalniz2019billionscale, + title={Billion-scale semi-supervised learning for image classification}, + author={I. Zeki Yalniz and Hervé Jégou and Kan Chen and Manohar Paluri and Dhruv Mahajan}, + year={2019}, + eprint={1905.00546}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +``` + diff --git a/notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb b/notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..8d43838e058c0477b904d561a2db0a6fcab0a0da --- /dev/null +++ b/notebooks/image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb @@ -0,0 +1,670 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3405d28d", + "metadata": {}, + "source": [ + "# Transfer Learning for Image Classification using PyTorch and the Intel® Transfer Learning Tool API\n", + "\n", + "This notebook uses the `tlt` library to do transfer learning for image classfication with a PyTorch pretrained model." + ] + }, + { + "cell_type": "markdown", + "id": "1d61b7ac", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions to setup a PyTorch environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0bf9fd0", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import PIL.Image as Image\n", + "import torch, torchvision\n", + "import requests\n", + "from io import BytesIO\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_and_extract_tar_file, download_file\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "8f1fc78c", + "metadata": {}, + "source": [ + "## 2. Get the model\n", + "\n", + "In this step, we call the model factory to list supported PyTorch image classification models. This is a list of pretrained models from [Torchvision](https://pytorch.org/vision/stable/models.html) and [PyTorch Hub](https://pytorch.org/hub/) that we tested with our API. Optionally, the `verbose=True` argument can be added to the `print_supported_models` function call to get more information about each model (such as the classification layer, image size, the original dataset, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad4aeafd", + "metadata": {}, + "outputs": [], + "source": [ + "# See a list of available models\n", + "model_factory.print_supported_models(use_case='image_classification', framework='pytorch')" + ] + }, + { + "cell_type": "markdown", + "id": "aa1d9d53", + "metadata": {}, + "source": [ + "Next, use the model factory to get one of the models listed in the previous cell. The `get_model` function returns a model object that will later be used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee4e43c0", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name='efficientnet_b0', framework='pytorch')\n", + "\n", + "print(\"Model name:\", model.model_name)\n", + "print(\"Framework:\", model.framework)\n", + "print(\"Use case:\", model.use_case)\n", + "print(\"Image size:\", model.image_size)" + ] + }, + { + "cell_type": "markdown", + "id": "dabd4183", + "metadata": {}, + "source": [ + "## 3. Get the dataset" + ] + }, + { + "cell_type": "markdown", + "id": "2d314ba0", + "metadata": {}, + "source": [ + "### Option A: Use your own dataset\n", + "\n", + "To use your own image dataset for transfer learning with the rest of this notebook, format your images as `.jpg` files and save them in folders named after the classes that you want the model to predict. To provide a working example using the correct layout, we will download a flower species dataset. After downloading and extracting, you will have the following subdirectories in your dataset directory. Each species subfolder will contain numerous `.jpg` files:\n", + "\n", + "```\n", + "flower_photos\n", + " └── daisy\n", + " └── dandelion\n", + " └── roses\n", + " └── sunflowers\n", + " └── tulips\n", + "```\n", + "\n", + "When using your own dataset, ensure that it is similarly organized with folders for each class. Change the `custom_dataset_path` variable to point to your dataset folder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ab17b6b", + "metadata": { + "tags": [ + "remove_for_tv_dataset" + ] + }, + "outputs": [], + "source": [ + "# For demonstration purposes, we download a flowers dataset. To instead use your own dataset, set the\n", + "# custom_dataset_path to point to your dataset's directory and comment out the download_and_extract_tar_file line.\n", + "custom_dataset_path = os.path.join(dataset_dir, \"flower_photos\")\n", + "\n", + "if not os.path.exists(custom_dataset_path):\n", + " download_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + " download_and_extract_tar_file(download_url, dataset_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "8cccbc99", + "metadata": {}, + "source": [ + "Call the dataset factory to load the dataset from the directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6706c01", + "metadata": { + "tags": [ + "remove_for_tv_dataset" + ] + }, + "outputs": [], + "source": [ + "# Load the dataset from the custom dataset path\n", + "dataset = dataset_factory.load_dataset(dataset_dir=custom_dataset_path,\n", + " use_case='image_classification', \n", + " framework='pytorch')\n", + "\n", + "print(\"Class names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "88897dc6", + "metadata": {}, + "source": [ + "Skip to the next step [4. Prepare the dataset](#4.-Prepare-the-dataset) to continue using the custom dataset." + ] + }, + { + "cell_type": "markdown", + "id": "c7e3cd9b", + "metadata": {}, + "source": [ + "### Option B: Use a dataset from the PyTorch's Torchvision Datasets catalog\n", + "\n", + "To use a Torchvision dataset, specify the name of the dataset in the `get_dataset` function. This example uses the `CIFAR10` dataset from the [Torchvision datasets for image classification](https://pytorch.org/vision/stable/datasets.html#image-classification), but you can choose from a variety of options. If the dataset is not found in the dataset directory it will be downloaded. Subsequent runs will reuse the already downloaded dataset.\n", + "\n", + "These Torchvision datasets are currently supported in the API:\n", + "* CIFAR10\n", + "* Country211\n", + "* DTD\n", + "* Food101\n", + "* FGVCAircraft\n", + "* RenderedSST2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba8bb0cb", + "metadata": { + "tags": [ + "remove_for_custom_dataset" + ] + }, + "outputs": [], + "source": [ + "dataset = dataset_factory.get_dataset(dataset_dir=dataset_dir,\n", + " use_case='image_classification', \n", + " framework='pytorch',\n", + " dataset_name='CIFAR10',\n", + " dataset_catalog='torchvision')\n", + "\n", + "print(dataset.info)\n", + "\n", + "print(\"Class names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "99f23249", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "Once you have your dataset from Option A or Option B above, use the following cells to split and preprocess the data. We split them into training and validation subsets, then resize the images to match the selected models, and then batch the images.\n", + "Data augmentation can be appplied by specifying the augmentations to be applied in add_aug parameter. Supported augmentations are given below\n", + "1. hflip - RandomHorizontalFlip\n", + "2. rotate - RandomRotate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd91fbcf", + "metadata": {}, + "outputs": [], + "source": [ + "# Split the dataset into training and validation subsets\n", + "dataset.shuffle_split(train_pct=.75, val_pct=.25)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7c95a70", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess the dataset with an image size that matches the model and a batch size of 32\n", + "batch_size = 32\n", + "dataset.preprocess(model.image_size, batch_size=batch_size, add_aug=['hflip','rotate'])" + ] + }, + { + "cell_type": "markdown", + "id": "3704772b", + "metadata": {}, + "source": [ + "## 5. Predict using the original model\n", + "\n", + "We get a single batch from our dataset, and use that to call predict on our model. Since we haven't done any training on the model yet, it will give us predictions using the original ImageNet trained model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd6782b0", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a single batch from the dataset\n", + "images, labels = dataset.get_batch()\n", + "labels = [dataset.class_names[id] for id in labels]\n", + "\n", + "# Download the ImageNet labels and load them into a list\n", + "labels_file = \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n", + "labels_file_path = os.path.join(dataset_dir, os.path.basename(labels_file))\n", + "\n", + "if not os.path.exists(labels_file_path):\n", + " download_file(labels_file, dataset_dir)\n", + "\n", + "with open(labels_file_path) as f:\n", + " imagenet_labels = f.readlines()\n", + " imagenet_classes = [l.strip() for l in imagenet_labels]\n", + " \n", + "# Predict using the original model\n", + "predictions = model.predict(images)\n", + "predictions = [imagenet_classes[id] for id in predictions]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d37b808f", + "metadata": {}, + "outputs": [], + "source": [ + "# Display the images with the predicted ImageNet label\n", + "plt.figure(figsize=(18,14))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " inp = images[n]\n", + " inp = inp.numpy().transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " inp = std * inp + mean\n", + " inp = np.clip(inp, 0, 1)\n", + " plt.imshow(inp)\n", + " correct_prediction = labels[n] == predictions[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predictions[n].title() if correct_prediction else \"{}\\n({})\".format(predictions[n], labels[n]) \n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"ImageNet predictions\", fontsize=20)\n", + "plt.show()\n", + "\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")" + ] + }, + { + "cell_type": "markdown", + "id": "a49ec7b7", + "metadata": {}, + "source": [ + "## 6. Transfer Learning\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the base model and add on a dense layer based on the number of classes in the dataset. The model is then compiled and trained based on the number of epochs specified in the argument. With the do_eval parameter set to True by default, this step will also show how the model can be evaluated. The model's evaluate function returns a list of metrics calculated from the dataset's validation subset." + ] + }, + { + "cell_type": "markdown", + "id": "ab510f51", + "metadata": {}, + "source": [ + "### Arguments\n", + "\n", + "#### Required\n", + "- **dataset** (ImageClassificationDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory for checkpoint files\n", + "- **epochs** (int): Number of epochs to train the model (default: 1)\n", + "\n", + "#### Optional\n", + "- **initial_checkpoints** (str): Path to checkpoint weights to load. If the path provided is a directory, the latest checkpoint will be used.\n", + "- **early_stopping** (bool): Enable early stopping if convergence is reached while training at the end of each epoch. (default: False)\n", + "- **lr_decay** (bool): If lr_decay is True and do_eval is True, learning rate decay on the validation loss is applied at the end of each epoch.\n", + "- **extra_layers** (list[int]): Optionally insert additional dense layers between the base model and output layer. This can help increase accuracy when fine-tuning a TFHub model. The input should be a list of integers representing the number and size of the layers, for example [1024, 512] will insert two dense layers, the first with 1024 neurons and the second with 512 neurons.\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b601fc", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.train(dataset, output_dir=output_dir, epochs=1)" + ] + }, + { + "cell_type": "markdown", + "id": "8582cfaf", + "metadata": {}, + "source": [ + "A complete model summary can be printed for all modules in case any need to be unfrozen:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a5e279b", + "metadata": {}, + "outputs": [], + "source": [ + "model.list_layers(verbose=True) " + ] + }, + { + "cell_type": "markdown", + "id": "9530b21d", + "metadata": {}, + "source": [ + "Layers can be unfrozen by passing their string names, such as the following:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "258662a6", + "metadata": {}, + "outputs": [], + "source": [ + "model.unfreeze_layer(\"features\") # Unfreezes the features layers\n", + "model.list_layers(verbose=True) " + ] + }, + { + "cell_type": "markdown", + "id": "707f17dd", + "metadata": {}, + "source": [ + "## 7. Predict\n", + "\n", + "Lastly, we predict using the same single batch that we used earlier with the ImageNet trained model to visualize the model's predictions after training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2bc447f", + "metadata": {}, + "outputs": [], + "source": [ + "# Predict with a single batch\n", + "predictions = model.predict(images)\n", + "\n", + "# Map the predicted ids to the class names\n", + "predictions = [dataset.class_names[id] for id in predictions]\n", + "\n", + "# Display the results\n", + "plt.figure(figsize=(16,16))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " inp = images[n]\n", + " inp = inp.numpy().transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " inp = std * inp + mean\n", + " inp = np.clip(inp, 0, 1)\n", + " plt.imshow(inp)\n", + " correct_prediction = labels[n] == predictions[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predictions[n].title() if correct_prediction else \"{}\\n({})\".format(predictions[n], labels[n]) \n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"Model predictions\", fontsize=16)\n", + "plt.show()\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")" + ] + }, + { + "cell_type": "markdown", + "id": "d5b03792", + "metadata": {}, + "source": [ + "### Custom Single Image Prediction" + ] + }, + { + "cell_type": "markdown", + "id": "6dc48f94", + "metadata": {}, + "source": [ + "We can also predict using a single image that wasn't part of our original dataset. We download a flower image from the [Open Images Dataset](https://storage.googleapis.com/openimages/web/index.html) and then resize it to match our model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5afd283f", + "metadata": {}, + "outputs": [], + "source": [ + "# Download an image from the web and resize it to match our model\n", + "image_url = 'https://c8.staticflickr.com/8/7095/7210797228_c7fe51c3cb_z.jpg'\n", + "\n", + "image_shape = (model.image_size, model.image_size)\n", + "daisy = Image.open(BytesIO(requests.get(image_url).content)).resize(image_shape)\n", + "daisy" + ] + }, + { + "cell_type": "markdown", + "id": "7b7a0227", + "metadata": {}, + "source": [ + "Then, we call predict by passing the np array for our image and add a dimension to our array to represent the batch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22146e35", + "metadata": {}, + "outputs": [], + "source": [ + "# Get the image as a np array and scale and normalize it \n", + "daisy = np.array(daisy)/255.0\n", + "daisy = (daisy - np.array([0.485, 0.456, 0.406])) / np.array([0.229, 0.224, 0.225])\n", + "\n", + "# Arrange the channels with a batch dimension first (np.newaxis) and RGB channels second (np.moveaxis)\n", + "daisy = torch.Tensor(np.moveaxis(daisy, -1, 0))[np.newaxis, ...]\n", + "\n", + "# Predict and print the class name\n", + "result = model.predict(daisy)\n", + "print(dataset.class_names[result[0]])" + ] + }, + { + "cell_type": "markdown", + "id": "26130983", + "metadata": {}, + "source": [ + "## 8. Export\n", + "\n", + "Lastly, we can call the model `export` function to generate a `saved_model.pb`. Each time the model is exported, a new numbered directory is created, which allows serving to pick up the latest model. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "391afe2b", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "40f5dc3e", + "metadata": {}, + "source": [ + "## 9. Post-training quantization\n", + "\n", + "In this section, the `tlt` API uses [Intel® Neural Compressor (INC)](https://github.com/intel/neural-compressor) to benchmark and quantize the model to get optimal inference performance.\n", + "\n", + "We use the Intel Neural Compressor to benchmark the full precision model to see how it performs, as our baseline.\n", + "\n", + "> Note that there is a known issue when running Intel Neural Compressor from a notebook that you may sometimes see the error \n", + "> `zmq.error.ZMQError: Address already in use`. If you see this error, rerun the cell again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8318885", + "metadata": {}, + "outputs": [], + "source": [ + "results = model.benchmark(dataset=dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "7997cc13", + "metadata": {}, + "source": [ + "Next we use Intel Neural Compressor to automatically search for the optimal quantization recipe for low-precision model inference within the accuracy loss constrains defined in the config. Running post training quantization may take several minutes, depending on your hardware and the exit policy (timeout and max trials).\n", + "\n", + "You can customize a config by passing these parameters to `get_inc_config()`:\n", + " * **approach** (str): The quantization approach (we recommend 'static' for image models and 'dynamic' for text models) \n", + " * **accuracy_criterion_relative** (float): Relative accuracy loss (default: 0.01, which is 1%)\n", + " * **exit_policy_timeout** (int): Tuning timeout in seconds (default: 0). Tuning processing finishes when the\n", + " timeout or max_trials is reached. A tuning timeout of 0 means that the tuning phase stops when the accuracy\n", + " criterion is met.\n", + "* **exit_policy_max_trials** (int): Maximum number of tuning trials (default: 50). Tuning processing finishes when\n", + " the timeout or or max_trials is reached." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce143e0f", + "metadata": {}, + "outputs": [], + "source": [ + "from tlt.utils.inc_utils import get_inc_config\n", + "\n", + "config = get_inc_config(approach='static',\n", + " accuracy_criterion_relative=0.01,\n", + " exit_policy_timeout=0,\n", + " exit_policy_max_trials=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ee3e1d7", + "metadata": {}, + "outputs": [], + "source": [ + "inc_output_dir = os.path.join(output_dir, 'quantized_models', model.model_name,\n", + " os.path.basename(saved_model_dir))\n", + "model.quantize(inc_output_dir, dataset=dataset, config=config)" + ] + }, + { + "cell_type": "markdown", + "id": "abb5d272", + "metadata": {}, + "source": [ + "Let's benchmark using the quantized model, so that we can compare the performance to the full precision model that was originally benchmarked." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a900100", + "metadata": {}, + "outputs": [], + "source": [ + "quantized_results = model.benchmark(dataset=dataset, saved_model_dir=inc_output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "0947915a", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "```\n", + "@ONLINE {tfflowers,\n", + "author = \"The TensorFlow Team\",\n", + "title = \"Flowers\",\n", + "month = \"jan\",\n", + "year = \"2019\",\n", + "url = \"http://download.tensorflow.org/example_images/flower_photos.tgz\" }\n", + "\n", + "@ONLINE {CIFAR10,\n", + "author = \"Alex Krizhevsky\",\n", + "title = \"CIFAR-10\",\n", + "year = \"2009\",\n", + "url = \"http://www.cs.toronto.edu/~kriz/cifar.html\" }\n", + "\n", + "@article{openimages,\n", + " title={OpenImages: A public dataset for large-scale multi-label and multi-class image classification.},\n", + " author={Krasin, Ivan and Duerig, Tom and Alldrin, Neil and Veit, Andreas and Abu-El-Haija, Sami\n", + " and Belongie, Serge and Cai, David and Feng, Zheyun and Ferrari, Vittorio and Gomes, Victor\n", + " and Gupta, Abhinav and Narayanan, Dhyanesh and Sun, Chen and Chechik, Gal and Murphy, Kevin},\n", + " journal={Dataset available from https://github.com/openimages},\n", + " year={2016}\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/image_classification/tlt_api_tf_image_classification/README.md b/notebooks/image_classification/tlt_api_tf_image_classification/README.md new file mode 100644 index 0000000000000000000000000000000000000000..441fd3153362194c7866b59c2aaffff8229a23ad --- /dev/null +++ b/notebooks/image_classification/tlt_api_tf_image_classification/README.md @@ -0,0 +1,51 @@ +# Transfer Learning for TensorFlow Image Classification using the Intel® Transfer Learning Tool API + +These notebooks demonstrate how to use the Intel Transfer Learning Tool API to do transfer learning for +image classification using TensorFlow and then quantize or optimize the graph for inference. + +`TLT_TF_Image_Classification_Transfer_Learning.ipynb` performs the following steps: +1. Import dependencies and setup parameters +1. Get the model +1. Get the dataset +1. Prepare the dataset +1. Predict using the original model +1. Transfer learning +1. Predict +1. Export +1. Post-training quantization + +`TLT_TF_Transfer_Learning_and_Graph_Optimization.ipynb` performs the following steps: +1. Import dependencies and setup parameters +1. Get the model +1. Get the dataset +1. Prepare the dataset +1. Evaluate using the original model +1. Transfer learning +1. Export +1. Graph Optimization + +## Running the notebooks + +To run the notebooks, follow the instructions to setup the [TensorFlow notebook environment](/notebooks/setup.md). + +## References + +Dataset citations +``` +@ONLINE {tfflowers, +author = "The TensorFlow Team", +title = "Flowers", +month = "jan", +year = "2019", +url = "http://download.tensorflow.org/example_images/flower_photos.tgz" } + +@article{openimages, + title={OpenImages: A public dataset for large-scale multi-label and multi-class image classification.}, + author={Krasin, Ivan and Duerig, Tom and Alldrin, Neil and Veit, Andreas and Abu-El-Haija, Sami + and Belongie, Serge and Cai, David and Feng, Zheyun and Ferrari, Vittorio and Gomes, Victor + and Gupta, Abhinav and Narayanan, Dhyanesh and Sun, Chen and Chechik, Gal and Murphy, Kevin}, + journal={Dataset available from https://github.com/openimages}, + year={2016} +} +``` + diff --git a/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb b/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c41a92d8e94cc6f17e718cf087bed41500021b35 --- /dev/null +++ b/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb @@ -0,0 +1,597 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "41a52dc9", + "metadata": {}, + "source": [ + "# Transfer Learning for Image Classification using TensorFlow and the Intel® Transfer Learning Tool API\n", + "\n", + "This notebook uses the `tlt` library to do transfer learning for image classfication with a TensorFlow pretrained model." + ] + }, + { + "cell_type": "markdown", + "id": "d1432fd2", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions to setup a TensorFlow environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b996490f", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import PIL.Image as Image\n", + "import tensorflow as tf\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_file, download_and_extract_tar_file\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "b18c4a5e", + "metadata": {}, + "source": [ + "## 2. Get the model\n", + "\n", + "In this step, we call the Intel Transfer Learning Tool model factory to list supported TensorFlow image classification models. This is a list of pretrained models from [TFHub](https://tfhub.dev) and [Keras Applications](https://keras.io/api/applications/) that we tested with our API. Optionally, the `verbose=True` argument can be added to the `print_supported_models` function call to get more information about each model (such as the image size, the original dataset, the preprocessor, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11a2d427", + "metadata": {}, + "outputs": [], + "source": [ + "# See a list of available models\n", + "model_factory.print_supported_models(use_case='image_classification', framework='tensorflow')" + ] + }, + { + "cell_type": "markdown", + "id": "384e7c0d", + "metadata": {}, + "source": [ + "Next, use the model factory to get one of the models listed in the previous cell. The `get_model` function returns a model object that will later be used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44d6f386", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name='resnet_v1_50', framework='tensorflow')\n", + "\n", + "print(\"Model name:\", model.model_name)\n", + "print(\"Framework:\", model.framework)\n", + "print(\"Use case:\", model.use_case)\n", + "print(\"Image size:\", model.image_size)" + ] + }, + { + "cell_type": "markdown", + "id": "e180d1fe", + "metadata": {}, + "source": [ + "## 3. Get the dataset\n", + "\n", + "We call the dataset factory to get a sample image classification dataset. For demonstration purposes, we are using the [tf_flowers](https://www.tensorflow.org/datasets/catalog/tf_flowers) dataset from the [TensorFlow Datasets catalog](https://www.tensorflow.org/datasets). This dataset contains images of flowers in 5 different classes." + ] + }, + { + "cell_type": "markdown", + "id": "681d42e8", + "metadata": {}, + "source": [ + "### Option A: Use your own dataset\n", + "\n", + "To use your own image dataset for transfer learning with the rest of this notebook, format your images as `.jpg` files and save them in folders named after the classes that you want the model to predict. To provide a working example using the correct layout, we will download a flower species dataset. After downloading and extracting, you will have the following subdirectories in your dataset directory. Each species subfolder will contain numerous `.jpg` files:\n", + "\n", + "```\n", + "flower_photos\n", + " └── daisy\n", + " └── dandelion\n", + " └── roses\n", + " └── sunflowers\n", + " └── tulips\n", + "```\n", + "\n", + "When using your own dataset, ensure that it is similarly organized with folders for each class. Change the `custom_dataset_path` variable to point to your dataset folder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01973de8", + "metadata": { + "tags": [ + "remove_for_tf_dataset" + ] + }, + "outputs": [], + "source": [ + "# For demonstration purposes, we download a flowers dataset. To instead use your own dataset, set the\n", + "# custom_dataset_path to point to your dataset's directory and comment out the download_and_extract_tar_file line.\n", + "custom_dataset_path = os.path.join(dataset_dir, \"flower_photos\")\n", + "\n", + "if not os.path.exists(custom_dataset_path):\n", + " download_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + " download_and_extract_tar_file(download_url, dataset_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "b20dc2db", + "metadata": {}, + "source": [ + "Call the dataset factory to load the dataset from the directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce99cbb3", + "metadata": { + "tags": [ + "remove_for_tf_dataset" + ] + }, + "outputs": [], + "source": [ + "# Load the dataset from the custom dataset path\n", + "dataset = dataset_factory.load_dataset(dataset_dir=custom_dataset_path,\n", + " use_case='image_classification', \n", + " framework='tensorflow')\n", + "\n", + "print(\"Class names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "34be95ae", + "metadata": {}, + "source": [ + "Skip to the next step [4. Prepare the dataset](#4.-Prepare-the-dataset) to continue using the custom dataset." + ] + }, + { + "cell_type": "markdown", + "id": "2c68f367", + "metadata": {}, + "source": [ + "### Option B: Use a dataset from the TensorFlow Datasets catalog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6b4cc26", + "metadata": { + "tags": [ + "remove_for_custom_dataset" + ] + }, + "outputs": [], + "source": [ + "dataset = dataset_factory.get_dataset(dataset_dir=dataset_dir,\n", + " use_case='image_classification', \n", + " framework='tensorflow',\n", + " dataset_name='tf_flowers',\n", + " dataset_catalog='tf_datasets')\n", + "\n", + "print(dataset.info)\n", + "\n", + "print(\"\\nClass names:\", str(dataset.class_names))" + ] + }, + { + "cell_type": "markdown", + "id": "c3ec8cfe", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "\n", + "Once you have your dataset from Option A or Option B above, use the following cells to split and preprocess the data. We split them into training and validation subsets, then resize the images to match the selected models, and then batch the images. Data augmentation can be applied by specifying the augmentations to be applied in __add_aug__ parameter. Supported augmentations are:\n", + "1. hvflip - RandomHorizontalandVerticalFlip\n", + "2. hflip - RandomHorizontalFlip\n", + "3. vflip - RandomVerticalFlip\n", + "4. rotate - RandomRotate\n", + "5. zoom - RandomZoom" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4282456e", + "metadata": {}, + "outputs": [], + "source": [ + "# Split the dataset into training and validation subsets\n", + "dataset.shuffle_split(train_pct=.75, val_pct=.25)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9789ded", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess the dataset with an image size and preprocessor that match the model and a batch size of 32\n", + "batch_size = 32\n", + "dataset.preprocess(model.image_size, batch_size=batch_size, preprocessor=model.preprocessor)" + ] + }, + { + "cell_type": "markdown", + "id": "a5ff5539", + "metadata": {}, + "source": [ + "## 5. Predict using the original model\n", + "\n", + "We get a single batch from our dataset, and use that to call predict on our model. Since we haven't done any training on the model yet, it will give us predictions using the original ImageNet trained model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a620aec8", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a single batch from the dataset\n", + "images, labels = dataset.get_batch()\n", + "labels = [dataset.class_names[id] for id in labels]\n", + "\n", + "# Download the ImageNet labels and load them into a list\n", + "labels_file = \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n", + "downloaded_file = tf.keras.utils.get_file(\"labels.txt\", origin=labels_file)\n", + "imagenet_classes = []\n", + "\n", + "with open(downloaded_file) as f:\n", + " imagenet_labels = f.readlines()\n", + " imagenet_classes = [l.strip() for l in imagenet_labels]\n", + "\n", + "# Predict using the original model\n", + "predictions = model.predict(images)\n", + "predictions = [imagenet_classes[id] for id in predictions]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad8f845e", + "metadata": {}, + "outputs": [], + "source": [ + "# Display the images with the predicted ImageNet label\n", + "plt.figure(figsize=(18,14))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "\n", + "for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " norm_images = (images[n]-np.min(images[n]))/(np.max(images[n])-np.min(images[n]))\n", + " plt.imshow(norm_images, vmin=np.min(norm_images), vmax=np.max(norm_images))\n", + " correct_prediction = labels[n] == predictions[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predictions[n] if correct_prediction else \"{}\\n({})\".format(predictions[n], labels[n]) \n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"ImageNet predictions\", fontsize=20)\n", + "plt.show()\n", + "\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")" + ] + }, + { + "cell_type": "markdown", + "id": "d875f7ea", + "metadata": {}, + "source": [ + "## 6. Transfer Learning\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the base model and add on a dense layer based on the number of classes in the dataset. The model is then compiled and trained based on the number of epochs specified in the argument. With the do_eval paramter set to True by default, this step will also show how the model can be evaluated and will return a list of metrics calculated from the dataset's validation subset." + ] + }, + { + "cell_type": "markdown", + "id": "213faef5", + "metadata": {}, + "source": [ + "### Arguments\n", + "\n", + "#### Required\n", + "- **dataset** (ImageClassificationDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory for checkpoint files\n", + "- **epochs** (int): Number of epochs to train the model (default: 1)\n", + "\n", + "#### Optional\n", + "- **initial_checkpoints** (str): Path to checkpoint weights to load. If the path provided is a directory, the latest checkpoint will be used.\n", + "- **early_stopping** (bool): Enable early stopping if convergence is reached while training at the end of each epoch. (default: False)\n", + "- **lr_decay** (bool): If lr_decay is True and do_eval is True, learning rate decay on the validation loss is applied at the end of each epoch.\n", + "- **enable_auto_mixed_precision** (bool or None): Enable auto mixed precision for training. Mixed precision uses both 16-bit and 32-bit floating point types to make training run faster and use less memory. It is recommended to enable auto mixed precision training when running on platforms that support bfloat16 (Intel third or fourth generation Xeon processors). If it is enabled on a platform that does not support bfloat16, it can be detrimental to the training performance. If enable_auto_mixed_precision is set to None, auto mixed precision will be automatically enabled when running with Intel fourth generation Xeon processors, and disabled for other platforms.\n", + "- **extra_layers** (list[int]): Optionally insert additional dense layers between the base model and output layer. This can help increase accuracy when fine-tuning. The input should be a list of integers representing the number and size of the layers, for example [1024, 512] will insert two dense layers, the first with 1024 neurons and the second with 512 neurons.\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76041187", + "metadata": {}, + "outputs": [], + "source": [ + "enable_auto_mixed_precision = None\n", + "\n", + "# Train using the pretrained model with the new dataset\n", + "history = model.train(dataset, output_dir=output_dir, epochs=1,\n", + " enable_auto_mixed_precision=enable_auto_mixed_precision)" + ] + }, + { + "cell_type": "markdown", + "id": "d1c7e3af", + "metadata": {}, + "source": [ + "## 7. Predict\n", + "\n", + "Let's predict using the same single batch that we used earlier with the ImageNet trained model to visualize the model's predictions after training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "721253eb", + "metadata": {}, + "outputs": [], + "source": [ + "# Predict with a single batch\n", + "predictions = model.predict(images)\n", + "\n", + "# Map the predicted ids to the class names\n", + "predictions = [dataset.class_names[id] for id in predictions]\n", + "\n", + "# Display the results\n", + "plt.figure(figsize=(18,14))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "\n", + "for n in range(min(batch_size, 30)):\n", + " plt.subplot(6,5,n+1)\n", + " norm_images = (images[n]-np.min(images[n]))/(np.max(images[n])-np.min(images[n]))\n", + " plt.imshow(norm_images, vmin=np.min(norm_images), vmax=np.max(norm_images))\n", + " correct_prediction = labels[n] == predictions[n]\n", + " color = \"darkgreen\" if correct_prediction else \"crimson\"\n", + " title = predictions[n] if correct_prediction else \"{}\\n({})\".format(predictions[n], labels[n]) \n", + " plt.title(title, fontsize=14, color=color)\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"Model predictions\", fontsize=16)\n", + "plt.show()\n", + "\n", + "print(\"Correct predictions are shown in green\")\n", + "print(\"Incorrect predictions are shown in red with the actual label in parenthesis\")" + ] + }, + { + "cell_type": "markdown", + "id": "54df401b", + "metadata": {}, + "source": [ + "### Custom Single Image Prediction" + ] + }, + { + "cell_type": "markdown", + "id": "ff61b5ca", + "metadata": {}, + "source": [ + "We can also predict using a single image that wasn't part of our original dataset. We download a flower image from the [Open Images Dataset](https://storage.googleapis.com/openimages/web/index.html) and then resize it to match our model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14595bbc", + "metadata": {}, + "outputs": [], + "source": [ + "# Download an image from the web and resize it to match our model\n", + "image_url = \"https://c8.staticflickr.com/8/7095/7210797228_c7fe51c3cb_z.jpg\"\n", + "daisy = download_file(image_url, output_dir)\n", + "\n", + "image_shape = (model.image_size, model.image_size)\n", + "daisy = Image.open(daisy).resize(image_shape)\n", + "daisy" + ] + }, + { + "cell_type": "markdown", + "id": "a300e55c", + "metadata": {}, + "source": [ + "Then, we call predict by passing the np array for our image and add a dimension to our array to represent the batch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfaa39dd", + "metadata": {}, + "outputs": [], + "source": [ + "# Get the image as a np array and call predict while adding a batch dimension (with np.newaxis) \n", + "daisy = np.array(daisy)/255.0\n", + "result = model.predict(daisy[np.newaxis, ...])\n", + "\n", + "# Print the predicted class name\n", + "print(dataset.class_names[result[0]])" + ] + }, + { + "cell_type": "markdown", + "id": "9d4830f0", + "metadata": {}, + "source": [ + "## 8. Export\n", + "\n", + "Next, we can call the model `export` function to generate a `saved_model.pb`. The model is saved in a format that is ready to use with [TensorFlow Serving](https://github.com/tensorflow/serving). Each time the model is exported, a new numbered directory is created, which allows serving to pick up the latest model. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aafcd747", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "f66f4b1d", + "metadata": {}, + "source": [ + "## 9. Post-training quantization\n", + "\n", + "In this section, the `tlt` API uses [Intel® Neural Compressor (INC)](https://github.com/intel/neural-compressor) to benchmark and quantize the model to get optimal inference performance." + ] + }, + { + "cell_type": "markdown", + "id": "f0edeffe", + "metadata": {}, + "source": [ + "We use the Intel Neural Compressor config to benchmark the full precision model to see how it performs, as our baseline.\n", + "\n", + "> Note that there is a known issue when running Intel Neural Compressor from a notebook that you may sometimes see the error \n", + "> `zmq.error.ZMQError: Address already in use`. If you see this error, rerun the cell again.\n", + "\n", + "> Likewise, if the benchmark function returns an empty dictionary `{}`, run the cell again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ee783f7", + "metadata": {}, + "outputs": [], + "source": [ + "model.benchmark(dataset=dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "4761363e", + "metadata": {}, + "source": [ + "Next we use Intel Neural Compressor to automatically search for the optimal quantization recipe for low-precision model inference within the accuracy loss constrains defined in the config. Running post training quantization may take several minutes, depending on your hardware and the exit policy (timeout and max trials)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "902200ca", + "metadata": {}, + "outputs": [], + "source": [ + "inc_output_dir = os.path.join(output_dir, 'quantized_models', model.model_name,\n", + " os.path.basename(saved_model_dir))\n", + "model.quantize(inc_output_dir, dataset=dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "0a298a32", + "metadata": {}, + "source": [ + "Let's benchmark using the quantized model, so that we can compare the performance to the full precision model that was originally benchmarked." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec65e1ee", + "metadata": {}, + "outputs": [], + "source": [ + "model.benchmark(dataset=dataset, saved_model_dir=inc_output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "18a6c74f", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "```\n", + "@ONLINE {tfflowers,\n", + "author = \"The TensorFlow Team\",\n", + "title = \"Flowers\",\n", + "month = \"jan\",\n", + "year = \"2019\",\n", + "url = \"http://download.tensorflow.org/example_images/flower_photos.tgz\" }\n", + "\n", + "@article{openimages,\n", + " title={OpenImages: A public dataset for large-scale multi-label and multi-class image classification.},\n", + " author={Krasin, Ivan and Duerig, Tom and Alldrin, Neil and Veit, Andreas and Abu-El-Haija, Sami\n", + " and Belongie, Serge and Cai, David and Feng, Zheyun and Ferrari, Vittorio and Gomes, Victor\n", + " and Gupta, Abhinav and Narayanan, Dhyanesh and Sun, Chen and Chechik, Gal and Murphy, Kevin},\n", + " journal={Dataset available from https://github.com/openimages},\n", + " year={2016}\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Transfer_Learning_and_Graph_Optimization.ipynb b/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Transfer_Learning_and_Graph_Optimization.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..4f4a86181d3a0fad832eb5b1100803241179b2a3 --- /dev/null +++ b/notebooks/image_classification/tlt_api_tf_image_classification/TLT_TF_Transfer_Learning_and_Graph_Optimization.ipynb @@ -0,0 +1,411 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "ffc0d761", + "metadata": {}, + "source": [ + "# Transfer Learning and Graph Optimization using TensorFlow and the Intel® Transfer Learning Tool API\n", + "\n", + "This notebook uses the `tlt` library to do transfer learning and graph optimization for image classfication with a TensorFlow pretrained model." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "b266ca11", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [notebooks README.md](/notebooks/README.md) to setup a TensorFlow environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1b31cc75", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import PIL.Image as Image\n", + "import tensorflow as tf\n", + "\n", + "# tlt imports\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_and_extract_tar_file\n", + "\n", + "# Specify a directory for the dataset to be downloaded\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + " \n", + "# Specify a directory for output\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Dataset directory:\", dataset_dir)\n", + "print(\"Output directory:\", output_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e49dafe3", + "metadata": {}, + "source": [ + "## 2. Get the model\n", + "\n", + "In this step, we call the Intel Transfer Learning Tool model factory to list supported TensorFlow image classification models. This is a list of pretrained models from [TFHub](https://tfhub.dev) and [Keras Applications](https://keras.io/api/applications/) that we tested with our API. Optionally, the `verbose=True` argument can be added to the `print_supported_models` function call to get more information about each model (such as the image size, the original dataset, the preprocessor, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad49dc51", + "metadata": {}, + "outputs": [], + "source": [ + "# See a list of available models\n", + "model_factory.print_supported_models(use_case='image_classification', framework='tensorflow')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "764c7063", + "metadata": {}, + "source": [ + "Next, use the model factory to get one of the models listed in the previous cell. The `get_model` function returns a model object that will later be used for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f101e379", + "metadata": {}, + "outputs": [], + "source": [ + "model = model_factory.get_model(model_name='resnet_v1_50', framework='tensorflow')\n", + "\n", + "print(\"Model name:\", model.model_name)\n", + "print(\"Framework:\", model.framework)\n", + "print(\"Use case:\", model.use_case)\n", + "print(\"Image size:\", model.image_size)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "3351fbbc", + "metadata": {}, + "source": [ + "## 3. Get the dataset\n", + "\n", + "We call the dataset factory to load sample image classification dataset. For demonstration purposes, we will download a flower species dataset. After downloading and extracting, you will have the following subdirectories in your dataset directory. Each species subfolder will contain numerous `.jpg` files:\n", + "\n", + "```\n", + "flower_photos\n", + " └── daisy\n", + " └── dandelion\n", + " └── roses\n", + " └── sunflowers\n", + " └── tulips\n", + "```\n", + "\n", + "When using your own dataset, ensure that it is similarly organized with folders for each class. Change the `custom_dataset_path` variable to point to your dataset folder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8741e47e", + "metadata": {}, + "outputs": [], + "source": [ + "# For demonstration purposes, we download a flowers dataset. To instead use your own dataset, set the\n", + "# custom_dataset_path to point to your dataset's directory and comment out the download_and_extract_tar_file line.\n", + "custom_dataset_path = os.path.join(dataset_dir, \"flower_photos\")\n", + "\n", + "if not os.path.exists(custom_dataset_path):\n", + " download_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + " download_and_extract_tar_file(download_url, dataset_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "0ddf65ed", + "metadata": {}, + "source": [ + "Call the dataset factory to load the dataset from the directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94ebc8e1", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the dataset from the custom dataset path\n", + "dataset = dataset_factory.load_dataset(dataset_dir=custom_dataset_path,\n", + " use_case='image_classification', \n", + " framework='tensorflow')\n", + "\n", + "print(\"Class names:\", str(dataset.class_names))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "372a5bd2", + "metadata": {}, + "source": [ + "## 4. Prepare the dataset\n", + "\n", + "Once you have your dataset, use the following cells to split and preprocess the data. We split them into training and validation subsets, then resize the images to match the selected models, and then batch the images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b4dc950", + "metadata": {}, + "outputs": [], + "source": [ + "# Split the dataset into training and validation subsets\n", + "dataset.shuffle_split(train_pct=.75, val_pct=.25)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33c883c2", + "metadata": {}, + "outputs": [], + "source": [ + "# Preprocess the dataset with an image size and preprocessor that match the model and a batch size of 32\n", + "batch_size = 32\n", + "dataset.preprocess(model.image_size, batch_size=batch_size, preprocessor=model.preprocessor)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "05acac53", + "metadata": {}, + "source": [ + "## 5. Evaluate the model before training\n", + "\n", + "Since we haven't done any training on the model yet, it will evaluate using the original ImageNet trained model and accuracy on the new classes will be near zero." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12ab6b31", + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate(dataset)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "7208e757", + "metadata": {}, + "source": [ + "## 6. Transfer Learning\n", + "\n", + "This step calls the model's train function with the dataset that was just prepared. The training function will get the base model and add on a dense layer based on the number of classes in the dataset. The model is then compiled and trained based on the number of epochs specified in the argument. With the do_eval parameter set to True by default, this step will also show how the model can be evaluated and will return a list of metrics calculated from the dataset's validation subset." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "039db26f", + "metadata": {}, + "source": [ + "### Arguments\n", + "\n", + "#### Required\n", + "- **dataset** (ImageClassificationDataset, required): Dataset to use when training the model\n", + "- **output_dir** (str): Path to a writeable directory for checkpoint files\n", + "- **epochs** (int): Number of epochs to train the model (default: 1)\n", + "\n", + "#### Optional\n", + "- **initial_checkpoints** (str): Path to checkpoint weights to load. If the path provided is a directory, the latest checkpoint will be used.\n", + "- **early_stopping** (bool): Enable early stopping if convergence is reached while training at the end of each epoch. (default: False)\n", + "- **lr_decay** (bool): If lr_decay is True and do_eval is True, learning rate decay on the validation loss is applied at the end of each epoch.\n", + "- **enable_auto_mixed_precision** (bool or None): Enable auto mixed precision for training. Mixed precision uses both 16-bit and 32-bit floating point types to make training run faster and use less memory. It is recommended to enable auto mixed precision training when running on platforms that support bfloat16 (Intel third or fourth generation Xeon processors). If it is enabled on a platform that does not support bfloat16, it can be detrimental to the training performance. If enable_auto_mixed_precision is set to None, auto mixed precision will be automatically enabled when running with Intel fourth generation Xeon processors, and disabled for other platforms.\n", + "- **extra_layers** (list[int]): Optionally insert additional dense layers between the base model and output layer. This can help increase accuracy when fine-tuning a TFHub model. The input should be a list of integers representing the number and size of the layers, for example [1024, 512] will insert two dense layers, the first with 1024 neurons and the second with 512 neurons.\n", + "\n", + "Note: refer to release documentation for an up-to-date list of train arguments and their current descriptions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7d51156", + "metadata": {}, + "outputs": [], + "source": [ + "enable_auto_mixed_precision = None\n", + "\n", + "# Train using the pretrained model from TF Hub with the new dataset\n", + "history = model.train(dataset, output_dir=output_dir, epochs=1,\n", + " enable_auto_mixed_precision=enable_auto_mixed_precision)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "81f75fa0", + "metadata": {}, + "source": [ + "This time, the accuracy looks much better." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "007cc8fe", + "metadata": {}, + "source": [ + "## 7. Export\n", + "\n", + "Next, we can call the model `export` function to generate a `saved_model.pb`. The model is saved in a format that is ready to use with [TensorFlow Serving](https://github.com/tensorflow/serving). Each time the model is exported, a new numbered directory is created, which allows serving to pick up the latest model. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eed84d55", + "metadata": {}, + "outputs": [], + "source": [ + "saved_model_dir = model.export(output_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "b3e1df1f", + "metadata": {}, + "source": [ + "## 8. Graph Optimization\n", + "\n", + "The `tlt` API uses [Intel® Neural Compressor (INC)](https://github.com/intel/neural-compressor) to optimize the FP32 graph for improved inference performance. Graph optimization performs the following:\n", + "* Converting variables to constants\n", + "* Removing training-only operations like checkpoint saving\n", + "* Stripping out parts of the graph that are never reached\n", + "* Removing debug operations like CheckNumerics\n", + "* Folding batch normalization ops into the pre-calculated weights\n", + "* Fusing common operations into unified versions\n", + "\n", + "For benchmarking, we will use an auto-generated config. If you want more control over the configuration, you can create your own config using the Intel Neural Compressor or the `tlt.utils.inc_utils.get_inc_config()` method." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e27391d9", + "metadata": {}, + "source": [ + "We use the Intel Neural Compressor config to benchmark the full non-optimized model to see how it performs, as our baseline.\n", + "\n", + "> Note that there is a known issue when running Intel Neural Compressor from a notebook that you may sometimes see the error \n", + "> `zmq.error.ZMQError: Address already in use`. If you see this error, rerun the cell again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4dbc03f", + "metadata": {}, + "outputs": [], + "source": [ + "result = model.benchmark(dataset)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "64210d77", + "metadata": {}, + "source": [ + "Next, we do the FP32 graph optimization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0531bf13", + "metadata": {}, + "outputs": [], + "source": [ + "# Create an output directory for optimization output with the same base name as our saved model directory\n", + "optimization_output_dir = os.path.join(output_dir, 'optimized_models', model.model_name,\n", + " os.path.basename(saved_model_dir))\n", + "\n", + "model.optimize_graph(optimization_output_dir, overwrite_model=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77775e97", + "metadata": {}, + "outputs": [], + "source": [ + "optimized_result = model.benchmark(dataset, saved_model_dir=optimization_output_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "210fb6ff", + "metadata": {}, + "source": [ + "## Dataset Citations\n", + "\n", + "```\n", + "@ONLINE {tfflowers,\n", + "author = \"The TensorFlow Team\",\n", + "title = \"Flowers\",\n", + "month = \"jan\",\n", + "year = \"2019\",\n", + "url = \"http://download.tensorflow.org/example_images/flower_photos.tgz\" }\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/performance/hf_text_classification_performance.ipynb b/notebooks/performance/hf_text_classification_performance.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3caea7d2b8bd5174e0646ca4559b4ef68b2a3aad --- /dev/null +++ b/notebooks/performance/hf_text_classification_performance.ipynb @@ -0,0 +1,809 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "57b029cd", + "metadata": {}, + "source": [ + "# Performance Comparison: Text Classification Transfer Learning with Hugging Face* and the Intel® Transfer Learning Tool\n", + "\n", + "This notebook uses the Hugging Face Trainer to do transfer learning with a text classification model with PyTorch*. The model is trained, evaluated, and exported. The same sequence is also done using the Intel Transfer Learning Tool. The Intel Transfer Learning Tool has a flag to control whether the Hugging Face Trainer is used under the hood, or if just PyTorch libraries are used. Training and evaluation are run both ways, giving us three combinations to compare:\n", + "* Using the Hugging Face Trainer\n", + "* Using the Intel Transfer Learning Tool with the Hugging Face Trainer\n", + "* Using the Intel Transfer Learning Tool with torch\n", + "\n", + "After all of the models have been trained and evaluated, charts are displayed to visually compare the training and evaluation metrics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30be8bb1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import psutil\n", + "\n", + "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n", + "os.environ['TOKENIZERS_PARALLELISM'] = 'false'\n", + "\n", + "import datasets\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.ticker as mtick\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch\n", + "import transformers\n", + "from transformers import DataCollatorWithPadding, Trainer\n", + "from torch.utils.data import DataLoader as loader\n", + "\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.platform_util import CPUInfo, OptimizedPlatformUtil, PlatformUtil\n", + "\n", + "# Specify the the default dataset directory\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Specify a directory for output (saved models and checkpoints)\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "# Location where Hugging Face will locally store data\n", + "os.environ['HF_HOME'] = dataset_directory\n", + "\n", + "datasets.utils.logging.set_verbosity(datasets.logging.ERROR)\n", + "\n", + "# Data Frame styles\n", + "table_styles =[{\n", + " 'selector': 'caption',\n", + " 'props': [\n", + " ('text-align', 'center'),\n", + " ('color', 'black'),\n", + " ('font-size', '16px')\n", + " ]\n", + "}]\n", + "\n", + "# Colors used in charts\n", + "blue = '#0071c5'\n", + "dark_blue = '#003c71'\n", + "yellow = '#ffcc4d'" + ] + }, + { + "cell_type": "markdown", + "id": "f2730530", + "metadata": {}, + "source": [ + "## 1. Display Platform Information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e63bb467", + "metadata": {}, + "outputs": [], + "source": [ + "# Get and display CPU/platform information\n", + "cpu_info = CPUInfo()\n", + "platform_util = PlatformUtil()\n", + "print(\"{0} CPU Information {0}\".format(\"=\" * 20))\n", + "print(\"CPU family:\", platform_util.cpu_family)\n", + "print(\"CPU model:\", platform_util.cpu_model)\n", + "print(\"CPU type:\", platform_util.cpu_type)\n", + "print(\"Physical cores per socket:\", cpu_info.cores_per_socket)\n", + "print(\"Total physical cores:\", cpu_info.cores)\n", + "cpufreq = psutil.cpu_freq()\n", + "print(\"Max Frequency:\", cpufreq.max)\n", + "print(\"Min Frequency:\", cpufreq.min)\n", + "cpu_socket_count = cpu_info.sockets\n", + "print(\"Socket Number:\", cpu_socket_count)\n", + "\n", + "print(\"\\n{0} Memory Information {0}\".format(\"=\" * 20))\n", + "svmem = psutil.virtual_memory()\n", + "print(\"Total: \", int(svmem.total / (1024 ** 3)), \"GB\")\n", + "\n", + "# Display Hugging Face version information\n", + "print(\"\\n{0} Hugging Face Information {0}\".format(\"=\" * 20))\n", + "print(\"Hugging Face Transformers version:\", transformers.__version__)\n", + "print(\"Hugging Face Datasets version:\", datasets.__version__)\n", + "\n", + "# Display PyTorch version information\n", + "print(\"\\n{0} PyTorch Information {0}\".format(\"=\" * 20))\n", + "print(\"PyTorch version:\", torch.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "e0b7f5fb", + "metadata": {}, + "source": [ + "## 2. Select a model and define parameters to use during training and evaluation\n", + "\n", + "### Select a model\n", + "\n", + "See the list of supported PyTorch text classification models from Hugging Face in the Intel Transfer Learning Tool." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "069e58da", + "metadata": {}, + "outputs": [], + "source": [ + "framework = 'pytorch'\n", + "use_case = 'text_classification'\n", + "model_hub = 'huggingface'\n", + "supported_models = model_factory.get_supported_models(framework, use_case)\n", + "supported_models = supported_models[use_case]\n", + "\n", + "# Filter to only get relevant models\n", + "supported_models = { key:value for (key,value) in supported_models.items() if value[framework]['model_hub'] == model_hub}\n", + "\n", + "print(\"Supported {} models for {} from {}\".format(framework, use_case, model_hub))\n", + "print(\"=\" * 70)\n", + "for model_name in supported_models.keys():\n", + " print(model_name)" + ] + }, + { + "cell_type": "markdown", + "id": "8fc116a5", + "metadata": {}, + "source": [ + "Set the `model_name` to the model that will be used during this experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bda109e9", + "metadata": {}, + "outputs": [], + "source": [ + "# Select a model\n", + "model_name = \"bert-base-cased\"" + ] + }, + { + "cell_type": "markdown", + "id": "bd568624", + "metadata": {}, + "source": [ + "### Select a dataset\n", + "\n", + "For these experiments, we will be using text classification datasets from the [Hugging Face Datasets catalog](https://huggingface.co/datasets?task_categories=task_categories:text-classification&sort=downloads). Specify the name of the dataset to use with the `dataset_name` variable in the next cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6aa35cf0", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_name = 'imdb'" + ] + }, + { + "cell_type": "markdown", + "id": "409141c0", + "metadata": {}, + "source": [ + "### Define parameters\n", + "\n", + "For consistency between the model training experiments using Hugging Face and the Intel Transfer Learning Tool, the next cell defines parameters that will be used by both methods." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9588f07d", + "metadata": {}, + "outputs": [], + "source": [ + "# Number of training epochs\n", + "training_epochs = 2\n", + "\n", + "# Shuffle the files after each training epoch\n", + "shuffle_files = True\n", + "\n", + "# Define the name of the split to use for validation (i.e. 'validation' or 'test')\n", + "eval_split=None\n", + "# If eval_split=None, split the 'train' dataset\n", + "validation_split = 0.05\n", + "training_split = 0.1\n", + "\n", + "# Set seed for consistency between runs (or None)\n", + "seed = 10\n", + "\n", + "# List of batch size(s) to compare (maximum of 4 batch sizes to try)\n", + "batch_size_list = [ 16, 32 ]\n", + "\n", + "learning_rate=3e-5\n", + "\n", + "# Text preprocessing\n", + "max_sequence_length = 128\n", + "padding = 'max_length'\n", + "truncation = True\n", + "\n", + "# Use the Intel Extension for PyTorch\n", + "use_ipex = True" + ] + }, + { + "cell_type": "markdown", + "id": "c508355f", + "metadata": {}, + "source": [ + "Validate parameter values and then print out the parameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec3ec8be", + "metadata": {}, + "outputs": [], + "source": [ + "if not isinstance(training_epochs, int):\n", + " raise TypeError(\"The training_epochs parameter should be an integer, but found a {}\".format(type(training_epochs)))\n", + "\n", + "if training_epochs < 1:\n", + " raise ValueError(\"The training_epochs parameter should not be less than 1.\")\n", + " \n", + "if not isinstance(shuffle_files, bool):\n", + " raise TypeError(\"The shuffle_files parameter should be a bool, but found a {}\".format(type(shuffle_files)))\n", + "\n", + "if not eval_split:\n", + " if not isinstance(validation_split, float):\n", + " raise TypeError(\"The validation_split parameter should be a float, but found a {}\".format(type(validation_split)))\n", + "\n", + " if not isinstance(training_split, float):\n", + " raise TypeError(\"The training_split parameter should be a float, but found a {}\".format(type(training_split)))\n", + "\n", + "if validation_split + training_split > 1:\n", + " raise ValueError(\"The sum of validation_split and training_split should not be greater than 1.\")\n", + "\n", + "if seed and not isinstance(seed, int):\n", + " raise TypeError(\"The seed parameter should be a integer or None, but found a {}\".format(type(seed)))\n", + "\n", + "if len(batch_size_list) > 4 or len(batch_size_list) == 0:\n", + " raise ValueError(\"The batch_size_list should have at most 4 values, but found {} values ({})\".format(\n", + " len(batch_size_list), batch_size_list))\n", + " \n", + "print(\"Number of training epochs:\", training_epochs)\n", + "print(\"Shuffle files:\", shuffle_files)\n", + "print(\"Training split: {}%\".format('train' if eval_split else training_split*100))\n", + "print(\"Validation split: {}%\".format(eval_split if eval_split else validation_split*100))\n", + "print(\"Seed:\", str(seed))\n", + "print(\"Batch size list:\", batch_size_list)" + ] + }, + { + "cell_type": "markdown", + "id": "d102b97b", + "metadata": {}, + "source": [ + "## 3. Train and evaluate the models\n", + "\n", + "In this section, we will compare the time that it takes to fine tune the text classification model using the dataset that was selected in the previous section.\n", + "\n", + "The fine tuning will be done in two different ways:\n", + "* Using the Hugging Face python libraries\n", + "* Using the Intel Transfer Learning Tool\n", + "\n", + "### Fine tuning using the Hugging Face libraries\n", + "\n", + "First, we download and prepare the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad0b244c", + "metadata": {}, + "outputs": [], + "source": [ + "transformers.set_seed(seed)\n", + "\n", + "# Determine the splits to load\n", + "split = ['train']\n", + "if eval_split:\n", + " split.append(eval_split)\n", + "\n", + "# Load the dataset from the Hugging Face dataset catalog\n", + "hf_dataset = datasets.load_dataset(dataset_name, cache_dir=dataset_directory, split=split)\n", + "\n", + "# Load the tokenizer based on our selected model\n", + "hf_tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, cache_dir=output_directory)\n", + "\n", + "text_column_names = [col_name for col_name in hf_dataset[0].column_names if col_name != 'label' and\n", + " all(isinstance(s, str) for s in hf_dataset[0][col_name])]\n", + "\n", + "# Tokenize the dataset\n", + "def tokenize_function(examples):\n", + " args = (examples[text_column_name] for text_column_name in text_column_names)\n", + " return hf_tokenizer(*args, padding=padding, max_length=max_sequence_length, truncation=truncation)\n", + "\n", + "tokenized_hf_dataset = [d.map(tokenize_function, batched=True) for d in hf_dataset]\n", + "for tokenized in tokenized_hf_dataset:\n", + " tokenized.set_format('torch')\n", + "\n", + "# If eval_split is defined, that split will be used for validation. Otherwise, the 'train' dataset split will be\n", + "# split by the defined percentage to use for training and evaluation.\n", + "hf_train_dataset = tokenized_hf_dataset[0]\n", + "if eval_split:\n", + " print(\"Using 'train' and '{}' dataset splits\".format(eval_split))\n", + " hf_train_subset = hf_train_dataset\n", + " hf_eval_subset = tokenized_hf_dataset[1]\n", + "else:\n", + " dataset_length = len(hf_train_dataset)\n", + " train_size = int(training_split * dataset_length)\n", + " eval_size = int(validation_split * dataset_length)\n", + " generator = torch.Generator().manual_seed(seed) \n", + " dataset_indices = torch.randperm(dataset_length, generator=generator).tolist() if shuffle_files else range(dataset_length)\n", + " train_indices = dataset_indices[:train_size]\n", + " eval_indices = dataset_indices[train_size:train_size + eval_size]\n", + " print(\"Using {}% for training and {}% for validation\".format(training_split * 100, validation_split * 100))\n", + " print(\"Total dataset size:\", dataset_length)\n", + " print(\"Train size:\", train_size)\n", + " print(\"Eval size:\", eval_size)\n", + " hf_train_subset = hf_train_dataset.select(train_indices)\n", + " hf_eval_subset = hf_train_dataset.select(eval_indices)\n", + "\n", + "# Get the number of classes from the train dataset features (either called 'label' or 'labels')\n", + "class_names = hf_dataset[0].features['label'].names if 'label' in hf_dataset[0].features else \\\n", + " hf_dataset[0].features['labels'].names\n", + "print(\"Class names: {}\".format(class_names))\n", + "\n", + "hf_train_dataset_length = len(hf_train_subset)\n", + "hf_eval_dataset_length = len(hf_eval_subset)\n", + "\n", + "# Define function to compute accuracy to pass to the Hugging Face Trainer\n", + "def compute_metrics(p: transformers.EvalPrediction):\n", + " preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions\n", + " preds = np.argmax(preds, axis=1)\n", + " return {\"accuracy\": (preds == p.label_ids).astype(np.float32).mean().item()}" + ] + }, + { + "cell_type": "markdown", + "id": "75bc822a", + "metadata": {}, + "source": [ + "Next, we iterate through our list of batch sizes to train the model for each configuration with the dataset that was prepared in the previous cell. After training is complete, the model is evaluated and exported. The training and evaluation metrics are saved to lists." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b3037a8", + "metadata": {}, + "outputs": [], + "source": [ + "hf_saved_model_paths = []\n", + "hf_training_metrics = []\n", + "hf_eval_results = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 40)\n", + " print('Training using batch size: {}'.format(batch_size))\n", + " print('-' * 40)\n", + " \n", + " # Get the model from pretrained\n", + " model = transformers.AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=len(class_names))\n", + "\n", + " # Setup a directory to save the model output\n", + " saved_model_dir = os.path.join(output_directory, model_name, 'HF_model_bs{}'.format(batch_size))\n", + " \n", + " # Note: Even when setting do_eval=False, an error gets thrown if a eval_dataset is not provided\n", + " training_args = transformers.TrainingArguments(\n", + " output_dir=saved_model_dir,\n", + " do_eval=False,\n", + " do_train=True,\n", + " learning_rate=learning_rate,\n", + " per_device_train_batch_size=batch_size,\n", + " per_device_eval_batch_size=batch_size,\n", + " num_train_epochs=training_epochs,\n", + " evaluation_strategy=\"epoch\",\n", + " push_to_hub=False,\n", + " no_cuda=True,\n", + " overwrite_output_dir=True,\n", + " seed=seed,\n", + " data_seed=seed,\n", + " use_ipex=use_ipex\n", + " )\n", + "\n", + " trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " train_dataset=hf_train_subset,\n", + " eval_dataset=hf_eval_subset,\n", + " compute_metrics=compute_metrics,\n", + " tokenizer=hf_tokenizer,\n", + " )\n", + "\n", + " # Train the model\n", + " history = trainer.train()\n", + " \n", + " # Evaluate the model\n", + " eval_results = trainer.evaluate()\n", + " \n", + " # Export the trained model\n", + " trainer.save_model(saved_model_dir)\n", + " \n", + " # Save objects and metrics\n", + " hf_training_metrics.append(history)\n", + " hf_eval_results.append(eval_results)\n", + " hf_saved_model_paths.append(saved_model_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "86cafc34", + "metadata": {}, + "source": [ + "### Using the Intel Transfer Learning tool API\n", + "\n", + "Next, we train the same model using the same parameters using the Intel Transfer Learning Tool. The Intel Transfer Learning Tool training method has an argument called `use_trainer` to determine if the Hugging Face Trainer will be used. If `use_trainer=False`, the torch libraries will be used to train the model. The next section will do the training with and without the Hugging Face Trainer, so that we can gather metrics both ways. After the model is trained, it is evaluted. Again, we save the training and evaluation metrics to lists." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e08223da", + "metadata": {}, + "outputs": [], + "source": [ + "# Intel Transfer Learning Tool using the Hugging Face Trainer\n", + "tlt_trainer_saved_model_paths = []\n", + "tlt_trainer_training_metrics = []\n", + "tlt_trainer_eval_results = []\n", + "\n", + "# Intel Transfer Learning Tool training using torch libraries\n", + "tlt_torch_saved_model_paths = []\n", + "tlt_torch_training_metrics = []\n", + "tlt_torch_eval_results = []\n", + "\n", + "split_names = ['train'] if eval_split is None else ['train', eval_split]\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 40)\n", + " print('Training using batch size: {}'.format(batch_size))\n", + " print('-' * 40)\n", + "\n", + " # Get the dataset\n", + " dataset = dataset_factory.get_dataset(dataset_directory, use_case, framework, dataset_name,\n", + " dataset_catalog=\"huggingface\", shuffle_files=shuffle_files,\n", + " split=split_names)\n", + "\n", + " # Batch and tokenize the dataset\n", + " dataset.preprocess(model_name, batch_size=batch_size, max_length=max_sequence_length, padding=padding,\n", + " truncation=truncation)\n", + "\n", + " # If the dataset doesn't have a defined split, then split it by percentages\n", + " if eval_split is None:\n", + " dataset.shuffle_split(train_pct=training_split, val_pct=validation_split)\n", + " \n", + " tlt_train_dataset_length = len(dataset.train_subset)\n", + " \n", + " if eval_split == 'test':\n", + " eval_dataset = dataset.test_subset\n", + " tlt_eval_dataset_length = len(eval_dataset)\n", + " else:\n", + " eval_dataset = dataset.validation_subset\n", + " tlt_eval_dataset_length = len(eval_dataset)\n", + " \n", + " # Verify dataset length between the experiments\n", + " assert tlt_train_dataset_length == hf_train_dataset_length\n", + " assert tlt_eval_dataset_length == hf_eval_dataset_length\n", + " \n", + " for use_trainer in [True, False]:\n", + " print('\\nTraining using Hugging Face Trainer: {}'.format(use_trainer))\n", + " \n", + " # Get the model\n", + " model = model_factory.get_model(model_name, framework)\n", + "\n", + " # Train the model\n", + " history = model.train(dataset, output_directory, epochs=training_epochs, ipex_optimize=use_ipex,\n", + " use_trainer=use_trainer, do_eval=False, learning_rate=learning_rate, seed=seed)\n", + " \n", + " eval_metrics = model.evaluate(eval_dataset)\n", + "\n", + " # Save the model\n", + " saved_model_dir = model.export(output_directory)\n", + " \n", + " if use_trainer:\n", + " tlt_trainer_training_metrics.append(history)\n", + " tlt_trainer_saved_model_paths.append(saved_model_dir)\n", + " tlt_trainer_eval_results.append(eval_metrics)\n", + " else:\n", + " tlt_torch_training_metrics.append(history)\n", + " tlt_torch_saved_model_paths.append(saved_model_dir)\n", + " tlt_torch_eval_results.append(eval_metrics)" + ] + }, + { + "cell_type": "markdown", + "id": "c8a27d05", + "metadata": {}, + "source": [ + "## 4. Compare metrics\n", + "\n", + "This section compares metrics for training and evaluating the model for the following experiments:\n", + "* Using the Hugging Face Trainer\n", + "* Using the Intel Transfer Learning Tool with the Hugging Face Trainer\n", + "* Using the Intel Transfer Learning Tool with torch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "570f872d", + "metadata": {}, + "outputs": [], + "source": [ + "display_df = []\n", + "\n", + "# Display training metrics\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " df = pd.DataFrame({\n", + " 'Hugging Face Trainer': [hf_training_metrics[i].metrics['train_loss'], hf_training_metrics[i].metrics['train_samples_per_second'], hf_training_metrics[i].metrics['train_runtime']],\n", + " 'Intel Transfer Learning Tool
using the HF Trainer': [tlt_trainer_training_metrics[i].metrics['train_loss'], tlt_trainer_training_metrics[i].metrics['train_samples_per_second'], tlt_trainer_training_metrics[i].metrics['train_runtime']],\n", + " 'Intel Transfer Learning Tool
using Torch': [tlt_torch_training_metrics[i]['Loss'], tlt_torch_training_metrics[i]['train_samples_per_second'][0], tlt_torch_training_metrics[i]['train_runtime'][0]],\n", + " }, index = ['Loss', 'Samples per second', 'Train Runtime'])\n", + " df = df.style.set_table_styles(table_styles).set_caption(\"Training metrics with batch size {}\".format(batch_size))\n", + " display_df.append(df)\n", + "\n", + "# Display evaluation metrics\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " df = pd.DataFrame({\n", + " 'Eval using the
Hugging Face Trainer': [\"{0:.2%}\".format(hf_eval_results[i]['eval_accuracy']), hf_eval_results[i]['eval_loss'], hf_eval_results[i]['eval_samples_per_second'], hf_eval_results[i]['eval_runtime']],\n", + " 'Intel Transfer Learning Tool
eval using the HF Trainer': [\"{0:.2%}\".format(tlt_trainer_eval_results[i]['eval_accuracy']), tlt_trainer_eval_results[i]['eval_loss'], tlt_trainer_eval_results[i]['eval_samples_per_second'], tlt_trainer_eval_results[i]['eval_runtime']],\n", + " 'Intel Transfer Learning Tool
eval using Torch': [\"{0:.2%}\".format(tlt_torch_eval_results[i]['eval_accuracy']), tlt_torch_eval_results[i]['eval_loss'], tlt_torch_eval_results[i]['eval_samples_per_second'], tlt_torch_eval_results[i]['eval_runtime']],\n", + " }, index = ['Eval accuracy', 'Eval Loss', 'Samples per second', 'Train Runtime'])\n", + " df = df.style.set_table_styles(table_styles).set_caption(\"Training metrics with batch size {}\".format(batch_size))\n", + " display_df.append(df)\n", + " \n", + "for df in display_df:\n", + " display(df)" + ] + }, + { + "cell_type": "markdown", + "id": "89360585", + "metadata": {}, + "source": [ + "### Training metrics\n", + "\n", + "Generate charts to compare the time that it took to train the model in each experiment (lower is better) and the thoughput (higher is better)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0919f4dc", + "metadata": {}, + "outputs": [], + "source": [ + "# Bar chart group labels\n", + "groups = [\"batch size = {}\".format(bs) for bs in batch_size_list]\n", + "\n", + "hf_train_runtime = [x.metrics['train_runtime'] for x in hf_training_metrics]\n", + "tlt_trainer_train_runtime = [x.metrics['train_runtime'] for x in tlt_trainer_training_metrics]\n", + "tlt_torch_train_runtime = [x['train_runtime'][0] for x in tlt_torch_training_metrics]\n", + "\n", + "hf_train_throughput = [x.metrics['train_samples_per_second'] for x in hf_training_metrics]\n", + "tlt_trainer_train_throughput = [x.metrics['train_samples_per_second'] for x in tlt_trainer_training_metrics]\n", + "tlt_torch_train_thoughput = [x['train_samples_per_second'][0] for x in tlt_torch_training_metrics]\n", + "\n", + "x = np.arange(len(groups))\n", + "width = 0.2 # the width of the bars\n", + "multiplier = 0\n", + "\n", + "# Setup bars for training run times\n", + "fig, (ax1, ax2) = plt.subplots(2)\n", + "fig.set_figheight(15)\n", + "fig.set_figwidth(10)\n", + "rects_tf = ax1.bar(x, hf_train_runtime, width, label='HF trained', color=yellow)\n", + "rects_tlt_trainer = ax1.bar(x + width, tlt_trainer_train_runtime, width, label='TLT using Trainer', color=blue)\n", + "rects_tlt_torch = ax1.bar(x + width * 2, tlt_torch_train_runtime, width, label='TLT using Torch', color=dark_blue)\n", + "ax1.bar_label(rects_tf, padding=3)\n", + "ax1.bar_label(rects_tlt_trainer, padding=3)\n", + "ax1.bar_label(rects_tlt_torch, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax1.set_ylabel('Seconds')\n", + "ax1.set_title('Training Runtime')\n", + "ax1.set_xticks(x+width, groups)\n", + "ax1.set_ymargin(0.2) \n", + "ax1.legend(ncols=2)\n", + "\n", + "# Setup bars for throughput\n", + "rects_tf = ax2.bar(x, hf_train_throughput, width, label='HF trained', color=yellow)\n", + "rects_tlt_trainer = ax2.bar(x + width, tlt_trainer_train_throughput, width, label='TLT using Trainer', color=blue)\n", + "rects_tlt_torch = ax2.bar(x + width * 2, tlt_torch_train_thoughput, width, label='TLT using Torch', color=dark_blue)\n", + "ax2.bar_label(rects_tf, padding=3)\n", + "ax2.bar_label(rects_tlt_trainer, padding=3)\n", + "ax2.bar_label(rects_tlt_torch, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax2.set_ylabel('Samples per second')\n", + "ax2.set_title('Training Throughput')\n", + "ax2.set_xticks(x+width, groups)\n", + "ax2.set_ymargin(0.2) \n", + "ax2.legend(ncols=2)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "54559456", + "metadata": {}, + "source": [ + "### Evaluation Metrics\n", + "\n", + "Next, we generate charts to compare the evaluation metrics for the same three experiments that were used for training the model. We have a charts with the validation accuracy, total evaluation time (lower is better), and evaluation throughput (higher is better)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02473c34", + "metadata": {}, + "outputs": [], + "source": [ + "# Decimals used for rounding\n", + "decimals = 2\n", + "\n", + "# Get the evaluation metrics\n", + "hf_eval_acc = [round(x['eval_accuracy'] * 100, decimals) for x in hf_eval_results]\n", + "tlt_trainer_eval_acc = [round(x['eval_accuracy'] * 100, decimals) for x in tlt_trainer_eval_results]\n", + "tlt_torch_eval_acc = [round(x['eval_accuracy'] * 100, decimals) for x in tlt_torch_eval_results]\n", + "\n", + "hf_eval_runtime = [round(x['eval_runtime'], decimals) for x in hf_eval_results]\n", + "tlt_trainer_eval_runtime = [round(x['eval_runtime'], decimals) for x in tlt_trainer_eval_results]\n", + "tlt_torch_eval_runtime = [round(x['eval_runtime'], decimals) for x in tlt_torch_eval_results]\n", + "\n", + "hf_eval_throughput = [round(x['eval_samples_per_second'], decimals) for x in hf_eval_results]\n", + "tlt_trainer_eval_throughput = [round(x['eval_samples_per_second'], decimals) for x in tlt_trainer_eval_results]\n", + "tlt_torch_eval_thoughput = [round(x['eval_samples_per_second'], decimals) for x in tlt_torch_eval_results]\n", + "\n", + "x = np.arange(len(groups))\n", + "width = 0.2 # the width of the bars\n", + "multiplier = 0\n", + "\n", + "# Setup bars for training run times\n", + "fig, (ax1, ax2, ax3) = plt.subplots(3)\n", + "fig.set_figheight(20)\n", + "fig.set_figwidth(10)\n", + "rects_tf = ax1.bar(x, hf_eval_acc, width, label='HF evaluated', color=yellow)\n", + "rects_tlt_trainer = ax1.bar(x + width, tlt_trainer_eval_acc, width, label='TLT eval using Trainer', color=blue)\n", + "rects_tlt_torch = ax1.bar(x + width * 2, tlt_torch_eval_acc, width, label='TLT eval using Torch', color=dark_blue)\n", + "ax1.bar_label(rects_tf, padding=3)\n", + "ax1.bar_label(rects_tlt_trainer, padding=3)\n", + "ax1.bar_label(rects_tlt_torch, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax1.set_ylabel('Percentage (%)')\n", + "ax1.set_title('Evaluation Accuracy')\n", + "ax1.set_xticks(x+width, groups)\n", + "ax1.set_ymargin(0.2) \n", + "ax1.legend(ncols=2)\n", + "\n", + "rects_tf = ax2.bar(x, hf_eval_runtime, width, label='HF evaluated', color=yellow)\n", + "rects_tlt_trainer = ax2.bar(x + width, tlt_trainer_eval_runtime, width, label='TLT eval using Trainer', color=blue)\n", + "rects_tlt_torch = ax2.bar(x + width * 2, tlt_torch_eval_runtime, width, label='TLT eval using Torch', color=dark_blue)\n", + "ax2.bar_label(rects_tf, padding=3)\n", + "ax2.bar_label(rects_tlt_trainer, padding=3)\n", + "ax2.bar_label(rects_tlt_torch, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax2.set_ylabel('Seconds')\n", + "ax2.set_title('Evaluation Runtime')\n", + "ax2.set_xticks(x+width, groups)\n", + "ax2.set_ymargin(0.2) \n", + "ax2.legend(ncols=2)\n", + "\n", + "# Setup bars for throughput\n", + "rects_tf = ax3.bar(x, hf_eval_throughput, width, label='HF evaluated', color=yellow)\n", + "rects_tlt_trainer = ax3.bar(x + width, tlt_trainer_eval_throughput, width, label='TLT eval using Trainer', color=blue)\n", + "rects_tlt_torch = ax3.bar(x + width * 2, tlt_torch_eval_thoughput, width, label='TLT eval using Torch', color=dark_blue)\n", + "ax3.bar_label(rects_tf, padding=3)\n", + "ax3.bar_label(rects_tlt_trainer, padding=3)\n", + "ax3.bar_label(rects_tlt_torch, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax3.set_ylabel('Samples per second')\n", + "ax3.set_title('Evaluation Throughput')\n", + "ax3.set_xticks(x+width, groups)\n", + "ax3.set_ymargin(0.2) \n", + "ax3.legend(ncols=2)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "68f77b7d", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "This concludes our performance comparison using a Hugging Face model with the Hugging Face `Trainer` and the Intel Transfer Learning Tool. All of the models trained during these experiments have been saved to your output directory. Any of these can be loaded back to perform further experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c26d581", + "metadata": {}, + "outputs": [], + "source": [ + "for i, batch_size in enumerate(batch_size_list):\n", + " print(\"\\nUsing batch size {}\".format(batch_size))\n", + " print('-' * 25)\n", + " print(\"Model trained using the Hugging Face Trainer:\\n\\t\", hf_saved_model_paths[i])\n", + " print(\"Model trained using the Intel Transfer Learning tool and the Hugging Face Trainer:\\n\\t\", tlt_trainer_saved_model_paths[i])\n", + " print(\"Model trained using the Intel Transfer Learning tool and the PyTorch libraries:\\n\\t\", tlt_torch_saved_model_paths[i])" + ] + }, + { + "cell_type": "markdown", + "id": "13e228cd", + "metadata": {}, + "source": [ + "## Citations \n", + "\n", + "```\n", + "@misc{devlin2019bert,\n", + " title={BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding}, \n", + " author={Jacob Devlin and Ming-Wei Chang and Kenton Lee and Kristina Toutanova},\n", + " year={2019},\n", + " eprint={1810.04805},\n", + " archivePrefix={arXiv},\n", + " primaryClass={cs.CL}\n", + "}\n", + "```\n", + "\n", + "```\n", + "@InProceedings{maas-EtAl:2011:ACL-HLT2011,\n", + " author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher},\n", + " title = {Learning Word Vectors for Sentiment Analysis},\n", + " booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},\n", + " month = {June},\n", + " year = {2011},\n", + " address = {Portland, Oregon, USA},\n", + " publisher = {Association for Computational Linguistics},\n", + " pages = {142--150},\n", + " url = {http://www.aclweb.org/anthology/P11-1015}\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/performance/tf_image_classification_performance.ipynb b/notebooks/performance/tf_image_classification_performance.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..1c2414474dcbf6f0e6eff54b1af0c190e1e912d5 --- /dev/null +++ b/notebooks/performance/tf_image_classification_performance.ipynb @@ -0,0 +1,1347 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "924b27fa", + "metadata": {}, + "source": [ + "# Performance Comparison: Image Classification Transfer Learning with TensorFlow and the Intel® Transfer Learning Tool\n", + "\n", + "This notebook uses the TensorFlow libraries to do transfer learning with an image classification model. The model is exported, evaluated, and used to generate predictions. The same sequence is also done using the Intel Transfer Learning Tool. The Intel Transfer Learning Tool is also used to optimize and quantized the trained model.\n", + "\n", + "Graphs are generated to visually compare:\n", + "* Training metrics (time per epoch, accuracy by epoch, loss by epoch)\n", + "* Evaluation metrics (time to evaluate the validation dataset, accuracy using the validation data)\n", + "* Prediction time for a single batch\n", + "* Latency and throughput for the trained models, quantized model, and the optimized model.\n", + "\n", + "The notebook has variables to allow controlling parameters such as the model name, dataset, the number of training epochs, and the batch size(s)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f305fbd", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' \n", + "\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.ticker as mtick\n", + "import numpy as np\n", + "import pandas as pd\n", + "import psutil\n", + "import random\n", + "import tempfile\n", + "import tensorflow as tf\n", + "import tensorflow_hub as hub\n", + "import warnings\n", + "\n", + "from tlt.datasets import dataset_factory\n", + "from tlt.models import model_factory\n", + "from tlt.utils.file_utils import download_and_extract_tar_file\n", + "from tlt.utils.platform_util import CPUInfo, OptimizedPlatformUtil, PlatformUtil\n", + "from utils import inc_utils\n", + "\n", + "# Ignore all warnings\n", + "warnings.filterwarnings('ignore')\n", + "tf.get_logger().setLevel('ERROR')\n", + "\n", + "# Specify the the default dataset directory\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Specify a directory for output (saved models and checkpoints)\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "print(\"Output directory:\", output_directory)\n", + "\n", + "# TF Hub cache directory\n", + "os.environ[\"TFHUB_CACHE_DIR\"] = os.path.join(output_directory, \".cache\", \"tfhub_modules\")\n", + "\n", + "# Data Frame styles\n", + "table_styles =[{\n", + " 'selector': 'caption',\n", + " 'props': [\n", + " ('text-align', 'center'),\n", + " ('color', 'black'),\n", + " ('font-size', '16px')\n", + " ]\n", + "}]\n", + "\n", + "# Colors used in charts\n", + "orange = '#ff6f00'\n", + "blue = '#0071c5'\n", + "dark_blue = '#003c71'\n", + "yellow = '#f3d54e'\n", + "\n", + "# Caption style for DataFrames\n", + "caption_style = [dict(selector=\"caption\", props=[(\"text-align\", \"center\"), (\"font-size\", \"14pt\"), (\"color\", \"black\")])]\n", + "\n", + "# Line styles\n", + "line_styles = ['solid', 'dotted', 'dashed', 'dashdot']\n", + "\n", + "# Marker styles\n", + "marker_styles = ['o', 'D', 's', 'v']" + ] + }, + { + "cell_type": "markdown", + "id": "4270c3e8", + "metadata": {}, + "source": [ + "## 1. Display Platform Information\n", + "\n", + "Use the `CPUInfo` and `PlatformUtil` classes in the get and display information about the platform and TensorFlow version." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f3c3ee5", + "metadata": {}, + "outputs": [], + "source": [ + "# Get and display CPU/platform information\n", + "cpu_info = CPUInfo()\n", + "platform_util = PlatformUtil()\n", + "print(\"{0} CPU Information {0}\".format(\"=\" * 20))\n", + "print(\"CPU family:\", platform_util.cpu_family)\n", + "print(\"CPU model:\", platform_util.cpu_model)\n", + "print(\"CPU type:\", platform_util.cpu_type)\n", + "print(\"Physical cores per socket:\", cpu_info.cores_per_socket)\n", + "print(\"Total physical cores:\", cpu_info.cores)\n", + "cpufreq = psutil.cpu_freq()\n", + "print(\"Max Frequency:\", cpufreq.max)\n", + "print(\"Min Frequency:\", cpufreq.min)\n", + "cpu_socket_count = cpu_info.sockets\n", + "print(\"Socket Number:\", cpu_socket_count)\n", + "\n", + "print(\"\\n{0} Memory Information {0}\".format(\"=\" * 20))\n", + "svmem = psutil.virtual_memory()\n", + "print(\"Total: \", int(svmem.total / (1024 ** 3)), \"GB\")\n", + "\n", + "# Display TensorFlow version information\n", + "print(\"\\n{0} TensorFlow Information {0}\".format(\"=\" * 20))\n", + "print(\"TensorFlow version:\", tf.__version__)\n", + "print(\"TensorFlow Hub version:\", hub.__version__)\n", + "major_version = int(tf.__version__.split(\".\")[0])\n", + "minor_version = int(tf.__version__.split(\".\")[1])\n", + "if major_version >= 2:\n", + " onednn_enabled = 0\n", + " if minor_version < 5:\n", + " from tensorflow.python import _pywrap_util_port\n", + " else:\n", + " from tensorflow.python.util import _pywrap_util_port\n", + " onednn_enabled = int(os.environ.get('TF_ENABLE_ONEDNN_OPTS', '0'))\n", + " on_onednn = _pywrap_util_port.IsMklEnabled() or (onednn_enabled == 1)\n", + "else:\n", + " on_onednn = tf.pywrap_tensorflow.IsMklEnabled()\n", + "\n", + "print(\"oneDNN enabled:\", on_onednn)\n", + "\n", + "# Don't use the NVidia GPU, if there is one\n", + "os.environ['CUDA_VISIBLE_DEVICES'] = \"\"" + ] + }, + { + "cell_type": "markdown", + "id": "1d5a1252", + "metadata": {}, + "source": [ + "## 2. Select a model and define parameters to use during training and evaluation\n", + "\n", + "### Select a model\n", + "\n", + "See the list of supported image classification models from TensorFlow Hub in the Intel Transfer Learning Tool." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d9b894", + "metadata": {}, + "outputs": [], + "source": [ + "framework = 'tensorflow'\n", + "use_case = 'image_classification'\n", + "model_hub = 'TFHub'\n", + "supported_models = model_factory.get_supported_models(framework, use_case)\n", + "supported_models = supported_models[use_case]\n", + "\n", + "# Filter to only get relevant models\n", + "supported_models = { key:value for (key,value) in supported_models.items() if value[framework]['model_hub'] == model_hub}\n", + "\n", + "print(\"Supported {} models for {} from {}\".format(framework, use_case, model_hub))\n", + "print(\"=\" * 70)\n", + "for model_name in supported_models.keys():\n", + " print(model_name)" + ] + }, + { + "cell_type": "markdown", + "id": "fd14554e", + "metadata": {}, + "source": [ + "Set the `model_name` to the model that will be used for transfer learning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bca71035", + "metadata": {}, + "outputs": [], + "source": [ + "# Select a model\n", + "model_name = \"resnet_v1_50\"\n", + "\n", + "# Get information about the model (image size and the feature vector handle)\n", + "# This information will be used during transfer learning using the TensorFlow framework API\n", + "if model_name in supported_models.keys():\n", + " model_info = supported_models[model_name][framework]\n", + " image_size = model_info[\"image_size\"]\n", + " feature_vector_handle = model_info['feature_vector']\n", + " \n", + " print(\"Model Name: {}\".format(model_name))\n", + " print(\"TF Hub feature vector: {}\".format(feature_vector_handle))\n", + " print(\"Image size: {}\".format(image_size))\n", + "else:\n", + " raise ValueError(\"The specified model is unsupported. Please select a model from the list of supported models.\")" + ] + }, + { + "cell_type": "markdown", + "id": "e05fc52e", + "metadata": {}, + "source": [ + "### Select a dataset\n", + "\n", + "By default, the notebook will use the [TensorFlow Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers), which has flower images that belong to 5 categories.\n", + "\n", + "To use your own dataset, set the `dataset_subdir` variable to the dataset path. The dataset directory is expected to have folders of images for each class, where the name of the folder will be used as the class name.\n", + "\n", + "```\n", + "dataset_dir\n", + " ├── class_a\n", + " ├── class_b\n", + " └── class_c\n", + "```\n", + "\n", + "Optionally, the `dataset_subdir` directory can have `train` and `test`/`validation` subdirectories. For example:\n", + "```\n", + "dataset_dir\n", + " ├── train\n", + " | ├── class_a\n", + " | ├── class_b\n", + " | └── class_c\n", + " └── test\n", + " ├── class_a\n", + " ├── class_b\n", + " └── class_c\n", + "```\n", + "If the dataset does not have separate folders for train and test/validation, the dataset will be split by percentage." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a0d6c0d", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_subdir = os.path.join(dataset_directory, \"flower_photos\")\n", + "\n", + "# Download the flowers dataset, if the folder doesn't exist\n", + "if not os.path.exists(dataset_subdir):\n", + " os.makedirs(dataset_subdir)\n", + " dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + " download_and_extract_tar_file(dataset_url, dataset_directory)\n", + " \n", + "print(\"Dataset path:\", dataset_subdir)\n", + "\n", + "print(\"\\nFolders in the dataset directory:\")\n", + "for d in os.listdir(dataset_subdir):\n", + " if os.path.isdir(os.path.join(dataset_subdir, d)):\n", + " print(\"-\", d)" + ] + }, + { + "cell_type": "markdown", + "id": "e1aa58df", + "metadata": {}, + "source": [ + "### Define parameters\n", + "\n", + "For consistency between the model training using the TensorFlow framework API and the model training using the Intel Transfer Learning Tool API, the next cell defines parameters that will be used by both methods." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eee80253", + "metadata": {}, + "outputs": [], + "source": [ + "# Number of training epochs\n", + "training_epochs = 2\n", + "\n", + "# Shuffle the files after each training epoch\n", + "shuffle_files = True\n", + "\n", + "# Define training/validation splits for the dataset \n", + "# (if the dataset directory does not have subdirectories for train and test/validation)\n", + "validation_split = 0.25\n", + "training_split = 1 - validation_split\n", + "\n", + "# Set seed for consistency between runs (or None)\n", + "seed = 10\n", + "\n", + "# List of batch size(s) to compare (maximum of 4 batch sizes to try)\n", + "batch_size_list = [ 256, 512 ]" + ] + }, + { + "cell_type": "markdown", + "id": "4847ae1d", + "metadata": {}, + "source": [ + "Validate parameter values and then print out the parameter values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "051ea2a1", + "metadata": {}, + "outputs": [], + "source": [ + "if not isinstance(training_epochs, int):\n", + " raise TypeError(\"The training_epochs parameter should be an integer, but found a {}\".format(type(training_epochs)))\n", + "\n", + "if training_epochs < 1:\n", + " raise ValueError(\"The training_epochs parameter should not be less than 1.\")\n", + " \n", + "if not isinstance(shuffle_files, bool):\n", + " raise TypeError(\"The shuffle_files parameter should be a bool, but found a {}\".format(type(shuffle_files)))\n", + "\n", + "if not isinstance(validation_split, float):\n", + " raise TypeError(\"The validation_split parameter should be a float, but found a {}\".format(type(validation_split)))\n", + "\n", + "if not isinstance(training_split, float):\n", + " raise TypeError(\"The training_split parameter should be a float, but found a {}\".format(type(training_split)))\n", + "\n", + "if validation_split + training_split > 1:\n", + " raise ValueError(\"The sum of validation_split and training_split should not be greater than 1.\")\n", + "\n", + "if seed and not isinstance(seed, int):\n", + " raise TypeError(\"The seed parameter should be a integer or None, but found a {}\".format(type(seed)))\n", + "\n", + "if len(batch_size_list) > 4 or len(batch_size_list) == 0:\n", + " raise ValueError(\"The batch_size_list should have at most 4 values, but found {} values ({})\".format(\n", + " len(batch_size_list), batch_size_list))\n", + " \n", + "print(\"Number of training epochs:\", training_epochs)\n", + "print(\"Shuffle files:\", shuffle_files)\n", + "print(\"Training split: {}%\".format(training_split*100))\n", + "print(\"Validation split: {}%\".format(validation_split*100))\n", + "print(\"Seed:\", str(seed))\n", + "print(\"Batch size list:\", batch_size_list)" + ] + }, + { + "cell_type": "markdown", + "id": "fb2d786a", + "metadata": {}, + "source": [ + "Define a callback method that track the time that it took to run training epochs, evaluation, and batch predictions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "713f648f", + "metadata": {}, + "outputs": [], + "source": [ + "# Callback to track the training time for each epoch, evaluation time, or prediction time\n", + "class TimerCallback(tf.keras.callbacks.Callback):\n", + " def __init__(self):\n", + " self.epoch_times = []\n", + " self.eval_times = []\n", + " self.predict_times = []\n", + " def on_epoch_begin(self, batch, logs={}):\n", + " self.tf_timestamp = tf.timestamp()\n", + " def on_epoch_end(self,epoch,logs = {}):\n", + " self.epoch_times.append((tf.timestamp() - self.tf_timestamp).numpy())\n", + " def on_test_begin(self, batch, logs={}):\n", + " self.tf_timestamp = tf.timestamp()\n", + " def on_test_end(self,epoch,logs = {}):\n", + " self.eval_times.append((tf.timestamp() - self.tf_timestamp).numpy())\n", + " def on_predict_begin(self, batch, logs={}):\n", + " self.tf_timestamp = tf.timestamp()\n", + " def on_predict_end(self,epoch,logs = {}):\n", + " self.predict_times.append((tf.timestamp() - self.tf_timestamp).numpy())" + ] + }, + { + "cell_type": "markdown", + "id": "abd401a7", + "metadata": {}, + "source": [ + "## 3. Compare the training time for transfer learning\n", + "\n", + "In this section, we will compare the time it takes to retrain the image classification model using the dataset that was selected in the previous section.\n", + "\n", + "The training will be done in two different ways to compare:\n", + "* Transfer learning using the TensorFlow framework and TF Hub libraries\n", + "* Transfer learning using the Intel Transfer Learning Tool API\n", + "\n", + "### Transfer learning using the TensorFlow framework and TF Hub libraries\n", + "\n", + "This section goes through using the TensorFlow framework and TF Hub libraries to retrain the model using the selected dataset.\n", + "\n", + "First, the dataset is loaded in, which allows us to determine the number of classes in the dataset. The original ImageNet dataset that the image classification model was trained on has 1000 classes. To do transfer learning using the new dataset, we will get the feature vector from TF Hub and then add on a classification layer that matches the number of classes in the new dataset.\n", + "\n", + "If multiple batch sizes were set in the `batch_size_list`, training will be run for each batch size." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08cfd3ea", + "metadata": {}, + "outputs": [], + "source": [ + "# Set seed\n", + "if seed:\n", + " os.environ['PYTHONHASHSEED'] = str(seed)\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + " tf.random.set_seed(seed)\n", + "\n", + "# Lists to track callbacks, datasets, models, and saved model directory for each batch size experiment\n", + "tf_time_callback_list = []\n", + "tf_dataset_list = []\n", + "tf_model_list = []\n", + "tf_export_dir_list = []\n", + "tf_history_list = []\n", + "\n", + "# Check if the dataset directory has subdirectories for train/validation/test splits\n", + "val_dataset_dir = None\n", + "train_dataset_dir = dataset_subdir\n", + "if os.path.exists(os.path.join(dataset_subdir, 'train')):\n", + " train_dataset_dir = os.path.join(dataset_subdir, 'train')\n", + " val_dataset_dir = os.path.join(dataset_subdir, 'validation')\n", + " \n", + " if not os.path.exists(val_dataset_dir):\n", + " if os.path.exists(os.path.join(dataset_subdir, 'test')):\n", + " val_dataset_dir = os.path.join(dataset_subdir, 'test')\n", + " else:\n", + " raise ValueError('The dataset directory ({}) has a \"train\" directory, but no \"validation\" or \"test\" directory.')\n", + "\n", + " print(\"Using training data from {}\".format(train_dataset_dir))\n", + " print(\"Using validation data from {}\".format(val_dataset_dir))\n", + " \n", + "# Load the dataset\n", + "tf_dataset = tf.keras.utils.image_dataset_from_directory(train_dataset_dir, batch_size=None, seed=seed)\n", + "class_names = tf_dataset.class_names\n", + "\n", + "if shuffle_files:\n", + " tf_dataset = tf_dataset.shuffle(tf_dataset.cardinality(), reshuffle_each_iteration=False, seed=seed)\n", + "\n", + "if val_dataset_dir:\n", + " # Load the validation/test sub directory\n", + " train_ds = tf_dataset\n", + " val_ds = tf.keras.utils.image_dataset_from_directory(val_dataset_dir, batch_size=None, seed=seed) \n", + " if shuffle_files:\n", + " val_ds = val_ds.shuffle(val_ds.cardinality(), reshuffle_each_iteration=False, seed=seed)\n", + " train_size = len(train_ds)\n", + " val_size = len(val_ds)\n", + "else:\n", + " # Split the data into train/validation subsets (Note that image_dataset_from_directory can also do splitting but\n", + " # we are doing it this way to match what the Intel Transfer Learning Tool does to ensure the same sized splits)\n", + " train_size = int(training_split * len(tf_dataset))\n", + " val_size = int(validation_split * len(tf_dataset))\n", + " train_ds = tf_dataset.take(train_size)\n", + " val_ds = tf_dataset.skip(train_size).take(val_size)\n", + "\n", + "print(\"Training dataset size:\", train_size)\n", + "print(\"Validation dataset size:\", val_size)\n", + " \n", + "# Preprocess the dataset\n", + "normalization_layer = tf.keras.layers.Rescaling(1. / 255)\n", + "\n", + "def preprocess_image(image, label):\n", + " image = tf.image.resize_with_pad(image, image_size, image_size)\n", + " image = normalization_layer(image)\n", + " return (image, label)\n", + "\n", + "train_ds = train_ds.map(preprocess_image)\n", + "val_ds = val_ds.map(preprocess_image)\n", + "\n", + "for batch_size in batch_size_list:\n", + " print('-' * 40)\n", + " print('Training using batch size: {}'.format(batch_size))\n", + " print('-' * 40)\n", + " \n", + " # Batch the dataset \n", + " batched_train_ds = train_ds.batch(batch_size)\n", + " batched_val_ds = val_ds.batch(batch_size)\n", + " \n", + " # Get the feature extractor layer from TF Hub\n", + " feature_extractor_layer = hub.KerasLayer(feature_vector_handle,\n", + " input_shape=(image_size, image_size, 3),\n", + " trainable=False)\n", + "\n", + " # Add the dense layer sized according to the number of classes in our dataset\n", + " tf_model = tf.keras.Sequential([\n", + " feature_extractor_layer,\n", + " tf.keras.layers.Dense(len(class_names))\n", + " ])\n", + "\n", + " # Configure the model optimizer and loss function\n", + " tf_model.compile(\n", + " optimizer=tf.keras.optimizers.Adam(),\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['acc'])\n", + " \n", + " tf_model.summary()\n", + "\n", + " # Define the callback for tracking the time it takes to train each epoch\n", + " tf_time_callback = TimerCallback()\n", + "\n", + " # Train the model\n", + " tf_history_list.append(tf_model.fit(batched_train_ds, epochs=training_epochs, shuffle=shuffle_files,\n", + " callbacks=[tf_time_callback]))\n", + " \n", + " # Export the trained model\n", + " tf_export_dir = os.path.join(output_directory, \"tf_saved_models\", model_name)\n", + " if not os.path.exists(tf_export_dir):\n", + " os.makedirs(tf_export_dir)\n", + " tf_export_dir = tempfile.mkdtemp(prefix=tf_export_dir + '/')\n", + " print(\"Save model to:\", tf_export_dir)\n", + " tf_model.save(tf_export_dir)\n", + " \n", + " # Append to lists for each batch size\n", + " tf_time_callback_list.append(tf_time_callback)\n", + " tf_dataset_list.append((batched_train_ds, batched_val_ds))\n", + " tf_model_list.append(tf_model)\n", + " tf_export_dir_list.append(tf_export_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "160cf79a", + "metadata": {}, + "source": [ + "### Transfer learning using the Intel Transfer Learning Tool API\n", + "\n", + "This section uses the Intel Transfer Learning Tool API to retrain the model using the selected dataset. This API simplifies the transfer learning process, so there are less lines of code compared to directly using the TensorFlow and TensorFlow Hub libraries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2fd15b3f", + "metadata": {}, + "outputs": [], + "source": [ + "# Use the OptimizedPlatform Util class from the Intel Transfer Learning Tool API to set recommended settings\n", + "optimized_platform_util = OptimizedPlatformUtil(omp_num_threads=cpu_info.cores_per_socket,\n", + " kmp_blocktime=0,\n", + " kmp_affinity='granularity=fine,compact,1,0',\n", + " tf_num_intraop_threads=cpu_info.cores_per_socket,\n", + " tf_num_interop_threads=cpu_info.sockets,\n", + " force_reset_env_vars=True)\n", + "\n", + "for k, v in optimized_platform_util.env_vars_dict.items():\n", + " if v is not None:\n", + " print(\"{}: {}\".format(k, v))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fac3d6df", + "metadata": {}, + "outputs": [], + "source": [ + "# Lists to track callbacks, datasets, models, and saved model directory for each batch size experiment\n", + "tlt_time_callback_list = []\n", + "tlt_dataset_list = []\n", + "tlt_model_list = []\n", + "tlt_export_dir_list = []\n", + "tlt_history_list = []\n", + " \n", + "for batch_size in batch_size_list:\n", + " print('-' * 40)\n", + " print('Training using batch size: {}'.format(batch_size))\n", + " print('-' * 40)\n", + " \n", + " # Use the model factory to get the model\n", + " tlt_model = model_factory.get_model(model_name=model_name, framework=framework)\n", + " \n", + " # Load, split, and preprocess the dataset\n", + " tlt_dataset = dataset_factory.load_dataset(dataset_dir=dataset_subdir, use_case=use_case, framework=framework)\n", + " \n", + " if not tlt_dataset.train_subset:\n", + " tlt_dataset.shuffle_split(train_pct=training_split, val_pct=validation_split, seed=seed, shuffle_files=shuffle_files)\n", + " \n", + " tlt_dataset.preprocess(tlt_model.image_size, batch_size=batch_size)\n", + " \n", + " # Define the callback for tracking the time it takes to train each epoch\n", + " tlt_time_callback = TimerCallback()\n", + "\n", + " # Train the model\n", + " tlt_history_list.append(tlt_model.train(tlt_dataset, output_dir=output_directory, epochs=training_epochs,\n", + " shuffle_files=shuffle_files, do_eval=False, callbacks=tlt_time_callback,\n", + " seed=seed))\n", + "\n", + " # Export the trained model\n", + " tlt_export_dir = os.path.join(output_directory, \"tlt_saved_models\")\n", + " tlt_export_dir = tlt_model.export(tlt_export_dir)\n", + " \n", + " # Append to lists for each batch size\n", + " tlt_time_callback_list.append(tlt_time_callback)\n", + " tlt_dataset_list.append(tlt_dataset)\n", + " tlt_model_list.append(tlt_model)\n", + " tlt_export_dir_list.append(tlt_export_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "e5e52a50", + "metadata": {}, + "source": [ + "### Optimize the model using the Intel Transfer Learning Tool API\n", + "\n", + "After training, the Intel Transfer Learning Tool can optimize the model to improve inference performance. This is done using the [Intel® Neural Compressor](https://github.com/intel/neural-compressor) quantizing the model or optimizing the full precision model. \n", + "\n", + "First, we setup a configuration file that with parameters that will be used by the Intel Neural Compressor for quantization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df079cb0", + "metadata": {}, + "outputs": [], + "source": [ + "tlt_quantization_dir_list = []\n", + "tlt_optimized_dir_list = []\n", + "inc_config_list = []\n", + "\n", + "# Create a tuning workspace directory for INC\n", + "nc_workspace = os.path.join(output_directory, 'nc_workspace')\n", + "\n", + "# Relative accuracy loss (1%)\n", + "relative_accuracy_criterion = 0.01\n", + "\n", + "# Define the exit policy timeout (in seconds) and max number of trials. The tuning processing finishes when\n", + "# the timeout or max trials is reached. A tuning timeout of 0 means that the tuning phase stops when the\n", + "# accuracy criterion is met.\n", + "timeout = 0\n", + "max_trials=15\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " # Create an output directories for the quantized and optimized models\n", + " tlt_quantization_dir = os.path.join(output_directory, 'tlt_quantized_models', model_name, os.path.basename(tlt_export_dir_list[i]))\n", + " tlt_optimized_dir = os.path.join(output_directory, 'tlt_optimized_models', model_name, os.path.basename(tlt_export_dir_list[i]))\n", + "\n", + " # Create an Intel Neural Compressor config based on the inputs that we are using\n", + " inc_config_list.append(tlt_model.get_inc_config(accuracy_criterion_relative=relative_accuracy_criterion,\n", + " exit_policy_timeout=timeout, exit_policy_max_trials=max_trials))\n", + " \n", + " # Append to lists for each batch size\n", + " tlt_quantization_dir_list.append(tlt_quantization_dir)\n", + " tlt_optimized_dir_list.append(tlt_optimized_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "534de66e", + "metadata": {}, + "source": [ + "Next, we quantize the model using the config file that was just generated. Quantization aims to improve inference\n", + "performance by reducing the number of bits required, by maintaining close the the same amount of accuracy as the full precision model. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "103ffd57", + "metadata": {}, + "outputs": [], + "source": [ + "for i, batch_size in enumerate(batch_size_list):\n", + " # Quantize the model\n", + " tlt_model.quantize(tlt_quantization_dir_list[i], tlt_dataset_list[i], config=inc_config_list[i])" + ] + }, + { + "cell_type": "markdown", + "id": "cfeea7eb", + "metadata": {}, + "source": [ + "Another option to improve inference performance is using graph optimization through the Intel Neural Compressor which:\n", + "* Converts variables to constants\n", + "* Removes training-only operations like checkpoint saving\n", + "* Strips out parts of the graph that are never reached\n", + "* Removes debug operations like CheckNumerics\n", + "* Folds batch normalization ops into the pre-calculated weights\n", + "* Fuses common operations into unified versions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a2f7a8e", + "metadata": {}, + "outputs": [], + "source": [ + "for i, batch_size in enumerate(batch_size_list):\n", + " # Optimize the full precision model\n", + " tlt_model.optimize_graph(tlt_optimized_dir_list[i])" + ] + }, + { + "cell_type": "markdown", + "id": "304258ec", + "metadata": {}, + "source": [ + "### Compare training times\n", + "\n", + "The table below compares the time it took to train each epoch (in seconds) using the TensorFlow framework libraries directly versus the Intel Transfer Learning Tool API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16299291", + "metadata": {}, + "outputs": [], + "source": [ + "display_df = []\n", + "plt.figure(figsize=(10,6))\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " # Sanity check that both datasets had the same number of batches\n", + " if len(tf_dataset_list[i][0]) != len(tlt_dataset_list[i].train_subset):\n", + " print(\"WARNING: For batch size {}, the TF training dataset had {} batches and the TLT training dataset had \"\n", + " \"{} batches. These values should have been the same.\".format(batch_size, len(tf_dataset_list[i][0]), len(tlt_dataset_list[i].train_subset)))\n", + " \n", + " # Calculate images/second\n", + " tf_images_per_second = [train_size / t for t in tf_time_callback_list[i].epoch_times]\n", + " tlt_images_per_second = [train_size / t for t in tlt_time_callback_list[i].epoch_times]\n", + " performance_delta = [\"{0:.2f}%\".format((tlt-tf)/tf * 100) for tf, tlt in zip(tf_images_per_second, tlt_images_per_second)]\n", + "\n", + " # Graph the results\n", + " epoch_list = [str(i) for i in range(1, training_epochs + 1)]\n", + " tf_train_time = tf_time_callback_list[i].epoch_times\n", + " tlt_train_time = tlt_time_callback_list[i].epoch_times\n", + "\n", + " plt.plot(epoch_list, tf_train_time, label=\"Using TF libraries with batch size {}\".format(batch_size),\n", + " linestyle=line_styles[i], marker=marker_styles[i], color=orange)\n", + " plt.plot(epoch_list, tlt_train_time, label=\"Using TLT with batch size {}\".format(batch_size), \n", + " linestyle=line_styles[i], marker=marker_styles[i],color=blue)\n", + " \n", + " # Create a DataFrame to display the results in a table\n", + " df = pd.DataFrame({\n", + " 'TF epoch time
(seconds)': tf_time_callback_list[i].epoch_times,\n", + " 'TLT epoch time
(seconds)': tlt_time_callback_list[i].epoch_times,\n", + " 'TF throughput
(images/sec)': tf_images_per_second,\n", + " 'TLT throughput
(images/sec)': tlt_images_per_second,\n", + " 'Performance
Boost': performance_delta\n", + " })\n", + " df.index += 1 \n", + " df = df.style.set_table_styles(table_styles).set_caption(\"Epoch training times with batch size {}\".format(batch_size))\n", + " display_df.append(df)\n", + "\n", + "plt.title(\"Training time per epoch\")\n", + "plt.xlabel(\"Epoch\")\n", + "plt.ylabel(\"Seconds\")\n", + "plt.legend()\n", + "plt.show()\n", + "\n", + "# Display tables with epoch training time for each batch size\n", + "for df in display_df:\n", + " display(df)" + ] + }, + { + "cell_type": "markdown", + "id": "388a7675", + "metadata": {}, + "source": [ + "Next, visualize the accuracy and loss metrics collected during training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59598b38", + "metadata": {}, + "outputs": [], + "source": [ + "# Graph the training accuracy by epoch for each batch size\n", + "plt.figure(figsize=(10,6))\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " tf_acc_time = [i * 100 for i in tf_history_list[i].history['acc']]\n", + " tlt_acc_time = [i * 100 for i in tlt_history_list[i]['acc']]\n", + "\n", + " plt.plot(epoch_list, tf_acc_time, label = \"Using TF libraries (batch size = {})\".format(batch_size), linestyle=line_styles[i], marker=marker_styles[i], color=orange)\n", + " plt.plot(epoch_list, tlt_acc_time, label = \"Using TLT (batch size = {})\".format(batch_size), linestyle=line_styles[i], marker=marker_styles[i], color=blue)\n", + "\n", + "plt.title(\"Training Accuracy by Epoch\")\n", + "plt.xlabel(\"Epoch\")\n", + "plt.ylabel(\"Accuracy (%)\")\n", + "plt.legend()\n", + "plt.show()\n", + "\n", + "# Graph the training loss by epoch for each batch size\n", + "plt.figure(figsize=(10,6))\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " tf_loss_time = tf_history_list[i].history['loss']\n", + " tlt_loss_time = tlt_history_list[i]['loss']\n", + "\n", + " plt.plot(epoch_list, tf_loss_time, label = \"Using TF libraries (batch size = {})\".format(batch_size), linestyle=line_styles[i], marker=marker_styles[i], color=orange)\n", + " plt.plot(epoch_list, tlt_loss_time, label = \"Using TLT (batch size = {})\".format(batch_size), linestyle=line_styles[i], marker=marker_styles[i], color=blue)\n", + "\n", + "plt.title(\"Training Loss by Epoch\")\n", + "plt.xlabel(\"Epoch\")\n", + "plt.ylabel(\"Loss\")\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e58715d4", + "metadata": {}, + "source": [ + "## 4. Evaluate and predict\n", + "\n", + "This section calls evaluation and prediction methods for the models trained using the TensorFlow libraries and the Intel Transfer Learning Tool.\n", + "\n", + "### Evaluate the models using the validation data\n", + "\n", + "First, evaluate the models trained using the TensorFlow libraries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ba3aadb", + "metadata": {}, + "outputs": [], + "source": [ + "tf_eval_callback_list = []\n", + "tf_eval_metrics_list = []\n", + "\n", + "# Evaluate using the TensorFlow framework model for each batch size\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 40)\n", + " print('Evaluate using batch size: {}'.format(batch_size))\n", + " print('-' * 40)\n", + " \n", + " tf_eval_callback = TimerCallback()\n", + " \n", + " # Use the test split of the dataset to evaluate the model\n", + " tf_eval_metrics_list.append(tf_model_list[i].evaluate(tf_dataset_list[i][1], callbacks=tf_eval_callback))\n", + " tf_eval_callback_list.append(tf_eval_callback)" + ] + }, + { + "cell_type": "markdown", + "id": "de6dbc91", + "metadata": {}, + "source": [ + "Next, evaluate the models trained using the Intel Transfer Learning Tool." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ed9e1a0", + "metadata": {}, + "outputs": [], + "source": [ + "tlt_eval_callback_list = []\n", + "tlt_eval_metrics_list = []\n", + "\n", + "# Evaluate using the Intel Transfer Learning Tool model for each batch size\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 40)\n", + " print('Evaluate using batch size: {}'.format(batch_size))\n", + " print('-' * 40)\n", + " \n", + " use_test_set = tlt_dataset_list[i].validation_subset is None and tlt_dataset_list[i].test_subset is not None\n", + " \n", + " tlt_eval_callback = TimerCallback()\n", + " tlt_eval_metrics_list.append(tlt_model_list[i].evaluate(tlt_dataset_list[i], callbacks=tlt_eval_callback, use_test_set=use_test_set))\n", + " tlt_eval_callback_list.append(tlt_eval_callback)" + ] + }, + { + "cell_type": "markdown", + "id": "231f3731", + "metadata": {}, + "source": [ + "After all the models have been evaluated, visualize the results using charts that the display the time that it took to evaluate each model and the accuracy that was found when using the validation dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c31fee1", + "metadata": {}, + "outputs": [], + "source": [ + "# Bar chart group labels\n", + "groups = [\"batch size = {}\".format(bs) for bs in batch_size_list]\n", + "\n", + "# Create grouped bar chart for evaluation time\n", + "decimals = 3 # number of decimals to use for rounding\n", + "tf_eval_times = [round(callback.eval_times[0], decimals) for callback in tf_eval_callback_list]\n", + "tlt_eval_times = [round(callback.eval_times[0], decimals) for callback in tlt_eval_callback_list]\n", + "\n", + "x = np.arange(len(groups))\n", + "width = 0.24 # the width of the bars\n", + "multiplier = 0\n", + "\n", + "# Setup bars for evaluation times\n", + "fig, (ax1, ax2) = plt.subplots(2)\n", + "fig.set_figheight(10)\n", + "fig.set_figwidth(10)\n", + "rects_tf = ax1.bar(x - width/2, tf_eval_times, width, label='TF eval', color=orange)\n", + "rects_tlt = ax1.bar(x + width/2, tlt_eval_times, width, label='TLT eval', color=blue)\n", + "ax1.bar_label(rects_tf, padding=3)\n", + "ax1.bar_label(rects_tlt, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax1.set_ylabel('Seconds')\n", + "ax1.set_title('Evaluation time')\n", + "ax1.set_xticks(x, groups)\n", + "ax1.set_ymargin(0.2) \n", + "ax1.legend(ncols=2)\n", + "#plt.show()\n", + "\n", + "# Evaluation accuracy comparison\n", + "decimals = 2\n", + "tf_acc_index = tf_model_list[0].metrics_names.index('acc')\n", + "tlt_acc_index = tlt_model_list[0]._model.metrics_names.index('acc')\n", + "tf_eval_accuracy = [round(x[tf_acc_index] * 100, decimals) for x in tf_eval_metrics_list]\n", + "tlt_eval_accuracy = [round(x[tlt_acc_index] * 100, decimals) for x in tlt_eval_metrics_list]\n", + "\n", + "# Setup bars for evaluation accuracy\n", + "rects_tf = ax2.bar(x - width/2, tf_eval_accuracy, width, label='TF accuracy', color=orange)\n", + "rects_tlt = ax2.bar(x + width/2, tlt_eval_accuracy, width, label='TLT accuracy', color=blue)\n", + "ax2.bar_label(rects_tf, padding=3)\n", + "ax2.bar_label(rects_tlt, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax2.set_ylabel('Accuracy (%)')\n", + "ax2.yaxis.set_major_formatter(mtick.PercentFormatter())\n", + "ax2.set_title('Evaluation accuracy using the validation data')\n", + "ax2.set_xticks(x, groups)\n", + "ax2.set_ymargin(0.2) \n", + "ax2.legend(ncols=2)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "4d759b5d", + "metadata": {}, + "source": [ + "### Predict using a batch of images\n", + "\n", + "Use the TensorFlow libaries to get a batch of images and predict using the trained models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38132396", + "metadata": {}, + "outputs": [], + "source": [ + "tf_predict_callback_list = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 50)\n", + " print('Predict on a single batch (batch size = {})'.format(batch_size))\n", + " print('-' * 50)\n", + " \n", + " tf_predict_time = TimerCallback()\n", + " dataset_batch = next(iter(tf_dataset_list[i][0]))\n", + " tf_batch, _ = dataset_batch\n", + " batch_predictions = tf_model_list[i].predict(tf_batch, callbacks=tf_predict_time)\n", + " tf_predict_callback_list.append(tf_predict_time)" + ] + }, + { + "cell_type": "markdown", + "id": "0d3beb10", + "metadata": {}, + "source": [ + "Similarly, use the Intel Transfer Learning Tool API to get a batch of images and predict using the trained models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66be9e10", + "metadata": {}, + "outputs": [], + "source": [ + "tlt_predict_callback_list = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 50)\n", + " print('Predict on a single batch (batch size = {})'.format(batch_size))\n", + " print('-' * 50)\n", + " \n", + " tlt_predict_time = TimerCallback()\n", + "\n", + " tlt_batch, _ = tlt_dataset_list[i].get_batch(subset='train')\n", + " predictions = tlt_model_list[i].predict(tlt_batch, callbacks=tlt_predict_time)\n", + " tlt_predict_callback_list.append(tlt_predict_time)" + ] + }, + { + "cell_type": "markdown", + "id": "a594631f", + "metadata": {}, + "source": [ + "Visualize the time that it took to get predictions for a batch of images for each model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "309f636a", + "metadata": {}, + "outputs": [], + "source": [ + "# Create grouped bar chart for prediction time\n", + "decimals = 3 # number of decimals to use for rounding\n", + "tf_predict_times = [round(callback.predict_times[0], decimals) for callback in tf_predict_callback_list]\n", + "tlt_predict_times = [round(callback.predict_times[0], decimals) for callback in tlt_predict_callback_list]\n", + "\n", + "# Setup bars for evaluation times\n", + "fig, ax = plt.subplots()\n", + "fig.set_figheight(6)\n", + "fig.set_figwidth(10)\n", + "rects_tf = ax.bar(x - width/2, tf_predict_times, width, label='TF predict', color=orange)\n", + "rects_tlt = ax.bar(x + width/2, tlt_predict_times, width, label='TLT predict', color=blue)\n", + "ax.bar_label(rects_tf, padding=3)\n", + "ax.bar_label(rects_tlt, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax.set_ylabel('Seconds')\n", + "ax.set_title('Prediction time for a single batch')\n", + "ax.set_xticks(x, groups)\n", + "ax.set_ymargin(0.2) \n", + "ax.legend(ncols=2)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ca0771ec", + "metadata": {}, + "source": [ + "### Check performance using the Intel® Neural Compressor\n", + "\n", + "Use the [Intel Neural Compressor](https://github.com/intel/neural-compressor/tree/master) to determine the performance of the exported models. \n", + "\n", + "We will compare:\n", + "* The original model that was trained using the TensorFlow and TF Hub libaries\n", + "* The model trained using the Intel Transfer Learning Tool\n", + "* The model trained and quantized using the Intel Transfer Learning Tool\n", + "* The model trained and optimized using the Intel Transfer Learning Tool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41b6baee", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset_dir = dataset_subdir\n", + "\n", + "if os.path.exists(os.path.join(dataset_subdir, 'validation')):\n", + " test_dataset_dir = os.path.join(dataset_subdir, 'validation')\n", + "elif os.path.exists(os.path.join(dataset_subdir, 'test')):\n", + " test_dataset_dir = os.path.join(dataset_subdir, 'test')\n", + " \n", + "print(\"Test dataset directory:\", test_dataset_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "e89ccbfb", + "metadata": {}, + "source": [ + "Use the Intel Neural Compressor to get the performance of the model trained using the TensorFlow libraries.\n", + "\n", + "Note that you may see a `zmq.error.ZMQError: Address already in use` error in the output, which is a known issuen when running the Intel Neural Compressor from Jupyter notebooks. If this happens, rerun the cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80be966e", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "tf_latency_list = []\n", + "tf_throughput_list = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 90)\n", + " print('Check performance for TF model (batch size = {})'.format(batch_size))\n", + " print('Saved model directory: {}'.format(tf_export_dir_list[i]))\n", + " print('-' * 90)\n", + " \n", + " results = inc_utils.performance(tf_export_dir_list[i], batch_size, image_size, test_dataset_dir, framework)\n", + " tf_latency, tf_throughput = inc_utils.calculate_latency_and_throughput(results)\n", + " \n", + " tf_latency_list.append(tf_latency)\n", + " tf_throughput_list.append(tf_throughput)" + ] + }, + { + "cell_type": "markdown", + "id": "7ee14994", + "metadata": {}, + "source": [ + "Next, get the performance of the the model that was trained and exported by the Intel Transfer Learning Toolkit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb33d150", + "metadata": {}, + "outputs": [], + "source": [ + "tlt_latency_list = []\n", + "tlt_throughput_list = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " print('-' * 90)\n", + " print('Check performance for TLT model (batch size = {})'.format(batch_size))\n", + " print('Saved model directory: {}'.format(tlt_export_dir_list[i]))\n", + " print('-' * 90)\n", + " \n", + " tlt_results = inc_utils.performance(tlt_export_dir_list[i], batch_size, image_size, test_dataset_dir, framework)\n", + " tlt_latency, tlt_throughput = inc_utils.calculate_latency_and_throughput(tlt_results)\n", + " \n", + " tlt_latency_list.append(tlt_latency)\n", + " tlt_throughput_list.append(tlt_throughput)" + ] + }, + { + "cell_type": "markdown", + "id": "4aaf38ec", + "metadata": {}, + "source": [ + "Get the performance of the model that was quantized using the Intel Transfer Learning tool with the Intel Neural Compressor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7322f6a1", + "metadata": {}, + "outputs": [], + "source": [ + "quantized_latency_list = []\n", + "quantized_throughput_list = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " try:\n", + " tlt_quantized_latency = 0\n", + " tlt_quantized_throughput = 0\n", + " \n", + " print('-' * 90)\n", + " print('Check performance for TLT quantized model (batch size = {})'.format(batch_size))\n", + " print('Saved model directory: {}'.format(tlt_quantization_dir_list[i]))\n", + " print('-' * 90)\n", + " \n", + " if not os.path.exists(os.path.join(tlt_quantization_dir_list[i], 'saved_model.pb')):\n", + " raise FileNotFoundError(\"The quantized model was not found at: {}\\nQuantization may have failed for this model/batch size.\".format(tlt_quantization_dir_list[i],))\n", + " \n", + " tlt_quantized_results = inc_utils.performance(tlt_quantization_dir_list[i], batch_size, image_size, test_dataset_dir, framework)\n", + " tlt_quantized_latency, tlt_quantized_throughput = inc_utils.calculate_latency_and_throughput(tlt_quantized_results)\n", + " except Exception as e:\n", + " print(\"Error when trying to check the performance for the quantized model with batch size {}\".format(batch_size))\n", + " print(e)\n", + " finally:\n", + " quantized_latency_list.append(tlt_quantized_latency)\n", + " quantized_throughput_list.append(tlt_quantized_throughput)" + ] + }, + { + "cell_type": "markdown", + "id": "e262895d", + "metadata": {}, + "source": [ + "Finally, get the performance of the model that was optimized using the Intel Transfer Learning tool with the Intel Neural Compressor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c42c815", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "optimized_latency_list = []\n", + "optimized_throughput_list = []\n", + "\n", + "for i, batch_size in enumerate(batch_size_list):\n", + " try:\n", + " tlt_optimized_latency = 0\n", + " tlt_optimized_throughput = 0\n", + " \n", + " print('-' * 90)\n", + " print('Check performance for TLT optimized model (batch size = {})'.format(batch_size))\n", + " print('Saved model directory: {}'.format(tlt_optimized_dir_list[i]))\n", + " print('-' * 90)\n", + " \n", + " if not os.path.exists(os.path.join(tlt_optimized_dir_list[i], 'saved_model.pb')):\n", + " raise FileNotFoundError(\"The optimized model was not found at: {}\\nOptimization may have failed for this model/batch size.\".format(tlt_optimized_dir_list[i],))\n", + "\n", + " tlt_optimized_results = inc_utils.performance(tlt_optimized_dir_list[i], batch_size, image_size, test_dataset_dir, framework)\n", + " \n", + " tlt_optimized_latency, tlt_optimized_throughput = inc_utils.calculate_latency_and_throughput(tlt_optimized_results)\n", + " except Exception as e:\n", + " print(\"Error when trying to check the performance for the optimized model with batch size {}\".format(batch_size))\n", + " print(e)\n", + " finally:\n", + " optimized_latency_list.append(tlt_optimized_latency)\n", + " optimized_throughput_list.append(tlt_optimized_throughput)" + ] + }, + { + "cell_type": "markdown", + "id": "44b337ab", + "metadata": {}, + "source": [ + "Visualize the latency and throughput results for all of the models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b5ca789", + "metadata": {}, + "outputs": [], + "source": [ + "width = 0.18 # the width of the bars\n", + "\n", + "# Round the latency values\n", + "decimals = 2 # number of decimals to use for rounding\n", + "tf_latency_list = [0 if math.isnan(x) else round(x, decimals) for x in tf_latency_list]\n", + "tlt_latency_list = [0 if math.isnan(x) else round(x, decimals) for x in tlt_latency_list]\n", + "quantized_latency_list = [0 if math.isnan(x) else round(x, decimals) for x in quantized_latency_list]\n", + "optimized_latency_list = [0 if math.isnan(x) else round(x, decimals) for x in optimized_latency_list]\n", + "\n", + "# Setup the grouped bar chart for latency\n", + "fig, ax = plt.subplots()\n", + "fig.set_figheight(6)\n", + "fig.set_figwidth(10)\n", + "rects_tf = ax.bar(x, tf_latency_list, width, label='TF latency', color=orange)\n", + "rects_tlt = ax.bar(x + width, tlt_latency_list, width, label='TLT latency', color=blue)\n", + "rects_quant = ax.bar(x + width * 2, quantized_latency_list, width, label='TLT quantized latency', color=yellow)\n", + "rects_opt = ax.bar(x + width * 3, optimized_latency_list, width, label='TLT optimized latency', color=dark_blue)\n", + "ax.bar_label(rects_tf, padding=3)\n", + "ax.bar_label(rects_tlt, padding=3)\n", + "ax.bar_label(rects_quant, padding=3)\n", + "ax.bar_label(rects_opt, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax.set_ylabel('Milliseconds')\n", + "ax.set_title('Latency')\n", + "ax.set_xticks(x + width*1.5, groups)\n", + "ax.set_ymargin(0.2) \n", + "ax.legend(ncols=2)\n", + "plt.show()\n", + "\n", + "# Round the throughput values\n", + "decimals = 0 # number of decimals to use for rounding\n", + "tf_throughput_list = [round(x, decimals) for x in tf_throughput_list]\n", + "tlt_throughput_list = [round(x, decimals) for x in tlt_throughput_list]\n", + "quantized_throughput_list = [round(x, decimals) for x in quantized_throughput_list]\n", + "optimized_throughput_list = [round(x, decimals) for x in optimized_throughput_list]\n", + "\n", + "# Setup the grouped bar chart for throughput\n", + "fig, ax = plt.subplots()\n", + "fig.set_figheight(6)\n", + "fig.set_figwidth(10)\n", + "rects_tf = ax.bar(x, tf_throughput_list, width, label='TF throughput', color=orange)\n", + "rects_tlt = ax.bar(x + width, tlt_throughput_list, width, label='TLT throughput', color=blue)\n", + "rects_quant = ax.bar(x + width * 2, quantized_throughput_list, width, label='TLT quantized throughput', color=yellow)\n", + "rects_opt = ax.bar(x + width * 3, optimized_throughput_list, width, label='TLT optimized throughput', color=dark_blue)\n", + "ax.bar_label(rects_tf, padding=3)\n", + "ax.bar_label(rects_tlt, padding=3)\n", + "ax.bar_label(rects_quant, padding=3)\n", + "ax.bar_label(rects_opt, padding=3)\n", + "\n", + "# Add labels, title, and legend\n", + "ax.set_ylabel('images/second')\n", + "ax.set_title('Throughput')\n", + "ax.set_xticks(x + width*1.5, groups)\n", + "ax.set_ymargin(0.2) \n", + "ax.legend(ncols=2)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "aa9644c8", + "metadata": {}, + "source": [ + "The experiments done in this notebook allowed us to compare the training time and inference/evaluation metrics when using the TensorFlow libraries and the Intel Transfer Learning tool. We can also see how batch size affects performance. More experiments can be done by rerunning this notebook with a different model, different dataset, and/or different training parameters.\n", + "\n", + "Other related notebooks:\n", + "* [Transfer Learning for Image Classification using TensorFlow and the Intel® Transfer Learning Tool API](../image_classification/tlt_api_tf_image_classification/TLT_TF_Image_Classification_Transfer_Learning.ipynb)\n", + "* [Transfer Learning for Image Classification using PyTorch and the Intel® Transfer Learning Tool API](../image_classification/tlt_api_pyt_image_classification/TLT_PyTorch_Image_Classification_Transfer_Learning.ipynb)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/performance/utils/inc_utils.py b/notebooks/performance/utils/inc_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e8a78908a597f0060481dbedde73b227dcda9201 --- /dev/null +++ b/notebooks/performance/utils/inc_utils.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np + + +def performance(saved_model_dir, batch_size, image_size, dataset_dir, framework, warmup=10, iteration=100, + cores_per_instance=None, num_of_instance=None, inter_num_of_threads=None, intra_num_of_threads=None): + """ + Uses the Intel Neural Compressor to get performance metrics for the specified model. + + :param saved_model_dir: Model to load + :param batch_size: Batch size + :param image_size: Image input size + :param dataset_dir: Dataset directory (for a custom image classification dataset) + :param framework: Framework (i.e. tensorflow) + :param warmup: Number of warmup iterations before running performance tests + :param iteration: The number of iterations to run for the performance test + :param cores_per_instance: Number of CPU cores to use per instance + :param num_of_instance: Number of instances to use for performance testing + :param inter_num_of_threads: Number of threads to use for inter-thread operations + :param intra_num_of_threads: Number of threads to use for intra-thread operations + :return: accuracy, batch_size, result_list + """ + + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + from neural_compressor.utils.create_obj_from_config import create_dataloader + + dataloader_args = { + 'batch_size': batch_size, + 'dataset': {'ImageFolder': {'root': dataset_dir}}, + 'transform': {'PaddedCenterCrop': {'size': image_size, 'crop_padding': 32}, + 'Resize': {'size': image_size, 'interpolation': 'bicubic'}, + 'Rescale': {} + }, + 'filter': None + } + + eval_dataloader = create_dataloader(framework, dataloader_args) + + conf = BenchmarkConfig(warmup=warmup, iteration=iteration) + try: + return fit(model=saved_model_dir, config=conf, b_dataloader=eval_dataloader) + except Exception: + # Retry a second time due to the known ZQMError when running from Jupyter + print("Retrying benchmarking a second time") + return fit(model=saved_model_dir, config=conf, b_dataloader=eval_dataloader) + + +def calculate_latency_and_throughput(results): + """ + Parses the results from the benchmarking function and returns the latency (ms) and throughput (samples/sec) + + :param results: Return value from calling the performance util function + :param batch_size: batch size + :return: latency (ms) and throughput (images/sec) + """ + _, batch_size, result_list = results['performance'] + latency = np.array(result_list).mean() / batch_size + latency_ms = latency * 1000 + throughput = 1. / latency + return latency_ms, throughput diff --git a/notebooks/plot_utils.py b/notebooks/plot_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3d043bde461fe0cf91a6b007af311b401048d211 --- /dev/null +++ b/notebooks/plot_utils.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import matplotlib.pyplot as plt + + +def plot_curves(history, checkpoint_dir): + """ + Reads a pickle file and plots accuracy and loss curves + + :param history: Pickle file + :return: None + """ + if not history: + raise FileNotFoundError("The pickle file {} does not exist".format(history)) + + acc = history['acc'] + val_acc = history['val_acc'] + loss = history['loss'] + val_loss = history['val_loss'] + plt.figure(figsize=(7, 7)) + plt.subplot(2, 1, 1) + plt.plot(acc, label='Training Accuracy') + plt.plot(val_acc, label='Validation Accuracy') + plt.legend(loc='lower right') + plt.ylabel('Accuracy') + plt.title('Training and Validation Accuracy') + + plt.subplot(2, 1, 2) + plt.plot(loss, label='Training Loss') + plt.plot(val_loss, label='Validation Loss') + plt.legend(loc='upper right') + plt.ylabel('Cross Entropy') + plt.title('Training and Validation Loss') + plt.xlabel('epoch') + if not os.path.exists(os.path.join(checkpoint_dir, 'train_val_plot.png')): + print("Saving plot in checkpoint_dir:", checkpoint_dir) + plt.savefig(os.path.join(checkpoint_dir, 'train_val_plot.png')) diff --git a/notebooks/question_answering/tfhub_question_answering/BERT_Question_Answering.ipynb b/notebooks/question_answering/tfhub_question_answering/BERT_Question_Answering.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..a5b4d777d99122486d91db2170f243e463a44533 --- /dev/null +++ b/notebooks/question_answering/tfhub_question_answering/BERT_Question_Answering.ipynb @@ -0,0 +1,417 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BERT fine tuning for Question-Answering\n", + "\n", + "This notebook demonstrates fine tuning BERT models from TF Hub using the [SQuAD dataset](https://rajpurkar.github.io/SQuAD-explorer/). Scripts from the [TensorFlow Model Garden](https://github.com/tensorflow/models) are used for preprocessing the training dataset and fine tuning.\n", + "\n", + "The notebook performs the following steps:\n", + "1. [Import dependencies and setup parameters](#1.-Import-dependencies-and-setup-parameters)\n", + "2. [Prepare the dataset](#2.-Prepare-the-dataset)\n", + "3. [Fine tuning and evaluation](#3.-Fine-tuning-and-evaluation)\n", + "4. [Export the saved model](#4.-Export-the-saved-model)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [README.md](/notebooks/README.md) to setup a TensorFlow environment with all the dependencies required to run the notebook.\n", + "\n", + "It will run one of the supported [BERT models from TF Hub](https://tfhub.dev/google/collections/bert/1). The table below has a list of the available models and links to their URLs in TF Hub." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "\n", + "from bert_utils import get_model_map\n", + "from tlt.utils.file_utils import download_file\n", + "\n", + "tfhub_model_map, models_df = get_model_map(\"tfhub_bert_model_map_qa.json\", return_data_frame=True)\n", + "models_df.style.hide(axis=\"index\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the name of the BERT model to use. This string must match one of the models listed in the table above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"bert_en_wwm_uncased_L-24_H-1024_A-16\"\n", + "if model_name not in tfhub_model_map.keys():\n", + " raise ValueError(\"The specified model name ({}) is not supported\".format(model_name))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a directory to download the dataset\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Define an output directory for the saved model to be exported\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + " \n", + "# Directory for downloading the BERT config and vocab file\n", + "bert_dir = os.path.join(output_directory, model_name)\n", + "\n", + "# Output directory for logs and checkpoints generated during training\n", + "if not os.path.isdir(output_directory):\n", + " os.makedirs(output_directory)\n", + "\n", + "# Directory to download the bert checkpoint zip so to get the vocab.txt and bert_config.json\n", + "if not os.path.isdir(bert_dir):\n", + " os.makedirs(bert_dir)\n", + " \n", + "# Get the BERT TF Hub URL from the model map\n", + "tfhub_bert_encoder = tfhub_model_map[model_name][\"bert_encoder\"]\n", + "checkpoint_url = tfhub_model_map[model_name][\"checkpoint_zip\"]\n", + "\n", + "print(\"Using TF Hub model:\", model_name)\n", + "print(\"BERT encoder URL:\", tfhub_bert_encoder)\n", + "print(\"Dataset directory:\", dataset_directory)\n", + "print(\"Output directory:\", output_directory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Path where the https://github.com/tensorflow/models repo will be cloned\n", + "tf_models_dir = os.path.join(output_directory, \"tensorflow-models\")\n", + "os.environ[\"TF_MODELS_DIR\"] = tf_models_dir\n", + "tf_models_branch = \"v2.12.0\"\n", + "\n", + "# Clone the TensorFlow models repo\n", + "if not os.path.exists(tf_models_dir):\n", + " !git clone --depth=1 --branch=$tf_models_branch https://github.com/tensorflow/models.git $tf_models_dir\n", + "\n", + "# Add the TensorFlow models repo to the PYTHONPATH\n", + "os.environ[\"PYTHONPATH\"] = \"{}:{}\".format(os.getenv(\"PYTHONPATH\", \"\"), tf_models_dir)\n", + "\n", + "from bert_qa_utils import create_mini_dataset_file, \\\n", + " display_predictions, \\\n", + " get_config_and_vocab_from_zip, \\\n", + " predict_squad_customized\n", + "\n", + "# Extract the vocab.txt and bert_config.json from the checkpoint zip file\n", + "vocab_txt, bert_config = get_config_and_vocab_from_zip(checkpoint_url, bert_dir)\n", + "\n", + "if not os.path.exists(vocab_txt):\n", + " ValueError(\"The vocab file could not be found at:\", vocab_txt)\n", + "\n", + "if not os.path.exists(bert_config):\n", + " ValueError(\"The bert config could not be found at:\", bert_config)\n", + "\n", + "print(\"Vocab file:\", vocab_txt)\n", + "print(\"BERT config:\", bert_config)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Prepare the dataset\n", + "\n", + "Download the SQuAD dataset, create smaller json files with a subset of the dev and train datasets, and then create TF records for the mini training dataset. The SQuAD dataset has json files for a train and dev datasets. The json files are formatted like:\n", + "\n", + "```\n", + "{\n", + " \"data\": [\n", + " {\n", + " \"title\": \"...\",\n", + " \"paragraphs\": [\n", + " {\n", + " \"qas\": [\n", + " {\n", + " \"question\": \"...\",\n", + " \"id\": \"\",\n", + " \"answers\": [\n", + " {\n", + " \"text\": \"...\",\n", + " \"answer_start\": \n", + " },\n", + " {\n", + " \"text\": \"...\",\n", + " \"answer_start\": \n", + " },\n", + " {\n", + " \"text\": \"...\",\n", + " \"answer_start\": \n", + " }\n", + " ],\n", + " \"is_impossible\": \n", + " },\n", + " ...\n", + " ],\n", + " \"context\": \".....\"\n", + " },\n", + " ...\n", + " ]\n", + " }\n", + " ],\n", + " \"version\": \"v2.0\"\n", + "}\n", + "```\n", + "\n", + "Each item in the data list has a title, a list of paragraphs that with questions/answers and a context string. The answer to each question is a segment of text from the context paragraph (unless the question is impossible).\n", + "\n", + "For this example, we will be using a subset of the dev and train dataset in order to speed up the execution time. The size of the datasets can be increased (or the full dataset can be used) to try to improve accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Specify to use SQuAD v1.1 or v2.0\n", + "squad_version = \"v1.1\"\n", + "\n", + "# Maximum sequence length\n", + "max_seq_length = 384\n", + "\n", + "# Specify the number of dataset items to grab from the dev and train datasets.\n", + "# More dataset items can increase accuracy, but will also increase the training/evaluation time.\n", + "num_dev_dataset_items = 2\n", + "num_train_dataset_items = 12\n", + "\n", + "# Flag to overwrite previously generated mini dataset .json files and the TF records file\n", + "overwrite = False\n", + "\n", + "# Dataset download directory\n", + "squad_dir = os.path.join(dataset_directory, \"squad\")\n", + "\n", + "squad_dev_dataset = os.path.join(squad_dir, \"dev-{}.json\".format(squad_version))\n", + "squad_train_dataset = os.path.join(squad_dir, \"train-{}.json\".format(squad_version))\n", + "version_2_with_negative = squad_version == \"v2.0\"\n", + "\n", + "# Create a directory for the SQuAD files, if the folder does not exist\n", + "if not os.path.isdir(squad_dir):\n", + " os.makedirs(squad_dir)\n", + "\n", + "# Download the SQuAD dev dataset file, if it doesn't exist\n", + "if not os.path.exists(squad_dev_dataset):\n", + " squad_dev_url = \"https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-{}.json\".format(squad_version)\n", + " download_file(squad_dev_url, squad_dir)\n", + "\n", + "# Download the SQuAD train dataset file, if it doesn't exist\n", + "if not os.path.exists(squad_train_dataset):\n", + " squad_train_url = \"https://rajpurkar.github.io/SQuAD-explorer/dataset/train-{}.json\".format(squad_version)\n", + " download_file(squad_train_url, squad_dir)\n", + " \n", + "# Create a smaller version of the dev dataset\n", + "squad_mini_file = \"mini-dev-{}.json\".format(squad_version)\n", + "mini_dataset_path = os.path.join(squad_dir, squad_mini_file)\n", + "create_mini_dataset_file(squad_dev_dataset, mini_dataset_path, num_dev_dataset_items, overwrite=overwrite)\n", + "\n", + "# Create a smaller version of the train dataset\n", + "squad_mini_train_file = \"mini-train-{}.json\".format(squad_version)\n", + "mini_train_dataset_path = os.path.join(squad_dir, squad_mini_train_file)\n", + "create_mini_dataset_file(squad_train_dataset, mini_train_dataset_path, num_train_dataset_items, overwrite=overwrite)\n", + "\n", + "# Create TF Records for the mini training dataset\n", + "train_mini_tfrecords_path = os.path.join(squad_dir, \"squad_mini_{}_train.tf_record\".format(squad_version))\n", + "squad_metadata_path = os.path.join(squad_dir, \"squad_{}_meta_data\".format(squad_version))\n", + "\n", + "# Preprocess the dataset, if we don't already have the files\n", + "if not os.path.exists(train_mini_tfrecords_path) or not os.path.exists(squad_metadata_path) or overwrite:\n", + " !python $tf_models_dir/official/nlp/data/create_finetuning_data.py \\\n", + " --squad_data_file=$mini_train_dataset_path \\\n", + " --vocab_file=$vocab_txt \\\n", + " --version_2_with_negative=$version_2_with_negative \\\n", + " --train_data_output_path=$train_mini_tfrecords_path \\\n", + " --meta_data_file_path=$squad_metadata_path \\\n", + " --fine_tuning_task_type=squad \\\n", + " --max_seq_length=$max_seq_length\n", + " \n", + " if os.path.exists(train_mini_tfrecords_path):\n", + " print(\"Preprocessed dataset: \", train_mini_tfrecords_path)\n", + "else:\n", + " print(\"The preprocessed training dataset was found at:\", train_mini_tfrecords_path)\n", + " print(\"The SQuAD metadata file was found at:\", squad_metadata_path)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Fine tuning and evaluation\n", + "\n", + "Train the model using the `run_squad.py` script from the [TensorFlow Model Garden](https://github.com/tensorflow/models/blob/v2.7.0/official/nlp/bert/run_squad.py) with the mode set to `train_and_eval`. The [TF Hub](https://tfhub.dev) model URL is being passed as the `hub_module_url`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "# Learning rate\n", + "learning_rate = 8e-5\n", + "\n", + "# Number of training epochs\n", + "num_train_epochs=1\n", + "\n", + "# Batch sizes\n", + "train_batch_size = 4\n", + "predict_batch_size = 4\n", + "\n", + "# Directory for checkpoints\n", + "checkpoint_dir = os.path.join(output_directory, \"{}_checkpoints\".format(model_name))\n", + "\n", + "if os.path.exists(checkpoint_dir):\n", + " if len(os.listdir(checkpoint_dir)) > 0:\n", + " print(\"WARNING: The model checkpoint directory is not empty and fine tuning may pick up \" \n", + " \"previously generated checkpoint files.\\n\")\n", + "else:\n", + " os.makedirs(checkpoint_dir)\n", + "\n", + "os.environ[\"TFHUB_CACHE_DIR\"] = os.path.join(output_directory, \"tfhub_modules\")\n", + "!python $tf_models_dir/official/legacy/bert/run_squad.py \\\n", + " --mode=train_and_eval \\\n", + " --input_meta_data_path=$squad_metadata_path \\\n", + " --train_data_path=$train_mini_tfrecords_path \\\n", + " --predict_file=$mini_dataset_path \\\n", + " --vocab_file=$vocab_txt \\\n", + " --bert_config_file=$bert_config \\\n", + " --hub_module_url=$tfhub_bert_encoder \\\n", + " --train_batch_size=$train_batch_size \\\n", + " --predict_batch_size=$predict_batch_size \\\n", + " --learning_rate=$learning_rate \\\n", + " --num_train_epochs=$num_train_epochs \\\n", + " --model_dir=$checkpoint_dir \\\n", + " --distribution_strategy=one_device" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "display_predictions(mini_dataset_path, os.path.join(checkpoint_dir, \"predictions.json\"), n=25)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Export the saved model\n", + "\n", + "Using the TensorFlow Model Garden API, export the saved model using the checkpoint files that were generated during fine tuning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from official.legacy.bert import bert_models\n", + "from official.legacy.bert import configs as bert_configs\n", + "from official.legacy.bert import model_saving_utils\n", + "\n", + "tf.keras.mixed_precision.set_global_policy('float32')\n", + "bert_config_obj = bert_configs.BertConfig.from_json_file(bert_config)\n", + "squad_model, _ = bert_models.squad_model(bert_config_obj,\n", + " max_seq_length,\n", + " hub_module_url=tfhub_bert_encoder)\n", + "\n", + "saved_model_dir = os.path.join(output_directory, \"{}_saved_model\".format(model_name))\n", + "\n", + "if not os.path.exists(saved_model_dir):\n", + " os.makedirs(saved_model_dir)\n", + "\n", + "model_saving_utils.export_bert_model(saved_model_dir, model=squad_model, checkpoint_dir=checkpoint_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Citations\n", + "\n", + "```\n", + "@misc{tensorflowmodelgarden2020,\n", + " author = {Hongkun Yu and Chen Chen and Xianzhi Du and Yeqing Li and\n", + " Abdullah Rashwan and Le Hou and Pengchong Jin and Fan Yang and\n", + " Frederick Liu and Jaeyoun Kim and Jing Li},\n", + " title = {{TensorFlow Model Garden}},\n", + " howpublished = {\\url{https://github.com/tensorflow/models}},\n", + " year = {2020}\n", + "}\n", + "\n", + "@article{2016arXiv160605250R,\n", + " author = { {Rajpurkar}, Pranav and {Zhang}, Jian and {Lopyrev},\n", + " Konstantin and {Liang}, Percy},\n", + " title = \"{SQuAD: 100,000+ Questions for Machine Comprehension of Text}\",\n", + " journal = {arXiv e-prints},\n", + " year = 2016,\n", + " eid = {arXiv:1606.05250},\n", + " pages = {arXiv:1606.05250},\n", + "archivePrefix = {arXiv},\n", + " eprint = {1606.05250},\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/question_answering/tfhub_question_answering/README.md b/notebooks/question_answering/tfhub_question_answering/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41b15db625d8af39b446d4f21c8373707abac15c --- /dev/null +++ b/notebooks/question_answering/tfhub_question_answering/README.md @@ -0,0 +1,43 @@ +# Question Answering fine tuning using TensorFlow + +This notebook demonstrates fine tuning using various [BERT](https://arxiv.org/abs/1810.04805) models +from [TF Hub](https://tfhub.dev) using Intel® Optimization for TensorFlow with the SQuAD dataset. + +The notebook performs the following steps: +1. Import dependencies and setup parameters +1. Prepare the dataset +1. Fine tuning and evaluation +1. Export the saved model + +## Running the notebooks + +To run the notebook, follow the instructions to setup the [TensorFlow notebook environment](/notebooks/setup.md). + +## References + +Dataset citations: +``` +@article{2016arXiv160605250R, + author = { {Rajpurkar}, Pranav and {Zhang}, Jian and {Lopyrev}, + Konstantin and {Liang}, Percy}, + title = "{SQuAD: 100,000+ Questions for Machine Comprehension of Text}", + journal = {arXiv e-prints}, + year = 2016, + eid = {arXiv:1606.05250}, + pages = {arXiv:1606.05250}, +archivePrefix = {arXiv}, + eprint = {1606.05250}, +} +``` + +TensorFlow Model Garden citation: +``` +@misc{tensorflowmodelgarden2020, + author = {Hongkun Yu and Chen Chen and Xianzhi Du and Yeqing Li and + Abdullah Rashwan and Le Hou and Pengchong Jin and Fan Yang and + Frederick Liu and Jaeyoun Kim and Jing Li}, + title = {{TensorFlow Model Garden}}, + howpublished = {\url{https://github.com/tensorflow/models}}, + year = {2020} +} +``` diff --git a/notebooks/question_answering/tfhub_question_answering/bert_qa_utils.py b/notebooks/question_answering/tfhub_question_answering/bert_qa_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ad3b9a664191655bbca4d5ae6eeacac7270fd5cb --- /dev/null +++ b/notebooks/question_answering/tfhub_question_answering/bert_qa_utils.py @@ -0,0 +1,208 @@ +# +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import glob +import json +import os +import pandas as pd +import sys +import tensorflow as tf + +sys.path.append(os.environ["TF_MODELS_DIR"]) + +from official.common import distribute_utils +from official.legacy.bert.run_squad_helper import get_dataset_fn +from tlt.utils.file_utils import download_file +from zipfile import ZipFile + +def create_mini_dataset_file(original_file, output_file, num_dataset_items, overwrite=False): + """ + Creates a mini version of the specified json file. The original_file is expected to be in a format + similar to the SQuAD dataset. The number of dataset items represents the number of child elements + under the "data" tag that will be grabbed for the mini dataset. Dataset items will be randomly + selected from the original dataset. Each child element may contain several sets of articles with + questions/answers. The overwrite flag specifies whether or not to overwrite a mini dataset file + that already exists. If overwrite=False and the mini dataset file already exists, nothing will happen. + """ + if not os.path.exists(output_file) or overwrite: + import random + + with open(original_file) as f: + original_data = json.load(f) + + total_len = len(original_data["data"]) + + if num_dataset_items > total_len: + raise ValueError("The number of dataset items ({}) cannot be more than the total " + "dataset length ({}).".format(num_dataset_items, total_len)) + + item_indicies = random.sample(range(0, total_len), num_dataset_items) + print("Total dataset length:", total_len) + print("Randomly selected dataset indices:", item_indicies) + + articles = [] + + for data_index in item_indicies: + article = {} + article["paragraphs"] = original_data["data"][data_index]["paragraphs"] + article["title"] = original_data["data"][data_index]["title"] + + for p in article["paragraphs"]: + for qas in p["qas"]: + qas["id"] = str(qas["id"]) + + articles.append(article) + + # Add the article to a dictionary for the mini dataset + mini_data = {} + mini_data["data"] = articles + + # Add on a version + mini_data["version"] = original_data["version"] if "version" in original_data.keys() else "1.0" + + with open(output_file, "w") as f: + f.write(json.dumps(mini_data, indent=4)) + + if os.path.exists(output_file): + print("Wrote dataset file with {} articles to: {}".format(num_dataset_items, output_file)) + else: + print("Found existing dataset file:", output_file) + + +def display_predictions(predict_data_path, results_file_path, n=10): + """ Displays n number of predictions along with the actual value """ + + def get_data_list(): + count = 0 + data_list = [] + with open(predict_data_path, "r") as actual_data_file: + actual_data = json.load(actual_data_file)["data"] + with open(results_file_path, "r") as results_file: + results = json.load(results_file) + for actual_item in actual_data: + for actual_paragraph in actual_item["paragraphs"]: + for actual_qas in actual_paragraph["qas"]: + if "is_impossible" in actual_qas.keys() and actual_qas["is_impossible"]: + actual_answer = "is_impossible" + elif len(actual_qas["answers"]) >= 1: + answers_text = [x["text"] for x in actual_qas["answers"]] + actual_answer = "
".join(set(answers_text)) + else: + actual_answer = "Unknown" + question = actual_qas["question"] + prediction = results[actual_qas["id"]] + data_list.append([question, prediction, actual_answer]) + count += 1 + if count > n: + return data_list + + predict_df = pd.DataFrame(get_data_list(), + columns=["Question", + "Predicted Answer", + "Actual Answer(s)"]) + return predict_df.style.hide(axis="index") + + +def get_config_and_vocab_from_zip(zip_url, bert_dir): + """ + We are loading the trained BERT model from TF Hub, however the run_squad.py scripts still + require us to pass in a vocab.txt and bert config file. We can get these from the checkpoint + .zip files. The directory structure of the .zip files for each BERT model is not + consistent, so there's a glob search being done to locate the actual vocab.txt and + bert_config.json file after they're extracted from the zip (sometimes they are in + subdirectories). + :param zip_url: URL where the checkpoint zip can be downloaded + :param bert_dir: BERT directory where the vocab.txt and bert_config.json should be copied + :return: Paths to the vocab.txt and bert_config.json + """ + vocab_txt = os.path.join(bert_dir, "vocab.txt") + bert_config = os.path.join(bert_dir, "bert_config.json") + + if not os.path.exists(vocab_txt) or not os.path.exists(bert_config): + downloaded_file = download_file(zip_url, bert_dir) + with ZipFile(downloaded_file, "r") as checkpoint_zip: + def get_file_from_zip(file_path): + file_basename = os.path.basename(file_path) + for zipinfo in checkpoint_zip.infolist(): + if file_basename in zipinfo.filename: + checkpoint_zip.extract(member=zipinfo.filename, path=bert_dir) + + if not os.path.exists(file_path): + # the file isn't directly in the bert_dir, so search subfolders and move it + search_path = os.path.join(bert_dir, "**", file_basename) + matches = glob.glob(search_path, recursive=True) + if matches: + os.replace(matches[0], file_path) + break + + if not os.path.exists(vocab_txt): + get_file_from_zip(vocab_txt) + + if not os.path.exists(bert_config): + get_file_from_zip(bert_config) + + os.remove(downloaded_file) + + return vocab_txt, bert_config + + +# This function was taken from the TensorFlow Model Garden repo and adapted +# to be a utility function that has a string for the strategy, directly passes +# in the max_seq_length instead of a metadata object, and removes the need for FLAGS +# being defined (instead just passes in the predict_batch_size as an arg). + +# https://github.com/tensorflow/models/blob/v2.7.0/official/nlp/bert/run_squad_helper.py#L176 +def predict_squad_customized(strategy_str, max_seq_length, predict_batch_size, + predict_tfrecord_path, num_steps, squad_model): + + strategy = distribute_utils.get_distribution_strategy(distribution_strategy=strategy_str) + + """Make predictions using a Bert-based squad model.""" + predict_dataset_fn = get_dataset_fn( + predict_tfrecord_path, + max_seq_length, + predict_batch_size, + is_training=False) + predict_iterator = iter( + strategy.distribute_datasets_from_function(predict_dataset_fn)) + + @tf.function + def predict_step(iterator): + """Predicts on distributed devices.""" + + def _replicated_step(inputs): + """Replicated prediction calculation.""" + x, _ = inputs + unique_ids = x.pop('unique_ids') + start_logits, end_logits = squad_model(x, training=False) + return dict( + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits) + + outputs = strategy.run(_replicated_step, args=(next(iterator),)) + return tf.nest.map_structure(strategy.experimental_local_results, outputs) + + all_results = [] + for _ in range(num_steps): + predictions = predict_step(predict_iterator) + for result in get_raw_results(predictions): + all_results.append(result) + if len(all_results) % 100 == 0: + print('Made predictions for %d records.', len(all_results)) + return all_results diff --git a/notebooks/question_answering/tfhub_question_answering/bert_utils.py b/notebooks/question_answering/tfhub_question_answering/bert_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c9ebf090273dd0839ca5dfa26cf46bfcea05b6e1 --- /dev/null +++ b/notebooks/question_answering/tfhub_question_answering/bert_utils.py @@ -0,0 +1,57 @@ +# +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import os +import pandas as pd + +from zipfile import ZipFile + + +def get_model_map(json_path, return_data_frame=False): + """ + Gets the model map from the speified json path and loads it into a python dictionary. If the + data frame option is enabled, it will also return the list of models in a pandas data frame + with column headers so that it can be used to display in a notebook. + """ + with open(json_path) as json_file: + tfhub_model_map = json.load(json_file) + + if return_data_frame: + # Generate list of model names and URL links to TF Hub based on the model map + model_options = [[i, + tfhub_model_map[i]["num_hidden_layers"], + tfhub_model_map[i]["hidden_size"], + tfhub_model_map[i]["num_attention_heads"], + "{0}".format( + tfhub_model_map[i]["bert_encoder"])] + for i in tfhub_model_map.keys()] + + if len(model_options) == 0: + print("Warning: No models were found in the json file:", json_path) + + pd.set_option('display.max_colwidth', None) + models_df = pd.DataFrame(model_options, + columns=["Model", + "Hidden layers", + "Hidden size", + "Attention heads", + "TF Hub BERT encoder URL"]) + return tfhub_model_map, models_df + else: + return tfhub_model_map diff --git a/notebooks/question_answering/tfhub_question_answering/tfhub_bert_model_map_qa.json b/notebooks/question_answering/tfhub_question_answering/tfhub_bert_model_map_qa.json new file mode 100644 index 0000000000000000000000000000000000000000..e8206476426968e555567b018f1656740af76f2a --- /dev/null +++ b/notebooks/question_answering/tfhub_question_answering/tfhub_bert_model_map_qa.json @@ -0,0 +1,26 @@ +{ + "bert_en_wwm_uncased_L-24_H-1024_A-16": { + "preprocess": "https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3", + "bert_encoder": "https://tfhub.dev/tensorflow/bert_en_wwm_uncased_L-24_H-1024_A-16/2", + "hidden_size": 1024, + "num_hidden_layers": 24, + "num_attention_heads": 16, + "checkpoint_zip": "https://storage.googleapis.com/bert_models/2019_05_30/wwm_uncased_L-24_H-1024_A-16.zip" + }, + "bert_en_uncased_L-12_H-768_A-12": { + "preprocess": "https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3", + "bert_encoder": "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2", + "hidden_size": 768, + "num_hidden_layers": 12, + "num_attention_heads": 12, + "checkpoint_zip": "https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip" + }, + "bert_en_uncased_L-24_H-1024_A-16": { + "preprocess": "https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3", + "bert_encoder": "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/2", + "hidden_size": 1024, + "num_hidden_layers": 24, + "num_attention_heads": 16, + "checkpoint_zip": "https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-24_H-1024_A-16.zip" + } +} diff --git a/notebooks/requirements.txt b/notebooks/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5326874445f18eb04bfb2186434cf4db692fdf34 --- /dev/null +++ b/notebooks/requirements.txt @@ -0,0 +1,29 @@ +Pillow~=9.5.0 +PyYAML~=6.0 +charset-normalizer~=3.1.0 +datasets~=2.12.0 +gin-config~=0.5.0 +intel-extension-for-pytorch==1.13.100 +intel-tensorflow==2.12.0 +ipython-genutils~=0.2.0 +ipython~=8.13.2 +ipywidgets~=8.0.6 +jmespath~=1.0.1 +matplotlib-inline~=0.1.6 +matplotlib~=3.7.1 +notebook~=6.5.4 +numpy~=1.23.5 +opencv-python~=4.7.0.72 +pandas~=2.0.1 +psutil~=5.9.5 +pycocotools~=2.0.6 +scikit-learn~=1.2.2 +scipy~=1.10.1 +sentencepiece~=0.1.99 +tensorflow-addons~=0.20.0 +tensorflow-datasets~=4.9.2 +tensorflow-hub~=0.13.0 +torch==1.13.1 +torchvision==0.14.1 +transformers~=4.30.0 +urllib3~=2.0.2 diff --git a/notebooks/setup.md b/notebooks/setup.md new file mode 100644 index 0000000000000000000000000000000000000000..5d6542bd1fe926bc7d23c5f6d0125c8c697576b1 --- /dev/null +++ b/notebooks/setup.md @@ -0,0 +1,45 @@ +# Environment Setup and Running the Notebooks + +Use the instructions below to install the dependencies required to run the notebooks. + +Software Requirements: +1. Linux* system (validated on Ubuntu* 20.04/22.04 LTS) +2. Python3 (3.8, 3.9, or 3.10), Pip/Conda and Virtualenv +3. git + +## Set Up Notebook Environment + +1. Install Intel® Transfer Learning Tool using the Developer Installation option in the [Get Started](/GetStarted.md) Guide. + This is required for the Intel Transfer Learning Tool tutorial notebooks, E2E notebooks, and performance comparison. Follow the + instructions in the [Get Started Guide](/GetStarted.md). You can + skip this step if you are only running the native framework notebooks. + +2. Activate the virtualenv or conda environment used to install Intel Transfer Learning Tool, + then from inside the activated environment, run these steps: + ``` + pip install --upgrade pip + pip install -r notebooks/requirements.txt + ``` + +3. Set environment variables for the path to the dataset folder and an output directory. + The dataset and output directories can be empty. The notebook will download the dataset to + the dataset directory, if it is empty. Subsequent runs will reuse the dataset. + If the `DATASET_DIR` and `OUTPUT_DIR` variables are not defined, the notebooks will + default to use `~/dataset` and `~/output`. + ``` + export DATASET_DIR= + export OUTPUT_DIR= + + mkdir -p $DATASET_DIR + mkdir -p $OUTPUT_DIR + ``` +4. Navigate to the notebook directory in your clone of the Transfer Learning repo, and then start the + [notebook server](https://jupyter.readthedocs.io/en/latest/running.html#starting-the-notebook-server): + ``` + cd notebooks + jupyter notebook --port 8888 + ``` +5. Copy and paste the URL from the terminal to your browser to view and run the notebooks. + +Once you have the environment and dependencies set up, see the list of available +[notebooks](/notebooks/README.md). diff --git a/notebooks/text_classification/pytorch_text_classification/PyTorch_Text_Classifier_fine_tuning.ipynb b/notebooks/text_classification/pytorch_text_classification/PyTorch_Text_Classifier_fine_tuning.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..1fd3175a3605e870cdd390f5562eda433921165f --- /dev/null +++ b/notebooks/text_classification/pytorch_text_classification/PyTorch_Text_Classifier_fine_tuning.ipynb @@ -0,0 +1,905 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "b65f0c82", + "metadata": {}, + "source": [ + "# Text Classifier fine tuning using IMDb with PyTorch\n", + "\n", + "This notebook demonstrates fine tuning pretrained models from [Hugging Face](https://huggingface.co) using text classification datasets from the [Hugging Face Datasets catalog](https://huggingface.co/datasets) or a custom dataset. The notebook uses [Intel® Extension for PyTorch*](https://github.com/intel/intel-extension-for-pytorch), which extends PyTorch with optimizations for an extra performance boost on Intel hardware.\n", + "\n", + "Please install the dependencies from the [setup.md](../../setup.md) file before executing this notebook.\n", + "\n", + "The notebook performs the following steps:\n", + "1. [Import dependencies and setup parameters](#1.-Import-dependencies-and-setup-parameters)\n", + "2. [Prepare the dataset](#2.-Prepare-the-dataset)\n", + "3. [Prepare the model for fine tuning and evaluation](#3.-Prepare-the-model-for-fine-tuning-and-evaluation)\n", + "4. [Export the model](#4.-Export-the-model)\n", + "5. [Reload the model and make predictions](#5.-Reload-the-model-and-make-predictions)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "454a6685", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [setup.md](../../setup.md) to setup a PyTorch environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b2b3bf9", + "metadata": {}, + "outputs": [], + "source": [ + "import intel_extension_for_pytorch as ipex\n", + "import logging\n", + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import sys\n", + "import torch\n", + "import warnings\n", + "import typing\n", + "import pickle\n", + "\n", + "from tqdm.auto import tqdm\n", + "from torch.optim import AdamW\n", + "from torch.utils.data import DataLoader\n", + "from datasets import ClassLabel, load_dataset, load_metric, Split\n", + "from datasets import logging as datasets_logging\n", + "from transformers.utils import logging as transformers_logging\n", + "from transformers import (\n", + " AutoModelForSequenceClassification,\n", + " AutoTokenizer,\n", + " Trainer,\n", + " TrainingArguments,\n", + " get_scheduler\n", + ")\n", + "from tlt.utils.file_utils import download_and_extract_zip_file\n", + "\n", + "# Set the logging stream to stdout\n", + "for handler in transformers_logging._get_library_root_logger().handlers:\n", + " handler.setStream(sys.stdout)\n", + "\n", + "sh = datasets_logging.logging.StreamHandler(sys.stdout)\n", + "\n", + "datasets_logging.set_verbosity_error()\n", + "warnings.filterwarnings('ignore')\n", + "os.environ[\"TRANSFORMERS_NO_ADVISORY_WARNINGS\"] = \"1\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0fdf13ec", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the name of the Hugging Face pretrained model to use (https://huggingface.co/models)\n", + "# For example: \n", + "# albert-base-v2\n", + "# bert-base-uncased\n", + "# distilbert-base-uncased\n", + "# distilbert-base-uncased-finetuned-sst-2-english\n", + "# roberta-base\n", + "model_name = \"distilbert-base-uncased\"\n", + "\n", + "# Define an output directory\n", + "output_dir = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\", model_name)\n", + "\n", + "# Define a dataset directory\n", + "dataset_dir = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "print(\"Model name:\", model_name)\n", + "print(\"Output directory:\", output_dir)\n", + "print(\"Dataset directory:\", dataset_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "2f258d4f", + "metadata": {}, + "source": [ + "## 2. Prepare the dataset\n", + "\n", + "The notebook has two options for getting a dataset:\n", + "* Option A: Use a dataset from the [Hugging Face Datasets catalog](https://huggingface.co/datasets)\n", + "* Option B: Use a custom dataset (downloaded from another source or from your local system)\n", + "\n", + "In both cases, the code ends up defining [`datasets.Dataset`](https://huggingface.co/docs/datasets/package_reference/main_classes#datasets.Dataset) objects for the train and evaluation splits.\n", + "\n", + "Execute the following cell to load the tokenizer and declare the base class used for the dataset setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "649c5c22", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the tokenizer\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + "\n", + "class TextClassificationData():\n", + " \"\"\"\n", + " Base class used for defining the text classification dataset being used. Defines Hugging Face datasets.Dataset\n", + " objects for train and evaluations splits, along with helper functions for preprocessing the dataset.\n", + " \"\"\"\n", + "\n", + " def __init__(self, dataset_name, tokenizer, sentence1_key, sentence2_key, label_key):\n", + " self.tokenizer = tokenizer\n", + " self.dataset_name = dataset_name\n", + " self.class_labels = None\n", + " \n", + " # Tokenized train and eval ds\n", + " self.train_ds = None\n", + " self.eval_ds = None\n", + " \n", + " # Column keys\n", + " self.sentence1_key = sentence1_key\n", + " self.sentence2_key = sentence2_key\n", + " self.label_key = label_key\n", + " \n", + " def tokenize_function(self, examples):\n", + " # Define the tokenizer args, depending on if the data has 2 sentences or just 1\n", + " args = ((examples[self.sentence1_key],) if self.sentence2_key is None \\\n", + " else (examples[self.sentence1_key], examples[self.sentence2_key]))\n", + " return self.tokenizer(*args, padding=\"max_length\", truncation=True)\n", + " \n", + " def tokenize_dataset(self, dataset):\n", + " # Apply the tokenize function to the dataset\n", + " tokenized_dataset = dataset.map(self.tokenize_function, batched=True)\n", + "\n", + " # Remove the raw text from the tokenized dataset\n", + " raw_text_columns = [self.sentence1_key, self.sentence2_key] if self.sentence2_key else [self.sentence1_key]\n", + " return tokenized_dataset.remove_columns(raw_text_columns)\n", + " \n", + " def define_train_eval_splits(self, dataset, train_split_name, eval_split_name, train_size=None, eval_size=None):\n", + " self.train_ds = dataset[train_split_name].shuffle().select(range(train_size)) if train_size \\\n", + " else tokenized_dataset[train_split_name] \n", + " self.eval_ds = dataset[eval_split_name].shuffle().select(range(eval_size)) if eval_size \\\n", + " else tokenized_dataset[eval_split_name]\n", + " \n", + " def get_label_names(self):\n", + " if self.class_labels:\n", + " return self.class_labels.names\n", + " else:\n", + " raise ValueError(\"Class labels were not defined\")\n", + " \n", + " def display_sample(self, split_name=\"train\", sample_size=7):\n", + " # Display a sample of the raw data\n", + " sentence1_sample = self.dataset[split_name][self.sentence1_key][:sample_size]\n", + " sentence2_sample = self.dataset[split_name][self.sentence2_key][:sample_size] if self.sentence2_key else None\n", + " label_sample = self.dataset[split_name][self.label_key][:sample_size]\n", + " dataset_sample = zip(sentence1_sample, sentence2_sample, label_sample) if self.sentence2_key \\\n", + " else zip(sentence1_sample, label_sample)\n", + "\n", + " columns = [self.sentence1_key, self.sentence2_key, self.label_key] if self.sentence2_key else \\\n", + " [self.sentence1_key, self.label_key]\n", + "\n", + " # Display the sample using a dataframe\n", + " sample = pd.DataFrame(dataset_sample, columns=columns)\n", + " return sample.style.hide()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "fd8512e1", + "metadata": {}, + "source": [ + "Now that the base class is defined, either run [Option A to use the Hugging Face Dataset catalog](#Option-A:-Use-a-Hugging-Face-dataset) or [Option B for a custom dataset](#Option-B:-Use-a-custom-dataset) downloaded from online or from your local system." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "640e5611", + "metadata": {}, + "source": [ + "### Option A: Use a Hugging Face dataset\n", + "\n", + "[Hugging Face Datasets](https://huggingface.co/datasets) has a catalog of datasets that can be specified by name. Information about the dataset is available in the catalog (including information on the size of the dataset and the splits).\n", + "\n", + "The next cell gets the [IMDb movie review dataset](https://huggingface.co/datasets/imdb) using the Hugging Face datasets API. If the notebook is executed multiple times, the dataset will be used from the dataset directory, to speed up the time that it takes to run.\n", + "\n", + "The IMDb dataset in Hugging Face has 3 splits: `train`, `test`, and `unsupervised`. This notebook will be using data from the `train` split for training and data from the `test` split for evaluation. The data has 2 columns: `text` (string with the movie review) and `label` (integer class label). The code in the next cell is setup to run using the IMDb dataset, so note that if a different dataset is being used, you may need to change the split names and/or the column names." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a1d5fc0", + "metadata": {}, + "outputs": [], + "source": [ + "class HFDSTextClassificationData(TextClassificationData):\n", + " \"\"\"\n", + " Class used for loading and preprocessing text classification datasets from the Hugging Face datasets catalog\n", + " \"\"\"\n", + " \n", + " def __init__(self, tokenizer, dataset_dir, dataset_name, train_size, eval_size, train_split_name,\n", + " eval_split_name, sentence1_key, sentence2_key, label_key):\n", + " \"\"\"\n", + " Initialize the HFDSTextClassificationData class for a text classification dataset from Hugging Face.\n", + " \n", + " :param tokenizer: Tokenizer to preprocess the dataset\n", + " :param dataset_dir: Cache directory used when loading the dataset\n", + " :param dataset_name: Name of the dataset to load from the Hugging Face catalog\n", + " :param train_size: Size of the training dataset. For quicker training or debug, use a subset of the data.\n", + " Set to `None` to use all the data.\n", + " :param eval_size: Size of the evaluation dataset.\n", + " :param train_split_name: String specifying which split to load for training (e.g. \"train[:80%]\"). See the\n", + " https://www.tensorflow.org/datasets/splits documentation for more information on\n", + " defining splits.\n", + " :param eval_split_name: String specifying the split to load for evaluation.\n", + " :param sentence1_key: Name of the sentence1 column\n", + " :param sentence2_key: Name of the sentence2 column or `None` if there's only one text column\n", + " :param label_key: Name of the label column\n", + " \"\"\"\n", + "\n", + " # Init base class\n", + " TextClassificationData.__init__(self, dataset_name, tokenizer, sentence1_key, sentence2_key, label_key) \n", + " \n", + " # Load the dataset from the Hugging Face dataset API\n", + " self.dataset = load_dataset(dataset_name, cache_dir=dataset_dir)\n", + "\n", + " # Tokenize the dataset\n", + " tokenized_dataset = self.tokenize_dataset(self.dataset)\n", + "\n", + " # Get the training and eval dataset based on the specified dataset sizes\n", + " self.define_train_eval_splits(tokenized_dataset, train_split_name, eval_split_name, train_size, eval_size)\n", + "\n", + " # Save the class label information to use later when predicting\n", + " self.class_labels = self.dataset[train_split_name].features[label_key]\n", + "\n", + "# Name of the Hugging Face dataset\n", + "dataset_name = \"imdb\"\n", + "\n", + "# For quicker training and debug runs, use a subset of the dataset by specifying the size of the train/eval datasets.\n", + "# Set the sizes `None` to use the full dataset. The full IMDb dataset has 25,000 training and 25,000 test examples.\n", + "train_dataset_size = 1000\n", + "eval_dataset_size = 1000\n", + "\n", + "# Name of the columns in the dataset (the column names may vary if you are not using the IMDb dataset)\n", + "sentence1_key = \"text\"\n", + "sentence2_key = None\n", + "label_key = \"label\"\n", + "\n", + "dataset = HFDSTextClassificationData(tokenizer, dataset_dir, dataset_name, train_dataset_size, eval_dataset_size,\n", + " Split.TRAIN, Split.TEST, sentence1_key, sentence2_key, label_key)\n", + "\n", + "# Print a sample of the data\n", + "dataset.display_sample(Split.TRAIN, sample_size=5)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "362625ec", + "metadata": {}, + "source": [ + "Skip to Step 3 [Get the model and setup the Trainer](#3.-Get-the-model-and-setup-the-Trainer) to continue using the dataset from the Hugging Face catalog." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "28c0ba36", + "metadata": {}, + "source": [ + "### Option B: Use a custom dataset\n", + "\n", + "Instead of using a dataset from the Hugging Face dataset catalog, a custom dataset from your local system or a download can be used.\n", + "\n", + "In this example, we download the [SMS Spam Collection dataset](https://archive.ics.uci.edu/ml/datasets/sms+spam+collection). (Note: Please see this dataset's applicable license for terms and conditions. Intel Corporation does not own the rights to this data set and does not confer any rights to it.) The zip file has a single tab-separated value file with two columns. The first column is the label (`ham` or `spam`) and the second column is the text of the SMS message:\n", + "```\n", + "\t\n", + "\t\n", + "\t\n", + "...\n", + "```\n", + "If you are using a custom dataset that has a similarly formatted csv or tsv file, you can use the class defined below. Create your object by passing in custom values for csv file name, delimiter, the label map, mapping function, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c706bfbf", + "metadata": {}, + "outputs": [], + "source": [ + "class CustomCsvTextClassificationData(TextClassificationData):\n", + " \"\"\"\n", + " Class used for loading and preprocessing text classification datasets from CSV files\n", + " \"\"\"\n", + " \n", + " def __init__(self, tokenizer, dataset_name, dataset_dir, data_files, delimiter, label_names, sentence1_key, sentence2_key,\n", + " label_key, train_percent=0.8, eval_percent=0.2, train_size=None, eval_size=None, map_function=None):\n", + " \"\"\"\n", + " Intialize the CustomCsvTextClassificationData class for a text classification\n", + " dataset. The classes uses the Hugging Face datasets API to load the CSV file,\n", + " and split it into a train and eval datasets based on the specified percentages.\n", + " If train_size and eval_size are also defined, the datasets are reduced to the\n", + " specified number of examples.\n", + " \n", + " :param tokenizer: Tokenizer to preprocess the dataset\n", + " :param dataset_name: Dataset name for identification purposes\n", + " :param dataset_dir: Directory where the csv file(s) are located\n", + " :param data_files: List of data file names\n", + " :param delimiter: Delimited for the csv files\n", + " :param label_names: List of label names\n", + " :param sentence1_key: Name of the sentence1 column\n", + " :param sentence2_key: Name of the sentence2 column or `None` if there's only one text column\n", + " :param label_key: Name of the label column\n", + " :param train_percent: Decimal value for the percentage of the dataset that should be used for training\n", + " (e.g. 0.8 for 80%)\n", + " :param eval_percent: Decimal value for the percentage of the dataset that should used for validation\n", + " (e.g. 0.2 for 20%)\n", + " :param train_size: Size of the training dataset. For quicker training or debug, use a subset of the data.\n", + " Set to `None` to use all the data.\n", + " :param eval_size: Size of the eval dataset. Set to `None` to use all the data.\n", + " :param map_function: (Optional) Map function to apply to the dataset. For example, if the csv file has string\n", + " labels instead of numerical values, map function can do the conversion.\n", + " \"\"\"\n", + " # Init base class\n", + " TextClassificationData.__init__(self, dataset_name, tokenizer, sentence1_key, sentence2_key, label_key)\n", + " \n", + " if (train_percent + eval_percent) > 1:\n", + " raise ValueError(\"The combined value of the train percentage and eval percentage \" \\\n", + " \"cannot be greater than 1\")\n", + " \n", + " # Create a list of the column names\n", + " column_names = [label_key, sentence1_key, sentence2_key] if sentence2_key else [label_key, sentence1_key]\n", + " \n", + " # Load the dataset using the Hugging Face API\n", + " self.dataset = load_dataset(dataset_dir, delimiter=delimiter, data_files=data_files, column_names=column_names)\n", + " \n", + " # Optionally map the dataset labels using the map_function\n", + " if map_function:\n", + " self.dataset = self.dataset.map(map_function)\n", + " \n", + " # Setup the class labels\n", + " self.class_labels = ClassLabel(num_classes=len(label_names), names=label_names)\n", + " self.dataset[Split.TRAIN].features[label_key] = self.class_labels\n", + " \n", + " # Split the dataset based on the percentages defined\n", + " self.dataset = self.dataset[Split.TRAIN].train_test_split(train_size=train_percent, test_size=eval_percent)\n", + " \n", + " # Tokenize the dataset\n", + " tokenized_dataset = self.tokenize_dataset(self.dataset)\n", + "\n", + " # Get the training and eval dataset based on the specified dataset sizes\n", + " self.define_train_eval_splits(tokenized_dataset, Split.TRAIN, Split.TEST, train_size, eval_size)\n", + "\n", + "\n", + "# Modify the variables below to use a different dataset or a csv file on your local system.\n", + "# The csv_path variable should be pointing to a csv file with 2 columns (the label and the text)\n", + "dataset_url = \"https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip\"\n", + "dataset_dir = os.path.join(dataset_dir, \"smsspamcollection\")\n", + "csv_name = \"SMSSpamCollection\"\n", + "delimiter = \"\\t\"\n", + "label_names = [\"ham\", \"spam\"]\n", + "\n", + "# Rename the file to include the csv extension so that the dataset API knows how to load the file\n", + "renamed_csv = \"{}.csv\".format(csv_name)\n", + "\n", + "# If we don't already have the csv file, download and extract the zip file to get it.\n", + "if not os.path.exists(os.path.join(dataset_dir, csv_name)) and \\\n", + " not os.path.exists(os.path.join(dataset_dir, renamed_csv)):\n", + " download_and_extract_zip_file(dataset_url, dataset_dir)\n", + "\n", + "if not os.path.exists(os.path.join(dataset_dir, renamed_csv)):\n", + " os.rename(os.path.join(dataset_dir, csv_name), os.path.join(dataset_dir, renamed_csv))\n", + " \n", + "# Columns\n", + "sentence1_key = \"text\"\n", + "sentence2_key = None\n", + "label_key = \"label\"\n", + "\n", + "# Map function to translate labels in the csv file to numerical values when loading the dataset\n", + "def map_spam(example):\n", + " example[\"label\"] = int(example[\"label\"] == \"spam\")\n", + " return example\n", + "\n", + "dataset = CustomCsvTextClassificationData(tokenizer, \"smsspamcollection\", dataset_dir, [renamed_csv], delimiter,\n", + " label_names, sentence1_key, sentence2_key, label_key, train_size=1000,\n", + " eval_size=1000, map_function=map_spam)\n", + "\n", + "# Print a sample of the data\n", + "dataset.display_sample(Split.TRAIN, 10)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e3a24bcd", + "metadata": {}, + "source": [ + "## 3. Prepare the model for fine tuning and evaluation\n", + "\n", + "The notebook has two options to train the model.\n", + "\n", + "- Option A: Use the [`Trainer`](https://huggingface.co/docs/transformers/v4.16.2/en/main_classes/trainer#transformers.Trainer) API from Hugging Face.\n", + "- Option B: Use the native PyTorch API.\n", + "\n", + "In both cases, the model ends up being a transformers model and depending on the class constructor arguments, the appropriate API is selected.\n", + "\n", + "Execute the following cell to declare the base class used for the Text Classification Model setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "797485aa", + "metadata": {}, + "outputs": [], + "source": [ + "class TextClassificationModel():\n", + " \"\"\"\n", + " Class used for model loading, training and evaluation.\n", + " \"\"\"\n", + " def __init__(self, \n", + " model_name: str, \n", + " num_labels: int, \n", + " training_args: TrainingArguments = None, \n", + " ipex_optimize: bool = True, \n", + " device: str = \"cpu\"):\n", + " \"\"\"\n", + " Initialize the TextClassificationModel class for a text classification model with\n", + " PyTorch. The class uses the model_name to load the pre-trained PyTorch model from\n", + " Hugging Face. If the training_args are given then the Trainer API is selected for\n", + " training and evaluation of the model otherwise native PyTorch API is selected for\n", + " model training and evaluation\n", + " \n", + " :param model_name: Name of the pre-trained model to load from Hugging Face\n", + " :param num_labels: Number of class labels\n", + " :param training_args: A TrainingArguments object if using the Trainer API to train\n", + " the model. If None, native PyTorch API is used for training.\n", + " :param ipex_optimize: If True, then the model is optimized to run on intel hardware.\n", + " :param device: Device to run on the PyTorch model.\n", + " \"\"\"\n", + " self.model_name = model_name\n", + " self.num_labels = num_labels\n", + " self.training_args = training_args\n", + " self.device = device\n", + " self.trainer = None\n", + " \n", + " self.train_ds = dataset.train_ds\n", + " self.eval_ds = dataset.eval_ds\n", + " \n", + " # Load the model using the pretrained weights\n", + " self.model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)\n", + " \n", + " # Apply the ipex optimize function to the model\n", + " if ipex_optimize:\n", + " self.model = ipex.optimize(self.model)\n", + " \n", + " def train(self, \n", + " dataset: TextClassificationData,\n", + " optimizers: typing.Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR],\n", + " num_train_epochs: int = 1,\n", + " batch_size: int = 16,\n", + " compute_metrics: typing.Callable = None,\n", + " shuffle_samples: bool = True\n", + " ):\n", + "\n", + " # If training_args are given, we use the `Trainer` API to train the model\n", + " if self.training_args:\n", + " self.model.train()\n", + " self.trainer = Trainer(model=self.model,\n", + " args=self.training_args,\n", + " train_dataset=self.train_ds,\n", + " eval_dataset=self.eval_ds,\n", + " optimizers=optimizers,\n", + " compute_metrics=compute_metrics)\n", + " self.trainer.train()\n", + " \n", + " # If training_args are not given, we use native PyTorch API to train the model\n", + " else:\n", + " \n", + " # Rename the `label` column to `labels` because the model expects the argument to be named `labels`\n", + " self.train_ds = self.train_ds.rename_column(\"label\", \"labels\")\n", + " \n", + " # Set the format of the dataset to return PyTorch tensors instead of lists\n", + " self.train_ds.set_format(\"torch\")\n", + " \n", + " train_dataloader = DataLoader(self.train_ds, shuffle=shuffle_samples, batch_size=batch_size)\n", + " \n", + " # Unpack the `optimizers` parameter to get optimizer and lr_scheduler\n", + " optimizer, lr_scheduler = optimizers[0], optimizers[1]\n", + " \n", + " # Define number of training steps for the training progress bar\n", + " num_training_steps = num_train_epochs * len(train_dataloader)\n", + " progress_bar = tqdm(range(num_training_steps))\n", + " \n", + " # Training loop\n", + " self.model.to(self.device)\n", + " self.model.train()\n", + " for epoch in range(num_train_epochs):\n", + " for batch in train_dataloader:\n", + " batch = {k: v.to(self.device) for k, v in batch.items()}\n", + " outputs = self.model(**batch)\n", + " loss = outputs.loss\n", + " loss.backward()\n", + "\n", + " optimizer.step()\n", + " lr_scheduler.step()\n", + " optimizer.zero_grad()\n", + " progress_bar.update(1)\n", + " \n", + " def evaluate(self, batch_size=16):\n", + " \n", + " if self.trainer:\n", + " metrics = self.trainer.evaluate()\n", + " for key in metrics.keys():\n", + " print(\"{}: {}\".format(key, metrics[key]))\n", + " else:\n", + " # Rename the `label` column to `labels` because the model expects the argument to be named `labels`\n", + " self.eval_ds = self.eval_ds.rename_column(\"label\", \"labels\")\n", + " \n", + " # Set the format of the dataset to return PyTorch tensors instead of lists\n", + " self.eval_ds.set_format(\"torch\")\n", + " \n", + " eval_dataloader = DataLoader(self.eval_ds, batch_size=batch_size)\n", + " progress_bar = tqdm(range(len(eval_dataloader)))\n", + " \n", + " metric = load_metric(\"accuracy\")\n", + " self.model.eval()\n", + " for batch in eval_dataloader:\n", + " batch = {k: v.to(self.device) for k, v in batch.items()}\n", + " with torch.no_grad():\n", + " outputs = self.model(**batch)\n", + "\n", + " logits = outputs.logits\n", + " predictions = torch.argmax(logits, dim=-1)\n", + " metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n", + " progress_bar.update(1)\n", + "\n", + " print(metric.compute())\n", + " \n", + " def predict(self, raw_input_text):\n", + " if isinstance(raw_input_text, str):\n", + " raw_input_text = list(raw_input_text)\n", + " \n", + " # Encode the raw text using the tokenizer\n", + " encoded_input = tokenizer(raw_text_input, padding=True, return_tensors='pt')\n", + " \n", + " # Input the encoded text(s) to the model and get the predicted results\n", + " self.model.eval()\n", + " output = self.model(**encoded_input)\n", + " _, predictions = torch.max(output.logits, dim=1)\n", + " \n", + " # Translate the predictions to class label strings\n", + " prediction_labels = dataset.class_labels.int2str(predictions)\n", + "\n", + " # Create a dataframe to display the results\n", + " result_list = [list(x) for x in zip(raw_text_input, prediction_labels)]\n", + " result_df = pd.DataFrame(result_list, columns=[\"Input Text\", \"Predicted Label\"])\n", + " return result_df.style.hide()\n", + " \n", + " def parameters(self):\n", + " return self.model.parameters()\n", + " \n", + " def save(self, output_dir):\n", + " self.model.save_pretrained(output_dir)\n", + " \n", + " @classmethod\n", + " def load(cls, output_dir):\n", + " return cls(output_dir, num_labels=len(dataset.get_label_names()))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "3e8f1edd", + "metadata": {}, + "source": [ + "Now that the `TextClassificationModel` class is defined, either use Option A to use the [`Trainer`](https://huggingface.co/docs/transformers/v4.16.2/en/main_classes/trainer#transformers.Trainer) API from Hugging Face or Option B to use the native PyTorch API." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "1a606f16", + "metadata": {}, + "source": [ + "### Option A: Use the [`Trainer`](https://huggingface.co/docs/transformers/v4.16.2/en/main_classes/trainer#transformers.Trainer) API from Hugging Face\n", + "\n", + "This step gets the pretrained model from [Hugging Face](https://huggingface.co/models) and sets up the\n", + "[TrainingArguments](https://huggingface.co/docs/transformers/v4.16.2/en/main_classes/trainer#transformers.TrainingArguments) and the\n", + "[Trainer](https://huggingface.co/docs/transformers/v4.16.2/en/main_classes/trainer#transformers.Trainer). For simplicity, this example is using default values for most of the training args, but we are specifying our output directory and the number of training epochs. If your output directory already has checkpoints from a previous run,\n", + "training will resume from the last checkpoint. The `overwrite_output_dir` training argument can be set to\n", + "`True` if you want to instead overwrite previously generated checkpoints.\n", + "\n", + "> Note that it is expected to see a warning at this step about some weights not being used. This is because\n", + "> the pretraining head from the original model is being replaced with a classification head." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d70a4f8", + "metadata": {}, + "outputs": [], + "source": [ + "num_train_epochs = 2\n", + "batch_size = 16\n", + "num_labels = len(dataset.get_label_names())\n", + "\n", + "# Define a TrainingArguments object for the Trainer API to use.\n", + "training_args = TrainingArguments(output_dir=output_dir, num_train_epochs=num_train_epochs)\n", + "\n", + "# Get the model from Hugging Face. Since we are specifying training_args, the model is trained and\n", + "# evaluated with the Trainer API.\n", + "model = TextClassificationModel(model_name=model_name, num_labels=num_labels, training_args=training_args)\n", + "\n", + "# Define model training parameters\n", + "learning_rate = 5e-5\n", + "optimizer = AdamW(model.parameters(), lr=learning_rate)\n", + "num_training_steps = num_train_epochs * len(dataset.train_ds)\n", + "metric = load_metric(\"accuracy\")\n", + "lr_scheduler = get_scheduler(\n", + " name=\"linear\", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps\n", + " )\n", + "\n", + "# Helper function for the Trainer API to compute metrics\n", + "def compute_metrics(eval_pred):\n", + " logits, labels = eval_pred\n", + " predictions = np.argmax(logits, axis=-1)\n", + " return metric.compute(predictions=predictions, references=labels)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "5fcabd97", + "metadata": {}, + "source": [ + "**Train and evaluate the model with the Trainer API**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1256c40", + "metadata": {}, + "outputs": [], + "source": [ + "model.train(\n", + " dataset, \n", + " optimizers=(optimizer, lr_scheduler), \n", + " num_train_epochs=num_train_epochs, \n", + " batch_size=batch_size,\n", + " compute_metrics=compute_metrics\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a7afe59", + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "cce10679", + "metadata": {}, + "source": [ + "### Option B: Use the native PyTorch API\n", + "\n", + "This step gets the pretrained model from [Hugging Face](https://huggingface.co/models) and uses native PyTorch API to train and evaluate the model.\n", + "\n", + "> Note that it is expected to see a warning at this step about some weights not being used. This is because\n", + "> the pretraining head from the original model is being replaced with a classification head." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "202f6b01", + "metadata": {}, + "outputs": [], + "source": [ + "num_train_epochs = 2\n", + "batch_size = 16\n", + "num_labels = len(dataset.get_label_names())\n", + "\n", + "# Get the model from Hugging Face. Since we are not specifying training_args, the model is trained and\n", + "# evaluated with the native PyTorch API.\n", + "model = TextClassificationModel(model_name=model_name, num_labels=num_labels)\n", + "\n", + "# Define model training parameters\n", + "learning_rate = 5e-5\n", + "optimizer = AdamW(model.parameters(), lr=learning_rate)\n", + "num_training_steps = num_train_epochs * len(dataset.train_ds)\n", + "lr_scheduler = get_scheduler(\n", + " name=\"linear\", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps\n", + " )" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "f4821e46", + "metadata": {}, + "source": [ + "**Train and evaluate the model with the native PyTorch API**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eeb2e694", + "metadata": {}, + "outputs": [], + "source": [ + "model.train(\n", + " dataset, \n", + " optimizers=(optimizer, lr_scheduler), \n", + " num_train_epochs=num_train_epochs, \n", + " batch_size=batch_size\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ff3b884", + "metadata": {}, + "outputs": [], + "source": [ + "model.evaluate()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "b83b873f", + "metadata": {}, + "source": [ + "## 4. Export the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faa4beb3", + "metadata": {}, + "outputs": [], + "source": [ + "# Save the model to our output directory\n", + "model.save(output_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "49449342", + "metadata": {}, + "source": [ + "## 5. Reload the model and make predictions\n", + "\n", + "The output directory is used to reload the model. In the next cell, we evalute the reloaded model to verify that we are getting the same metrics that we saw after fine tuning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e339d50d", + "metadata": {}, + "outputs": [], + "source": [ + "reloaded_model = TextClassificationModel.load(output_dir)\n", + " \n", + "reloaded_model.evaluate()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c70a5386", + "metadata": {}, + "source": [ + "Next, we demonstrate how encode raw text input and get predictions from the reloaded model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40d231c0", + "metadata": {}, + "outputs": [], + "source": [ + "# Setup some raw text input\n", + "if dataset.dataset_name == \"imdb\":\n", + " raw_text_input = [\"It was okay. I finished it, but wouldn't watch it again.\",\n", + " \"So bad\",\n", + " \"Definitely not my favorite\",\n", + " \"Highly recommended\"]\n", + "elif dataset.dataset_name == \"smsspamcollection\":\n", + " raw_text_input = [\"Happy Birthday!\",\n", + " \"Thank you for your order, please click the following link for tracking info 12345678\",\n", + " \"Congratulations! You have won a free trip to Australia!!! Reply back with your full name and address.\",\n", + " \"Can you get some milk while you're at the store?\",\n", + " \"On my way\",\n", + " \"OMG LOL :D\",\n", + " \"Urgent! The IRS has been trying to contact you regarding your tax return. Please call 555-555-5555 immediately\"]\n", + "else:\n", + " # Define your own input text when using another dataset\n", + " raw_text_input = []\n", + "\n", + "\n", + "model.predict(raw_text_input)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "bee40324", + "metadata": {}, + "source": [ + "## Citations\n", + "\n", + "```\n", + "@InProceedings{maas-EtAl:2011:ACL-HLT2011,\n", + " author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher},\n", + " title = {Learning Word Vectors for Sentiment Analysis},\n", + " booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},\n", + " month = {June},\n", + " year = {2011},\n", + " address = {Portland, Oregon, USA},\n", + " publisher = {Association for Computational Linguistics},\n", + " pages = {142--150},\n", + " url = {http://www.aclweb.org/anthology/P11-1015}\n", + "}\n", + "\n", + "@misc{misc_sms_spam_collection_228,\n", + " author = {Almeida, Tiago},\n", + " title = {{SMS Spam Collection}},\n", + " year = {2012},\n", + " howpublished = {UCI Machine Learning Repository}\n", + "}\n", + "```\n", + "Please see this dataset's applicable license for terms and conditions. Intel Corporation does not own the rights to this data set and does not confer any rights to it." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/text_classification/pytorch_text_classification/README.md b/notebooks/text_classification/pytorch_text_classification/README.md new file mode 100644 index 0000000000000000000000000000000000000000..da9c58fd836b60a843cc4f7a701b6e446a37a632 --- /dev/null +++ b/notebooks/text_classification/pytorch_text_classification/README.md @@ -0,0 +1,48 @@ +# Text Classifier fine tuning with PyTorch + +This notebook demonstrates fine tuning [pretrained models from Hugging Face](https://huggingface.co/models) +using text classification datasets from the [Hugging Face Datasets catalog](https://huggingface.co/datasets) or +a custom dataset. The [IMDb Larget Movie Review dataset](https://ai.stanford.edu/~amaas/data/sentiment/) is used +from the Hugging Face Datasets catalog, and the [SMS Spam Collection dataset](https://archive.ics.uci.edu/ml/datasets/sms+spam+collection) +is used as an example of a custom dataset being loaded from a csv file. + +The notebook uses +[Intel® Extension for PyTorch\*](https://intel.github.io/intel-extension-for-pytorch) which extends PyTorch +with optimizations for extra performance boost on Intel hardware. + +The notebook performs the following steps: +1. Import dependencies and setup parameters +2. Prepare the dataset +3. Prepare the Model for Fine Tuning and Evaluation +4. Export the model +5. Reload the model and make predictions + +## Running the notebook + +To run the notebook, follow the instructions to setup the [PyTorch notebook environment](/notebooks/setup.md). + +## References + +Dataset Citations +``` +@InProceedings{maas-EtAl:2011:ACL-HLT2011, + author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher}, + title = {Learning Word Vectors for Sentiment Analysis}, + booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies}, + month = {June}, + year = {2011}, + address = {Portland, Oregon, USA}, + publisher = {Association for Computational Linguistics}, + pages = {142--150}, + url = {http://www.aclweb.org/anthology/P11-1015} +} + +@misc{misc_sms_spam_collection_228, + author = {Almeida, Tiago}, + title = {{SMS Spam Collection}}, + year = {2012}, + howpublished = {UCI Machine Learning Repository} +} +``` +Please see this dataset's applicable license for terms and conditions. Intel Corporation does not own the rights to this data set and does not confer any rights to it. + diff --git a/notebooks/text_classification/tfhub_text_classification/BERT_Binary_Text_Classification.ipynb b/notebooks/text_classification/tfhub_text_classification/BERT_Binary_Text_Classification.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6877c1d4ebb34c463ab1821a1e36abb67c5415ea --- /dev/null +++ b/notebooks/text_classification/tfhub_text_classification/BERT_Binary_Text_Classification.ipynb @@ -0,0 +1,667 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Binary text classification using BERT models from TF Hub\n", + "\n", + "This notebook demonstrates fine tuning BERT models from [TF Hub](https://tfhub.dev) with binary text classification datasets.\n", + "\n", + "The notebook performs the following steps:\n", + "1. [Import dependencies and setup parameters](#1.-Import-dependencies-and-setup-parameters)\n", + "2. [Prepare the dataset](#2.-Prepare-the-dataset)\n", + "3. [Build the model](#3.-Build-the-model)\n", + "4. [Fine tuning and evaluation](#4.-Fine-tuning-and-evaluation)\n", + "5. [Export the model](#5.-Export-the-model)\n", + "6. [Reload the model and make predictions](#6.-Reload-the-model-and-make-predictions)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [README.md](/notebooks/README.md) to setup a TensorFlow environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "import tensorflow_hub as hub\n", + "import tensorflow_datasets as tfds\n", + "\n", + "from bert_utils import get_model_map\n", + "from tlt.utils.file_utils import download_and_extract_zip_file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Note that tensorflow_text isn't used directly but the import is required to register ops used by the\n", + "# BERT text preprocessor\n", + "! pip3 install tensorflow-text==2.12 --no-deps\n", + "import tensorflow_text" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook will run one of the supported [BERT models from TF Hub](https://tfhub.dev/google/collections/bert/1). The table below has a list of the available models and links to their URLs in TF Hub." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the TF Hub model map from json and print a list of the supported models\n", + "tfhub_model_map, models_df = get_model_map(\"tfhub_bert_model_map_classifier.json\", return_data_frame=True)\n", + "models_df.style.hide(axis=\"index\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the name of the BERT model to use. This string must match one of the models listed in the table above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"small_bert/bert_en_uncased_L-2_H-128_A-2\"\n", + "if model_name not in tfhub_model_map.keys():\n", + " raise ValueError(\"The specified model name ({}) is not supported\".format(model_name))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a directory to download the dataset\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Define an output directory for the saved model to be exported\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "# Output directory for logs and checkpoints generated during training\n", + "if not os.path.isdir(output_directory):\n", + " os.makedirs(output_directory)\n", + " \n", + "tfhub_preprocess = tfhub_model_map[model_name][\"preprocess\"]\n", + "tfhub_bert_encoder = tfhub_model_map[model_name][\"bert_encoder\"]\n", + "\n", + "print(\"Using TF Hub model:\", model_name)\n", + "print(\"BERT encoder URL:\", tfhub_bert_encoder)\n", + "print(\"Preprocessor URL:\", tfhub_preprocess)\n", + "print(\"Dataset directory:\", dataset_directory)\n", + "print(\"Output directory:\", output_directory)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Prepare the dataset\n", + "\n", + "The notebook has two options for getting a dataset:\n", + "* Option A: Use a dataset from the [TensorFlow Datasets catalog](https://www.tensorflow.org/datasets/catalog/overview)\n", + "* Option B: Use a custom dataset (downloaded from another source or from your local system)\n", + "\n", + "In both cases, the code ends up defining [`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) objects for each split (train, validation, and test) and a map for the translating the numerical to string label.\n", + "\n", + "Execute the following cell to set the batch size and declare the base class used for the dataset setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the dataset batch size\n", + "batch_size = 32\n", + "\n", + "# Base class used for defining the binary text classification dataset being used\n", + "class BinaryTextClassificationData():\n", + " def __init__(self, batch_size, label_map):\n", + " self.batch_size = batch_size\n", + " self.label_map = label_map\n", + " self.reverse_label_map = {}\n", + " self.train_ds = None\n", + " self.val_ds = None\n", + " self.test_ds = None\n", + " self.dataset_name = \"\"\n", + " \n", + " for k, v in self.label_map.items():\n", + " self.reverse_label_map[v] = k\n", + " \n", + " def get_str_label(self, numerical_value):\n", + " if not isinstance(numerical_value, int):\n", + " numerical_value = int(tf.math.round(numerical_value))\n", + " \n", + " if numerical_value in self.label_map.keys():\n", + " return label_map[numerical_value]\n", + " else:\n", + " raise ValueError(\"The key {} was not found in the label map\".format(numerical_value))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that the base class is defined, either run [Option A to use the TensorFlow Dataset catalog](#Option-A:-Use-a-TensorFlow-dataset) or [Option B for a custom dataset](#Option-B:-Use-a-custom-dataset) downloaded from online or from your local system." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option A: Use a TensorFlow dataset\n", + "\n", + "[TensorFlow Datasets](https://www.tensorflow.org/datasets) has a [catalog of datasets](https://www.tensorflow.org/datasets/catalog/overview) that can be specified by name. Information about the dataset is available in the catalog (including information on the size of the dataset and the splits).\n", + "\n", + "The next cell demonstrates using the [`imdb_reviews`](https://www.tensorflow.org/datasets/catalog/imdb_reviews) dataset from the TensorFlow datasets catalog to get splits for training, validation, and test. Skip the next cell if you would like to instead use \"Option B\" for a custom dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class TFDSBinaryTextClassificationData(BinaryTextClassificationData):\n", + " def __init__(self, dataset_dir, tfds_name, train_split, val_split, test_split, label_map, batch_size):\n", + " \"\"\"\n", + " Intialize the TFDSBinaryTextClassificationData class for a dataset binary text classification dataset\n", + " from the TensorFlow dataset catalog.\n", + " \n", + " :param dataset_dir: Path to a dataset directory to read/write data\n", + " :param tfds_name: String name of the TensorFlow dataset to load\n", + " :param train_split: String specifying which split to load for training (e.g. \"train[:80%]\"). See the\n", + " https://www.tensorflow.org/datasets/splits documentation for more information on\n", + " defining splits.\n", + " :param val_split: String specifying the split to load for validation.\n", + " :param test_split: String specifying the split to load for test.\n", + " :param label_map: Dictionary where the key is a numerical value and the value is the string label\n", + " :param batch_size: Batch size\n", + " \"\"\"\n", + " # Init base class\n", + " BinaryTextClassificationData.__init__(self, batch_size, label_map) \n", + " \n", + " [self.train_ds, self.val_ds, self.test_ds], info = tfds.load(tfds_name,\n", + " data_dir=dataset_dir,\n", + " split=[train_split, val_split, test_split],\n", + " batch_size=batch_size,\n", + " as_supervised=True,\n", + " shuffle_files=True,\n", + " with_info=True)\n", + " self.dataset_name = tfds_name\n", + " print(info)\n", + "\n", + "\n", + "# Name of the TFDS to use\n", + "tfds_name=\"imdb_reviews\"\n", + "\n", + "# Location where the dataset will be downloaded\n", + "dataset_directory = os.path.join(dataset_directory, tfds_name)\n", + "if not os.path.isdir(dataset_directory):\n", + " os.makedirs(dataset_directory)\n", + "\n", + "# Label map for sentiment analysis\n", + "label_map = {\n", + " 1: \"Positive\",\n", + " 0: \"Negative\"\n", + "}\n", + " \n", + "# Initialize the dataset splits using a dataset from the TensorFlow datasets catalog\n", + "dataset = TFDSBinaryTextClassificationData(dataset_dir=dataset_directory,\n", + " tfds_name=tfds_name,\n", + " train_split=\"train[:50%]\",\n", + " val_split=\"train[:20%]\",\n", + " test_split=\"test[:20%]\",\n", + " label_map=label_map,\n", + " batch_size=batch_size)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Skip to the next step [3. Build the model](#3.-Build-the-model) to continue using the TF dataset." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option B: Use a custom dataset\n", + "\n", + "Instead of using a dataset from TensorFlow datasets, another dataset from your local system or a download can be used. \n", + "\n", + "In this example, we download the [SMS Spam Collection dataset](https://archive.ics.uci.edu/ml/datasets/sms+spam+collection). (Note: Please see this dataset's applicable license for terms and conditions. Intel Corporation does not own the rights to this data set and does not confer any rights to it.) The zip file has a single tab-separated value file with two columns. The first column is the label (`ham` or `spam`) and the second column is the text of the SMS message:\n", + "```\n", + "\t\n", + "\t\n", + "\t\n", + "...\n", + "```\n", + "If you are using a custom dataset that has a similarly formatted csv or tsv file, you can still use the class defined below. Just create your object passing in custom values for delimiter, header (whether the file has a header row), the label map, mapping function, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CustomCsvBinaryTextClassificationData(BinaryTextClassificationData):\n", + " def __init__(self, csv_file, delimiter, header, train_percent, val_percent,\n", + " test_percent, label_map, batch_size, dataset_name, map_function=None):\n", + " \"\"\"\n", + " Intialize the CustomCsvBinaryTextClassificationData class for a dataset binary text\n", + " classification dataset that uses a single csv file.\n", + " \n", + " :param csv_file: Path to the csv file\n", + " :param delimiter: String character that separates the fields in each row\n", + " :param header: Boolean indicating whether or not the csv file has a header line that should be skipped\n", + " :param train_percent: Decimal value for the percentage of the dataset that should be used for training\n", + " (e.g. 0.8 for 80%)\n", + " :param val_percent: Decimal value for the percentage of the dataset that should be used for validation\n", + " (e.g. 0.1 for 10%)\n", + " :param test_percent: Decimal value for the percentage of the dataset that should be used for test\n", + " (e.g. 0.1 for 10%)\n", + " :param label_map: Dictionary where the key is a numerical value and the value is the string label\n", + " :param batch_size: Batch size\n", + " :param dataset_name: Name of the dataset. This is used later in this notebook for naming the saved model\n", + " export folder and determining which input strings to use when testing the reloaded model\n", + " :param map_function: (Optional) If the csv file has string labels instead of the numerical values, provide a\n", + " map function to apply on the dataset\n", + " \"\"\"\n", + " # Init base class\n", + " BinaryTextClassificationData.__init__(self, batch_size, label_map)\n", + " \n", + " self.dataset_name = dataset_name\n", + " \n", + " if (train_percent + val_percent + test_percent) > 1:\n", + " raise ValueError(\"The combined value of the train percentage, validation percentage, and \" \\\n", + " \"test percentage cannot be greater than 1\")\n", + " \n", + " if not os.path.exists(csv_file):\n", + " raise FileNotFoundError(\"Unable to find the csv file at\", csv_file)\n", + " \n", + " custom_dataset = tf.data.experimental.CsvDataset(filenames=csv_file,\n", + " record_defaults=[tf.string, tf.string],\n", + " field_delim=delimiter,\n", + " use_quote_delim=False,\n", + " header=header)\n", + " \n", + " # Count the number of lines in the csv file to get the dataset length\n", + " custom_dataset_len = sum(1 for line in open(csv_file))\n", + " \n", + " if header:\n", + " custom_dataset_len -= 1\n", + " \n", + " # Optionally map the dataset labels using the map_function\n", + " if map_function:\n", + " custom_dataset = custom_dataset.map(lambda x, y: (y, map_function(x)))\n", + " \n", + " # Create batches based on the specified batch size\n", + " custom_dataset = custom_dataset.batch(batch_size)\n", + " \n", + " # Calculate sizes for the splits\n", + " total_num_batches = int(custom_dataset_len / batch_size)\n", + " train_size = int(train_percent * total_num_batches)\n", + " val_size = int(val_percent * total_num_batches)\n", + " test_size = int(test_percent * total_num_batches)\n", + "\n", + " # Create the train, validation, and test splits\n", + " self.train_ds = custom_dataset.take(train_size) \n", + " self.val_ds = custom_dataset.skip(train_size).take(val_size)\n", + " self.test_ds = custom_dataset.skip(train_size).skip(val_size)\n", + "\n", + " # Set the cardinality so that progress bars will work properly\n", + " self.train_ds = self.train_ds.apply(tf.data.experimental.assert_cardinality(train_size))\n", + " self.val_ds = self.val_ds.apply(tf.data.experimental.assert_cardinality(val_size))\n", + " self.test_ds = self.test_ds.apply(tf.data.experimental.assert_cardinality(test_size))\n", + "\n", + "\n", + "# Modify the variables below to use a different dataset or a csv file on your local system.\n", + "# The csv_path variable should be pointing to a csv file with 2 columns (the label and the text)\n", + "dataset_url = \"https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip\"\n", + "dataset_directory = os.path.join(dataset_directory, \"smsspamcollection\")\n", + "csv_name = \"SMSSpamCollection\"\n", + "delimiter = \"\\t\"\n", + "header = False # Set to true if the csv file has a header row\n", + "csv_path = os.path.join(dataset_directory, csv_name)\n", + "\n", + "# If we don't already have the csv file, download and extract the zip file to get it.\n", + "if not os.path.exists(csv_path):\n", + " download_and_extract_zip_file(dataset_url, dataset_directory)\n", + "\n", + "# Define the label map for your dataset. The label map below is for the SMS Spam Collection dataset.\n", + "# The labels defined in this dictionary should match the labels in the csv file.\n", + "label_map = {\n", + " 1: \"spam\",\n", + " 0: \"ham\"\n", + "}\n", + "\n", + "# Map function to translate labels in the csv file to numerical values when loading the dataset\n", + "def map_spam(x):\n", + " if x == \"spam\":\n", + " return 1\n", + " else:\n", + " return 0\n", + "\n", + "# Initialize the dataset splits using the custom dataset\n", + "dataset = CustomCsvBinaryTextClassificationData(csv_file=csv_path,\n", + " delimiter=delimiter,\n", + " header=header,\n", + " train_percent=0.8,\n", + " val_percent=0.1,\n", + " test_percent=0.1,\n", + " label_map=label_map,\n", + " batch_size=batch_size,\n", + " dataset_name=csv_name,\n", + " map_function=map_spam)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Build the model\n", + "\n", + "Create the BERT model to fine tune using a input layer, the preprocessing layer (from TF Hub), the BERT encoder layer (from TF Hub), one dense layer, and a dropout layer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = tf.keras.layers.Input(shape=(), dtype=tf.string, name='input_layer')\n", + "preprocessing_layer = hub.KerasLayer(tfhub_preprocess, name='preprocessing')\n", + "encoder_inputs = preprocessing_layer(input_layer)\n", + "encoder_layer = hub.KerasLayer(tfhub_bert_encoder, trainable=True, name='encoder')\n", + "outputs = encoder_layer(encoder_inputs)\n", + "net = outputs['pooled_output']\n", + "net = tf.keras.layers.Dropout(0.1)(net)\n", + "net = tf.keras.layers.Dense(1, activation=None, name='classifier')(net)\n", + "classifier_model = tf.keras.Model(input_layer, net)\n", + "\n", + "classifier_model.summary()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Fine tuning and evaluation\n", + "\n", + "Train the model for the specified number of epochs, then evaluate the model using the test dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "# The number of training epochs to run\n", + "num_train_epochs = 2\n", + "\n", + "# Learning rate\n", + "learning_rate = 3e-5\n", + "\n", + "# Maximum total input sequence length after WordPiece tokenization (longer sequences will be truncated)\n", + "max_seq_length = 128\n", + "\n", + "classifier_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate,epsilon=1e-08),\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=tf.metrics.BinaryAccuracy())\n", + "\n", + "history = classifier_model.fit(dataset.train_ds,\n", + " validation_data=dataset.val_ds,\n", + " epochs=num_train_epochs)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evaluate the accuracy using the test dataset. If the accuracy does not meet your expectations, try to increasing the size of the training dataset split or the number of training epochs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "loss, accuracy = classifier_model.evaluate(dataset.test_ds)\n", + "\n", + "print(f'Loss: {loss}')\n", + "print(f'Accuracy: {accuracy}')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Predict using a single batch from the test dataset, and then display the results along with the input text and the actual label." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "num_steps = 1\n", + "predictions = classifier_model.predict(dataset.test_ds, batch_size=batch_size, steps=num_steps)\n", + "\n", + "prediction_list = []\n", + "step_count = 0\n", + "\n", + "for batch in dataset.test_ds:\n", + " label_list = list(batch[1].numpy())\n", + " text_list = list(batch[0].numpy())\n", + " \n", + " for i, (text, actual_label) in enumerate(zip(text_list, label_list)):\n", + " score = tf.math.sigmoid(predictions[i])\n", + " prediction = int(tf.math.round(score))\n", + " prediction = dataset.get_str_label(prediction)\n", + " prediction_list.append([text.decode('utf-8'),\n", + " tf.get_static_value(score)[0],\n", + " prediction,\n", + " dataset.get_str_label(actual_label)])\n", + " \n", + " step_count += 1\n", + " if num_steps <= step_count:\n", + " break\n", + " \n", + "result_df = pd.DataFrame(prediction_list, columns=[\"Input Text\", \"Score\", \"Predicted Label\", \"Actual Label\"])\n", + "result_df.style.hide(axis=\"index\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Export the model\n", + "\n", + "Since training has completed, export the `saved_model.pb` to the output directory in a folder with the model and dataset name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_dir = \"{}_{}\".format(model_name, dataset.dataset_name)\n", + "model_dir = os.path.join(output_directory, model_dir)\n", + "classifier_model.save(model_dir, include_optimizer=False)\n", + "\n", + "saved_model_path = os.path.join(model_dir, \"saved_model.pb\")\n", + "if os.path.exists(saved_model_path):\n", + " print(\"Saved model location:\", saved_model_path)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Reload the model and make predictions\n", + "\n", + "Reload from the `saved_model.pb` in the output directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reloaded_model = tf.saved_model.load(model_dir)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The next section defines a list of strings to send as input to the reloaded model. If you are using a dataset other than the [IMDB movie reviews](https://www.tensorflow.org/datasets/catalog/imdb_reviews) or the [SMS Spam Collection](https://archive-beta.ics.uci.edu/ml/datasets/sms+spam+collection), you can update the snippet below with your own list of input text." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if dataset.dataset_name == \"imdb_reviews\":\n", + " input_text = [\"Awesome movie\",\n", + " \"It was entertaining, but completely predictable.\",\n", + " \"Wasn't what I expected, but I still enjoyed it\",\n", + " \"I wouldn't recommend this movie to my worst enemy\",\n", + " \"I'm not sure how good the movie was, because I fell asleep\"]\n", + "elif dataset.dataset_name == \"SMSSpamCollection\":\n", + " input_text = [\"Happy Birthday!\",\n", + " \"Thank you for your order, please click the following link for tracking info 12345678\",\n", + " \"Congratulations! You have won a free trip to Australia!!! Reply back with your full name and address.\",\n", + " \"Can you get some milk while you're at the store?\",\n", + " \"On my way\",\n", + " \"OMG LOL :D\",\n", + " \"Urgent! The IRS has been trying to contact you regarding your tax return. Please call 555-555-5555 immediately\"]\n", + "else:\n", + " # Define your own list of input text for another dataset\n", + " input_text = []\n", + " \n", + "if not input_text:\n", + " raise ValueError(\"Please define the list of input_text strings.\")\n", + "\n", + "# Send the input text to the reloaded model\n", + "predict_results = tf.sigmoid(reloaded_model(tf.constant(input_text)))\n", + "\n", + "# Get the results into a data frame to display\n", + "result_list = [[input_text[i],\n", + " tf.get_static_value(predict_results[i])[0],\n", + " dataset.get_str_label(tf.get_static_value(predict_results[i])[0])] for i in range(len(input_text))]\n", + "result_df = pd.DataFrame(result_list, columns=[\"Input Text\", \"Score\", \"Predicted Label\"])\n", + "result_df.style.hide(axis=\"index\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Citations\n", + "\n", + "```\n", + "@InProceedings{maas-EtAl:2011:ACL-HLT2011,\n", + " author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher},\n", + " title = {Learning Word Vectors for Sentiment Analysis},\n", + " booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},\n", + " month = {June},\n", + " year = {2011},\n", + " address = {Portland, Oregon, USA},\n", + " publisher = {Association for Computational Linguistics},\n", + " pages = {142--150},\n", + " url = {http://www.aclweb.org/anthology/P11-1015}\n", + "}\n", + "\n", + "@misc{misc_sms_spam_collection_228,\n", + " author = {Almeida, Tiago},\n", + " title = {{SMS Spam Collection}},\n", + " year = {2012},\n", + " howpublished = {UCI Machine Learning Repository}\n", + "}\n", + "```\n", + "\n", + "Please see this dataset's applicable license for terms and conditions. Intel Corporation does not own the rights to this data set and does not confer any rights to it." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/text_classification/tfhub_text_classification/BERT_Multi_Text_Classification.ipynb b/notebooks/text_classification/tfhub_text_classification/BERT_Multi_Text_Classification.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6a25165bf9078f564e6e0e689d63c1ca95bd3b2c --- /dev/null +++ b/notebooks/text_classification/tfhub_text_classification/BERT_Multi_Text_Classification.ipynb @@ -0,0 +1,660 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multiclass text classification using BERT models from TF Hub\n", + "\n", + "This notebook demonstrates fine tuning BERT models from [TF Hub](https://tfhub.dev) with multiclass text classification datasets.\n", + "\n", + "The notebook performs the following steps:\n", + "1. [Import dependencies and setup parameters](#1.-Import-dependencies-and-setup-parameters)\n", + "2. [Prepare the dataset](#2.-Prepare-the-dataset)\n", + "3. [Build the model](#3.-Build-the-model)\n", + "4. [Fine tuning and evaluation](#4.-Fine-tuning-and-evaluation)\n", + "5. [Export the model](#5.-Export-the-model)\n", + "6. [Reload the model and make predictions](#6.-Reload-the-model-and-make-predictions)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Import dependencies and setup parameters\n", + "\n", + "This notebook assumes that you have already followed the instructions in the [README.md](/notebooks/README.md) to setup a TensorFlow environment with all the dependencies required to run the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "import tensorflow_hub as hub\n", + "import tensorflow_datasets as tfds\n", + "\n", + "from bert_utils import get_model_map\n", + "from tlt.utils.file_utils import download_and_extract_zip_file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Note that tensorflow_text isn't used directly but the import is required to register ops used by the\n", + "# BERT text preprocessor\n", + "! pip3 install tensorflow-text==2.12.0 --no-deps\n", + "import tensorflow_text" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook will run one of the supported [BERT models from TF Hub](https://tfhub.dev/google/collections/bert/1). The table below has a list of the available models and links to their URLs in TF Hub." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the TF Hub model map from json and print a list of the supported models\n", + "tfhub_model_map, models_df = get_model_map(\"tfhub_bert_model_map_classifier.json\", return_data_frame=True)\n", + "models_df.style.hide(axis=\"index\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the name of the BERT model to use. This string must match one of the models listed in the table above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"small_bert/bert_en_uncased_L-2_H-128_A-2\"\n", + "if model_name not in tfhub_model_map.keys():\n", + " raise ValueError(\"The specified model name ({}) is not supported\".format(model_name))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a directory to download the dataset\n", + "dataset_directory = os.environ[\"DATASET_DIR\"] if \"DATASET_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"dataset\")\n", + "\n", + "# Define an output directory for the saved model to be exported\n", + "output_directory = os.environ[\"OUTPUT_DIR\"] if \"OUTPUT_DIR\" in os.environ else \\\n", + " os.path.join(os.environ[\"HOME\"], \"output\")\n", + "\n", + "# Output directory for logs and checkpoints generated during training\n", + "if not os.path.isdir(output_directory):\n", + " os.makedirs(output_directory)\n", + " \n", + "tfhub_preprocess = tfhub_model_map[model_name][\"preprocess\"]\n", + "tfhub_bert_encoder = tfhub_model_map[model_name][\"bert_encoder\"]\n", + "\n", + "print(\"Using TF Hub model:\", model_name)\n", + "print(\"BERT encoder URL:\", tfhub_bert_encoder)\n", + "print(\"Preprocessor URL:\", tfhub_preprocess)\n", + "print(\"Dataset directory:\", dataset_directory)\n", + "print(\"Output directory:\", output_directory)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Prepare the dataset\n", + "\n", + "This notebook gets the dataset from a text file or from the [TensorFlow Datasets catalog](https://www.tensorflow.org/datasets/catalog/overview).\n", + "\n", + "The code ends up defining [`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) objects for each split (train, validation, and test) and a map for the translating the numerical to string label.\n", + "\n", + "Execute the following cell to set the batch size and declare the base class used for the dataset setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the dataset batch size\n", + "batch_size = 32\n", + "\n", + "# Base class used for defining the multi text classification dataset being used\n", + "class MultiTextClassificationData():\n", + " def __init__(self, batch_size, label_map):\n", + " self.batch_size = batch_size\n", + " self.label_map = label_map\n", + " self.reverse_label_map = {}\n", + " self.train_ds = None\n", + " self.val_ds = None\n", + " self.test_ds = None\n", + " self.dataset_name = \"\"\n", + " \n", + " for k, v in self.label_map.items():\n", + " self.reverse_label_map[v] = k\n", + " \n", + " def get_str_label(self, numerical_value):\n", + " if not isinstance(numerical_value, int):\n", + " numerical_value = int(tf.math.round(numerical_value))\n", + " \n", + " if numerical_value in self.label_map.keys():\n", + " return self.label_map[numerical_value]\n", + " else:\n", + " raise ValueError(\"The key {} was not found in the label map\".format(numerical_value))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option A: Use a TensorFlow dataset\n", + "\n", + "[TensorFlow Datasets](https://www.tensorflow.org/datasets) has a [catalog of datasets](https://www.tensorflow.org/datasets/catalog/overview) that can be specified by name. Information about the dataset is available in the catalog (including information on the size of the dataset and the splits).\n", + "\n", + "The next cell demonstrates using the [`ag_news_subset`](https://www.tensorflow.org/datasets/catalog/ag_news_subset) dataset from the TensorFlow datasets catalog to get splits for training, validation, and test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class TFDSMultiTextClassificationData(MultiTextClassificationData):\n", + " def __init__(self, dataset_dir, tfds_name, train_split, val_split, test_split, label_map, batch_size):\n", + " \"\"\"\n", + " Intialize the TFDSMultiTextClassificationData class for a dataset multi text classification dataset\n", + " from the TensorFlow dataset catalog.\n", + " \n", + " :param dataset_dir: Path to a dataset directory to read/write data\n", + " :param tfds_name: String name of the TensorFlow dataset to load\n", + " :param train_split: String specifying which split to load for training (e.g. \"train[:80%]\"). See the\n", + " https://www.tensorflow.org/datasets/splits documentation for more information on\n", + " defining splits.\n", + " :param val_split: String specifying the split to load for validation.\n", + " :param test_split: String specifying the split to load for test.\n", + " :param label_map: Dictionary where the key is a numerical value and the value is the string label\n", + " :param batch_size: Batch size\n", + " \"\"\"\n", + " # Init base class\n", + " MultiTextClassificationData.__init__(self, batch_size, label_map) \n", + " \n", + " [self.train_ds, self.val_ds, self.test_ds], info = tfds.load(tfds_name,\n", + " data_dir=dataset_dir,\n", + " split=[train_split, val_split, test_split],\n", + " batch_size=batch_size,\n", + " as_supervised=True,\n", + " shuffle_files=True,\n", + " with_info=True)\n", + " self.dataset_name = tfds_name\n", + " print(info)\n", + "\n", + "\n", + "# Name of the TFDS to use\n", + "tfds_name=\"ag_news_subset\"\n", + "\n", + "# Location where the dataset will be downloaded\n", + "dataset_directory = os.path.join(dataset_directory, tfds_name)\n", + "if not os.path.isdir(dataset_directory):\n", + " os.makedirs(dataset_directory)\n", + "\n", + "# Label map for sentiment analysis\n", + "label_map = {\n", + " 0: \"World\",\n", + " 1: \"Sports\",\n", + " 2: \"Business\",\n", + " 3: \"Sci/Tech\"\n", + "}\n", + " \n", + "# Initialize the dataset splits using a dataset from the TensorFlow datasets catalog\n", + "dataset = TFDSMultiTextClassificationData(dataset_dir=dataset_directory,\n", + " tfds_name=tfds_name,\n", + " train_split=\"train[:50%]\",\n", + " val_split=\"train[:20%]\",\n", + " test_split=\"test[:20%]\",\n", + " label_map=label_map,\n", + " batch_size=batch_size)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Skip to the next step [3. Build the model](#3.-Build-the-model) to continue using the TF dataset." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Option B: Use your own dataset\n", + "Instead of using a dataset from TensorFlow datasets, another dataset from your local system or a download can be used.\n", + "\n", + "In this example, we download the Conference Title dataset. This is a single tab-separated value file with two columns. The first column is the conference title and the second column is the label (VLDB, ISCAS, SIGGRAPH, INFOCOM, WWW):\n", + "\n", + "```\n", + "\t