Merge pull request #93 from borisdayma/inference

dalle_mini/text.py

@@ -98,23 +98,8 @@ def replace_person_token(t):
 
 
 def fix_html(t):
-    "Adapted from fastai"
-    t = (
-        t.replace("#39;", "'")
-        .replace("&", "&")
-        .replace("amp;", "&")
-        .replace("#146;", "'")
-        .replace("nbsp;", " ")
-        .replace("#36;", "$")
-        .replace("\\n", "\n")
-        .replace("quot;", "'")
-        .replace("<br />", "\n")
-        .replace('\\"', '"')
-        .replace("<unk>", " ")
-        .replace(" @.@ ", ".")
-        .replace(" @-@ ", "-")
-    )
-    return html.unescape(t)
+    # from OpenAI CLIP
+    return html.unescape(html.unescape(t))
 
 
 def replace_punctuation_with_commas(t):

dev/inference/samples.txt

@@ -1,103 +1,112 @@
 white snow covered mountain under blue sky during daytime
-aerial view of the beach at night
 aerial view of the beach during daytime
+aerial view of the beach at night
 a beautiful sunset at a beach with a shell on the shore
 a farmhouse surrounded by beautiful flowers
-a photo of a fantasy version of New York City
-a picture of fantasy kingdoms
-a volcano erupting in the middle of San Francisco
-big wave destroying a city
-Paris in a far future, futuristic Paris
 sunset over green mountains
-the last sunrise on earth
-underwater cathedral
+a photo of san francisco golden gate bridge
 painting of an oniric forest glade surrounded by tall trees
-real painting of an alien from Monet
 a graphite sketch of a gothic cathedral
 a graphite sketch of Elon Musk
 still life in the style of Kandinsky
 still life in the style of Picasso
 a colorful stairway to heaven
 a background consisting of colors blue, green, and red
+Mohammed Ali and Mike Tyson in a match
+Pele and Maradona in a match
+view of Mars from space
+a picture of the Eiffel tower on the moon
+a picture of the Eiffel tower on the moon, Earth is in the background
+watercolor of the Eiffel tower on the moon
+the moon is a skull
+epic sword fight
+underwater cathedral
+a photo of a fantasy version of New York City
+a picture of fantasy kingdoms
+a volcano erupting next to San Francisco golden gate bridge
+big wave destroying a city
+Paris in a far future, futuristic Paris
+real painting of an alien from Monet
 the communist statue of liberty
 robots taking control over humans
-epic sword fight
-an avocado armchair
-an armchair in the shape of an avocado
-logo of an avocado armchair
-an avocado armchair flying into space
-a cute avocado armchair singing karaoke on stage in front of a crowd of strawberry shaped lamps
-an illustration of an avocado in a christmas sweater staring at its reflection in a mirror
-illustration of an avocado armchair
-illustration of an avocado armchair getting married to a pineapple
-a muscular banana sitting upright on a bench smoking watching a banana on television, high definition photography
-Mohammed Ali and Mike Tyson in a hypothetical match
-Pele and Maradona in a hypothetical match
-view of mars from space
 illustration of an astronaut in a space suit playing guitar
 a clown wearing a spacesuit floating in space
-a picture of the eiffel tower on the moon
-watercolor of the Eiffel tower on the moon
-a photo of the French flag on the planet Saturn
-the moon is a skull
 a dog playing with a ball
 a cat sits on top of an alligator
 a rat holding a red lightsaber in a white background
 A unicorn is passing by a rainbow in a field of flowers
-a dog eating worthlessness
 an elephant made of carrots
 an elephant on a unicycle during a circus
 photography of a penguin watching television
+a penguin is walking on the Moon, Earth is in the background
+a penguin standing on a tower of books holds onto a rope from a helicopter
 rat wearing a crown
-a portrait of a nightmare creature watching at you
-a white room full of a black substance
-happy, happiness
-sad, sadness
-the representation of infinity
-a cute pikachu teapot
+looking into the sky, 10 airplanes are seen overhead
+shelves filled with books and alchemy potion bottles
+this is a detailed high-resolution scan of a human brain
+a restaurant menu
+a bottle of coca-cola on a table
+a peanut
+a cross-section view of a walnut
+a living room with two white armchairs and a painting of the collosseum. The painting is mounted above a modern fireplace.
+a long line of alternating green and red blocks
+a long line of green blocks on a beach at subset
+a long line of peaches on a beach at sunset
 a picture of a castle from minecraft
+a cute pikachu teapot
 an illustration of pikachu sitting on a bench
-mario eating an avocado while walking his baby koala
+mario is jumping over a zebra during the sunset
+famous anime hero
 star wars concept art
 a cartoon of a superhero bear
 an illustration of a cute skeleton wearing a blue hoodie
 illustration of a baby shark swimming around corals
 Cartoon of a carrot with big eyes
 logo of a robot wearing glasses and reading a book
-a bottle of coca-cola on a table
 a cactus lifting weights
-a living room with two white armchairs and a painting of the collosseum. The painting is mounted above a modern fireplace.
-a long line of alternating green and red blocks
-a long line of green blocks on a beach at subset
-a long line of peaches on a beach at sunset
-a peanut
+illustration of a cactus lifting weigths
+logo of a cactus lifting weights
 a photo of a camera from the future
-a restaurant menu
 a skeleton with the shape of a spider
-looking into the sky, 10 airplanes are seen overhead
-shelves filled with books and alchemy potion bottles
-this is a detailed high-resolution scan of a human brain
 a collection of glasses is sitting on a table
-a cross-section view of a walnut
 a painting of a capybara sitting on a mountain during fall in surrealist style
 a pentagonal green clock
-a photo of san francisco golden gate bridge
 a pixel art illustration of an eagle sitting in a field in the afternoon
 a professional high-quality emoji of a lovestruck cup of boba
 a small red block sitting on a large green block
 a storefront that has the word 'openai' written on it
 a tatoo of a black broccoli
 a variety of clocks is sitting on a table
-an emoji of a baby fox wearing a blue hat, blue gloves, red shirt, and red pants
+an emoji of a baby fox wearing a blue hat, green gloves, red shirt, and yellow pants
 an emoji of a baby penguin wearing a blue hat, blue gloves, red shirt, and green pants
 an extreme close-up view of a capybara sitting in a field
 an illustration of a baby cucumber with a mustache playing chess
 an illustration of a baby daikon radish in a tutu walking a dog
 an illustration of a baby hedgehog in a cape staring at its reflection in a mirror
 an illustration of a baby panda with headphones holding an umbrella in the rain
-an illustration of an avocado in a beanie riding a motorcycle
 urinals are lined up in a jungle
+a muscular banana sitting upright on a bench smoking watching a banana on television, high definition photography
 a human face
 a person is holding a phone and a waterbottle, running a marathon
 a photograph of Ellen G. White
 Young woman riding her bike through the forest
+a portrait of a nightmare creature watching at you
+a white room full of a black substance
+the best soccer team of the world
+the best basketball team of the world
+the best football team of the world
+happy, happiness
+sad, sadness
+the representation of infinity
+the end of the world
+the last sunrise on earth
+an avocado armchair
+an armchair in the shape of an avocado
+illustration of an avocado armchair
+illustration of an armchair in the shape of an avocado
+logo of an avocado armchair
+an avocado armchair flying into space
+a cute avocado armchair singing karaoke on stage in front of a crowd of strawberry shaped lamps
+an illustration of an avocado in a christmas sweater staring at its reflection in a mirror
+illustration of an avocado armchair getting married to a pineapple
+an illustration of an avocado in a beanie riding a motorcycle

dev/inference/wandb-backend.ipynb

@@ -32,12 +32,15 @@
    "outputs": [],
    "source": [
     "run_ids = ['3kaut6e8']\n",
+    "# Alamy - 3kaut6e8\n",
+    "# YFCC - to do\n",
+    "# HF spaces - 4oh3u7ca\n",
     "ENTITY, PROJECT = 'wandb', 'hf-flax-dalle-mini'\n",
     "VQGAN_REPO, VQGAN_COMMIT_ID = 'dalle-mini/vqgan_imagenet_f16_16384', None\n",
     "normalize_text = False\n",
     "latest_only = True   # log only latest or all versions\n",
     "suffix = ''           # mainly for duplicate inference runs with a deleted version\n",
-    "add_clip_32 = True"
+    "add_clip_32 = False"
    ]
   },
   {
@@ -47,7 +50,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "run_ids = ['k76r0v39']\n",
+    "run_ids = ['2u5lk3uw']\n",
+    "# poorly shuffled 1nj161cl\n",
+    "# well shuffled he9rrc3q\n",
+    "# non normalized 1fwxpyfh ! requires changing normalize_text\n",
     "ENTITY, PROJECT = 'dalle-mini', 'dalle-mini'  # used only for training run\n",
     "VQGAN_REPO, VQGAN_COMMIT_ID = 'dalle-mini/vqgan_imagenet_f16_16384', None\n",
     "normalize_text = True\n",
@@ -56,6 +62,19 @@
     "add_clip_32 = False"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "71f27b96-7e6c-4472-a2e4-e99a8fb67a72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model.generate parameters - Not used yet\n",
+    "gen_top_k = None\n",
+    "gen_top_p = None\n",
+    "temperature = None"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -115,16 +134,6 @@
     "        return logits"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ebf4f7bf-2efa-46cc-b3f4-2d7a54f7b2cb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "clip_params['logit_scale']"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -204,7 +213,9 @@
     "            do_sample=True,\n",
     "            num_beams=1,\n",
     "            prng_key=key,\n",
-    "            params=params\n",
+    "            params=params,\n",
+    "            top_k=gen_top_k,\n",
+    "            top_p=gen_top_p\n",
     "        )\n",
     "    \n",
     "    return _generate"
@@ -286,7 +297,8 @@
     "\n",
     "            # generate images\n",
     "            images = []\n",
-    "            for i in tqdm(range(num_images // jax.device_count()), desc='Generating Images'):\n",
+    "            pbar = tqdm(range(num_images // jax.device_count()), desc='Generating Images', leave=None)\n",
+    "            for i in pbar:\n",
     "                key, subkey = jax.random.split(key)\n",
     "                encoded_images = p_generate(tokenized_prompt, shard_prng_key(subkey), model_params)\n",
     "                encoded_images = encoded_images.sequences[..., 1:]\n",
@@ -296,7 +308,7 @@
     "                    images.append(Image.fromarray(np.asarray(img * 255, dtype=np.uint8)))\n",
     "\n",
     "            # get clip scores\n",
-    "            print('Calculating CLIP scores')\n",
+    "            pbar.set_description('Calculating CLIP scores')\n",
     "            clip_inputs = processor(text=batch, images=images, return_tensors='np', padding='max_length', max_length=77, truncation=True).data\n",
     "            # each shard will have one prompt, images need to be reorganized to be associated to the correct shard\n",
     "            images_per_prompt_indices = np.asarray(range(0, len(images), batch_size))\n",
@@ -333,6 +345,7 @@
     "                    top_images = [wandb.Image(cur_images[x]) for x in idx]\n",
     "                    top_scores = [scores[x] for x in idx]\n",
     "                    results32.append([sample] + top_images + top_scores)\n",
+    "            pbar.close()\n",
     "\n",
     "    # log results\n",
     "    table = wandb.Table(columns=columns, data=results)\n",