{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Instructions\n",
        "\n",
        "To run Collage Diffusion, we need to specify a sequence of RGBA image layers as input--under \"Set asset paths\", point to the location in Google Drive where you have these assets stored. For improved performance, we would also like textual representations specialized to the individual collage layers--we have provided these assets for several test scenes from the paper, add these assets to your Google Drive and point \"asset_folder\" to the correct location.\n",
        "\n",
        "After \"Code Setup\", we have illustrated an iterative image generation and editing pipeline for the three test scenes provided. There are inline comments describing the chosen parameters in each section. "
      ],
      "metadata": {
        "id": "jUiDSCGaNMNs"
      },
      "id": "jUiDSCGaNMNs"
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set asset paths"
      ],
      "metadata": {
        "id": "CM99Cbv2NGg0"
      },
      "id": "CM99Cbv2NGg0"
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "os.listdir('/content/gdrive/')\n",
        "scene_dir = \"/content/gdrive/My Drive/Colab Notebooks/image_layers/\"\n",
        "asset_folder = \"/content/gdrive/My Drive/Colab Notebooks/asset_library\""
      ],
      "metadata": {
        "id": "udkckwxdOzXn"
      },
      "id": "udkckwxdOzXn",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ff1T7VpWr2A-"
      },
      "source": [
        "# Code Setup"
      ],
      "id": "Ff1T7VpWr2A-"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OHn5g9RIb9JD"
      },
      "outputs": [],
      "source": [
        "%pip install diffusers==0.10.0\n",
        "%pip install accelerate\n",
        "%pip install transformers\n",
        "%pip install einops\n",
        "%pip install pytorch-lightning\n",
        "%pip install python-multipart==0.0.5"
      ],
      "id": "OHn5g9RIb9JD"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b2e6fb82"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from transformers import CLIPModel, CLIPTextModel, CLIPTokenizer\n",
        "import numpy as np\n",
        "import random\n",
        "from PIL import Image, ImageFilter\n",
        "from torch import autocast\n",
        "from difflib import SequenceMatcher\n",
        "import pickle\n",
        "import math\n",
        "from einops import rearrange, repeat\n",
        "import matplotlib.pyplot as plt\n",
        "import os"
      ],
      "id": "b2e6fb82"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ba12aff2"
      },
      "outputs": [],
      "source": [
        "import torchvision.transforms as transforms\n",
        "\n",
        "class ImageLayer:\n",
        "    def __init__(self, rgba, pos, image_str, ftc = None):\n",
        "        self.rgba = rgba\n",
        "        self.pos = pos\n",
        "        self.image_str = image_str\n",
        "        self.ftc = ftc # Optional \n",
        "    \n",
        "    def get_mask(self):\n",
        "        return (np.array(self.rgba)[:,:,3]/255)\n",
        "    \n",
        "    def get_mask(image):\n",
        "        return (np.array(image)[:,:,3]/255)\n",
        "    \n",
        "    def get_bbox(self):\n",
        "        img = self.get_mask() > 0.5\n",
        "        rows = np.any(img, axis=1)\n",
        "        cols = np.any(img, axis=0)\n",
        "        rmin, rmax = np.where(rows)[0][[0, -1]]\n",
        "        cmin, cmax = np.where(cols)[0][[0, -1]]\n",
        "\n",
        "        return rmin, cmin, rmax, cmax\n",
        "    \n",
        "    def get_bbox_from_mask(img):\n",
        "        rows = np.any(img, axis=1)\n",
        "        cols = np.any(img, axis=0)\n",
        "        rmin, rmax = np.where(rows)[0][[0, -1]]\n",
        "        cmin, cmax = np.where(cols)[0][[0, -1]]\n",
        "        \n",
        "        return rmin, cmin, rmax, cmax\n",
        "    \n",
        "    def get_box_mask(img):\n",
        "        print(img.shape)\n",
        "        init_mask = np.zeros(img.shape,dtype=float)\n",
        "        bbox = ImageLayer.get_bbox_from_mask(img)\n",
        "        init_mask[bbox[0]:bbox[2]+1,bbox[1]:bbox[3]+1] = 1.\n",
        "        return init_mask\n",
        "    \n",
        "    def get_init_mask(self, bbox=False):\n",
        "        if bbox:\n",
        "            init_mask = np.zeros(self.rgba.size,dtype=float)\n",
        "            bbox = self.get_bbox()\n",
        "            init_mask[bbox[0]:bbox[2]+1,bbox[1]:bbox[3]+1] = 1.\n",
        "        else:\n",
        "            init_mask = self.get_mask()\n",
        "        return init_mask\n",
        "        \n",
        "    def get_pyramid(init_mask, res_list, device):\n",
        "        # Now we have a float mask for all the regions we want to emphasize\n",
        "        torch_mask = torch.tensor(init_mask).unsqueeze(0)\n",
        "        return_dict = {}\n",
        "        for res in res_list:\n",
        "            temp_im = transforms.Resize(size=res)(torch_mask).to(device)\n",
        "            return_dict[res] = temp_im\n",
        "        return return_dict\n",
        "    \n",
        "    def bottom_visible(bottom, top):\n",
        "        return (bottom*(1. - top))\n",
        "    \n",
        "    def blur_mask(img, blur_size):\n",
        "        img = np.array(Image.fromarray((img*255).astype(np.uint8), 'L').filter(ImageFilter.GaussianBlur(blur_size)))/255.\n",
        "        return img\n",
        "    \n",
        "    def add_layers(layer_list):\n",
        "        # Return the composite image, and also return a mask per image\n",
        "        composite_image = Image.new(\"RGB\", (512,512), (150,150,150))\n",
        "        num_layers = len(layer_list)\n",
        "        for i in range(0, num_layers):\n",
        "            composite_image.paste(layer_list[i].rgba, layer_list[i].pos, layer_list[i].rgba)\n",
        "        ftb_composites = []\n",
        "        for i in range(1, num_layers):\n",
        "            new_composite = Image.new(\"RGBA\", (512,512), (150,150,150,0))\n",
        "            for j in range(i, num_layers):\n",
        "                new_composite.paste(layer_list[j].rgba, layer_list[j].pos, layer_list[j].rgba)\n",
        "            ftb_composites.append(new_composite)\n",
        "        ftb_composites.append(Image.new(\"RGBA\", (512,512), (150,150,150,0)))\n",
        "        mask_layers = []\n",
        "        for i in range(0, num_layers):\n",
        "            bottom_mask = Image.new(\"RGBA\", (512,512), (150,150,150,0))\n",
        "            bottom_mask.paste(layer_list[i].rgba, layer_list[i].pos, layer_list[i].rgba)\n",
        "            bottom_mask = ImageLayer.get_mask(bottom_mask)\n",
        "            top_mask = ImageLayer.get_mask(ftb_composites[i])\n",
        "            mask_layers.append(ImageLayer.bottom_visible(bottom_mask, top_mask))\n",
        "        # Return\n",
        "        return composite_image, mask_layers"
      ],
      "id": "ba12aff2"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cefbc13a"
      },
      "outputs": [],
      "source": [
        "class FineTuneConcept:\n",
        "    def __init__(self, image_name, image, instance_prompt, init_token):\n",
        "        self.image_name = image_name\n",
        "        self.image = image\n",
        "        self.instance_prompt = instance_prompt\n",
        "        self.init_token = init_token\n",
        "\n",
        "    def build_ti_script(self):\n",
        "        # Concat model name\n",
        "        model_string = 'export MODEL_NAME=\"stabilityai/stable-diffusion-2-1-base\"\\n'\n",
        "        input_dir = asset_folder + '/' + self.image_name + '/'\n",
        "        os.makedirs(input_dir, exist_ok=True)\n",
        "        self.image.save(input_dir + \"image.jpg\")\n",
        "        input_string = 'export INSTANCE_DIR=\"' + input_dir + '\"\\n'\n",
        "        output_dir = asset_folder + '/' + self.image_name + '/'\n",
        "        output_string = 'export OUTPUT_DIR=\"' + output_dir + '\"\\n'\n",
        "        prompt_string = 'export INSTANCE_PROMPT=\"\\'' + self.instance_prompt + '\\'\"\\n'\n",
        "        init_string = 'export INIT_TOKEN=\"\\'' + self.init_token + '\\'\"\\n'\n",
        "        cmd_string = Path('../lorpt_template.txt').read_text()\n",
        "        ti_script = model_string + input_string + output_string + prompt_string + init_string + cmd_string\n",
        "        ti_script = ti_script.replace(\"$INSTANCE_PROMPT\", \"\\\"\" + self.instance_prompt + \"\\\"\")\n",
        "        ti_script = ti_script.replace(\"$INIT_TOKEN\", \"\\\"\" + self.init_token + \"\\\"\")\n",
        "        bash_dir = asset_folder + '/'\n",
        "        os.makedirs(bash_dir, exist_ok=True)\n",
        "        with open(bash_dir + self.image_name + '.sh', \"w\") as text_file:\n",
        "            text_file.write(ti_script)\n",
        "    \n",
        "    def run_ti_script(self, device_id):\n",
        "        bash_script = asset_folder + '/' + self.image_name + '.sh'\n",
        "        subprocess.Popen(\"CUDA_VISIBLE_DEVICES=\" + str(device_id) + \" bash \" + bash_script, shell=True,\n",
        "             stdin=None, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, close_fds=True, start_new_session=True)\n",
        "        \n",
        "    def test_ti(self, device_id):\n",
        "        ckpt_path = asset_folder + '/' + self.image_name # Flat image directory \n",
        "        run_ti(self.instance_prompt, ckpt_path, \"cuda:\" + str(device_id))\n",
        "        \n",
        "    def get_ckpt_path(self, step):\n",
        "        output_dir = asset_folder + '/' + self.image_name + '/'\n",
        "        ckpt_path = output_dir + 'lora_weight_e' + str(step - 1) + '_s' + str(step) + '.ti.pt'\n",
        "        return ckpt_path\n",
        "    \n",
        "    def training_complete(self, step):\n",
        "        path = self.get_ckpt_path(step)\n",
        "        return os.path.isfile(path)\n",
        "    \n",
        "    def all_ti(self, device_id):\n",
        "        self.build_ti_script()\n",
        "        self.run_ti_script(device_id)\n",
        "        self.test_ti(device_id)"
      ],
      "id": "cefbc13a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f206430e"
      },
      "outputs": [],
      "source": [
        "def init_attention_func_nvda(unet, tokens, masks, strengths, neg_strengths, sigma, orig_sched=False):\n",
        "    def new_attention(self, query, key, value):\n",
        "        if self.upcast_attention:\n",
        "            query = query.float()\n",
        "            key = key.float()\n",
        "\n",
        "        attention_scores = torch.baddbmm(\n",
        "            torch.empty(query.shape[0], query.shape[1], key.shape[1], dtype=query.dtype, device=query.device),\n",
        "            query,\n",
        "            key.transpose(-1, -2),\n",
        "            beta=0,\n",
        "            alpha=self.scale,\n",
        "        )\n",
        "        \n",
        "        ##############################################\n",
        "        attention_probs = attention_scores\n",
        "        b = 2\n",
        "        bh, n, d = attention_probs.shape\n",
        "        h = bh // b\n",
        "        num_iter = bh//(h*2)\n",
        "        attention_probs = rearrange(attention_probs,'(b h) n d -> b h n d', b=b)\n",
        "        n1 = (int)(np.sqrt(n))\n",
        "        attention_probs = rearrange(attention_probs,'b h (n1 n2) d -> b h n1 n2 d', n1 = n1)\n",
        "        \n",
        "        for i in range(num_iter):\n",
        "            for j in range(len(tokens)):\n",
        "                mask = masks[j][n1] # Mask pyramid is a dict so we can do this. Shape: (1,n1,n1)\n",
        "                if orig_sched:\n",
        "                    max_val = attention_probs[num_iter+i,:,:,:,tokens[j]].max()\n",
        "                    attention_probs[num_iter+i,:,:,:,tokens[j]] += mask*strengths[j]*math.log(sigma + 1.01)*max_val\n",
        "                    attention_probs[num_iter+i,:,:,:,tokens[j]] -= (1. - mask)*neg_strengths[j]*math.log(sigma + 1.01)*max_val\n",
        "                else:\n",
        "                    max_val = attention_probs[num_iter+i,:,:,:,tokens[j]].max()\n",
        "                    attention_probs[num_iter+i,:,:,:,tokens[j]] += mask*strengths[j]*math.log(math.log(sigma + 1.01) + 1.)*max_val\n",
        "                    attention_probs[num_iter+i,:,:,:,tokens[j]] -= (1. - mask)*neg_strengths[j]*math.log(math.log(sigma + 1.01) + 1.)*max_val\n",
        "        attention_probs = rearrange(attention_probs,'b h n1 n2 d -> b h (n1 n2) d')\n",
        "        attention_probs = rearrange(attention_probs,'b h n d -> (b h) n d', b=b)\n",
        "        attention_scores = attention_probs\n",
        "        #################################################\n",
        "        \n",
        "        \n",
        "        attention_probs = attention_scores.softmax(dim=-1)\n",
        "\n",
        "        # cast back to the original dtype\n",
        "        attention_probs = attention_probs.to(value.dtype)\n",
        "\n",
        "        # compute attention output\n",
        "        hidden_states = torch.bmm(attention_probs, value)\n",
        "\n",
        "        # reshape hidden_states\n",
        "        hidden_states = self.reshape_batch_dim_to_heads(hidden_states)\n",
        "        return hidden_states\n",
        "    \n",
        "    for name, module in unet.named_modules():\n",
        "        module_name = type(module).__name__\n",
        "        if module_name == \"CrossAttention\" and \"attn2\" in name:\n",
        "            module.last_attn_slice = None\n",
        "            module.use_last_attn_slice = False\n",
        "            module.use_last_attn_weights = False\n",
        "            module.save_last_attn_slice = False\n",
        "            module._attention = new_attention.__get__(module, type(module))"
      ],
      "id": "f206430e"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mZ5ZnykndYYW"
      },
      "outputs": [],
      "source": [
        "# Copyright 2022 The HuggingFace Team. All rights reserved.\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License.\n",
        "\n",
        "import os\n",
        "import inspect\n",
        "import copy\n",
        "import math\n",
        "from typing import Callable, List, Optional, Union, Dict\n",
        "from PIL import Image, ImageFilter\n",
        "from urllib.request import urlopen\n",
        "from io import BytesIO\n",
        "import numpy as np\n",
        "from einops import rearrange, repeat\n",
        "import io\n",
        "import requests\n",
        "import base64\n",
        "\n",
        "import PIL\n",
        "import torch\n",
        "\n",
        "from diffusers.utils import is_accelerate_available\n",
        "from packaging import version\n",
        "from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer\n",
        "from pytorch_lightning import seed_everything\n",
        "\n",
        "from diffusers.configuration_utils import FrozenDict\n",
        "from diffusers.models import AutoencoderKL, UNet2DConditionModel\n",
        "from diffusers.pipeline_utils import DiffusionPipeline\n",
        "from diffusers.schedulers import (\n",
        "    DDIMScheduler,\n",
        "    DPMSolverMultistepScheduler,\n",
        "    EulerAncestralDiscreteScheduler,\n",
        "    EulerDiscreteScheduler,\n",
        "    LMSDiscreteScheduler,\n",
        "    PNDMScheduler,\n",
        ")\n",
        "from diffusers.utils import PIL_INTERPOLATION, deprecate, logging\n",
        "from diffusers.utils import deprecate, logging\n",
        "from diffusers.pipelines.stable_diffusion import (\n",
        "    StableDiffusionPipelineOutput,\n",
        "    StableDiffusionSafetyChecker,\n",
        ")\n",
        "\n",
        "\n",
        "logger = logging.get_logger(__name__)  # pylint: disable=invalid-name\n",
        "\n",
        "\n",
        "def preprocess_image(image):\n",
        "    w, h = image.size\n",
        "    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32\n",
        "    image = image.resize((w, h), resample=PIL_INTERPOLATION[\"lanczos\"])\n",
        "    image = np.array(image).astype(np.float32) / 255.0\n",
        "    image = image[None].transpose(0, 3, 1, 2)\n",
        "    image = torch.from_numpy(image)\n",
        "    return 2.0 * image - 1.0\n",
        "\n",
        "\n",
        "def preprocess_mask(mask, scale_factor=8):\n",
        "    mask = mask.convert(\"L\")\n",
        "    w, h = mask.size\n",
        "    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32\n",
        "    mask = mask.resize(\n",
        "        (w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION[\"nearest\"]\n",
        "    )\n",
        "    mask = np.array(mask).astype(np.float32) / 255.0\n",
        "    mask = np.tile(mask, (4, 1, 1))\n",
        "    mask = mask[None].transpose(0, 1, 2, 3)  # what does this step do?\n",
        "    mask = 1 - mask  # repaint white, keep black\n",
        "    mask = torch.from_numpy(mask)\n",
        "    return mask\n",
        "\n",
        "\n",
        "def load_learned_embed_in_clip(\n",
        "    learned_embeds_path, text_encoder, tokenizer, new_token=None, idempotent=False\n",
        "):\n",
        "    loaded_learned_embeds = torch.load(learned_embeds_path, map_location=\"cpu\")\n",
        "\n",
        "    # separate token and the embeds\n",
        "    token_list = list(loaded_learned_embeds.keys())\n",
        "    print(\"Token list\", token_list)\n",
        "    for trained_token in token_list:\n",
        "        embeds = loaded_learned_embeds[trained_token]\n",
        "\n",
        "        # cast to dtype of text_encoder\n",
        "        dtype = text_encoder.get_input_embeddings().weight.dtype\n",
        "\n",
        "        # add the token in tokenizer\n",
        "        token = new_token if new_token is not None else trained_token\n",
        "        num_added_tokens = tokenizer.add_tokens(token)\n",
        "        i = 1\n",
        "        if num_added_tokens == 0 and idempotent:\n",
        "            print(\"Token add failed!\")\n",
        "            return token\n",
        "\n",
        "        while num_added_tokens == 0:\n",
        "            print(f\"The tokenizer already contains the token {token}.\")\n",
        "            token = f\"{token[:-1]}-{i}>\"\n",
        "            print(f\"Attempting to add the token {token}.\")\n",
        "            num_added_tokens = tokenizer.add_tokens(token)\n",
        "            i += 1\n",
        "\n",
        "        # resize the token embeddings\n",
        "        text_encoder.resize_token_embeddings(len(tokenizer))\n",
        "\n",
        "        # get the id for the token and assign the embeds\n",
        "        token_id = tokenizer.convert_tokens_to_ids(token)\n",
        "        text_encoder.get_input_embeddings().weight.data[token_id] = embeds\n",
        "        print(token)\n",
        "    return\n",
        "\n",
        "\n",
        "class StableDiffusionPipeline(DiffusionPipeline):\n",
        "    r\"\"\"\n",
        "    Pipeline for text-to-image generation using Stable Diffusion.\n",
        "    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the\n",
        "    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)\n",
        "    Args:\n",
        "        vae ([`AutoencoderKL`]):\n",
        "            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.\n",
        "        text_encoder ([`CLIPTextModel`]):\n",
        "            Frozen text-encoder. Stable Diffusion uses the text portion of\n",
        "            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically\n",
        "            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.\n",
        "        tokenizer (`CLIPTokenizer`):\n",
        "            Tokenizer of class\n",
        "            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).\n",
        "        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.\n",
        "        scheduler ([`SchedulerMixin`]):\n",
        "            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of\n",
        "            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].\n",
        "        safety_checker ([`StableDiffusionSafetyChecker`]):\n",
        "            Classification module that estimates whether generated images could be considered offensive or harmful.\n",
        "            Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.\n",
        "        feature_extractor ([`CLIPFeatureExtractor`]):\n",
        "            Model that extracts features from generated images to be used as inputs for the `safety_checker`.\n",
        "    \"\"\"\n",
        "    _optional_components = [\"safety_checker\", \"feature_extractor\"]\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        vae: AutoencoderKL,\n",
        "        text_encoder: CLIPTextModel,\n",
        "        tokenizer: CLIPTokenizer,\n",
        "        unet: UNet2DConditionModel,\n",
        "        scheduler: Union[\n",
        "            DDIMScheduler,\n",
        "            PNDMScheduler,\n",
        "            LMSDiscreteScheduler,\n",
        "            EulerDiscreteScheduler,\n",
        "            EulerAncestralDiscreteScheduler,\n",
        "            DPMSolverMultistepScheduler,\n",
        "        ],\n",
        "        safety_checker: StableDiffusionSafetyChecker,\n",
        "        feature_extractor: CLIPFeatureExtractor,\n",
        "        requires_safety_checker: bool = True,\n",
        "    ):\n",
        "        super().__init__()\n",
        "\n",
        "        if (\n",
        "            hasattr(scheduler.config, \"steps_offset\")\n",
        "            and scheduler.config.steps_offset != 1\n",
        "        ):\n",
        "            deprecation_message = (\n",
        "                f\"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`\"\n",
        "                f\" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure \"\n",
        "                \"to update the config accordingly as leaving `steps_offset` might led to incorrect results\"\n",
        "                \" in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,\"\n",
        "                \" it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`\"\n",
        "                \" file\"\n",
        "            )\n",
        "            deprecate(\n",
        "                \"steps_offset!=1\", \"1.0.0\", deprecation_message, standard_warn=False\n",
        "            )\n",
        "            new_config = dict(scheduler.config)\n",
        "            new_config[\"steps_offset\"] = 1\n",
        "            scheduler._internal_dict = FrozenDict(new_config)\n",
        "\n",
        "        if (\n",
        "            hasattr(scheduler.config, \"clip_sample\")\n",
        "            and scheduler.config.clip_sample is True\n",
        "        ):\n",
        "            deprecation_message = (\n",
        "                f\"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`.\"\n",
        "                \" `clip_sample` should be set to False in the configuration file. Please make sure to update the\"\n",
        "                \" config accordingly as not setting `clip_sample` in the config might lead to incorrect results in\"\n",
        "                \" future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very\"\n",
        "                \" nice if you could open a Pull request for the `scheduler/scheduler_config.json` file\"\n",
        "            )\n",
        "            deprecate(\n",
        "                \"clip_sample not set\", \"1.0.0\", deprecation_message, standard_warn=False\n",
        "            )\n",
        "            new_config = dict(scheduler.config)\n",
        "            new_config[\"clip_sample\"] = False\n",
        "            scheduler._internal_dict = FrozenDict(new_config)\n",
        "\n",
        "        if safety_checker is None and requires_safety_checker:\n",
        "            logger.warning(\n",
        "                f\"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure\"\n",
        "                \" that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered\"\n",
        "                \" results in services or applications open to the public. Both the diffusers team and Hugging Face\"\n",
        "                \" strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling\"\n",
        "                \" it only for use-cases that involve analyzing network behavior or auditing its results. For more\"\n",
        "                \" information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\"\n",
        "            )\n",
        "\n",
        "        if safety_checker is not None and feature_extractor is None:\n",
        "            raise ValueError(\n",
        "                \"Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety\"\n",
        "                \" checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead.\"\n",
        "            )\n",
        "\n",
        "        is_unet_version_less_0_9_0 = hasattr(\n",
        "            unet.config, \"_diffusers_version\"\n",
        "        ) and version.parse(\n",
        "            version.parse(unet.config._diffusers_version).base_version\n",
        "        ) < version.parse(\n",
        "            \"0.9.0.dev0\"\n",
        "        )\n",
        "        is_unet_sample_size_less_64 = (\n",
        "            hasattr(unet.config, \"sample_size\") and unet.config.sample_size < 64\n",
        "        )\n",
        "        if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:\n",
        "            deprecation_message = (\n",
        "                \"The configuration file of the unet has set the default `sample_size` to smaller than\"\n",
        "                \" 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the\"\n",
        "                \" following: \\n- CompVis/stable-diffusion-v1-4 \\n- CompVis/stable-diffusion-v1-3 \\n-\"\n",
        "                \" CompVis/stable-diffusion-v1-2 \\n- CompVis/stable-diffusion-v1-1 \\n- runwayml/stable-diffusion-v1-5\"\n",
        "                \" \\n- runwayml/stable-diffusion-inpainting \\n you should change 'sample_size' to 64 in the\"\n",
        "                \" configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`\"\n",
        "                \" in the config might lead to incorrect results in future versions. If you have downloaded this\"\n",
        "                \" checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for\"\n",
        "                \" the `unet/config.json` file\"\n",
        "            )\n",
        "            deprecate(\n",
        "                \"sample_size<64\", \"1.0.0\", deprecation_message, standard_warn=False\n",
        "            )\n",
        "            new_config = dict(unet.config)\n",
        "            new_config[\"sample_size\"] = 64\n",
        "            unet._internal_dict = FrozenDict(new_config)\n",
        "\n",
        "        self.register_modules(\n",
        "            vae=vae,\n",
        "            text_encoder=text_encoder,\n",
        "            tokenizer=tokenizer,\n",
        "            unet=unet,\n",
        "            scheduler=scheduler,\n",
        "            safety_checker=safety_checker,\n",
        "            feature_extractor=feature_extractor,\n",
        "        )\n",
        "        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)\n",
        "        self.register_to_config(requires_safety_checker=requires_safety_checker)\n",
        "\n",
        "    def enable_vae_slicing(self):\n",
        "        r\"\"\"\n",
        "        Enable sliced VAE decoding.\n",
        "        When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several\n",
        "        steps. This is useful to save some memory and allow larger batch sizes.\n",
        "        \"\"\"\n",
        "        self.vae.enable_slicing()\n",
        "\n",
        "    def disable_vae_slicing(self):\n",
        "        r\"\"\"\n",
        "        Disable sliced VAE decoding. If `enable_vae_slicing` was previously invoked, this method will go back to\n",
        "        computing decoding in one step.\n",
        "        \"\"\"\n",
        "        self.vae.disable_slicing()\n",
        "\n",
        "    def enable_sequential_cpu_offload(self, gpu_id=0):\n",
        "        r\"\"\"\n",
        "        Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,\n",
        "        text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a\n",
        "        `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.\n",
        "        \"\"\"\n",
        "        if is_accelerate_available():\n",
        "            from accelerate import cpu_offload\n",
        "        else:\n",
        "            raise ImportError(\"Please install accelerate via `pip install accelerate`\")\n",
        "\n",
        "        device = torch.device(f\"cuda:{gpu_id}\")\n",
        "\n",
        "        for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:\n",
        "            if cpu_offloaded_model is not None:\n",
        "                cpu_offload(cpu_offloaded_model, device)\n",
        "\n",
        "        if self.safety_checker is not None:\n",
        "            # TODO(Patrick) - there is currently a bug with cpu offload of nn.Parameter in accelerate\n",
        "            # fix by only offloading self.safety_checker for now\n",
        "            cpu_offload(self.safety_checker.vision_model, device)\n",
        "\n",
        "    @property\n",
        "    def _execution_device(self):\n",
        "        r\"\"\"\n",
        "        Returns the device on which the pipeline's models will be executed. After calling\n",
        "        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module\n",
        "        hooks.\n",
        "        \"\"\"\n",
        "        if self.device != torch.device(\"meta\") or not hasattr(self.unet, \"_hf_hook\"):\n",
        "            return self.device\n",
        "        for module in self.unet.modules():\n",
        "            if (\n",
        "                hasattr(module, \"_hf_hook\")\n",
        "                and hasattr(module._hf_hook, \"execution_device\")\n",
        "                and module._hf_hook.execution_device is not None\n",
        "            ):\n",
        "                return torch.device(module._hf_hook.execution_device)\n",
        "        return self.device\n",
        "\n",
        "    def _encode_prompt(\n",
        "        self,\n",
        "        prompt,\n",
        "        device,\n",
        "        num_images_per_prompt,\n",
        "        do_classifier_free_guidance,\n",
        "        negative_prompt,\n",
        "    ):\n",
        "        r\"\"\"\n",
        "        Encodes the prompt into text encoder hidden states.\n",
        "        Args:\n",
        "            prompt (`str` or `list(int)`):\n",
        "                prompt to be encoded\n",
        "            device: (`torch.device`):\n",
        "                torch device\n",
        "            num_images_per_prompt (`int`):\n",
        "                number of images that should be generated per prompt\n",
        "            do_classifier_free_guidance (`bool`):\n",
        "                whether to use classifier free guidance or not\n",
        "            negative_prompt (`str` or `List[str]`):\n",
        "                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored\n",
        "                if `guidance_scale` is less than `1`).\n",
        "        \"\"\"\n",
        "        batch_size = len(prompt) if isinstance(prompt, list) else 1\n",
        "\n",
        "        text_inputs = self.tokenizer(\n",
        "            prompt,\n",
        "            padding=\"max_length\",\n",
        "            max_length=self.tokenizer.model_max_length,\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "        )\n",
        "        text_input_ids = text_inputs.input_ids\n",
        "        untruncated_ids = self.tokenizer(\n",
        "            prompt, padding=\"longest\", return_tensors=\"pt\"\n",
        "        ).input_ids\n",
        "\n",
        "        if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(\n",
        "            text_input_ids, untruncated_ids\n",
        "        ):\n",
        "            removed_text = self.tokenizer.batch_decode(\n",
        "                untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]\n",
        "            )\n",
        "            logger.warning(\n",
        "                \"The following part of your input was truncated because CLIP can only handle sequences up to\"\n",
        "                f\" {self.tokenizer.model_max_length} tokens: {removed_text}\"\n",
        "            )\n",
        "\n",
        "        if (\n",
        "            hasattr(self.text_encoder.config, \"use_attention_mask\")\n",
        "            and self.text_encoder.config.use_attention_mask\n",
        "        ):\n",
        "            attention_mask = text_inputs.attention_mask.to(device)\n",
        "        else:\n",
        "            attention_mask = None\n",
        "\n",
        "        text_embeddings = self.text_encoder(\n",
        "            text_input_ids.to(device),\n",
        "            attention_mask=attention_mask,\n",
        "        )\n",
        "        text_embeddings = text_embeddings[0]\n",
        "\n",
        "        # duplicate text embeddings for each generation per prompt, using mps friendly method\n",
        "        bs_embed, seq_len, _ = text_embeddings.shape\n",
        "        text_embeddings = text_embeddings.repeat(1, num_images_per_prompt, 1)\n",
        "        text_embeddings = text_embeddings.view(\n",
        "            bs_embed * num_images_per_prompt, seq_len, -1\n",
        "        )\n",
        "\n",
        "        # get unconditional embeddings for classifier free guidance\n",
        "        if do_classifier_free_guidance:\n",
        "            uncond_tokens: List[str]\n",
        "            if negative_prompt is None:\n",
        "                uncond_tokens = [\"\"] * batch_size\n",
        "            elif type(prompt) is not type(negative_prompt):\n",
        "                raise TypeError(\n",
        "                    f\"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=\"\n",
        "                    f\" {type(prompt)}.\"\n",
        "                )\n",
        "            elif isinstance(negative_prompt, str):\n",
        "                uncond_tokens = [negative_prompt]\n",
        "            elif batch_size != len(negative_prompt):\n",
        "                raise ValueError(\n",
        "                    f\"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:\"\n",
        "                    f\" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches\"\n",
        "                    \" the batch size of `prompt`.\"\n",
        "                )\n",
        "            else:\n",
        "                uncond_tokens = negative_prompt\n",
        "\n",
        "            max_length = text_input_ids.shape[-1]\n",
        "            uncond_input = self.tokenizer(\n",
        "                uncond_tokens,\n",
        "                padding=\"max_length\",\n",
        "                max_length=max_length,\n",
        "                truncation=True,\n",
        "                return_tensors=\"pt\",\n",
        "            )\n",
        "\n",
        "            if (\n",
        "                hasattr(self.text_encoder.config, \"use_attention_mask\")\n",
        "                and self.text_encoder.config.use_attention_mask\n",
        "            ):\n",
        "                attention_mask = uncond_input.attention_mask.to(device)\n",
        "            else:\n",
        "                attention_mask = None\n",
        "\n",
        "            uncond_embeddings = self.text_encoder(\n",
        "                uncond_input.input_ids.to(device),\n",
        "                attention_mask=attention_mask,\n",
        "            )\n",
        "            uncond_embeddings = uncond_embeddings[0]\n",
        "\n",
        "            # duplicate unconditional embeddings for each generation per prompt, using mps friendly method\n",
        "            seq_len = uncond_embeddings.shape[1]\n",
        "            uncond_embeddings = uncond_embeddings.repeat(1, num_images_per_prompt, 1)\n",
        "            uncond_embeddings = uncond_embeddings.view(\n",
        "                batch_size * num_images_per_prompt, seq_len, -1\n",
        "            )\n",
        "\n",
        "            # For classifier free guidance, we need to do two forward passes.\n",
        "            # Here we concatenate the unconditional and text embeddings into a single batch\n",
        "            # to avoid doing two forward passes\n",
        "            text_embeddings = torch.cat([uncond_embeddings, text_embeddings])\n",
        "\n",
        "        return text_embeddings\n",
        "\n",
        "    def run_safety_checker(self, image, device, dtype):\n",
        "        has_nsfw_concept = None\n",
        "        return image, has_nsfw_concept\n",
        "\n",
        "    def decode_latents(self, latents):\n",
        "        latents = 1 / 0.18215 * latents\n",
        "        image = self.vae.decode(latents).sample\n",
        "        image = (image / 2 + 0.5).clamp(0, 1)\n",
        "        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16\n",
        "        image = image.cpu().permute(0, 2, 3, 1).float().numpy()\n",
        "        return image\n",
        "\n",
        "    def prepare_extra_step_kwargs(self, generator, eta):\n",
        "        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature\n",
        "        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.\n",
        "        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502\n",
        "        # and should be between [0, 1]\n",
        "\n",
        "        accepts_eta = \"eta\" in set(\n",
        "            inspect.signature(self.scheduler.step).parameters.keys()\n",
        "        )\n",
        "        extra_step_kwargs = {}\n",
        "        if accepts_eta:\n",
        "            extra_step_kwargs[\"eta\"] = eta\n",
        "\n",
        "        # check if the scheduler accepts generator\n",
        "        accepts_generator = \"generator\" in set(\n",
        "            inspect.signature(self.scheduler.step).parameters.keys()\n",
        "        )\n",
        "        if accepts_generator:\n",
        "            extra_step_kwargs[\"generator\"] = generator\n",
        "        return extra_step_kwargs\n",
        "\n",
        "    def check_inputs(self, prompt, strength, callback_steps):\n",
        "        if not isinstance(prompt, str) and not isinstance(prompt, list):\n",
        "            raise ValueError(\n",
        "                f\"`prompt` has to be of type `str` or `list` but is {type(prompt)}\"\n",
        "            )\n",
        "\n",
        "        if strength < 0 or strength > 1:\n",
        "            raise ValueError(\n",
        "                f\"The value of strength should in [1.0, 1.0] but is {strength}\"\n",
        "            )\n",
        "\n",
        "        if (callback_steps is None) or (\n",
        "            callback_steps is not None\n",
        "            and (not isinstance(callback_steps, int) or callback_steps <= 0)\n",
        "        ):\n",
        "            raise ValueError(\n",
        "                f\"`callback_steps` has to be a positive integer but is {callback_steps} of type\"\n",
        "                f\" {type(callback_steps)}.\"\n",
        "            )\n",
        "\n",
        "    def prepare_latents(\n",
        "        self,\n",
        "        batch_size,\n",
        "        num_channels_latents,\n",
        "        height,\n",
        "        width,\n",
        "        dtype,\n",
        "        device,\n",
        "        generator,\n",
        "        latents=None,\n",
        "    ):\n",
        "        shape = (\n",
        "            batch_size,\n",
        "            num_channels_latents,\n",
        "            height // self.vae_scale_factor,\n",
        "            width // self.vae_scale_factor,\n",
        "        )\n",
        "        if isinstance(generator, list) and len(generator) != batch_size:\n",
        "            raise ValueError(\n",
        "                f\"You have passed a list of generators of length {len(generator)}, but requested an effective batch\"\n",
        "                f\" size of {batch_size}. Make sure the batch size matches the length of the generators.\"\n",
        "            )\n",
        "\n",
        "        if latents is None:\n",
        "            rand_device = \"cpu\" if device.type == \"mps\" else device\n",
        "\n",
        "            if isinstance(generator, list):\n",
        "                shape = (1,) + shape[1:]\n",
        "                latents = [\n",
        "                    torch.randn(\n",
        "                        shape, generator=generator[i], device=rand_device, dtype=dtype\n",
        "                    )\n",
        "                    for i in range(batch_size)\n",
        "                ]\n",
        "                latents = torch.cat(latents, dim=0).to(device)\n",
        "            else:\n",
        "                latents = torch.randn(\n",
        "                    shape, generator=generator, device=rand_device, dtype=dtype\n",
        "                ).to(device)\n",
        "        else:\n",
        "            if latents.shape != shape:\n",
        "                raise ValueError(\n",
        "                    f\"Unexpected latents shape, got {latents.shape}, expected {shape}\"\n",
        "                )\n",
        "            latents = latents.to(device)\n",
        "\n",
        "        # scale the initial noise by the standard deviation required by the scheduler\n",
        "        latents = latents * self.scheduler.init_noise_sigma\n",
        "        return latents\n",
        "\n",
        "    def prepare_latents_legacy(\n",
        "        self,\n",
        "        image,\n",
        "        timestep,\n",
        "        batch_size,\n",
        "        num_images_per_prompt,\n",
        "        dtype,\n",
        "        device,\n",
        "        generator,\n",
        "    ):\n",
        "        image = image.to(device=self.device, dtype=dtype)\n",
        "        init_latent_dist = self.vae.encode(image).latent_dist\n",
        "        init_latents = init_latent_dist.sample(generator=generator)\n",
        "        init_latents = 0.18215 * init_latents\n",
        "\n",
        "        # Expand init_latents for batch_size and num_images_per_prompt\n",
        "        init_latents = torch.cat(\n",
        "            [init_latents] * batch_size * num_images_per_prompt, dim=0\n",
        "        )\n",
        "        init_latents_orig = init_latents\n",
        "\n",
        "        # add noise to latents using the timesteps\n",
        "        noise = torch.randn(\n",
        "            init_latents.shape, generator=generator, device=self.device, dtype=dtype\n",
        "        )\n",
        "        init_latents = self.scheduler.add_noise(init_latents, noise, timestep)\n",
        "        latents = init_latents\n",
        "        return latents, init_latents_orig, noise\n",
        "\n",
        "    def generate_mask(\n",
        "        self, \n",
        "        mask_layers, \n",
        "        img2img_strength, \n",
        "        t_is, \n",
        "        noise_blur = 10,\n",
        "    ):\n",
        "        collage_mask = Image.new(\"L\", (512,512), ((int)(img2img_strength * 255)))\n",
        "        for i in range(len(t_is)):\n",
        "            layer_image = Image.fromarray((mask_layers[i]*t_is[i]*255).astype(np.uint8), 'L')\n",
        "            mask_image = Image.fromarray((mask_layers[i]*255).astype(np.uint8), 'L')\n",
        "            collage_mask.paste(layer_image, (0,0), mask_image)\n",
        "        collage_mask = collage_mask.filter(ImageFilter.GaussianBlur(noise_blur))\n",
        "        return collage_mask\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def __call__(\n",
        "        self,\n",
        "        prompt: Union[str, List[str]],\n",
        "        height: Optional[int] = None,\n",
        "        width: Optional[int] = None,\n",
        "        num_inference_steps: int = 50,\n",
        "        guidance_scale: float = 7.5,\n",
        "        negative_prompt: Optional[Union[str, List[str]]] = None,\n",
        "        num_images_per_prompt: Optional[int] = 1,\n",
        "        eta: float = 0.0,\n",
        "        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,\n",
        "        latents: Optional[torch.FloatTensor] = None,\n",
        "        output_type: Optional[str] = \"pil\",\n",
        "        return_dict: bool = True,\n",
        "        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,\n",
        "        callback_steps: Optional[int] = 1,\n",
        "    ):\n",
        "        r\"\"\"\n",
        "        Function invoked when calling the pipeline for generation.\n",
        "        Args:\n",
        "            prompt (`str` or `List[str]`):\n",
        "                The prompt or prompts to guide the image generation.\n",
        "            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):\n",
        "                The height in pixels of the generated image.\n",
        "            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):\n",
        "                The width in pixels of the generated image.\n",
        "            num_inference_steps (`int`, *optional*, defaults to 50):\n",
        "                The number of denoising steps. More denoising steps usually lead to a higher quality image at the\n",
        "                expense of slower inference.\n",
        "            guidance_scale (`float`, *optional*, defaults to 7.5):\n",
        "                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).\n",
        "                `guidance_scale` is defined as `w` of equation 2. of [Imagen\n",
        "                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >\n",
        "                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,\n",
        "                usually at the expense of lower image quality.\n",
        "            negative_prompt (`str` or `List[str]`, *optional*):\n",
        "                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored\n",
        "                if `guidance_scale` is less than `1`).\n",
        "            num_images_per_prompt (`int`, *optional*, defaults to 1):\n",
        "                The number of images to generate per prompt.\n",
        "            eta (`float`, *optional*, defaults to 0.0):\n",
        "                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to\n",
        "                [`schedulers.DDIMScheduler`], will be ignored for others.\n",
        "            generator (`torch.Generator`, *optional*):\n",
        "                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)\n",
        "                to make generation deterministic.\n",
        "            latents (`torch.FloatTensor`, *optional*):\n",
        "                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image\n",
        "                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents\n",
        "                tensor will ge generated by sampling using the supplied random `generator`.\n",
        "            output_type (`str`, *optional*, defaults to `\"pil\"`):\n",
        "                The output format of the generate image. Choose between\n",
        "                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.\n",
        "            return_dict (`bool`, *optional*, defaults to `True`):\n",
        "                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a\n",
        "                plain tuple.\n",
        "            callback (`Callable`, *optional*):\n",
        "                A function that will be called every `callback_steps` steps during inference. The function will be\n",
        "                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.\n",
        "            callback_steps (`int`, *optional*, defaults to 1):\n",
        "                The frequency at which the `callback` function will be called. If not specified, the callback will be\n",
        "                called at every step.\n",
        "        Returns:\n",
        "            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:\n",
        "            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.\n",
        "            When returning a tuple, the first element is a list with the generated images, and the second element is a\n",
        "            list of `bool`s denoting whether the corresponding generated image likely represents \"not-safe-for-work\"\n",
        "            (nsfw) content, according to the `safety_checker`.\n",
        "        \"\"\"\n",
        "        # 0. Default height and width to unet\n",
        "        height = height or self.unet.config.sample_size * self.vae_scale_factor\n",
        "        width = width or self.unet.config.sample_size * self.vae_scale_factor\n",
        "\n",
        "        # 1. Check inputs. Raise error if not correct\n",
        "        # self.check_inputs(prompt, height, width, callback_steps) # TODO see what the old function did (updated for collage)\n",
        "\n",
        "        # 2. Define call parameters\n",
        "        batch_size = 1 if isinstance(prompt, str) else len(prompt)\n",
        "        device = self._execution_device\n",
        "        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)\n",
        "        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`\n",
        "        # corresponds to doing no classifier free guidance.\n",
        "        do_classifier_free_guidance = guidance_scale > 1.0\n",
        "\n",
        "        # 3. Encode input prompt\n",
        "        text_embeddings = self._encode_prompt(\n",
        "            prompt,\n",
        "            device,\n",
        "            num_images_per_prompt,\n",
        "            do_classifier_free_guidance,\n",
        "            negative_prompt,\n",
        "        )\n",
        "\n",
        "        # 4. Prepare timesteps\n",
        "        self.scheduler.set_timesteps(num_inference_steps, device=device)\n",
        "        timesteps = self.scheduler.timesteps\n",
        "\n",
        "        # 5. Prepare latent variables\n",
        "        num_channels_latents = self.unet.in_channels\n",
        "        latents = self.prepare_latents(\n",
        "            batch_size * num_images_per_prompt,\n",
        "            num_channels_latents,\n",
        "            height,\n",
        "            width,\n",
        "            text_embeddings.dtype,\n",
        "            device,\n",
        "            generator,\n",
        "            latents,\n",
        "        )\n",
        "\n",
        "        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline\n",
        "        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)\n",
        "\n",
        "        # 7. Denoising loop\n",
        "        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order\n",
        "        with self.progress_bar(total=num_inference_steps) as progress_bar:\n",
        "            for i, t in enumerate(timesteps):\n",
        "                # expand the latents if we are doing classifier free guidance\n",
        "                latent_model_input = (\n",
        "                    torch.cat([latents] * 2) if do_classifier_free_guidance else latents\n",
        "                )\n",
        "                latent_model_input = self.scheduler.scale_model_input(\n",
        "                    latent_model_input, t\n",
        "                )\n",
        "\n",
        "                # predict the noise residual\n",
        "                noise_pred = self.unet(\n",
        "                    latent_model_input, t, encoder_hidden_states=text_embeddings\n",
        "                ).sample\n",
        "\n",
        "                # perform guidance\n",
        "                if do_classifier_free_guidance:\n",
        "                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)\n",
        "                    noise_pred = noise_pred_uncond + guidance_scale * (\n",
        "                        noise_pred_text - noise_pred_uncond\n",
        "                    )\n",
        "\n",
        "                # compute the previous noisy sample x_t -> x_t-1\n",
        "                latents = self.scheduler.step(\n",
        "                    noise_pred, t, latents, **extra_step_kwargs\n",
        "                ).prev_sample\n",
        "\n",
        "                # call the callback, if provided\n",
        "                if i == len(timesteps) - 1 or (\n",
        "                    (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0\n",
        "                ):\n",
        "                    progress_bar.update()\n",
        "                    if callback is not None and i % callback_steps == 0:\n",
        "                        callback(i, t, latents)\n",
        "\n",
        "        # 8. Post-processing\n",
        "        image = self.decode_latents(latents)\n",
        "\n",
        "        # 9. Run safety checker\n",
        "        image, has_nsfw_concept = self.run_safety_checker(\n",
        "            image, device, text_embeddings.dtype\n",
        "        )\n",
        "\n",
        "        # 10. Convert to PIL\n",
        "        if output_type == \"pil\":\n",
        "            image = self.numpy_to_pil(image)\n",
        "\n",
        "        if not return_dict:\n",
        "            return (image, has_nsfw_concept)\n",
        "\n",
        "        return StableDiffusionPipelineOutput(\n",
        "            images=image, nsfw_content_detected=has_nsfw_concept\n",
        "        )\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def preprocess_layers(\n",
        "        self,\n",
        "        layers: List[Dict[str, any]],\n",
        "        w_pos: List[float],\n",
        "        w_neg: List[float],\n",
        "        collage_prompt,\n",
        "        use_ca: bool = True,\n",
        "        collage_src: str = None,\n",
        "    ):\n",
        "        layer_list = []\n",
        "        for layer in layers:\n",
        "            layer_rgba = Image.open(urlopen(layer[\"originalImgUrl\"])).convert(\"RGBA\")\n",
        "\n",
        "            # print(f\"Word embedding is {layer['wordEmbedding']}\")\n",
        "            image_layer = ImageLayer(\n",
        "                rgba=layer_rgba,\n",
        "                image_str=layer[\"textPrompt\"],\n",
        "                pos=(\n",
        "                    int(512 * layer[\"transform\"][\"position\"][\"x\"]),\n",
        "                    int(512 * layer[\"transform\"][\"position\"][\"y\"]),\n",
        "                ),\n",
        "                scale=512 * layer[\"transform\"][\"scale\"],\n",
        "                noise_strength=layer[\"noiseStrength\"],\n",
        "                rotation=layer[\"transform\"][\"rotation\"],\n",
        "                ftc=layer[\"wordEmbedding\"]\n",
        "                if layer[\"wordEmbedding\"] is not None\n",
        "                else None,\n",
        "            )\n",
        "\n",
        "            layer_list.append(image_layer)\n",
        "\n",
        "        ### Finetuning ###\n",
        "        new_token_options = [\n",
        "            \"<krk>\",\n",
        "            \"<zxz>\",\n",
        "            \"<fda>\",\n",
        "            \"<xtc>\",\n",
        "            \"<kyt>\",\n",
        "            \"<tlw>\",\n",
        "            \"<lxt>\",\n",
        "        ]\n",
        "        # Check if we need to finetune\n",
        "        textual_inversion = False\n",
        "        for layer in layers:\n",
        "            if layer[\"wordEmbedding\"] is not None:\n",
        "                textual_inversion = True\n",
        "                break\n",
        "\n",
        "        if textual_inversion:\n",
        "            # Modify full-image string\n",
        "            i = 0\n",
        "            for layer in layer_list:\n",
        "                if layer.ftc is not None:\n",
        "                    print(\"collage prompt: \", collage_prompt)\n",
        "                    print(f\"layer image str: {layer.image_str}\")\n",
        "                    match_token = max(\n",
        "                        collage_prompt.find(layer.image_str + \" \"),\n",
        "                        collage_prompt.find(layer.image_str + \",\"),\n",
        "                    )\n",
        "                    if match_token >= 0:\n",
        "                        print(\"found match\")\n",
        "                        collage_prompt = (\n",
        "                            collage_prompt[:match_token]\n",
        "                            + new_token_options[i]\n",
        "                            + \" \"\n",
        "                            + collage_prompt[match_token:]\n",
        "                        )\n",
        "                i += 1\n",
        "\n",
        "        ###\n",
        "        composite_image, mask_layers = ImageLayer.add_layers(layer_list, background=collage_src)\n",
        "        mask_pyramids = [\n",
        "            ImageLayer.get_pyramid(mask_layer, [64, 32, 16, 8], self.device)\n",
        "            for mask_layer in mask_layers\n",
        "        ]\n",
        "\n",
        "        ### Finetuning part 2 ###\n",
        "        if textual_inversion:\n",
        "            for i in range(len(layer_list)):\n",
        "                if layer_list[i].ftc is None:\n",
        "                    continue\n",
        "                print(\"Finetuning layer \", i)\n",
        "                learned_path = layer_list[i].ftc  # I'm treating ftc as a path here\n",
        "                load_learned_embed_in_clip(\n",
        "                    learned_path,\n",
        "                    self.text_encoder,\n",
        "                    self.tokenizer,\n",
        "                    new_token_options[i],\n",
        "                    idempotent=True,\n",
        "                )\n",
        "\n",
        "        ### TOKEN STUFF, TODO: pull out into another function ###\n",
        "        if use_ca:\n",
        "            # Tokenize collage prompt\n",
        "            prompt_tokens = self.tokenizer(\n",
        "                collage_prompt,\n",
        "                max_length=self.tokenizer.model_max_length,\n",
        "                truncation=True,\n",
        "                return_tensors=\"pt\",\n",
        "                return_overflowing_tokens=True,\n",
        "            ).input_ids[0]\n",
        "\n",
        "            mask_token_list = []\n",
        "\n",
        "            for layer in layer_list:\n",
        "                layer_tokens = self.tokenizer(\n",
        "                    layer.image_str,\n",
        "                    max_length=self.tokenizer.model_max_length,\n",
        "                    truncation=True,\n",
        "                    return_tensors=\"pt\",\n",
        "                    return_overflowing_tokens=True,\n",
        "                ).input_ids[0]\n",
        "                token_match = (prompt_tokens == layer_tokens[1]).nonzero()\n",
        "\n",
        "                if token_match.shape[0] == 0:\n",
        "                    continue\n",
        "\n",
        "                # String match\n",
        "                # TODO: pull out this logic into a separate function\n",
        "                match_index = 1\n",
        "                elim_set = set()\n",
        "                if token_match.shape[0] == 1:\n",
        "                    layer_token_index = (\n",
        "                        token_match.item()\n",
        "                    )  # Index with 1 to ignore start token\n",
        "                else:\n",
        "                    while len(elim_set) + 1 < token_match.shape[0]:\n",
        "                        for i in range(token_match.shape[0]):\n",
        "                            if (\n",
        "                                prompt_tokens[token_match[i] + match_index]\n",
        "                                != layer_tokens[1 + match_index]\n",
        "                            ):\n",
        "                                elim_set.add(i)\n",
        "                    for i in range(token_match.shape[0]):\n",
        "                        if i not in elim_set:\n",
        "                            layer_token_index = token_match[i]\n",
        "                            break\n",
        "                num_tokens = len(layer_tokens) - 2  # Remove start and end token\n",
        "                if textual_inversion:\n",
        "                    num_tokens += 1\n",
        "                    layer_token_index -= 1\n",
        "                mask_token_list.append(\n",
        "                    np.arange(layer_token_index, layer_token_index + num_tokens)\n",
        "                )\n",
        "\n",
        "            # Flattening lists\n",
        "            print(\"About to flatten lists\")\n",
        "            all_tokens = []\n",
        "            all_masks = []\n",
        "            all_strengths = []\n",
        "            all_negative_strengths = []\n",
        "            for i in range(len(mask_token_list)):\n",
        "                for j in range(len(mask_token_list[i])):\n",
        "                    all_tokens.append(mask_token_list[i][j])\n",
        "                    all_masks.append(mask_pyramids[i])\n",
        "                    all_strengths.append(w_pos[i])\n",
        "                    all_negative_strengths.append(w_neg[i])\n",
        "\n",
        "        ### Update attention for ca, TODO pull out into another function ###\n",
        "        if use_ca:\n",
        "            print(f\"{all_tokens=}, {all_strengths=}\")\n",
        "\n",
        "            def attention_mod(unet, sigma):\n",
        "                return init_attention_func_nvda(\n",
        "                    unet,\n",
        "                    all_tokens,\n",
        "                    all_masks,\n",
        "                    all_strengths,\n",
        "                    all_negative_strengths,\n",
        "                    sigma,\n",
        "                )\n",
        "\n",
        "        else:\n",
        "            attention_mod = None\n",
        "\n",
        "        return composite_image, mask_layers, attention_mod, collage_prompt\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def partial(\n",
        "        self,\n",
        "        prompt: Union[str, List[str]],\n",
        "        start_step: int,\n",
        "        end_step: int,\n",
        "        job_outdir: str,\n",
        "        save_all_steps: bool = False,\n",
        "        height: Optional[int] = None,\n",
        "        width: Optional[int] = None,\n",
        "        num_inference_steps: int = 50,\n",
        "        guidance_scale: float = 7.5,\n",
        "        negative_prompt: Optional[Union[str, List[str]]] = None,\n",
        "        num_images_per_prompt: Optional[int] = 1,\n",
        "        eta: float = 0.0,\n",
        "        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,\n",
        "        latents: Optional[torch.FloatTensor] = None,\n",
        "        output_type: Optional[str] = \"pil\",\n",
        "        return_dict: bool = True,\n",
        "        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,\n",
        "        callback_steps: Optional[int] = 1,\n",
        "    ):\n",
        "        r\"\"\"\n",
        "        Function invoked when calling the pipeline for generation.\n",
        "        Args:\n",
        "            prompt (`str` or `List[str]`):\n",
        "                The prompt or prompts to guide the image generation.\n",
        "            height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):\n",
        "                The height in pixels of the generated image.\n",
        "            width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):\n",
        "                The width in pixels of the generated image.\n",
        "            num_inference_steps (`int`, *optional*, defaults to 50):\n",
        "                The number of denoising steps. More denoising steps usually lead to a higher quality image at the\n",
        "                expense of slower inference.\n",
        "            guidance_scale (`float`, *optional*, defaults to 7.5):\n",
        "                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).\n",
        "                `guidance_scale` is defined as `w` of equation 2. of [Imagen\n",
        "                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >\n",
        "                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,\n",
        "                usually at the expense of lower image quality.\n",
        "            negative_prompt (`str` or `List[str]`, *optional*):\n",
        "                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored\n",
        "                if `guidance_scale` is less than `1`).\n",
        "            num_images_per_prompt (`int`, *optional*, defaults to 1):\n",
        "                The number of images to generate per prompt.\n",
        "            eta (`float`, *optional*, defaults to 0.0):\n",
        "                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to\n",
        "                [`schedulers.DDIMScheduler`], will be ignored for others.\n",
        "            generator (`torch.Generator`, *optional*):\n",
        "                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)\n",
        "                to make generation deterministic.\n",
        "            latents (`torch.FloatTensor`, *optional*):\n",
        "                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image\n",
        "                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents\n",
        "                tensor will ge generated by sampling using the supplied random `generator`.\n",
        "            output_type (`str`, *optional*, defaults to `\"pil\"`):\n",
        "                The output format of the generate image. Choose between\n",
        "                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.\n",
        "            return_dict (`bool`, *optional*, defaults to `True`):\n",
        "                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a\n",
        "                plain tuple.\n",
        "            callback (`Callable`, *optional*):\n",
        "                A function that will be called every `callback_steps` steps during inference. The function will be\n",
        "                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.\n",
        "            callback_steps (`int`, *optional*, defaults to 1):\n",
        "                The frequency at which the `callback` function will be called. If not specified, the callback will be\n",
        "                called at every step.\n",
        "        Returns:\n",
        "            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:\n",
        "            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.\n",
        "            When returning a tuple, the first element is a list with the generated images, and the second element is a\n",
        "            list of `bool`s denoting whether the corresponding generated image likely represents \"not-safe-for-work\"\n",
        "            (nsfw) content, according to the `safety_checker`.\n",
        "        \"\"\"\n",
        "        assert (\n",
        "            self.setup_complete is not None and self.setup_complete is True\n",
        "        ), \"Please call self.setup() before using this function.\"\n",
        "        self.reset_attentions()\n",
        "\n",
        "        # 0. Default height and width to unet\n",
        "        height = height or self.unet.config.sample_size * self.vae_scale_factor\n",
        "        width = width or self.unet.config.sample_size * self.vae_scale_factor\n",
        "\n",
        "        # 1. Check inputs. Raise error if not correct\n",
        "        # self.check_inputs(prompt, height, width, callback_steps)  # TODO see what the old function did (updated for collage)\n",
        "\n",
        "        # 2. Define call parameters\n",
        "        batch_size = 1 if isinstance(prompt, str) else len(prompt)\n",
        "        device = self._execution_device\n",
        "        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)\n",
        "        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`\n",
        "        # corresponds to doing no classifier free guidance.\n",
        "        do_classifier_free_guidance = guidance_scale > 1.0\n",
        "\n",
        "        # 3. Encode input prompt\n",
        "        text_embeddings = self._encode_prompt(\n",
        "            prompt,\n",
        "            device,\n",
        "            num_images_per_prompt,\n",
        "            do_classifier_free_guidance,\n",
        "            negative_prompt,\n",
        "        )\n",
        "\n",
        "        # 4. Prepare timesteps\n",
        "        self.scheduler.set_timesteps(num_inference_steps, device=device)\n",
        "        timesteps = self.scheduler.timesteps\n",
        "        subjob_timesteps = timesteps[start_step : end_step + 1]\n",
        "\n",
        "        # 5. Prepare latent variables\n",
        "        num_channels_latents = self.unet.in_channels\n",
        "\n",
        "        if start_step > 0:\n",
        "            self.logger.info(\n",
        "                f\"Loading in latents from step {start_step - 1} ({start_step} - {end_step}).\"\n",
        "            )\n",
        "            latents = torch.load(\n",
        "                os.path.join(\n",
        "                    job_outdir,\n",
        "                    f\"intermediate_{start_step - 1}.pt\",\n",
        "                )\n",
        "            )\n",
        "        else:\n",
        "            self.logger.info(f\"Preparing new latents for step 0.\")\n",
        "            latents = self.prepare_latents(\n",
        "                batch_size * num_images_per_prompt,\n",
        "                num_channels_latents,\n",
        "                height,\n",
        "                width,\n",
        "                text_embeddings.dtype,\n",
        "                device,\n",
        "                generator,\n",
        "                latents,\n",
        "            )\n",
        "\n",
        "        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline\n",
        "        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)\n",
        "\n",
        "        # 7. Denoising loop\n",
        "        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order\n",
        "        self.logger.info(\n",
        "            f\"Starting denoising loop for steps {start_step} to {end_step}.\"\n",
        "        )\n",
        "        with self.progress_bar(total=num_inference_steps) as progress_bar:\n",
        "            for i, t in enumerate(subjob_timesteps):\n",
        "                # expand the latents if we are doing classifier free guidance\n",
        "                latent_model_input = (\n",
        "                    torch.cat([latents] * 2) if do_classifier_free_guidance else latents\n",
        "                )\n",
        "                latent_model_input = self.scheduler.scale_model_input(\n",
        "                    latent_model_input, t\n",
        "                )\n",
        "\n",
        "                # predict the noise residual\n",
        "                noise_pred = self.unet(\n",
        "                    latent_model_input, t, encoder_hidden_states=text_embeddings\n",
        "                ).sample\n",
        "\n",
        "                # perform guidance\n",
        "                if do_classifier_free_guidance:\n",
        "                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)\n",
        "                    noise_pred = noise_pred_uncond + guidance_scale * (\n",
        "                        noise_pred_text - noise_pred_uncond\n",
        "                    )\n",
        "\n",
        "                # compute the previous noisy sample x_t -> x_t-1\n",
        "                latents = self.scheduler.step(\n",
        "                    noise_pred, t, latents, **extra_step_kwargs\n",
        "                ).prev_sample\n",
        "\n",
        "                # call the callback, if provided\n",
        "                if i == len(timesteps) - 1 or (\n",
        "                    (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0\n",
        "                ):\n",
        "                    progress_bar.update()\n",
        "                    if callback is not None and i % callback_steps == 0:\n",
        "                        callback(i, t, latents)\n",
        "\n",
        "                if save_all_steps:\n",
        "                    image = self.decode_latents(latents)\n",
        "                    self.numpy_to_pil(image)[0].save(\n",
        "                        os.path.join(job_outdir, f\"{start_step + i}.png\")\n",
        "                    )\n",
        "\n",
        "        torch.save(\n",
        "            latents,\n",
        "            os.path.join(job_outdir, f\"intermediate_{end_step}.pt\"),\n",
        "        )\n",
        "\n",
        "        # 8. Post-processing\n",
        "        image = self.decode_latents(latents)\n",
        "        self.numpy_to_pil(image)[0].save(os.path.join(job_outdir, f\"{end_step}.png\"))\n",
        "\n",
        "        # 9. Run safety checker\n",
        "        image, has_nsfw_concept = self.run_safety_checker(\n",
        "            image, device, text_embeddings.dtype\n",
        "        )\n",
        "\n",
        "        # 10. Convert to PIL\n",
        "        if output_type == \"pil\":\n",
        "            image = self.numpy_to_pil(image)\n",
        "\n",
        "        if not return_dict:\n",
        "            return (image, has_nsfw_concept)\n",
        "\n",
        "        return StableDiffusionPipelineOutput(\n",
        "            images=image, nsfw_content_detected=has_nsfw_concept\n",
        "        )\n",
        "\n",
        "    def get_timesteps(self, num_inference_steps, strength, device):\n",
        "        # get the original timestep using init_timestep\n",
        "        offset = self.scheduler.config.get(\"steps_offset\", 0)\n",
        "        init_timestep = int(num_inference_steps * strength) + offset\n",
        "        init_timestep = min(init_timestep, num_inference_steps)\n",
        "\n",
        "        t_start = max(num_inference_steps - init_timestep + offset, 0)\n",
        "        timesteps = self.scheduler.timesteps[t_start:]\n",
        "\n",
        "        return timesteps, num_inference_steps - t_start\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def collage(\n",
        "        self,\n",
        "        prompt: Union[str, List[str]],\n",
        "        image: Union[torch.FloatTensor, PIL.Image.Image],\n",
        "        mask_image: Union[torch.FloatTensor, PIL.Image.Image],\n",
        "        strength: float = 0.8,\n",
        "        num_inference_steps: Optional[int] = 50,\n",
        "        guidance_scale: Optional[float] = 7.5,\n",
        "        negative_prompt: Optional[Union[str, List[str]]] = None,\n",
        "        num_images_per_prompt: Optional[int] = 1,\n",
        "        eta: Optional[float] = 0.0,\n",
        "        generator: Optional[torch.Generator] = None,\n",
        "        output_type: Optional[str] = \"pil\",\n",
        "        return_dict: bool = True,\n",
        "        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,\n",
        "        callback_steps: Optional[int] = 1,\n",
        "        attention_mod: Optional[Callable] = None,\n",
        "        **kwargs,\n",
        "    ):\n",
        "        r\"\"\"\n",
        "        Function invoked when calling the pipeline for generation.\n",
        "        Hello\n",
        "\n",
        "        Args:\n",
        "            prompt (`str` or `List[str]`):\n",
        "                The prompt or prompts to guide the image generation.\n",
        "            image (`torch.FloatTensor` or `PIL.Image.Image`):\n",
        "                `Image`, or tensor representing an image batch, that will be used as the starting point for the\n",
        "                process. This is the image whose masked region will be inpainted.\n",
        "            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):\n",
        "                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be\n",
        "                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a\n",
        "                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should\n",
        "                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.\n",
        "            strength (`float`, *optional*, defaults to 0.8):\n",
        "                Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength`\n",
        "                is 1, the denoising process will be run on the masked area for the full number of iterations specified\n",
        "                in `num_inference_steps`. `image` will be used as a reference for the masked area, adding more noise to\n",
        "                that region the larger the `strength`. If `strength` is 0, no inpainting will occur.\n",
        "            num_inference_steps (`int`, *optional*, defaults to 50):\n",
        "                The reference number of denoising steps. More denoising steps usually lead to a higher quality image at\n",
        "                the expense of slower inference. This parameter will be modulated by `strength`, as explained above.\n",
        "            guidance_scale (`float`, *optional*, defaults to 7.5):\n",
        "                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).\n",
        "                `guidance_scale` is defined as `w` of equation 2. of [Imagen\n",
        "                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >\n",
        "                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,\n",
        "                usually at the expense of lower image quality.\n",
        "            negative_prompt (`str` or `List[str]`, *optional*):\n",
        "                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored\n",
        "                if `guidance_scale` is less than `1`).\n",
        "            num_images_per_prompt (`int`, *optional*, defaults to 1):\n",
        "                The number of images to generate per prompt.\n",
        "            eta (`float`, *optional*, defaults to 0.0):\n",
        "                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to\n",
        "                [`schedulers.DDIMScheduler`], will be ignored for others.\n",
        "            generator (`torch.Generator`, *optional*):\n",
        "                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation\n",
        "                deterministic.\n",
        "            output_type (`str`, *optional*, defaults to `\"pil\"`):\n",
        "                The output format of the generate image. Choose between\n",
        "                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.\n",
        "            return_dict (`bool`, *optional*, defaults to `True`):\n",
        "                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a\n",
        "                plain tuple.\n",
        "            callback (`Callable`, *optional*):\n",
        "                A function that will be called every `callback_steps` steps during inference. The function will be\n",
        "                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.\n",
        "            callback_steps (`int`, *optional*, defaults to 1):\n",
        "                The frequency at which the `callback` function will be called. If not specified, the callback will be\n",
        "                called at every step.\n",
        "\n",
        "        Returns:\n",
        "            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:\n",
        "            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.\n",
        "            When returning a tuple, the first element is a list with the generated images, and the second element is a\n",
        "            list of `bool`s denoting whether the corresponding generated image likely represents \"not-safe-for-work\"\n",
        "            (nsfw) content, according to the `safety_checker`.\n",
        "        \"\"\"\n",
        "        # message = \"Please use `image` instead of `init_image`.\"\n",
        "        # init_image = deprecate(\"init_image\", \"0.12.0\", message, take_from=kwargs)\n",
        "        # image = init_image or image\n",
        "\n",
        "        # 1. Check inputs\n",
        "        self.check_inputs(prompt, strength, callback_steps)\n",
        "\n",
        "        # 2. Define call parameters\n",
        "        batch_size = 1 if isinstance(prompt, str) else len(prompt)\n",
        "        device = self._execution_device\n",
        "        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)\n",
        "        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`\n",
        "        # corresponds to doing no classifier free guidance.\n",
        "        do_classifier_free_guidance = guidance_scale > 1.0\n",
        "\n",
        "        # 3. Encode input prompt\n",
        "        text_embeddings = self._encode_prompt(\n",
        "            prompt,\n",
        "            device,\n",
        "            num_images_per_prompt,\n",
        "            do_classifier_free_guidance,\n",
        "            negative_prompt,\n",
        "        )\n",
        "\n",
        "        # 4. Preprocess image and mask\n",
        "        if not isinstance(image, torch.FloatTensor):\n",
        "            image = preprocess_image(image)\n",
        "\n",
        "        if not isinstance(mask_image, torch.FloatTensor):\n",
        "            mask_image = preprocess_mask(mask_image, self.vae_scale_factor)\n",
        "\n",
        "        # 5. set timesteps\n",
        "        self.scheduler.set_timesteps(num_inference_steps, device=device)\n",
        "        timesteps, num_inference_steps = self.get_timesteps(\n",
        "            num_inference_steps, strength, device\n",
        "        )\n",
        "        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)\n",
        "\n",
        "        # 6. Prepare latent variables\n",
        "        # encode the init image into latents and scale the latents\n",
        "        latents, init_latents_orig, noise = self.prepare_latents_legacy(\n",
        "            image,\n",
        "            latent_timestep,\n",
        "            batch_size,\n",
        "            num_images_per_prompt,\n",
        "            text_embeddings.dtype,\n",
        "            device,\n",
        "            generator,\n",
        "        )\n",
        "\n",
        "        # 7. Prepare mask latent\n",
        "        mask = mask_image.to(device=self.device, dtype=latents.dtype)\n",
        "        mask = torch.cat([mask] * batch_size * num_images_per_prompt)\n",
        "\n",
        "        # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline\n",
        "        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)\n",
        "\n",
        "        # 9. Denoising loop\n",
        "        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order\n",
        "        with self.progress_bar(total=num_inference_steps) as progress_bar:\n",
        "            for i, t in enumerate(timesteps):\n",
        "                # expand the latents if we are doing classifier free guidance\n",
        "                latent_model_input = (\n",
        "                    torch.cat([latents] * 2) if do_classifier_free_guidance else latents\n",
        "                )\n",
        "                latent_model_input = self.scheduler.scale_model_input(\n",
        "                    latent_model_input, t\n",
        "                )\n",
        "\n",
        "                if attention_mod is not None:\n",
        "                    sigma = self.scheduler.sigmas[i]\n",
        "                    attention_mod(self.unet, sigma)\n",
        "\n",
        "                # predict the noise residual\n",
        "                noise_pred = self.unet(\n",
        "                    latent_model_input, t, encoder_hidden_states=text_embeddings\n",
        "                ).sample\n",
        "\n",
        "                # perform guidance\n",
        "                if do_classifier_free_guidance:\n",
        "                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)\n",
        "                    noise_pred = noise_pred_uncond + guidance_scale * (\n",
        "                        noise_pred_text - noise_pred_uncond\n",
        "                    )\n",
        "                # compute the previous noisy sample x_t -> x_t-1\n",
        "                latents = self.scheduler.step(\n",
        "                    noise_pred, t, latents, **extra_step_kwargs\n",
        "                ).prev_sample\n",
        "                # masking\n",
        "\n",
        "                noise = torch.randn(\n",
        "                    latents.shape,\n",
        "                    generator=generator,\n",
        "                    device=self.device,\n",
        "                    dtype=text_embeddings.dtype,\n",
        "                )\n",
        "                init_latents_proper = self.scheduler.add_noise(\n",
        "                    init_latents_orig, noise, torch.tensor([t])\n",
        "                )\n",
        "\n",
        "                mask_t = (mask > (1 - t/1000)).type(mask.dtype) # So when t is high, most of the mask is 1 (fixed), but when t is low, most of the mask is 0 (variable)\n",
        "                dilate_size = (int)(4*t/1000)\n",
        "                mask_t_dilated = torch.nn.functional.max_pool2d(mask_t, dilate_size*2 + 1, stride=1, padding=dilate_size)\n",
        "                latents = (init_latents_proper * mask_t_dilated) + (latents * (1 - mask_t_dilated))\n",
        "\n",
        "                # call the callback, if provided\n",
        "                if i == len(timesteps) - 1 or (\n",
        "                    (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0\n",
        "                ):\n",
        "                    progress_bar.update()\n",
        "                    if callback is not None and i % callback_steps == 0:\n",
        "                        callback(i, t, latents)\n",
        "\n",
        "        # 10. Post-processing\n",
        "        image = self.decode_latents(latents)\n",
        "\n",
        "        # 11. Run safety checker\n",
        "        image, has_nsfw_concept = self.run_safety_checker(\n",
        "            image, device, text_embeddings.dtype\n",
        "        )\n",
        "\n",
        "        # 12. Convert to PIL\n",
        "        if output_type == \"pil\":\n",
        "            image = self.numpy_to_pil(image)\n",
        "\n",
        "        if not return_dict:\n",
        "            return (image, has_nsfw_concept)\n",
        "\n",
        "        return StableDiffusionPipelineOutput(\n",
        "            images=image, nsfw_content_detected=has_nsfw_concept\n",
        "        )\n",
        "\n",
        "    def set_logger(self, logger):\n",
        "        self.logger = logger\n",
        "\n",
        "    def setup(self):\n",
        "        # Saving attention\n",
        "        self.original_attentions = {}\n",
        "        for name, module in self.unet.named_modules():\n",
        "            module_name = type(module).__name__\n",
        "            if module_name == \"CrossAttention\" and \"attn2\" in name:\n",
        "                self.original_attentions[name] = copy.deepcopy(module)\n",
        "        self.setup_complete = True\n",
        "\n",
        "    def reset_attentions(self):\n",
        "        assert (\n",
        "            self.setup_complete is not None and self.setup_complete is True\n",
        "        ), \"Please call self.setup() before using this function.\"\n",
        "        for name, module in self.unet.named_modules():\n",
        "            module_name = type(module).__name__\n",
        "            if module_name == \"CrossAttention\" and \"attn2\" in name:\n",
        "                module = copy.deepcopy(self.original_attentions[name])"
      ],
      "id": "mZ5ZnykndYYW"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7a07646d"
      },
      "outputs": [],
      "source": [
        "new_token_options = [\"<krk>\",\"<lol>\",\"<fda>\",\"<xtc>\",\"<kyt>\",\"<tlw>\",\"<lxt>\",\"<xlw>\",\"<xzv>\",\"<vxx>\",\"<xvc>\"]\n",
        "model_path_diffusion = \"stabilityai/stable-diffusion-2-1-base\"\n",
        "\n",
        "def all_collage(experiment_name, collage_prompt, layer_list, ca, textual_inversion, inversion_step_list, device_id, input_seed,\n",
        "                w_pos, w_neg, t_is, img2img_strength=0.7, num_results=10, collage_mask = None, guidance_scale = 7.5, \n",
        "                orig_sched=None, show_all=False, noise_blur=10, negative_prompt=\"\",\n",
        "               seed_offset=0):\n",
        "    device = \"cuda:\" + str(device_id)\n",
        "    \n",
        "    #############################\n",
        "    # Step 1: textual inversion #\n",
        "    #############################\n",
        "    \n",
        "    collage_prompt = collage_prompt + \" \"\n",
        "    collage_prompt_orig = collage_prompt + \"\"\n",
        "    if textual_inversion:\n",
        "        tune_step = np.max(inversion_step_list)\n",
        "        print(tune_step)\n",
        "        # First check that all the desired models have been generated, run if necessary\n",
        "        for layer in layer_list:\n",
        "            if layer.ftc is not None:\n",
        "                if not layer.ftc.training_complete(tune_step):   \n",
        "                    raise Exception(\"Fine tuning not complete at step\", tune_step)\n",
        "        # Modify full-image string\n",
        "        i = 0\n",
        "        for layer in layer_list:\n",
        "            if layer.ftc is not None:\n",
        "                match_token = max(collage_prompt.find(layer.image_str + \" \"),collage_prompt.find(layer.image_str + \",\"))\n",
        "                if match_token >= 0:\n",
        "                    collage_prompt = collage_prompt[:match_token] + new_token_options[i] + \" \" + collage_prompt[match_token:]\n",
        "            i += 1\n",
        "            print(\"Collage prompt: \", collage_prompt)\n",
        "    \n",
        "    ###################################\n",
        "    # Step 2,3: image seed, ca masks #\n",
        "    ###################################\n",
        "    layer_composite, mask_layers = ImageLayer.add_layers(layer_list)\n",
        "    \n",
        "    # Build collage_mask here\n",
        "    if t_is is not None:\n",
        "        collage_mask = Image.new(\"L\", (512,512), ((int)(img2img_strength * 255)))\n",
        "        for i in range(len(t_is)):\n",
        "            layer_image = Image.fromarray((mask_layers[i]*t_is[i]*255).astype(np.uint8), 'L')\n",
        "            mask_image = Image.fromarray((mask_layers[i]*255).astype(np.uint8), 'L')\n",
        "            collage_mask.paste(layer_image, (0,0), mask_image)\n",
        "        collage_mask = collage_mask.filter(ImageFilter.GaussianBlur(noise_blur))\n",
        "    display(collage_mask)\n",
        "    \n",
        "    if input_seed is None:\n",
        "        input_seed = layer_composite\n",
        "    if ca:\n",
        "        mask_pyramids = [ImageLayer.get_pyramid(mask_layer, [64,32,16,8], device) for mask_layer in mask_layers]\n",
        "    \n",
        "    ###############################\n",
        "    # Step 4: token mapping       #\n",
        "    # We also load the model here #\n",
        "    ###############################\n",
        "    pipe = StableDiffusionPipeline.from_pretrained(model_path_diffusion, torch_dtype=torch.float16).to(\n",
        "        device\n",
        "    )\n",
        "    pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)\n",
        "    if textual_inversion:\n",
        "        for i in range(len(layer_list)):\n",
        "            if layer_list[i].ftc is None:\n",
        "                continue\n",
        "            learned_path = layer_list[i].ftc.get_ckpt_path(inversion_step_list[i])\n",
        "            print(\"learned path\", learned_path, \"step\", inversion_step_list[i])\n",
        "            load_learned_embed_in_clip(\n",
        "                learned_path,\n",
        "                pipe.text_encoder,\n",
        "                pipe.tokenizer,\n",
        "                new_token_options[i],\n",
        "                idempotent=True,\n",
        "            )\n",
        "    # With tokenizer updated, we can use the tokenizer to get token positions for everything\n",
        "    if ca:\n",
        "        prompt_tokens = get_tokens(pipe, collage_prompt)\n",
        "        mask_token_list = []\n",
        "        for layer in layer_list:\n",
        "            layer_tokens = get_tokens(pipe, layer.image_str)\n",
        "            token_match = (prompt_tokens == layer_tokens[1]).nonzero()\n",
        "            if token_match.shape[0] == 0:\n",
        "                continue\n",
        "            token_options = token_match\n",
        "            match_index = 1\n",
        "            elim_set = set()\n",
        "            if token_match.shape[0] == 1:\n",
        "                layer_token_index = token_match.item() # Index with 1 to ignore start token\n",
        "            else:\n",
        "                while len(elim_set) + 1 < token_match.shape[0]:\n",
        "                    for i in range(token_match.shape[0]):\n",
        "                        if prompt_tokens[token_match[i] + match_index] != layer_tokens[1 + match_index]:\n",
        "                            elim_set.add(i)\n",
        "                for i in range(token_match.shape[0]):\n",
        "                    if i not in elim_set:\n",
        "                        layer_token_index = token_match[i]\n",
        "                        break\n",
        "            num_tokens = len(layer_tokens) - 2 # Remove start and end token\n",
        "            if textual_inversion:\n",
        "                num_tokens += 1\n",
        "                layer_token_index -= 1\n",
        "            mask_token_list.append(np.arange(layer_token_index, layer_token_index + num_tokens))\n",
        "\n",
        "        old_style_boxes = [ImageLayer.get_bbox_from_mask(mask_layer) for mask_layer in mask_layers]\n",
        "\n",
        "        all_tokens = []\n",
        "        all_masks = []\n",
        "        all_strengths = []\n",
        "        all_negative_strengths = []\n",
        "        all_old_boxes = []\n",
        "        for i in range(len(mask_token_list)):\n",
        "            for j in range(len(mask_token_list[i])):\n",
        "                all_tokens.append(mask_token_list[i][j])\n",
        "                all_masks.append(mask_pyramids[i])\n",
        "                all_strengths.append(w_pos[i])\n",
        "                all_negative_strengths.append(w_neg[i])\n",
        "                all_old_boxes.append(old_style_boxes[i])\n",
        "\n",
        "        \n",
        "        print(\"All tokens\", all_tokens, [prompt_token(pipe, collage_prompt, index) for index in all_tokens])\n",
        "        print(\"All strengths\", all_strengths)\n",
        "            \n",
        "    display(input_seed)\n",
        "            \n",
        "    # Update attention if ca\n",
        "    if ca:\n",
        "        if orig_sched is None:\n",
        "            orig_sched = False\n",
        "        def attention_mod(unet, sigma):\n",
        "            return init_attention_func_nvda(unet, all_tokens, all_masks, all_strengths, all_negative_strengths, sigma)\n",
        "    else:\n",
        "        attention_mod = None\n",
        "        \n",
        "    return_list = []\n",
        "    if collage_mask is not None:\n",
        "        for i in range(num_results):\n",
        "            torch.manual_seed(i + seed_offset)\n",
        "            image = pipe.collage(prompt=collage_prompt, image=input_seed, mask_image=collage_mask, strength=img2img_strength, guidance_scale=guidance_scale, negative_prompt=negative_prompt, attention_mod=attention_mod).images[0]\n",
        "            if show_all:\n",
        "                display(image)\n",
        "            return_list.append(image)\n",
        "    # Time to save images in return_list\n",
        "    '''\n",
        "    save_root = \"./\"\n",
        "    text_flag = \"img2img\"\n",
        "    ca_flag = \"with_ca\" if ca else \"no_ca\"\n",
        "    ft_flag = \"with_ft\" if textual_inversion else \"no_ft\"\n",
        "    mask_flag = \"with_mask\" if collage_mask else \"no_mask\"\n",
        "    save_dir = save_root + experiment_name + \"/\" + \"-\".join([text_flag, ca_flag, ft_flag, mask_flag]) + \"/\"\n",
        "    os.makedirs(save_dir, exist_ok=True)\n",
        "    for i in range(num_results):\n",
        "        return_list[i].save(save_dir + str(i) + \".jpg\")\n",
        "    '''\n",
        "    return return_list"
      ],
      "id": "7a07646d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "34d2019f"
      },
      "outputs": [],
      "source": [
        "def get_tokens(pipe, prompt): # No pad\n",
        "    return pipe.tokenizer(prompt, max_length=pipe.tokenizer.model_max_length, truncation=True, return_tensors=\"pt\", return_overflowing_tokens=True).input_ids[0]\n",
        "\n",
        "def prompt_token(pipe, prompt, index):\n",
        "    tokens = pipe.tokenizer(prompt, padding=\"max_length\", max_length=pipe.tokenizer.model_max_length, truncation=True, return_tensors=\"pt\", return_overflowing_tokens=True).input_ids[0]\n",
        "    return pipe.tokenizer.decode(tokens[index:index+1])"
      ],
      "id": "34d2019f"
    },
    {
      "cell_type": "code",
      "source": [
        "def import_scene(curr_scene):\n",
        "  image_dir = scene_dir + curr_scene + \"/layers/\"\n",
        "  source_files = os.listdir(image_dir)\n",
        "\n",
        "  source_files = [file if file.find('.png') >= 0 else None for file in source_files]\n",
        "\n",
        "  source_files = sorted([file for file in source_files if file is not None])[::-1]\n",
        "\n",
        "  layer_list = []\n",
        "  alt = False\n",
        "  alt_elem = None\n",
        "  for layer_id in np.arange(len(source_files)):\n",
        "      layer_str = (\" \".join(source_files[layer_id].split('_')[2:])).split('.')[0].replace('-',' ')\n",
        "      layer_str_und = (\"_\".join(source_files[layer_id].split('_')[2:])).split('.')[0].replace('-',' ')\n",
        "      if \"alt\" in layer_str:\n",
        "          alt_elem = layer_str[:-4]\n",
        "          layer_str = alt_elem\n",
        "          if not alt:\n",
        "              continue\n",
        "      else:\n",
        "          if alt_elem is not None and alt and layer_str == alt_elem:\n",
        "              continue\n",
        "      layer_image = Image.open(image_dir + source_files[layer_id]).convert(\"RGBA\")\n",
        "      ftc = FineTuneConcept(curr_scene + \"-\" + layer_str_und, None, None, None)\n",
        "      if not ftc.training_complete(800):\n",
        "          print(\"Not complete: \", layer_str)\n",
        "          ftc = None\n",
        "      layer_list.append(ImageLayer(layer_image,(0,0),layer_str, ftc = ftc))\n",
        "\n",
        "  print([layer.image_str for layer in layer_list])\n",
        "  return layer_list"
      ],
      "metadata": {
        "id": "JzHMI2rNOU7q"
      },
      "id": "JzHMI2rNOU7q",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "afe3db09"
      },
      "source": [
        "#Toy Box Scene"
      ],
      "id": "afe3db09"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fcdd5841"
      },
      "outputs": [],
      "source": [
        "layer_list = import_scene(\"toyBox2\")"
      ],
      "id": "fcdd5841"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d9a2656b"
      },
      "outputs": [],
      "source": [
        "layer_list[1].image_str = \"american football\"\n",
        "layer_list[1].ftc = None"
      ],
      "id": "d9a2656b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "032970e9"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"a teddy bear, a wood train, and an american football, in front of a tan background\""
      ],
      "id": "032970e9"
    },
    {
      "cell_type": "markdown",
      "source": [
        "Key parameters: \"collage prompt\" is the overall collage text string, containing text corresponding to each layer. ca=True indicates that we manipulate cross-attention layers, and w_pos and w_neg govern the weights for manipulating cross-attention. textual_inversion=True indicates that we use learned representations per layer, and inversion_step_list indicates the checkpoint step from inversion used for each layer. t_is indicates the noise level $t_i$ added to each layer. "
      ],
      "metadata": {
        "id": "CHwes8g64Gdo"
      },
      "id": "CHwes8g64Gdo"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3df61161",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images = all_collage(\"toyBox2_step1\", collage_prompt, layer_list, ca=True, textual_inversion=True, inversion_step_list=[1000]*4, device_id=0, \n",
        "            input_seed=None, w_pos = [0.0,0.5,0.2,0.2], w_neg = [0.5,0.5,0.5,0.5], img2img_strength=0.8, show_all=True,\n",
        "                         t_is = [0.8,0.2,0.8,0.5], noise_blur=20, negative_prompt=\"collage\")"
      ],
      "id": "3df61161"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c0ebeffd"
      },
      "source": [
        "## I like image 3, but update the train"
      ],
      "id": "c0ebeffd"
    },
    {
      "cell_type": "code",
      "source": [
        "len(room_images)"
      ],
      "metadata": {
        "id": "sYT8eiQ2Wm66"
      },
      "id": "sYT8eiQ2Wm66",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We create a new collage, where the first layer is the image we are editing, and the second layer re-pastes the content from the original collage for the desired layer to change. The $t_i$ values are set appropriately to allow minimal change in the background layer but a high level of change in the foreground layer. "
      ],
      "metadata": {
        "id": "q4D9SiyQ5HMi"
      },
      "id": "q4D9SiyQ5HMi"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3af82138"
      },
      "outputs": [],
      "source": [
        "curr_image = room_images[3]\n",
        "curr_image.putalpha(255)\n",
        "layer_list_2 = [ImageLayer(curr_image,(0,0),\"teddy bear, and an american football, in front of a tan background\"),\n",
        "             layer_list[3]]"
      ],
      "id": "3af82138"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cd15dc40"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"a wood train, a teddy bear, and an american football, in front of a tan background\""
      ],
      "id": "cd15dc40"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "493b1055",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images_2 = all_collage(\"toyBox2_step2\", collage_prompt, layer_list_2, ca=True, textual_inversion=True, inversion_step_list=[1000]*2, device_id=0, \n",
        "            input_seed=None, w_pos = [0.0,0.0], w_neg = [0.2,0.0], img2img_strength=0.8, show_all=True,\n",
        "                         t_is = [0.05,1.0], noise_blur=50, negative_prompt=\"collage, warped\")"
      ],
      "id": "493b1055"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aa2ec569"
      },
      "source": [
        "# Bento Box Scene"
      ],
      "id": "aa2ec569"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dca1bba2"
      },
      "outputs": [],
      "source": [
        "layer_list = import_scene(\"bentoBox\")"
      ],
      "id": "dca1bba2"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b8e11ba8"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"a bento box with rice, edamame, ginger, and sushi\""
      ],
      "id": "b8e11ba8"
    },
    {
      "cell_type": "markdown",
      "source": [
        "Key parameters: \"collage prompt\" is the overall collage text string, containing text corresponding to each layer. ca=True indicates that we manipulate cross-attention layers, and w_pos and w_neg govern the weights for manipulating cross-attention. textual_inversion=True indicates that we use learned representations per layer, and inversion_step_list indicates the checkpoint step from inversion used for each layer. t_is indicates the noise level $t_i$ added to each layer. "
      ],
      "metadata": {
        "id": "CIbNFQI-46zp"
      },
      "id": "CIbNFQI-46zp"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d1aa6fba",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images = all_collage(\"bentoBox_step1final\", collage_prompt, layer_list, ca=True, textual_inversion=True, inversion_step_list=[1000,500,1000,1000,1000], device_id=0, \n",
        "            input_seed=None, w_pos = [0.0,0.4,0.0,0.6,0.2], w_neg = [0.5,0.5,0.5,0.5,0.5], img2img_strength=0.8, show_all=True,\n",
        "                         t_is=[0.5,0.5,0.6,0.8,0.8], noise_blur=20, negative_prompt=\"collage\")"
      ],
      "id": "d1aa6fba"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5b37512d"
      },
      "source": [
        "## Everything except sushi great in image 5: let's try variants"
      ],
      "id": "5b37512d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e591f580"
      },
      "outputs": [],
      "source": [
        "curr_image = room_images[5]\n",
        "curr_image.putalpha(255)\n",
        "layer_list_2 = [ImageLayer(curr_image,(0,0),\"bento box with rice, edamame, ginger\"),\n",
        "             layer_list[4]]"
      ],
      "id": "e591f580"
    },
    {
      "cell_type": "code",
      "source": [
        "collage_prompt = \"a bento box with rice, edamame, ginger, and sushi\""
      ],
      "metadata": {
        "id": "B7oWUzIGTRwr"
      },
      "id": "B7oWUzIGTRwr",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We create a new collage, where the first layer is the image we are editing, and the second layer re-pastes the content from the original collage for the desired layer to change. The $t_i$ values are set appropriately to allow minimal change in the background layer but a high level of change in the foreground layer. "
      ],
      "metadata": {
        "id": "J2WSqnTM5b_T"
      },
      "id": "J2WSqnTM5b_T"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0c2152f3",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images_2 = all_collage(\"bentoBox_step2final\", collage_prompt, layer_list_2, ca=True, textual_inversion=True, inversion_step_list=[1000]*2, device_id=0, \n",
        "            input_seed=None, w_pos = [0.0,0.2], w_neg = [0.2,0.0], img2img_strength=0.8, show_all=True,\n",
        "                         t_is=[0.05,1.0], noise_blur=30, negative_prompt=\"collage, warped, ugly\")"
      ],
      "id": "0c2152f3"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8bef5631"
      },
      "source": [
        "## Now swap the ginger"
      ],
      "id": "8bef5631"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0e5c17dd"
      },
      "outputs": [],
      "source": [
        "curr_image = room_images_2[2]\n",
        "curr_image.putalpha(255)\n",
        "layer_list_3 = [ImageLayer(curr_image,(0,0),\"bento box with rice, edamame, sushi\"),\n",
        "             layer_list[3]]"
      ],
      "id": "0e5c17dd"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bf0da473"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"a bento box with rice, edamame, sushi, and ginger\""
      ],
      "id": "bf0da473"
    },
    {
      "cell_type": "markdown",
      "source": [
        "We create a new collage, where the first layer is the image we are editing, and the second layer re-pastes the content from the original collage for the desired layer to change. The $t_i$ values are set appropriately to allow minimal change in the background layer but a high level of change in the foreground layer. "
      ],
      "metadata": {
        "id": "1ZE4Rj785d__"
      },
      "id": "1ZE4Rj785d__"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "49d0f9b3",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images_3 = all_collage(\"bentoBox_step3final\", collage_prompt, layer_list_3, ca=True, textual_inversion=True, inversion_step_list=[1000]*2, device_id=0, \n",
        "            input_seed=None, w_pos = [0.0,0.5], w_neg = [0.5,0.0], img2img_strength=0.7, show_all=True,\n",
        "                         t_is=[0.05,1.0], noise_blur=30, negative_prompt=\"collage\")"
      ],
      "id": "49d0f9b3"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5f3ce8da"
      },
      "source": [
        "# Cake Scene"
      ],
      "id": "5f3ce8da"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e11be8d8"
      },
      "outputs": [],
      "source": [
        "layer_list = import_scene(\"cakeTable2\")"
      ],
      "id": "e11be8d8"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fd0042b9"
      },
      "outputs": [],
      "source": [
        "layer_list[4].image_str = \"pink cakes\""
      ],
      "id": "fd0042b9"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "01c11c9b"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"a wood table with two white chairs behind, two pink cakes on top, maroon bookshelves behind, and winter window\""
      ],
      "id": "01c11c9b"
    },
    {
      "cell_type": "markdown",
      "source": [
        "Key parameters: \"collage prompt\" is the overall collage text string, containing text corresponding to each layer. ca=True indicates that we manipulate cross-attention layers, and w_pos and w_neg govern the weights for manipulating cross-attention. textual_inversion=True indicates that we use learned representations per layer, and inversion_step_list indicates the checkpoint step from inversion used for each layer. t_is indicates the noise level $t_i$ added to each layer. "
      ],
      "metadata": {
        "id": "alxGS-JV5CVV"
      },
      "id": "alxGS-JV5CVV"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9b47cb13",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images = all_collage(\"cakeTable3_step1final\", collage_prompt, layer_list, ca=True, textual_inversion=True, inversion_step_list=[500,1000,1000,1000,100], device_id=0, \n",
        "            input_seed=None, w_pos = [0.45,0.35,0.5,0.1,0.3], w_neg = [1.0,1.0,1.0,1.0,2.0], img2img_strength=0.75, show_all=True,\n",
        "                         t_is=[0.8,0.8,0.8,0.6,0.6], noise_blur=10, negative_prompt=\"collage, warped, cupcake\")"
      ],
      "id": "9b47cb13"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "92515464"
      },
      "source": [
        "## Let's build on image 1"
      ],
      "id": "92515464"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "440f923e"
      },
      "outputs": [],
      "source": [
        "curr_image = room_images[5]\n",
        "curr_image.putalpha(255)\n",
        "layer_list_2 = [ImageLayer(curr_image,(0,0),\"wood table with two white chairs behind, maroon bookshelves behind, and winter window\"),\n",
        "             layer_list[4]]"
      ],
      "id": "440f923e"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "754ec7ac"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"two pink cakes on a wood table with two white chairs behind, maroon bookshelves behind, and winter window \""
      ],
      "id": "754ec7ac"
    },
    {
      "cell_type": "markdown",
      "source": [
        "We create a new collage, where the first layer is the image we are editing, and the second layer re-pastes the content from the original collage for the desired layer to change. The $t_i$ values are set appropriately to allow minimal change in the background layer but a high level of change in the foreground layer. "
      ],
      "metadata": {
        "id": "r1o6JAkI5ssN"
      },
      "id": "r1o6JAkI5ssN"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f4cfc5b6",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images_2 = all_collage(\"cakeTable3_step2final\", collage_prompt, layer_list_2, ca=True, textual_inversion=True, inversion_step_list=[100]*2, device_id=0, \n",
        "            input_seed=None, w_pos = [0.0,0.0], w_neg = [0.5,0.0], img2img_strength=0.75, show_all=True,\n",
        "                         t_is=[0.05,1.0], noise_blur=20, negative_prompt=\"collage, warped\", seed_offset=0)"
      ],
      "id": "f4cfc5b6"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "370be18e"
      },
      "source": [
        "## Now let's swap the window"
      ],
      "id": "370be18e"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "636179e5"
      },
      "outputs": [],
      "source": [
        "curr_image = room_images_2[6]\n",
        "_, masks = ImageLayer.add_layers(layer_list)\n",
        "blurred_mask = np.array(Image.fromarray((masks[0]*255).astype(np.uint8), 'L').filter(ImageFilter.GaussianBlur(10)))/255.\n",
        "curr_image.putalpha(255)\n",
        "curr_np = np.array(curr_image)/255.\n",
        "curr_np[:,:,3] *= (1.0 - (blurred_mask > 0.6))\n",
        "curr_image = Image.fromarray((curr_np*255).astype(np.uint8), 'RGBA')"
      ],
      "id": "636179e5"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "70dccf8c"
      },
      "outputs": [],
      "source": [
        "curr_image"
      ],
      "id": "70dccf8c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6d416542"
      },
      "outputs": [],
      "source": [
        "layer_list_3 = [layer_list[0], ImageLayer(curr_image,(0,0),\"two pink decorated cakes on a wood table with two white chairs behind, maroon bookshelf behind,\")]"
      ],
      "id": "6d416542"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a7455a62"
      },
      "outputs": [],
      "source": [
        "collage_prompt = \"two pink decorated cakes on a wood table with two white chairs behind, maroon bookshelf behind, and winter window\""
      ],
      "id": "a7455a62"
    },
    {
      "cell_type": "markdown",
      "source": [
        "We create a new collage, where the first layer is the image we are editing, and the second layer re-pastes the content from the original collage for the desired layer to change. The $t_i$ values are set appropriately to allow minimal change in the background layer but a high level of change in the foreground layer. "
      ],
      "metadata": {
        "id": "_qfugkjT51Uc"
      },
      "id": "_qfugkjT51Uc"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "861fd526",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "room_images_3 = all_collage(\"cakeTable3_step3final\", collage_prompt, layer_list_3, ca=True, textual_inversion=True, inversion_step_list=[500,500], device_id=0, \n",
        "            input_seed=None, w_pos = [0.4,0.0], w_neg = [0.0,0.0], img2img_strength=0.75, show_all=True,\n",
        "                         t_is=[1.0,0.05], noise_blur=10, negative_prompt=\"collage, warped\", seed_offset=10)"
      ],
      "id": "861fd526"
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python [conda env:sd2_fresher]",
      "language": "python",
      "name": "conda-env-sd2_fresher-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}