Doohickey_Beta.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Doohickey V0.3.0 (BETA)\n",
        "---\n",
        "by [aicrumb](https://twitter.com/aicrumb) <br>\n",
        "\n",
        "Wait for the first cell to finish to log in, then you can run the rest. Check out [johnowhitaker](https://twitter.com/johnowhitaker)'s \"Grokking Stable Diffusion\" to see how sampling from Stable Diffusion works in more detail. It helped a lot in the development of this notebook. <br>\n",
        "**warning** *messy code ahead, model juggling*\n",
        "\n",
        "I wanna see what people make! if you use this feel free to tag me"
      ],
      "metadata": {
        "id": "HZfxL5A9rA7W"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "5dbNsZ38fPsy",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 290,
          "referenced_widgets": [
            "04be0d7f1bda49e1bcba2699e48e50b9",
            "e86b6f1f629b4db49c25cd33937bdf05",
            "732cfbfe6ac54755a71d6021cc6c5716",
            "d3d226f37ecc4d3aa304de0880c4fb49",
            "3f036597057c4a729b5de8515f9201d5",
            "713e1c4bf02f47e0a94859ccbf9f2487",
            "6d74176a5e4d4bbe801c233abd841be3",
            "1786a074991f4d9dbc5717c2503b191d",
            "149213d3c2aa4632bbadee0fc129eea0",
            "297105181dbb4a7b8c1f652d0901d783",
            "d79c203fa65a45098e2673293d693612",
            "373ae7d7fc2547d0aa1b79113322b61f",
            "90f35f1fa0ee4f0f89e707fd05695fca",
            "d7245bf56f424e0e811b65f2b957b1c1"
          ]
        },
        "outputId": "82c67447-f114-430d-d6ca-929946abd4bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Libraries already installed.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "04be0d7f1bda49e1bcba2699e48e50b9"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "#@title Install required libraries / Log in to the 🤗 hub\n",
        "import os\n",
        "from IPython.display import clear_output\n",
        "import time\n",
        "if not os.path.exists(\"installed.txt\"):\n",
        "    # red lines, it's fines, that's what i always say\n",
        "    !pip install transformers diffusers lpips -q\n",
        "    # !pip install git+https://github.com/openai/CLIP -q\n",
        "    !pip install open_clip_torch -q\n",
        "    !pip install wget -q\n",
        "    !sudo apt-get install git-lfs\n",
        "    !cat \"test\" > installed.txt\n",
        "    !mkdir /content/output\n",
        "    print(\"Installed libraries\")\n",
        "    time.sleep(1) # just so that the user can see a glimpse of the print to know it went succesfuly\n",
        "    clear_output(wait=False)\n",
        "else:\n",
        "    print(\"Libraries already installed.\")\n",
        "\n",
        "#@markdown Base stable diffusion is \"CompVis/stable-diffusion-v1-4\". <br>\n",
        "#@markdown CompVis type (.ckpt files) should be loaded as \"user/id/filename\" (on huggingface)\n",
        "model_name = \"runwayml/stable-diffusion-v1-5\" #@param {\"type\":\"string\"}\n",
        "model_type = \"compvis\" if \".ckpt\" in model_name else \"diffusers\"\n",
        "\n",
        "traced = False \n",
        "# i'll add some traced models as time goes on, i'm just focused on getting the code working right now\n",
        "\n",
        "unet_path = None\n",
        "\n",
        "#@markdown not tested yet because i needed to push a fix really fast\n",
        "token = \"\" #@param {type:\"string\"}\n",
        "\n",
        "if token==\"\":\n",
        "    from huggingface_hub import notebook_login\n",
        "    notebook_login()\n",
        "else:\n",
        "    !mkdir -p ~/.huggingface\n",
        "    !echo -n \"$token\" > ~/.huggingface/token"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setup"
      ],
      "metadata": {
        "id": "0mKBMENAXu7F"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "5jxDUlP7fyld",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Import libraries\n",
        "import torch\n",
        "torch.manual_seed(0)\n",
        "from transformers import CLIPTextModel, CLIPTokenizer\n",
        "from diffusers import AutoencoderKL, UNet2DConditionModel\n",
        "from diffusers import LMSDiscreteScheduler, DDIMScheduler, KarrasVeScheduler, PNDMScheduler, DDPMScheduler\n",
        "from IPython.display import Image, display\n",
        "from tqdm.auto import tqdm, trange\n",
        "from torch import autocast\n",
        "import PIL.Image as PImage\n",
        "import numpy\n",
        "from torchvision import transforms\n",
        "import torchvision.transforms.functional as f\n",
        "import random\n",
        "import requests\n",
        "from io import BytesIO\n",
        "\n",
        "# Set device\n",
        "torch_device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "offload_device = \"cpu\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load Models / Enable Attention Slicing\n",
        "\n",
        "#@markdown <details> attention slicing makes it so that, in pipelines, generating only requries 3.2GB of vram, at a 10% speed decrease <br>\n",
        "#@markdown reported here: <a href=\"https://huggingface.co/blog/diffusers-2nd-month#optimizations-for-smaller-gpus\">link</a> <br>\n",
        "#@markdown attention slicing is enabled by default on the traced models <br>\n",
        "#@markdown `slice_size` integer 1-8\n",
        "\n",
        "# default behavior\n",
        "delete_previous_model = True #@param {\"type\":\"boolean\"}\n",
        "if delete_previous_model:\n",
        "    !rm -r /content/output-model\n",
        "\n",
        "if model_type == \"compvis\" and not os.path.exists(\"/content/output-model\"):\n",
        "    repo_id = \"/\".join(model_name.split(\"/\")[:2])\n",
        "    filename = \"/\".join(model_name.split(\"/\")[2:])\n",
        "\n",
        "    from huggingface_hub import hf_hub_download\n",
        "    print(\"downloading model...\")\n",
        "    compvis_path = hf_hub_download(repo_id=repo_id, filename=filename)\n",
        "    print(\"downloading conversion scripts...\")\n",
        "    !pip install OmegaConf -q\n",
        "    !curl https://raw.githubusercontent.com/huggingface/diffusers/main/scripts/convert_original_stable_diffusion_to_diffusers.py > convert.py\n",
        "    print(\"creating diffusers-style model... (this will take a while)\")\n",
        "    !python convert.py --checkpoint_path \"$compvis_path\" --dump_path \"output-model\"\n",
        "else:\n",
        "    print(\"Model already downloaded!\")\n",
        "\n",
        "if model_type==\"compvis\":\n",
        "    model_name = \"output-model\"\n",
        "\n",
        "\n",
        "attention_slicing = True #@param {\"type\":\"boolean\"}\n",
        "# slicing_factor = 12 #@param\n",
        "vae = AutoencoderKL.from_pretrained(model_name, subfolder=\"vae\", use_auth_token=True)\n",
        "\n",
        "\n",
        "try:\n",
        "    tokenizer = CLIPTokenizer.from_pretrained(model_name, subfolder=\"tokenizer\")\n",
        "    text_encoder = CLIPTextModel.from_pretrained(model_name, subfolder=\"text_encoder\", use_auth_token=True)\n",
        "except:\n",
        "    print(\"Text encoder could not be loaded from the repo specified for some reason, falling back to the vit-l repo\")\n",
        "    text_encoder = CLIPTextModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
        "    tokenizer = CLIPTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
        "if unet_path!=None:\n",
        "    # unet = UNet2DConditionModel.from_pretrained(unet_path)\n",
        "    from huggingface_hub import hf_hub_download\n",
        "    model_name = hf_hub_download(repo_id=unet_path, filename=\"unet.pt\")\n",
        "    unet = torch.jit.load(model_name)\n",
        "else:\n",
        "    unet = UNet2DConditionModel.from_pretrained(model_name, subfolder=\"unet\", use_auth_token=True)\n",
        "    if attention_slicing:\n",
        "        # slice_size = unet.config.attention_head_dim // slicing_factor\n",
        "        slice_size = 1 #@param\n",
        "        unet.set_attention_slice(slice_size)\n",
        "\n",
        "scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule=\"scaled_linear\", num_train_timesteps=1000)\n",
        "\n",
        "vae = vae.to(offload_device).half()\n",
        "text_encoder = text_encoder.to(offload_device).half()\n",
        "unet = unet.to(torch_device).half()\n",
        "\n",
        "def requires_grad(model, val=False):\n",
        "    for param in model.parameters():\n",
        "        param.requires_grad = val\n",
        "\n",
        "requires_grad(vae)\n",
        "requires_grad(text_encoder)\n",
        "requires_grad(unet)\n",
        "\n",
        "clear_output(wait=False)\n"
      ],
      "metadata": {
        "id": "oJRhI9mXOS9Y",
        "cellView": "form"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load Stable Inversion\n",
        "from huggingface_hub import hf_hub_download\n",
        "\n",
        "stable_inversion = \"\" #@param {type:\"string\"}\n",
        "if len(stable_inversion)>1:\n",
        "    g = hf_hub_download(repo_id=stable_inversion, filename=\"token_embeddings.pt\")\n",
        "    text_encoder.text_model.embeddings.token_embedding.weight = torch.load(g)"
      ],
      "metadata": {
        "id": "5aS15TZKk1QF",
        "cellView": "form"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title load textual-inversion concepts from 🤗 hub\n",
        "\n",
        "#@markdown `load_full_concepts_library` if turned on will take a While, it loads every single stable diffusion concept on https://huggingface.co/sd-concepts-library <br>\n",
        "#@markdown `specific_concepts` can be a list of strings, containing the ids of your concepts (from sd-concepts-library or your own repos, example `[\"sd-concepts-library/my-first-inversion\", \"sd-concepts-library/my-second-inversion\"]` etc.)\n",
        "\n",
        "load_full_concepts_library = False #@param {\"type\":\"boolean\"}\n",
        "\n",
        "from huggingface_hub import HfApi\n",
        "import wget\n",
        "import os\n",
        "api = HfApi()\n",
        "def load_learned_embed_in_clip(learned_embeds_path, text_encoder, tokenizer, token=None):\n",
        "        loaded_learned_embeds = torch.load(learned_embeds_path, map_location=\"cpu\")\n",
        "        \n",
        "        # separate token and the embeds\n",
        "        trained_token = list(loaded_learned_embeds.keys())[0]\n",
        "        embeds = loaded_learned_embeds[trained_token]\n",
        "\n",
        "        # cast to dtype of text_encoder\n",
        "        dtype = text_encoder.get_input_embeddings().weight.dtype\n",
        "        embeds.to(dtype)\n",
        "\n",
        "        # add the token in tokenizer\n",
        "        token = token if token is not None else trained_token\n",
        "        num_added_tokens = tokenizer.add_tokens(token)\n",
        "        i = 1\n",
        "        # while(num_added_tokens == 0):\n",
        "        #     print(f\"The tokenizer already contains the token {token}.\")\n",
        "        #     token = f\"{token[:-1]}-{i}>\"\n",
        "        #     print(f\"Attempting to add the token {token}.\")\n",
        "        #     num_added_tokens = tokenizer.add_tokens(token)\n",
        "        #     i+=1\n",
        "        \n",
        "        # resize the token embeddings\n",
        "        text_encoder.resize_token_embeddings(len(tokenizer))\n",
        "        \n",
        "        # get the id for the token and assign the embeds\n",
        "        token_id = tokenizer.convert_tokens_to_ids(token)\n",
        "        text_encoder.get_input_embeddings().weight.data[token_id] = embeds\n",
        "        return token\n",
        "\n",
        "\n",
        "if load_full_concepts_library:\n",
        "    models_list = api.list_models(author=\"sd-concepts-library\", sort=\"likes\", direction=-1)\n",
        "    models = []\n",
        "\n",
        "    print(\"Setting up the public library\")\n",
        "    for model in models_list:\n",
        "        model_content = {}\n",
        "        model_id = model.modelId\n",
        "        model_content[\"id\"] = model_id\n",
        "        embeds_url = f\"https://huggingface.co/{model_id}/resolve/main/learned_embeds.bin\"\n",
        "        os.makedirs(model_id,exist_ok = True)\n",
        "        if not os.path.exists(f\"{model_id}/learned_embeds.bin\"):\n",
        "            try:\n",
        "                wget.download(embeds_url, out=model_id)\n",
        "            except:\n",
        "                continue\n",
        "        token_identifier = f\"https://huggingface.co/{model_id}/raw/main/token_identifier.txt\"\n",
        "        response = requests.get(token_identifier)\n",
        "        token_name = response.text\n",
        "        print(f\"added {token_name}\")\n",
        "        concept_type = f\"https://huggingface.co/{model_id}/raw/main/type_of_concept.txt\"\n",
        "        response = requests.get(concept_type)\n",
        "        concept_name = response.text\n",
        "        model_content[\"concept_type\"] = concept_name\n",
        "        images = []\n",
        "        model_content[\"images\"] = images\n",
        "\n",
        "        learned_token = load_learned_embed_in_clip(f\"{model_id}/learned_embeds.bin\", text_encoder, tokenizer, token_name)\n",
        "        model_content[\"token\"] = learned_token\n",
        "        models.append(model_content)\n",
        "\n",
        "specific_concepts = [\"sd-concepts-library/cat-toy\"] #@param\n",
        "\n",
        "models = []\n",
        "for model in specific_concepts:\n",
        "    model_content = {}\n",
        "    model_content[\"id\"] = model\n",
        "    embeds_url = f\"https://huggingface.co/{model}/resolve/main/learned_embeds.bin\"\n",
        "    os.makedirs(model,exist_ok = True)\n",
        "    if not os.path.exists(f\"{model}/learned_embeds.bin\"):\n",
        "        try:\n",
        "            wget.download(embeds_url, out=model)\n",
        "        except:\n",
        "            continue\n",
        "    token_identifier = f\"https://huggingface.co/{model}/raw/main/token_identifier.txt\"\n",
        "    response = requests.get(token_identifier)\n",
        "    token_name = response.text\n",
        "    print(f\"added {token_name}\")\n",
        "\n",
        "    concept_type = f\"https://huggingface.co/{model}/raw/main/type_of_concept.txt\"\n",
        "    response = requests.get(concept_type)\n",
        "    concept_name = response.text\n",
        "    model_content[\"concept_type\"] = concept_name\n",
        "    images = []\n",
        "    model_content[\"images\"] = images\n",
        "\n",
        "    learned_token = load_learned_embed_in_clip(f\"{model}/learned_embeds.bin\", text_encoder, tokenizer, token_name)\n",
        "    model_content[\"token\"] = learned_token\n",
        "    models.append(model_content)\n",
        "\n"
      ],
      "metadata": {
        "id": "v4_YYtPhO8Do",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d68bf458-2cd4-41b7-8249-743ff0a1b052"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "added <cat-toy>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Convert Old Embedding\n",
        "\n",
        "#@markdown convert original textual-inversion embeds to huggingface-style ones <br> You can run this cell however many times is needed to input all of your inversions.\n",
        "\n",
        "from IPython.display import FileLink\n",
        "import torch\n",
        "\n",
        "input_file = \"\" #@param {\"type\":\"string\"}\n",
        "placeholder_token = \"<my-style>\" #@param {\"type\":\"string\"}\n",
        "\n",
        "def convert_and_load(input_file, placeholder_token):\n",
        "    x = torch.load(input_file, map_location=torch.device('cpu'))\n",
        "\n",
        "    params_dict = {\n",
        "        placeholder_token: torch.tensor(list(x['string_to_param'].items())[0][1])\n",
        "    }\n",
        "    torch.save(params_dict, \"learned_embeds.bin\")\n",
        "    load_learned_embed_in_clip(\"learned_embeds.bin\", text_encoder, tokenizer, placeholder_token)\n",
        "    print(\"loaded\", placeholder_token)\n",
        "\n",
        "if input_file != \"\":\n",
        "    convert_and_load(input_file, placeholder_token)"
      ],
      "metadata": {
        "id": "BdYuNYcjlQhf",
        "cellView": "form"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown load a few midjourney styles\n",
        "import gdown\n",
        "gdown.download_folder(url=\"https://drive.google.com/drive/u/9/folders/1whqzuBtiAIo9V12I20I1EVkfE9TLb1hS\", quiet=True)\n",
        "\n",
        "try:\n",
        "    folder = \"/content/midj textual inversion\"\n",
        "    files = [folder+\"/\"+i for i in os.listdir(folder) if \".pt\" in i]\n",
        "except: # WHY DOES IT SOMETIMES DO THIS AND SOMETIMES DO IT THE OTHER WAY????? IM SO ANGRY\n",
        "    folder = \"/content\"\n",
        "    files = [folder+\"/\"+i for i in os.listdir(folder) if \".pt\" in i]\n",
        "names = [\"<\"+i.split(\"/\")[-1].split(\".\")[0]+\">\" for i in files]\n",
        "\n",
        "for i,j in zip(files, names):\n",
        "    convert_and_load(i,j)"
      ],
      "metadata": {
        "id": "Fymo27KWmEur",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "68217242-e061-4ddc-ed57-616d8defb301"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:15: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  from ipykernel import kernelapp as app\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "loaded <midj-portrait>\n",
            "loaded <midj-anthro>\n",
            "loaded <midj-strong>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "if you have learned_embeds.bin for your concept, you can just uncomment this line (remove the hash sign and space before it), and change \"<my-concept>\" to be whatever your token is, like \"<among-us-imposter>\" or whatever you called it\n",
        "\"\"\"\n",
        "\n",
        "# load_learned_embed_in_clip(\"learned_embeds.bin\", text_encoder, tokenizer, \"<my-concept>\")\n"
      ],
      "metadata": {
        "id": "H7aolcwI5kJN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "87467976-fc19-4fb9-c17f-8d664e94709b"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nif you have learned_embeds.bin for your concept, you can just uncomment this line (remove the hash sign and space before it), and change \"<my-concept>\" to be whatever your token is, like \"<among-us-imposter>\" or whatever you called it\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown trace / tensorrt\n",
        "tensorrt = traced\n",
        "\n",
        "in_channels = 4 # for later, since the traced version doesn't have this attribute\n",
        "for param in unet.parameters():\n",
        "    param.requires_grad = False\n",
        "\n",
        "if traced and unet_path==None:\n",
        "    with torch.no_grad():\n",
        "        with torch.autocast(torch_device):\n",
        "            dummy_latent = torch.randn((1,4,512//8,512//8), device='cuda', requires_grad=False, dtype=torch.float16)\n",
        "            dummy_time = torch.randn((), device='cuda', requires_grad=False, dtype=torch.float32)\n",
        "            dummy_txt_emb = torch.randn((1,77,768), device='cuda', requires_grad=False, dtype=torch.float16)\n",
        "            unet = torch.jit.trace(lambda a,b,c:  unet(a,b,c)['sample'], (dummy_latent, dummy_time, dummy_txt_emb))\n",
        "\n",
        "    unet.save(\"traced-unet.pt\")\n",
        "\n",
        "    del unet \n",
        "    import gc\n",
        "    gc.collect()\n",
        "    torch.cuda.empty_cache()\n",
        "    \n",
        "    unet_path=\"yeah theres a unet\"\n",
        "    print(\"traced\")\n",
        "if tensorrt and traced and unet_path!=None:\n",
        "    if unet_path==\"yeah theres a unet\":\n",
        "        unet = torch.jit.load(\"traced-unet.pt\").eval().to(torch_device)\n",
        "    else:\n",
        "        unet.eval().to(torch_device)\n",
        "    unet.qconfig = torch.ao.quantization.get_default_qconfig('fx2trt')\n",
        "    torch.ao.quantization.prepare(unet, inplace=True)\n",
        "    torch.ao.quantization.convert(unet, inplace=True)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "_YvnN5JE7CRn"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "ihwTrK4xg-38",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Set up generation loop\n",
        "\n",
        "to_tensor_tfm = transforms.ToTensor()\n",
        "\n",
        "# mismatch of tons of image encoding / decoding / loading functions i cant be asked to clean up right now\n",
        "\n",
        "def latents_to_pil(latents):\n",
        "  # bath of latents -> list of images\n",
        "  latents = (1 / 0.18215) * latents\n",
        "  with torch.no_grad():\n",
        "    image = vae.decode(latents)\n",
        "  image = (image / 2 + 0.5).clamp(0, 1)\n",
        "  image = image.detach().cpu().permute(0, 2, 3, 1).numpy()\n",
        "  images = (image * 255).round().astype(\"uint8\")\n",
        "  pil_images = [Image.fromarray(image) for image in images]\n",
        "  return pil_images\n",
        "\n",
        "def get_latent_from_url(url, size=(512,512)):\n",
        "    import PIL\n",
        "    response = requests.get(url)\n",
        "    img = PImage.open(BytesIO(response.content))\n",
        "    img = img.resize(size, resample=PIL.Image.LANCZOS).convert(\"RGB\")\n",
        "    img = np.array(img).astype(np.float32) / 255.0\n",
        "    img = img[None].transpose(0, 3, 1, 2)\n",
        "    img = torch.from_numpy(img)\n",
        "    img = 2.0 * img - 1.0\n",
        "    with torch.no_grad():\n",
        "        with autocast(\"cuda\"):\n",
        "            init_image = img.to(device=torch_device)\n",
        "            init_latent_dist = vae.encode(init_image).latent_dist\n",
        "            init_latents = init_latent_dist.sample()\n",
        "            init_latents = 0.18215 * init_latents\n",
        "            return init_latents\n",
        "\n",
        "def scale_and_decode(latents):\n",
        "    with autocast(\"cuda\"):\n",
        "        # scale and decode the image latents with vae\n",
        "        latents = 1 / 0.18215 * latents\n",
        "        with torch.no_grad():\n",
        "            image = vae.decode(latents).sample.squeeze(0)\n",
        "        image = f.to_pil_image((image / 2 + 0.5).clamp(0, 1))\n",
        "        return image\n",
        "\n",
        "def fetch(url_or_path):\n",
        "    import io\n",
        "    if str(url_or_path).startswith('http://') or str(url_or_path).startswith('https://'):\n",
        "        r = requests.get(url_or_path)\n",
        "        r.raise_for_status()\n",
        "        fd = io.BytesIO()\n",
        "        fd.write(r.content)\n",
        "        fd.seek(0)\n",
        "        return PImage.open(fd).convert('RGB')\n",
        "    return PImage.open(open(url_or_path, 'rb')).convert('RGB')\n",
        "\n",
        "\"\"\"\n",
        "grabs all text up to the first occurrence of ':' \n",
        "uses the grabbed text as a sub-prompt, and takes the value following ':' as weight\n",
        "if ':' has no value defined, defaults to 1.0\n",
        "repeats until no text remaining\n",
        "\"\"\"\n",
        "def split_weighted_subprompts(text, split=\":\"):\n",
        "    remaining = len(text)\n",
        "    prompts = []\n",
        "    weights = []\n",
        "    while remaining > 0:\n",
        "        if split in text:\n",
        "            idx = text.index(split) # first occurrence from start\n",
        "            # grab up to index as sub-prompt\n",
        "            prompt = text[:idx]\n",
        "            remaining -= idx\n",
        "            # remove from main text\n",
        "            text = text[idx+1:]\n",
        "            # find value for weight \n",
        "            if \" \" in text:\n",
        "                idx = text.index(\" \") # first occurence\n",
        "            else: # no space, read to end\n",
        "                idx = len(text)\n",
        "            if idx != 0:\n",
        "                try:\n",
        "                    weight = float(text[:idx])\n",
        "                except: # couldn't treat as float\n",
        "                    print(f\"Warning: '{text[:idx]}' is not a value, are you missing a space?\")\n",
        "                    weight = 1.0\n",
        "            else: # no value found\n",
        "                weight = 1.0\n",
        "            # remove from main text\n",
        "            remaining -= idx\n",
        "            text = text[idx+1:]\n",
        "            # append the sub-prompt and its weight\n",
        "            prompts.append(prompt)\n",
        "            weights.append(weight)\n",
        "        else: # no : found\n",
        "            if len(text) > 0: # there is still text though\n",
        "                # take remainder as weight 1\n",
        "                prompts.append(text)\n",
        "                weights.append(1.0)\n",
        "            remaining = 0\n",
        "    # print(prompts, weights)\n",
        "    return prompts, weights \n",
        "\n",
        "\n",
        "# from some stackoverflow comment\n",
        "import numpy as np\n",
        "def lerp(a, b, x):\n",
        "    \"linear interpolation\"\n",
        "    return a + x * (b - a)\n",
        "def fade(t):\n",
        "    \"6t^5 - 15t^4 + 10t^3\"\n",
        "    return 6 * t**5 - 15 * t**4 + 10 * t**3\n",
        "def gradient(h, x, y):\n",
        "    \"grad converts h to the right gradient vector and return the dot product with (x,y)\"\n",
        "    vectors = np.array([[0, 1], [0, -1], [1, 0], [-1, 0]])\n",
        "    g = vectors[h % 4]\n",
        "    return g[:, :, 0] * x + g[:, :, 1] * y\n",
        "def perlin(x, y, seed=0):\n",
        "    # permutation table\n",
        "    np.random.seed(seed)\n",
        "    p = np.arange(256, dtype=int)\n",
        "    np.random.shuffle(p)\n",
        "    p = np.stack([p, p]).flatten()\n",
        "    # coordinates of the top-left\n",
        "    xi, yi = x.astype(int), y.astype(int)\n",
        "    # internal coordinates\n",
        "    xf, yf = x - xi, y - yi\n",
        "    # fade factors\n",
        "    u, v = fade(xf), fade(yf)\n",
        "    # noise components\n",
        "    n00 = gradient(p[p[xi] + yi], xf, yf)\n",
        "    n01 = gradient(p[p[xi] + yi + 1], xf, yf - 1)\n",
        "    n11 = gradient(p[p[xi + 1] + yi + 1], xf - 1, yf - 1)\n",
        "    n10 = gradient(p[p[xi + 1] + yi], xf - 1, yf)\n",
        "    # combine noises\n",
        "    x1 = lerp(n00, n10, u)\n",
        "    x2 = lerp(n01, n11, u)  # FIX1: I was using n10 instead of n01\n",
        "    return lerp(x1, x2, v)  # FIX2: I also had to reverse x1 and x2 here\n",
        "\n",
        "clip_model = None\n",
        "def sample(args):\n",
        "    global in_channels\n",
        "    global text_encoder # uugghhhghhghgh\n",
        "    global vae # UUGHGHHGHGH\n",
        "    global unet # .hggfkgjks;ldjf\n",
        "    global clip_model\n",
        "\n",
        "    prompts, weights = split_weighted_subprompts(args.prompt)\n",
        "    negative_prompts, negative_weights = split_weighted_subprompts(args.negative)\n",
        "\n",
        "    h,w = args.size\n",
        "    steps = args.steps\n",
        "    scale = args.scale\n",
        "    classifier_guidance = args.classifier_guidance\n",
        "    use_init = len(args.init_img)>1\n",
        "    if args.seed!=-1:\n",
        "        seed = args.seed\n",
        "        generator = torch.manual_seed(seed)\n",
        "    else:\n",
        "        seed = random.randint(0,10_000)\n",
        "        generator = torch.manual_seed(seed)\n",
        "    print(f\"Generating with seed {seed}...\")\n",
        "    \n",
        "    # tokenize / encode text\n",
        "    print(\"prompts\", prompts)\n",
        "    tokens = [tokenizer(prompt, padding=\"max_length\", max_length=tokenizer.model_max_length, truncation=True, return_tensors=\"pt\") for prompt in prompts]\n",
        "    neg_tokens = [tokenizer(prompt, padding=\"max_length\", max_length=tokenizer.model_max_length, truncation=True, return_tensors=\"pt\") for prompt in negative_prompts]\n",
        "    with torch.no_grad():\n",
        "        # move CLIP to cuda\n",
        "        text_encoder = text_encoder.to(torch_device)\n",
        "        text_embeddings = [text_encoder(tok.input_ids.to(torch_device))[0].unsqueeze(0) for tok in tokens]\n",
        "        text_embeddings = [text_embeddings[i]*weights[i] for i in range(len(text_embeddings))]\n",
        "        text_embeddings = torch.cat(text_embeddings, 0).sum(0)\n",
        "\n",
        "        neg_text_embeddings = [text_encoder(tok.input_ids.to(torch_device))[0].unsqueeze(0) for tok in neg_tokens]\n",
        "        neg_text_embeddings = [neg_text_embeddings[i]*negative_weights[i] for i in range(len(neg_text_embeddings))]\n",
        "        neg_text_embeddings = torch.cat(neg_text_embeddings, 0).sum(0)\n",
        "\n",
        "        # max_length = 77\n",
        "        # uncond_input = tokenizer(\n",
        "        #     [\"\"], padding=\"max_length\", max_length=max_length, return_tensors=\"pt\"\n",
        "        # )\n",
        "        # uncond_embeddings = text_encoder(uncond_input.input_ids.to(torch_device))[0] \n",
        "        text_embeddings = torch.cat([neg_text_embeddings, text_embeddings])\n",
        "        # move it back to CPU so there's more vram for generating\n",
        "        text_encoder = text_encoder.to(offload_device)\n",
        "    images = []\n",
        "\n",
        "    if args.lpips_guidance:\n",
        "        import lpips\n",
        "        lpips_model = lpips.LPIPS(net='vgg').to(torch_device)\n",
        "        init = to_tensor_tfm(fetch(args.init_img).resize(args.size)).to(torch_device)\n",
        "\n",
        "    for batch_n in trange(args.batches):\n",
        "        with autocast(\"cuda\"):\n",
        "            scheduler.set_timesteps(steps)\n",
        "            if not use_init or args.start_step==0:\n",
        "                latents = torch.randn(\n",
        "                    (1, in_channels, h // 8, w // 8),\n",
        "                )\n",
        "                latents = latents.to(torch_device)\n",
        "                latents = latents * scheduler.sigmas[0]\n",
        "                start_step = 0\n",
        "            else:\n",
        "                # Start step\n",
        "                start_step = args.start_step - 1\n",
        "                start_sigma = scheduler.sigmas[start_step]\n",
        "                start_timestep = int(scheduler.timesteps[start_step])\n",
        "\n",
        "                # Prep latents\n",
        "                vae = vae.to(torch_device)\n",
        "                encoded = get_latent_from_url(args.init_img, (h,w))\n",
        "                if not classifier_guidance:\n",
        "                    vae = vae.to(offload_device)\n",
        "\n",
        "                # ???????????????????????????????????????\n",
        "                offset = scheduler.config.get(\"steps_offset\", 0)\n",
        "                init_timestep = int(steps * (start_step / steps)) + offset\n",
        "                init_timestep = min(init_timestep, steps)\n",
        "                timesteps = scheduler.timesteps[init_timestep]\n",
        "                timesteps = torch.tensor([timesteps], device=torch_device)\n",
        "\n",
        "                # add noise to latents using the timesteps\n",
        "                noise = torch.randn_like(encoded)\n",
        "                latents = scheduler.add_noise(encoded, noise, timesteps)\n",
        "\n",
        "            if args.perlin_multi != 0 and args.start_step==0:\n",
        "                linx = np.linspace(0, 5, h // 8, endpoint=False)\n",
        "                liny = np.linspace(0, 5, w // 8, endpoint=False)\n",
        "                x, y = np.meshgrid(liny, linx)\n",
        "                p = [np.expand_dims(perlin(x, y, seed=i), 0) for i in range(4)] # reproducable seed\n",
        "                p = np.concatenate(p, 0)\n",
        "                p = torch.tensor(p).unsqueeze(0).cuda()\n",
        "                # latents = latents + (p * args.perlin_multi).to(torch_device).half()\n",
        "                latents = latents*(1-(args.perlin_multi*0.1)) + (p*args.perlin_multi).to(torch_device).half()\n",
        "\n",
        "            vae = vae.to(offload_device)\n",
        "            if classifier_guidance and args.clip_scale!=0:\n",
        "                clip_model = clip_model.to(offload_device)\n",
        "            for i, t in tqdm(enumerate(scheduler.timesteps), total=steps):\n",
        "                if i > args.start_step:\n",
        "                    latent_model_input = torch.cat([latents] * 2)\n",
        "                    sigma = scheduler.sigmas[i]\n",
        "                    latent_model_input = latent_model_input / ((sigma**2 + 1) ** 0.5)\n",
        "                    uncond_input, cond_input = latent_model_input.chunk(2)\n",
        "\n",
        "                    with torch.no_grad():\n",
        "                        noise_pred_uncond = unet(uncond_input, t, encoder_hidden_states=text_embeddings[0].unsqueeze(0))[\"sample\"]\n",
        "                    if classifier_guidance:\n",
        "                        cond_input = cond_input.requires_grad_()\n",
        "                        noise_pred_cond = unet(cond_input, t, encoder_hidden_states=text_embeddings[1].unsqueeze(0))[\"sample\"]\n",
        "                    else:\n",
        "                        with torch.no_grad():\n",
        "                            noise_pred_cond = unet(cond_input, t, encoder_hidden_states=text_embeddings[1].unsqueeze(0))[\"sample\"]\n",
        "                    \n",
        "                    noise_pred = noise_pred_uncond + scale * (noise_pred_cond - noise_pred_uncond)\n",
        "\n",
        "                    # classifier guidance\n",
        "                    if classifier_guidance:\n",
        "                        latents = latents.detach().requires_grad_()\n",
        "                        latents_x0 = latents - sigma * noise_pred\n",
        "\n",
        "                        vae = vae.cuda()\n",
        "                        denoised_images = vae.decode((1 / 0.18215) * latents_x0).sample / 2 + 0.5\n",
        "\n",
        "                        loss = 0\n",
        "                        if args.clip_scale != 0:\n",
        "                            loss = args.loss_fn(denoised_images, \"clip\") * args.clip_scale\n",
        "                        if args.tv_scale != 0:\n",
        "                            loss = args.loss_fn(denoised_images, \"tv\") * args.tv_scale\n",
        "                        if args.lpips_scale != 0:\n",
        "                            denoised_images = f.resize(denoised_images, (512,512))\n",
        "                            init = f.resize(init, (512,512))\n",
        "                            init_losses = lpips_model(denoised_images, init)\n",
        "                            loss = loss + init_losses.sum() * args.lpips_scale\n",
        "                        loss = loss.cuda()\n",
        "                        cond_input = cond_input.cuda()\n",
        "                        if args.clip_scale != 0: clip_model.cuda().float(); unet.cuda(); vae.cuda()\n",
        "                        cond_grad = -torch.autograd.grad(loss.float(), cond_input.float())[0]\n",
        "                        cond_grad = torch.nan_to_num(cond_grad)\n",
        "                        magnitude = cond_grad.square().mean().sqrt()\n",
        "                        cond_grad = cond_grad * magnitude.clamp(max=args.clamp_max) / magnitude\n",
        "                        cond_grad = torch.nan_to_num(cond_grad)\n",
        "                        latents = latents.detach() + cond_grad.detach() * sigma**2\n",
        "                    latents = scheduler.step(noise_pred, i, latents)[\"prev_sample\"]\n",
        "        # yaaaaay juggling but guess what it DOESNT WORK!!!!\n",
        "        vae = vae.to(torch_device).half()\n",
        "        unet = unet.to(offload_device)\n",
        "        text_encoder = text_encoder.to(offload_device)\n",
        "\n",
        "        output_image = scale_and_decode(latents.detach().requires_grad_(False).half())\n",
        "\n",
        "        vae = vae.to(offload_device)\n",
        "        unet = unet.to(torch_device)\n",
        "        text_encoder = text_encoder.to(torch_device)\n",
        "        images.append(output_image)\n",
        "\n",
        "        import gc\n",
        "        gc.collect()\n",
        "        torch.cuda.empty_cache()\n",
        "\n",
        "        images[-1].save(f\"output/{batch_n}.png\")\n",
        "        display(Image(f\"output/{batch_n}.png\"))\n",
        "        if args.notif:\n",
        "            from google.colab import output\n",
        "            output.eval_js('new Audio(\"https://freesound.org/data/previews/80/80921_1022651-lq.ogg\").play()')\n",
        "    return images\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generate\n",
        "<p>Explanation for each parameter + Tips<p>\n",
        "<hr>\n",
        "Sections:\n",
        "<details> <summary> <b> 1. General Settings </b> </summary>\n",
        "<details> <summary> <code> prompt </code> </summary>\n",
        "    The prompt for your image. Can be just one string or multiple strings, separated by their respective weights. (example: <code>frog:0.5 dog:0.7</code>) The weights control how much the prompt is supposed to be paid attention to. Try to stick around 0.7-1.3 when all weights are added up for \"normal\" results. <br>\n",
        "    Another way to weight prompts is to wrap things you want to focus on in parenthesis, and things you want to de-focus with square brackets. (example: <code>[[red]] (frog)</code>)\n",
        "    Also included are prompt tags for easy prompt design, beginner prompt..alchemists may not have an intuition for how to change something simple (like \"red bird\") into a prompt that works well. Included are the tags: <br>\n",
        "    <code>{artstation}</code>: for that \"trending on artstation\" feel <br>\n",
        "    <code>{overwatch}</code>: a painterly overwatch-like fanart style <br>\n",
        "    <code>{ghibli}</code>: like a ghibli movie <br>\n",
        "    <code>{intricate}</code>: self explanatory. <br> <br>\n",
        "    if you're using a Doodad model from Doohickey, an additional \"<doodad>\" tag is added for pushing the aesthetic even further. Prompt weighting is disabled for Doodad models because of some behind-the-scenes prompt-editing going on, however you can re-enable it by commenting out the first few lines in the generation cell that pertain to editing the prompt.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> init_img </code> </summary>\n",
        "    An image to use as a starting point to generate from. There's two ways to use this. One is to set it as a url to the image you want to start from and turn \"start_step\" so some number below \"steps\". Another is to set it as a url, but keep start_step at 0 and change these settings: <br>\n",
        "    <code>\n",
        "    classifier_guidance = True <br>\n",
        "    lpips_guidance = True <br>\n",
        "    lpips_scale = 8 <br>\n",
        "    loss_scale = 0 <br>\n",
        "    </code>\n",
        "    this will push the model to use that sort of pose but not necessarily keep every detail about it, this can give higher fidelity results but at some time cost. (Also it doesn't always .. keep the pose but it still increases fidelity anyway? I don't know how to explain it 100%) (example: <a href=\"https://twitter.com/aicrumb/status/1571292327871750146\">link</a>) <br>\n",
        "    try using <a href=\"https://www.pexels.com/\">pexels</a> or <a href=\"https://reference.pictures/free/\">reference.pictures</a> for reference poses here\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> size </code> </summary>\n",
        "    The size should be formatted as <code>[height, width]</code><br>\n",
        "    around 768 is the highest either will go without putting the machine out of memory. Each number should be a multiple of 64 (448, 512, 576, 640, 704, 768, etc)\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> steps </code> </summary>\n",
        "    The steps variable is how long the image should take to render. A good jumping off point if you don't know how it works intuitively yet would be 50-75 (my personal favorite being 65) <br>\n",
        "    The only way to really know how it effects the generation is to try it.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> start_step </code> </summary>\n",
        "    Like said in the <code>init_img</code> section, <code>start_step</code> should be a number below whatever you set <code>steps</code> to. It just skips that far into the generation and should be used when you have an <code>init_img</code> set.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> perlin_multi </code> </summary>\n",
        "    This variable adds perlin noise to the starting point of generation. It can help the model latch onto bigger shapes and can make things a little more coherent when using larger sizes than 512x512. Good starting points are anywhere from 0-0.72. (<a href=\"https://twitter.com/aicrumb/status/1562268451078537216\">example</a>)\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> scale </code> </summary>\n",
        "    The 'Classifier Free Guidance Scale'. It tries to push the image more into the prompt's direction. A good value is 7.5. Lower lets the model imagine more details and higher forces it to adhere to the prompt more strictly, but it doesn't always work.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> seed </code> </summary>\n",
        "    For creating re-producable results. Set to -1 for a random seed. (The seed will be printed out at the beginning of generation so if you end up liking one of the results and want to change the settings a little to run it again and see how that effects it, you can put the seed to whatever is printed before generation.)\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> batches </code> </summary>\n",
        "    This controls how many images it makes.\n",
        "</details>\n",
        "</details>\n",
        "\n",
        "\n",
        "<!--                    -->\n",
        "<!--                    -->\n",
        "\n",
        "<details> <summary> <b> 2. Classifier Guidance </b> </summary>\n",
        "<details> <summary> <code> lpips_scale </code> </summary>\n",
        "    Perceptual loss to <code>init_img</code>. Pushes the image to be structurally similar to the <code>init_img</code>. If 0 the lpips model will not be loaded, and the generation time will be faster. See the <code>init_img</code> documentation for tricks with this.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> clip_scale </code> </summary>\n",
        "    CLIP similarity to <code>clip_text_prompt</code> and <code>clip_image_prompt</code>. If 0 the CLIP model will not be loaded, and the generation time will be faster. I recommend a max of 0.2 and an average around 0.1. CLIP guidance is NOT DETERMINISTIC! Outputs are not 100% reproducible.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> tv_scale </code> </summary>\n",
        "    Total variance loss. Higher makes the image smoother. Doesn't work wonderfully\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> clamp_max </code> </summary>\n",
        "    Applied in this formula to the gradient <br>\n",
        "    <code>\n",
        "    magnitude = cond_grad.square().mean().sqrt()<br>\n",
        "    cond_grad = cond_grad * magnitude.clamp(max=args.clamp_max) / magnitude\n",
        "    </code>\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> quick_guidance </code> </summary>\n",
        "    not great for clip guidance, guiding on this <a href=\"https://twitter.com/anton_lozhkov/status/1572906509335498752\"> twitter post </a> instead of decoding with the vae\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> clip_text_prompt </code> </summary>\n",
        "    The prompt for CLIP to push the image towards. Has the same weighting scheme as <code>prompt</code>.\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> clip_image_prompt </code> </summary>\n",
        "    Image urls for CLIP to push the generated image towards. Instead of denoting image weights with :weight like in <code>clip_text_prompt</code> and <code>prompt</code>, it should be denoted with a vertical bar. (url|weight)\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> clip_model_name </code> and <code> clip_model_pretrained </code> </summary>\n",
        "    The CLIP model to load assuming <code>clip_scale</code> isn't equal to 0. <br> models larger than ViT-B-16-plus-240 are too big for the free tier of Colab <br>\n",
        "    Current models that can be loaded are: <br>\n",
        "    <code>[('ViT-B-32', 'openai'), <br>\n",
        "    ('ViT-B-32', 'laion400m_e31'),  <br>\n",
        "    ('ViT-B-32', 'laion400m_e32'), <br>\n",
        "    ('ViT-B-32', 'laion2b_e16'), <br>\n",
        "    ('ViT-B-32', 'laion2b_s34b_b79k'),  <br>\n",
        "    ('ViT-B-32-quickgelu', 'openai'), <br>\n",
        "    ('ViT-B-32-quickgelu', 'laion400m_e31'),  <br>\n",
        "    ('ViT-B-32-quickgelu', 'laion400m_e32'), <br>\n",
        "    ('ViT-B-16', 'openai'), <br>\n",
        "    ('ViT-B-16', 'laion400m_e31'), <br>\n",
        "    ('ViT-B-16', 'laion400m_e32'), <br>\n",
        "    ('ViT-B-16-plus-240', 'laion400m_e31'), <br>\n",
        "    ('ViT-B-16-plus-240', 'laion400m_e32'), <br>\n",
        "    ('ViT-L-14', 'openai'), <br>\n",
        "    ('ViT-L-14', 'laion400m_e31'), <br>\n",
        "    ('ViT-L-14', 'laion400m_e32'), <br>\n",
        "    ('ViT-L-14', 'laion2b_s32b_b82k'), <br>\n",
        "    ('ViT-L-14-336', 'openai'), <br>\n",
        "    ('ViT-H-14', 'laion2b_s32b_b79k'), <br>\n",
        "    ('ViT-g-14', 'laion2b_s12b_b42k')] <br></code>\n",
        "    <p>I would suggest <code> 'ViT-B-32', 'laion2b_s34b_b79k' </code></p>\n",
        "</details>\n",
        "\n",
        "<details> <summary> <code> cutn </code> </summary>\n",
        "    How many permutations of the image to show to CLIP to base the scoring off of. If using large models (vit-h, vit-l) cutn should be as low as 1-3 to avoid going out of memory. Smaller ones (vit-b) can be around 8 for optimal performance, though.\n",
        "</details>\n",
        "</details>"
      ],
      "metadata": {
        "id": "LDgzNksxMlhy"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PytCwKXCmPid",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ---\n",
        "\n",
        "midjourney_style = False #@param {type:\"boolean\"}\n",
        "# idk how people normally do this and i cba to look\n",
        "prompt = \"brutalist architecture by beksinski \\u003Cmidj-portrait>\" #@param {\"type\":\"string\"}\n",
        "negative_prompt = \"\" #@param {\"type\":\"string\"}\n",
        "if midjourney_style:\n",
        "    if model_name == \"doohickey/doodad-v1-1\":\n",
        "        prompt = f\"imv <midj-strong>{prompt}:0.5 imv {prompt}:0.5\"\n",
        "    elif model_name == \"doohickey/doodad-v1-2\":\n",
        "        prompt = f\"<doodad> <midj-strong> {prompt}:0.55 <doodad> {prompt}:0.55 text, words, graffiti:-0.2\"\n",
        "    elif model_name == \"doohickey/doodad-v1-3\":\n",
        "        # no need for doodad token, i distributed it 5% over all other tokens LOL its still there if you want to use it though to make it even more stylized\n",
        "        prompt = f\"<midj-strong> {prompt}:0.55 {prompt}:0.55 text, words, graffiti:-0.1\"\n",
        "    else:\n",
        "        prompt = f\"<midj-strong> {prompt}:0.55 {prompt}:0.55 text, words, graffiti:-0.1\"\n",
        "\n",
        "bracket_base = 0.0\n",
        "bracket_multiplier = 1.\n",
        "init_img = \"\" #@param {\"type\":\"string\"}\n",
        "size = [1920, 1024] #@param\n",
        "steps = 100 #@param\n",
        "start_step = 90 #@param\n",
        "perlin_multi = 0. #@param\n",
        "scale = 8 #@param\n",
        "seed = -1 #@param\n",
        "batches = 8 #@param\n",
        "\n",
        "# a few \"styles\" from prompts i      stole      from lexica that I know work well, for easy prompt building if you don't have an idea of what to do to improve your prompt\n",
        "prompt_suffix_map = {\n",
        "    \"{artstation}\": \"by ross tran, greg rutkowski, trending on artstation, photograph, hyperreal, octane render, oil on canvas\",\n",
        "    \"{overwatch}\": \"from overwatch, character portrait, close up, concept art, intricate details, highly detailed photorealistic in the style of marco plouffe, keos masons, joel torres, seseon yoon, artgerm and warren louw\",\n",
        "    \"{ghibli}\": \"still from studio ghibli movie; very detailed, focused, colorful, antoine pierre mongin, trending on artstation, 8 k\",\n",
        "    \"{intricate}\": \"4 k resolution, trending on artstation, very very detailed, masterpiece, stunning, intricate\"\n",
        "}\n",
        "def add_suffixes(prompt):\n",
        "    for i in prompt_suffix_map.keys():\n",
        "        prompt = prompt.replace(i,prompt_suffix_map[i])\n",
        "    return prompt\n",
        "prompt = add_suffixes(prompt)\n",
        "\n",
        "\n",
        "def count(string, start=\"(\", end=\")\", negative=True):\n",
        "    temp_string = \"\"\n",
        "    temp_multiplier = bracket_base\n",
        "    mode = \"neutral\"\n",
        "    extension = \"\"\n",
        "    for char in string:\n",
        "        if char == start and mode == \"neutral\":\n",
        "            mode = \"writing\"\n",
        "            temp_multiplier = bracket_base if not negative else -bracket_base\n",
        "        if char == start and mode == \"writing\":\n",
        "            temp_multiplier *= bracket_multiplier\n",
        "        if char == end and mode == \"writing\":\n",
        "            extension += f\" {temp_string}:{str(temp_multiplier)}\"\n",
        "            mode = \"neutral\"\n",
        "            temp_multiplier = bracket_base if not negative else -bracket_base\n",
        "            temp_string = \"\"\n",
        "        if char not in [start, end] and mode == \"writing\":\n",
        "            temp_string += char\n",
        "    for char in [start, end]:\n",
        "        string = string.replace(char, \"\")\n",
        "    return string, extension\n",
        "\n",
        "def add_brackets(prompt):\n",
        "    if \":\" not in prompt[-5:]:\n",
        "        prompt += \":1\"\n",
        "    clean, ext_p = count(prompt, start=\"(\", end=\")\", negative=False)\n",
        "    clean, ext_n = count(clean, start=\"[\", end=\"]\", negative=True)\n",
        "    return prompt + ext_p + ext_n #  make it work more like automatics so the prompts are more cross-compatible\n",
        "\n",
        "prompt = add_brackets(prompt)\n",
        "negative_prompt = add_brackets(negative_prompt)\n",
        "#@markdown ---\n",
        "\n",
        "# classifier_guidance = True #@param {\"type\":\"boolean\"}\n",
        "# lpips_guidance = True #@param {\"type\":\"boolean\"}\n",
        "lpips_scale = 0. #@param\n",
        "clip_scale = 0. #@param\n",
        "tv_scale = 0. #@param\n",
        "clamp_max = 0.01 #@param\n",
        "quick_guidance = False #@param {\"type\":\"boolean\"}\n",
        "classifier_guidance = (lpips_scale!=0) or (clip_scale!=0) or (tv_scale!=0)\n",
        "lpips_guidance = lpips_scale!=0\n",
        "\n",
        "\n",
        "class BlankClass():\n",
        "    def __init__(self):\n",
        "        bruh = 'BRUH'\n",
        "args = BlankClass()\n",
        "args.prompt = prompt\n",
        "args.negative = negative_prompt\n",
        "args.init_img = init_img\n",
        "args.size = size \n",
        "args.steps = steps \n",
        "args.start_step = start_step \n",
        "args.scale = scale\n",
        "args.perlin_multi = perlin_multi\n",
        "args.seed = seed\n",
        "args.batches = batches \n",
        "args.classifier_guidance = classifier_guidance\n",
        "args.lpips_guidance = lpips_guidance\n",
        "args.lpips_scale = lpips_scale\n",
        "# args.loss_scale = clip_scale\n",
        "args.clip_scale = clip_scale\n",
        "args.tv_scale = tv_scale\n",
        "args.clamp_max = clamp_max\n",
        "args.quick_guidance = quick_guidance\n",
        "\n",
        "if args.classifier_guidance:\n",
        "    # import clip\n",
        "    import open_clip as clip\n",
        "    from torch import nn\n",
        "    import torch.nn.functional as F\n",
        "    import io\n",
        "\n",
        "    class MakeCutouts(nn.Module):\n",
        "        def __init__(self, cut_size, cutn, cut_pow=1.):\n",
        "            super().__init__()\n",
        "            self.cut_size = cut_size\n",
        "            self.cutn = cutn\n",
        "            self.cut_pow = cut_pow\n",
        "\n",
        "        def forward(self, input):\n",
        "            sideY, sideX = input.shape[2:4]\n",
        "            max_size = min(sideX, sideY)\n",
        "            min_size = min(sideX, sideY, self.cut_size)\n",
        "            cutouts = []\n",
        "            for _ in range(self.cutn):\n",
        "                size = int(torch.rand([])**self.cut_pow * (max_size - min_size) + min_size)\n",
        "                offsetx = torch.randint(0, sideX - size + 1, ())\n",
        "                offsety = torch.randint(0, sideY - size + 1, ())\n",
        "                cutout = input[:, :, offsety:offsety + size, offsetx:offsetx + size]\n",
        "                cutouts.append(F.adaptive_avg_pool2d(cutout, self.cut_size))\n",
        "            return torch.cat(cutouts)\n",
        "    # make_cutouts = MakeCutouts(224, 16)\n",
        "    \n",
        "    clip_text_prompt = \"\" #@param {\"type\":\"string\"}\n",
        "    if clip_scale != 0:\n",
        "        clip_text_prompt = add_suffixes(clip_text_prompt)\n",
        "        clip_text_prompt = add_brackets(clip_text_prompt)\n",
        "\n",
        "    clip_image_prompt = \"\" #@param {\"type\":\"string\"}\n",
        "\n",
        "    if clip_scale != 0:\n",
        "        # clip_model = clip.load(\"ViT-B/32\", jit=False)[0].eval().requires_grad_(False).to(torch_device)\n",
        "        clip_model_name = \"ViT-B-32\" #@param {\"type\":\"string\"}\n",
        "        clip_model_pretrained = \"laion2b_s34b_b79k\" #@param {\"type\":\"string\"}\n",
        "        clip_model, _, preprocess = clip.create_model_and_transforms(clip_model_name, pretrained=clip_model_pretrained)\n",
        "        clip_model = clip_model.eval().requires_grad_(False).to(torch_device)\n",
        "\n",
        "        cutn = 1 #@param\n",
        "        make_cutouts = MakeCutouts(clip_model.visual.image_size if type(clip_model.visual.image_size)!= tuple else clip_model.visual.image_size[0], cutn)\n",
        "\n",
        "    target = None\n",
        "    if len(clip_text_prompt) > 1 and clip_scale != 0:\n",
        "        clip_text_prompt, clip_text_weights = split_weighted_subprompts(clip_text_prompt)\n",
        "        target = clip_model.encode_text(clip.tokenize(clip_text_prompt).to(torch_device)) * torch.tensor(clip_text_weights).view(len(clip_text_prompt), 1).to(torch_device)\n",
        "    if len(clip_image_prompt) > 1 and clip_scale != 0:\n",
        "        clip_image_prompt, clip_image_weights = split_weighted_subprompts(clip_image_prompt, split=\"|\")\n",
        "        # pesky spaces\n",
        "        clip_image_prompt = [p.replace(\" \", \"\") for p in clip_image_prompt]\n",
        "        images = [fetch(image) for image in clip_image_prompt]\n",
        "        images = [f.to_tensor(i).unsqueeze(0) for i in images]\n",
        "        images = [make_cutouts(i) for i in images]\n",
        "        encodings = [clip_model.encode_image(i.to(torch_device)).mean(0) for i in images]\n",
        "        \n",
        "        for i in range(len(encodings)):\n",
        "            encodings[i] = (encodings[i] * clip_image_weights[i]).unsqueeze(0)\n",
        "        # print(encodings.shape)\n",
        "        encodings = torch.cat(encodings, 0)\n",
        "        encoding = encodings.sum(0)\n",
        "\n",
        "        if target!=None:\n",
        "            target = target + encoding\n",
        "        else:\n",
        "            target = encoding\n",
        "        target = target.half().to(torch_device)\n",
        "\n",
        "    # free a little memory, we dont use the text encoder after this so just delete it\n",
        "    if clip_scale != 0:\n",
        "        clip_model.transformer = None\n",
        "        import gc\n",
        "        gc.collect()\n",
        "        torch.cuda.empty_cache()\n",
        "    def spherical_distance(x, y):\n",
        "        x = F.normalize(x, dim=-1)\n",
        "        y = F.normalize(y, dim=-1)\n",
        "        l = (x - y).norm(dim=-1).div(2).arcsin().pow(2).mul(2).mean()\n",
        "        return l \n",
        "    def tv_loss(input):\n",
        "        input = F.pad(input, (0, 1, 0, 1), 'replicate')\n",
        "        return ((input[..., :-1, 1:] - input[..., :-1, :-1])**2 + (input[..., 1:, :-1] - input[..., :-1, :-1])**2).mean()\n",
        "    def loss_fn(x,mode):\n",
        "        global clip_model\n",
        "        global unet\n",
        "        global vae\n",
        "        # crappy way of handling it, i know\n",
        "        if mode==\"clip\":\n",
        "            # with torch.autocast(\"cuda\"):\n",
        "            with torch.amp.autocast(device_type='cuda', dtype=torch.float16):\n",
        "                cutouts = make_cutouts(x)\n",
        "                # oh my god there's something that requires clip to be at full precision\n",
        "                unet = unet.cpu()\n",
        "                vae = vae.cpu()\n",
        "                \n",
        "                clip_model = clip_model.float().cuda()\n",
        "                encoding = clip_model.encode_image(cutouts.float().cuda()).half()\n",
        "                clip_model = clip_model.half().cpu()\n",
        "\n",
        "                loss = spherical_distance(encoding, target)\n",
        "                unet = unet.cuda()\n",
        "                # vae = vae.cuda()\n",
        "                return loss.mean().cuda()\n",
        "        if mode==\"tv\":\n",
        "            return tv_loss(x).mean()\n",
        "\n",
        "    args.loss_fn = loss_fn\n",
        "#@markdown ---\n",
        "notify_me_on_every_image = True #@param {\"type\":\"boolean\"}\n",
        "args.notif = notify_me_on_every_image\n",
        "dtype = torch.float16\n",
        "\n",
        "try:\n",
        "    with torch.amp.autocast(device_type=torch_device, dtype=dtype):\n",
        "        output = sample(args)\n",
        "except KeyboardInterrupt:\n",
        "    print('Interrupting generation..')\n",
        "else:\n",
        "    print('No errors caught!')\n",
        "print(\"Done!\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown 🔔\n",
        "from google.colab import output\n",
        "output.eval_js('new Audio(\"https://freesound.org/data/previews/80/80921_1022651-lq.ogg\").play()')"
      ],
      "metadata": {
        "cellView": "form",
        "id": "ja_qUMLCxZY7"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Post-Processing"
      ],
      "metadata": {
        "id": "umTlv-Qru-Sq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ### Flavor your image\n",
        "# LOL this is no different than the cell below for matching\n",
        "!pip install color-matcher -q\n",
        "!mkdir flavor\n",
        "from color_matcher import ColorMatcher\n",
        "from color_matcher.io_handler import load_img_file, save_img_file, FILE_EXTS\n",
        "from color_matcher.normalizer import Normalizer\n",
        "\n",
        "\n",
        "color_map = {\n",
        "    \"dust\": \"https://onecms-res.cloudinary.com/image/upload/s--z1xo9sBq--/c_fill%2Cg_auto%2Ch_676%2Cw_1200/f_auto%2Cq_auto/v1/mediacorp/tdy/image/2022/09/07/20220907_nyt_aiart.jpg?itok=8LHesjBa\",\n",
        "    \"jungle\": \"https://medialist.info/wp-content/uploads/2022/06/2022_06_02_medialist_midjourney.jpg\",\n",
        "    \"muddy-gold\": \"https://miro.medium.com/max/1200/1*XoVCoIeNJ16PYOmWKOs0mg.png\",\n",
        "    \"diamond\": \"https://i.pinimg.com/564x/b7/b3/6a/b7b36a7efdce4e9c53bbbb2c8fbe5c30.jpg\",\n",
        "    \"rose\": \"https://i.pinimg.com/564x/21/60/d5/2160d5efafff9b910859e2657268aaa2.jpg\",\n",
        "    \"deep-ocean\": \"https://i.pinimg.com/564x/93/1c/8b/931c8b2b184ba175686ebe815cec22b0.jpg\",\n",
        "    \"yellow\": \"https://i.pinimg.com/564x/62/ba/d7/62bad7f5cbc3740c149c16b3f2aedf3d.jpg\",\n",
        "    \"sakura\": \"https://i.pinimg.com/564x/aa/a5/fa/aaa5fa9feaf11434187f6c11e8b9fa3b.jpg\"\n",
        "}\n",
        "\n",
        "flavor = \"sakura\" #@param ['deep-ocean', 'diamond', 'dust', 'jungle', 'muddy-gold', 'random', 'rose', 'sakura', 'yellow']\n",
        "if flavor!=\"random\":\n",
        "    print(f\"Using '{flavor}' flavor\")\n",
        "    flavor = color_map[flavor]\n",
        "else:\n",
        "    import random\n",
        "    flavor = random.choice(list(color_map.keys()))\n",
        "    print(f\"Using '{flavor}' flavor\")\n",
        "    flavor = color_map[flavor]\n",
        "reference_image = fetch(flavor)\n",
        "reference_image.save(\"ref.png\")\n",
        "\n",
        "img_ref = load_img_file('ref.png')\n",
        "\n",
        "src_path = '/content/output'\n",
        "filenames = [os.path.join(src_path, f) for f in os.listdir(src_path)\n",
        "                     if f.lower().endswith(FILE_EXTS)]\n",
        "\n",
        "cm = ColorMatcher()\n",
        "for i, fname in enumerate(filenames):\n",
        "    img_src = load_img_file(fname) * 0.95\n",
        "    # ('default', 'hm', 'reinhard', 'mvgd', 'mkl', 'hm-mvgd-hm', 'hm-mkl-hm')\n",
        "    img_res = cm.transfer(src=img_src, ref=img_ref, method='mkl')\n",
        "    img_res = Normalizer(img_res).uint8_norm()\n",
        "    save_img_file(img_res, os.path.join(f'flavor/{i}.png'))\n",
        "    display(Image(f'flavor/{i}.png'))"
      ],
      "metadata": {
        "cellView": "form",
        "id": "c6XuWtK0U1Dy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5f7e642b-26af-4274-dacd-6974a5c84f20"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mkdir: cannot create directory ‘flavor’: File exists\n",
            "Using 'sakura' flavor\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ### Color Correction\n",
        "!pip install git+https://github.com/shunsukeaihara/colorcorrect.git -q\n",
        "# via https://github.com/shunsukeaihara/colorcorrect\n",
        "!mkdir correct-output\n",
        "\n",
        "# why does it do this\n",
        "!rm -r /content/output/.ipynb_checkpoints\n",
        "import colorcorrect.algorithm as cca\n",
        "from colorcorrect.util import from_pil, to_pil\n",
        "from colorcorrect.algorithm import *\n",
        "algorithm = automatic_color_equalization #@param [\"grey_world\", \"max_white\", \"retinex\", \"retinex_with_adjust\", \"standard_deviation_weighted_grey_world\", \"standard_deviation_and_luminance_weighted_grey_world\", \"luminance_weighted_grey_world\", \"automatic_color_equalization\"] {type:\"raw\"}\n",
        "\n",
        "folders_map = {\n",
        "    \"raw\": \"/content/output\",\n",
        "    \"flavored\": \"/content/flavor\"\n",
        "}\n",
        "\n",
        "folder = \"flavored\" #@param [\"raw\", \"flavored\"]\n",
        "\n",
        "src_path = folders_map[folder]\n",
        "filenames = [os.path.join(src_path, f) for f in os.listdir(src_path)]\n",
        "\n",
        "for i, fname in enumerate(filenames):\n",
        "    image = PImage.open(fname)\n",
        "    to_pil(algorithm(from_pil(image))).save(f'correct-output/{i}.png')\n",
        "    display(Image(f'correct-output/{i}.png'))"
      ],
      "metadata": {
        "cellView": "form",
        "id": "xjFEXM26u_q2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ### Match colors to a reference image\n",
        "#@markdown fun tip: you can use https://colorhunt.co/ to find a color palette to match your image to, download the palette as an image, upload it to colab, set reference_url to the file path\n",
        "# !pip install color-matcher -q\n",
        "!mkdir color-output\n",
        "from color_matcher import ColorMatcher\n",
        "from color_matcher.io_handler import load_img_file, save_img_file, FILE_EXTS\n",
        "from color_matcher.normalizer import Normalizer\n",
        "\n",
        "reference_url = \"https://media.discordapp.net/attachments/1012719433745186976/1021101572953997423/unknown.png?width=286&height=272\" #@param {\"type\":\"string\"}\n",
        "reference_image = fetch(reference_url)\n",
        "reference_image.save(\"ref.png\")\n",
        "\n",
        "img_ref = load_img_file('ref.png')\n",
        "\n",
        "print(\"#\"*20)\n",
        "print(\"Reference image\")\n",
        "display(Image(\"ref.png\"))\n",
        "print(\"\\n\" + \"#\"*20)\n",
        "print(\"Color Matched outputs\")\n",
        "\n",
        "src_path = '/content/output'\n",
        "filenames = [os.path.join(src_path, f) for f in os.listdir(src_path)\n",
        "                     if f.lower().endswith(FILE_EXTS)]\n",
        "\n",
        "cm = ColorMatcher()\n",
        "for i, fname in enumerate(filenames):\n",
        "    img_src = load_img_file(fname) * 0.95\n",
        "    # ('default', 'hm', 'reinhard', 'mvgd', 'mkl', 'hm-mvgd-hm', 'hm-mkl-hm')\n",
        "    img_res = cm.transfer(src=img_src, ref=img_ref, method='mkl')\n",
        "    img_res = Normalizer(img_res).uint8_norm()\n",
        "    save_img_file(img_res, os.path.join(f'color-output/{i}.png'))\n",
        "    display(Image(f'color-output/{i}.png'))"
      ],
      "metadata": {
        "cellView": "form",
        "id": "H0ae60HUkrB_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown 🔔\n",
        "from google.colab import output\n",
        "output.eval_js('new Audio(\"https://freesound.org/data/previews/80/80921_1022651-lq.ogg\").play()')"
      ],
      "metadata": {
        "cellView": "form",
        "id": "crU68Wv15gMs"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "0mKBMENAXu7F"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "04be0d7f1bda49e1bcba2699e48e50b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e86b6f1f629b4db49c25cd33937bdf05",
              "IPY_MODEL_732cfbfe6ac54755a71d6021cc6c5716",
              "IPY_MODEL_d3d226f37ecc4d3aa304de0880c4fb49",
              "IPY_MODEL_3f036597057c4a729b5de8515f9201d5"
            ],
            "layout": "IPY_MODEL_713e1c4bf02f47e0a94859ccbf9f2487"
          }
        },
        "e86b6f1f629b4db49c25cd33937bdf05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d74176a5e4d4bbe801c233abd841be3",
            "placeholder": "​",
            "style": "IPY_MODEL_1786a074991f4d9dbc5717c2503b191d",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "732cfbfe6ac54755a71d6021cc6c5716": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_149213d3c2aa4632bbadee0fc129eea0",
            "placeholder": "​",
            "style": "IPY_MODEL_297105181dbb4a7b8c1f652d0901d783",
            "value": ""
          }
        },
        "d3d226f37ecc4d3aa304de0880c4fb49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_d79c203fa65a45098e2673293d693612",
            "style": "IPY_MODEL_373ae7d7fc2547d0aa1b79113322b61f",
            "tooltip": ""
          }
        },
        "3f036597057c4a729b5de8515f9201d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90f35f1fa0ee4f0f89e707fd05695fca",
            "placeholder": "​",
            "style": "IPY_MODEL_d7245bf56f424e0e811b65f2b957b1c1",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "713e1c4bf02f47e0a94859ccbf9f2487": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "6d74176a5e4d4bbe801c233abd841be3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1786a074991f4d9dbc5717c2503b191d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "149213d3c2aa4632bbadee0fc129eea0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "297105181dbb4a7b8c1f652d0901d783": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d79c203fa65a45098e2673293d693612": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "373ae7d7fc2547d0aa1b79113322b61f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "90f35f1fa0ee4f0f89e707fd05695fca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7245bf56f424e0e811b65f2b957b1c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}