text-to-sound.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.append('./Diffsound/')\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "import soundfile\n",
    "import datetime\n",
    "from pathlib import Path\n",
    "\n",
    "from Diffsound.prompt2prompt import Controllers\n",
    "\n",
    "from Diffsound.evaluation.generate_samples_batch import Diffsound\n",
    "from Diffsound.sound_synthesis.utils.misc import seed_everything\n",
    "import librosa\n",
    "import librosa.display\n",
    "import IPython.display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Diffsound_(Diffsound):\n",
    "    def __init__(self, config, path, ckpt_vocoder, seed=0):\n",
    "        super(Diffsound_, self).__init__(config, path, ckpt_vocoder)\n",
    "        self.seed = seed\n",
    "\n",
    "    def generate_sample(self, prompts, truncation_rate, save_root, content_token=None, controller=None, fast=False):\n",
    "        batch_size = 8\n",
    "        if fast != False:\n",
    "            add_string = 'r,fast'+str(fast-1)\n",
    "        else:\n",
    "            add_string = 'r'\n",
    "\n",
    "        generate_num = 0\n",
    "        base_name_ = 'sample_'\n",
    "        data_i = {}\n",
    "        data_i['text'] = prompts\n",
    "        data_i['image'] = None\n",
    "\n",
    "        if controller is None:\n",
    "            controller = Controllers.AttentionStore()\n",
    "\n",
    "        with torch.no_grad():\n",
    "            model_out = self.run_and_generate(\n",
    "                data_i=data_i,\n",
    "                filter_ratio=0,\n",
    "                controller=controller,\n",
    "                content_token=content_token,\n",
    "                replicate=1,\n",
    "                content_ratio=1,\n",
    "                return_att_weight=False,\n",
    "                sample_type=\"top\"+str(truncation_rate)+add_string,\n",
    "            )\n",
    "            content = model_out['decode_content']\n",
    "            content_token = model_out['content_token']\n",
    "            # spec to sound\n",
    "            os.makedirs(save_root, exist_ok=True)\n",
    "            for b in range(content.shape[0]):\n",
    "                print(f\"text: {data_i['text'][generate_num]}\")\n",
    "\n",
    "                save_base_name = base_name_ + str(generate_num)\n",
    "                spec = content[b]\n",
    "                spec = spec.squeeze(0).cpu().numpy()\n",
    "                spec = (spec + 1) / 2\n",
    "\n",
    "                save_png_root = os.path.join(save_root, 'png')\n",
    "                os.makedirs(save_png_root, exist_ok=True)\n",
    "                save_png_name = os.path.join(save_png_root, save_base_name + '.png')\n",
    "#                 print_spec(spec, save_png_name)\n",
    "\n",
    "                # save spec\n",
    "                save_spec_root = os.path.join(save_root, 'spec')\n",
    "                os.makedirs(save_spec_root, exist_ok=True)\n",
    "                save_spec_path = os.path.join(save_spec_root, save_base_name + '.npy')\n",
    "                np.save(save_spec_path, spec)\n",
    "\n",
    "                # save wav\n",
    "                if self.vocoder is not None:\n",
    "                    wave_from_vocoder = self.vocoder(torch.from_numpy(spec).unsqueeze(0).to(self.device)).cpu().squeeze().detach().numpy()\n",
    "                    save_sound_root = os.path.join(save_root, 'sound')\n",
    "                    os.makedirs(save_sound_root, exist_ok=True)\n",
    "                    save_wav_path = os.path.join(save_sound_root, save_base_name + '.wav')\n",
    "                    soundfile.write(save_wav_path, wave_from_vocoder, 22050, 'PCM_24')\n",
    "\n",
    "                    wav_to_spectrogram(save_wav_path)\n",
    "\n",
    "                    IPython.display.display(IPython.display.Audio(save_wav_path))\n",
    "\n",
    "                generate_num += 1\n",
    "        return content_token\n",
    "\n",
    "\n",
    "def print_spec(spec, save_path):\n",
    "    fig, ax = plt.subplots(figsize=(8, 3))\n",
    "    img = librosa.display.specshow(\n",
    "        spec[:, :512],\n",
    "        sr=22050,\n",
    "        x_axis='time',\n",
    "        y_axis='linear',\n",
    "        ax=ax,\n",
    "        cmap=None)\n",
    "    fig.colorbar(img, ax=ax, format=\"%+2.1f dB\")\n",
    "    plt.savefig(save_path)\n",
    "    plt.show()\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import librosa\n",
    "import soundfile as sf\n",
    "\n",
    "def wav_read(path):\n",
    "    wave, fs = sf.read(path) #音データと周波数を読み込む\n",
    "    return wave, fs\n",
    "\n",
    "def wav_to_spectrogram(wav_file, n_mels=128, n_fft=2048, hop_length=512):\n",
    "    plt.figure(figsize=(8,2))\n",
    "    wave, fs = wav_read(wav_file)\n",
    "    n_mels = 128\n",
    "    fmax = 8000\n",
    "    mel = librosa.feature.melspectrogram(y=wave, sr=fs, n_mels=n_mels, fmax=fmax)\n",
    "    mel_dB = librosa.power_to_db(mel, ref=np.max) #dBに変換\n",
    "    frq = librosa.mel_frequencies(n_mels=n_mels, fmax=fmax) #周波数軸\n",
    "    t = librosa.frames_to_time(np.arange(mel_dB.shape[1]), sr=fs)  # 時間軸\n",
    "    plt.pcolormesh(t, frq, mel_dB, cmap = 'jet')\n",
    "    plt.show()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that cap_text.yaml includes the config of vagan, we must choose the right path for it.\n",
    "def run(\n",
    "    Diffsound,\n",
    "    prompts,\n",
    "    save_root_ = './Diffsound/OUT',\n",
    "    save_root = None,\n",
    "    controller=Controllers.AttentionStore(),\n",
    "    content_token = None):\n",
    "    seed_everything(Diffsound.seed)\n",
    "    if save_root is None:\n",
    "        key_words = 'Real_vgg_pre_399'\n",
    "        random_seconds_shift = datetime.timedelta(seconds=np.random.randint(60))\n",
    "        now = (datetime.datetime.now() - random_seconds_shift).strftime('%Y-%m-%dT%H-%M-%S')\n",
    "        save_root = os.path.join(save_root_, key_words + '_samples_'+now, 'caps_validation')\n",
    "    content_token = Diffsound.generate_sample(prompts=prompts, content_token=content_token, truncation_rate=0.85, save_root=save_root, controller=controller, fast=False)\n",
    "    return content_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "config_path = './Diffsound/evaluation/caps_text.yaml'\n",
    "pretrained_model_path = './Diffsound/pre_model/diffsound_audiocaps.pth'\n",
    "ckpt_vocoder = './Diffsound/vocoder/logs/vggsound/'\n",
    "diffsound = Diffsound_(config=config_path, path=pretrained_model_path, ckpt_vocoder=ckpt_vocoder, seed=0)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 通常の生成"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    \"Rain falls heavily on a metal roof and windows\"\n",
    "    ]\n",
    "content_token = run(Diffsound=diffsound,prompts=prompts)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 通常の生成とWordSwapを用いた生成"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    \"A piano music plays with whistling noise\",\n",
    "    \"A jazz music plays with whistling noise\",\n",
    "    \"A sweet music plays with whistling noise\",\n",
    "    \"A soft music plays with whistling noise\",\n",
    "]\n",
    "_ = run(diffsound, prompts, content_token=content_token)\n",
    "controller = Controllers.AttentionReplace(prompts, 100, cross_replace_steps=.6, self_replace_steps=.4)\n",
    "_ = run(diffsound, prompts, controller=controller, content_token=content_token)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "# FaderControlを用いた生成"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\"A man speaks before and after a gunshot\"] * 4\n",
    "equalizer = Controllers.get_equalizer(prompts[0], word_select=(\"gunshot\",), values=(10, .0, -4))\n",
    "controller = Controllers.AttentionReweight(prompts, 100, cross_replace_steps=1., self_replace_steps=.2, equalizer=equalizer)\n",
    "\n",
    "_ = run(diffsound, prompts, content_token=content_token, controller=controller)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "# PromptRefinementを用いた生成"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    \"A music plays with whistling noise\",\n",
    "    \"A jazz music plays with whistling noise\",\n",
    "    \"A sweet music plays with whistling noise\",\n",
    "    \"A soft music plays with whistling noise\",\n",
    "]\n",
    "_ = run(diffsound, prompts=prompts, content_token=content_token)\n",
    "\n",
    "lb = Controllers.LocalBlend(prompts, (\"music\", (\"jazz\", \"music\"), (\"sweet\", \"music\"), (\"soft\", \"music\")))\n",
    "\n",
    "controller =Controllers.AttentionRefine(prompts, 100, cross_replace_steps={\"default_\": 1., \"music\": .05},\n",
    "                             self_replace_steps=.4, local_blend=lb)\n",
    "_ = run(diffsound,prompts=prompts, content_token=content_token, controller=controller)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ef472dfb43bfe56ebc2afcee31d2935d7d56374708cd2b3e6583bb9f9d250286"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}