langchainchatbot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2b51f77c-bce1-4c4a-a7e6-3a8e757d14c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: LANGCHAIN_TRACING_V2=true\n",
      "env: LANGCHAIN_ENDPOINT=\"https://api.smith.langchain.com\"\n",
      "env: LANGCHAIN_API_KEY=\"\"\n",
      "env: LANGCHAIN_PROJECT=\"\"\n"
     ]
    }
   ],
   "source": [
    "%env LANGCHAIN_TRACING_V2=true\n",
    "%env LANGCHAIN_ENDPOINT=\"https://api.smith.langchain.com\"\n",
    "%env LANGCHAIN_API_KEY=\"\"\n",
    "%env LANGCHAIN_PROJECT=\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "334e0391-3599-47a3-a4e4-2c1be7562c9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 20 key-value pairs and 291 tensors from /Users/kevinrvaz/.cache/huggingface/hub/models--TheBloke--Mistral-7B-OpenOrca-GGUF/snapshots/fbd9cd059e5fa0bba72a0abe8aea7e127d7994cb/mistral-7b-openorca.Q4_K_M.gguf (version GGUF V2)\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = open-orca_mistral-7b-openorca\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 32768\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32              = 10000.000000\n",
      "llama_model_loader: - kv  11:                          general.file_type u32              = 15\n",
      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr[str,32002]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr[f32,32002]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr[i32,32002]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 32000\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens cache size = 5\n",
      "llm_load_vocab: token to piece cache size = 0.1637 MB\n",
      "llm_load_print_meta: format           = GGUF V2\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32002\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: vocab_only       = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_swa            = 0\n",
      "llm_load_print_meta: n_embd_head_k    = 128\n",
      "llm_load_print_meta: n_embd_head_v    = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
      "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: causal attn      = 1\n",
      "llm_load_print_meta: pooling type     = 0\n",
      "llm_load_print_meta: rope type        = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_ctx_orig_yarn  = 32768\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: ssm_d_conv       = 0\n",
      "llm_load_print_meta: ssm_d_inner      = 0\n",
      "llm_load_print_meta: ssm_d_state      = 0\n",
      "llm_load_print_meta: ssm_dt_rank      = 0\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 7.24 B\n",
      "llm_load_print_meta: model size       = 4.07 GiB (4.83 BPW) \n",
      "llm_load_print_meta: general.name     = open-orca_mistral-7b-openorca\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 32000 '<dummy32000>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_print_meta: max token length = 48\n",
      "llm_load_tensors: ggml ctx size =    0.27 MiB\n",
      "ggml_backend_metal_log_allocated_size: allocated buffer, size =   132.52 MiB, (  132.58 / 10922.67)\n",
      "llm_load_tensors: offloading 1 repeating layers to GPU\n",
      "llm_load_tensors: offloaded 1/33 layers to GPU\n",
      "llm_load_tensors:        CPU buffer size =  4165.38 MiB\n",
      "llm_load_tensors:      Metal buffer size =   132.50 MiB\n",
      ".................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 2048\n",
      "llama_new_context_with_model: n_batch    = 512\n",
      "llama_new_context_with_model: n_ubatch   = 512\n",
      "llama_new_context_with_model: flash_attn = 0\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M2 Pro\n",
      "ggml_metal_init: picking default device: Apple M2 Pro\n",
      "ggml_metal_init: using embedded metal library\n",
      "ggml_metal_init: GPU name:   Apple M2 Pro\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple8  (1008)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyCommon3 (3003)\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyMetal3  (5001)\n",
      "ggml_metal_init: simdgroup reduction support   = true\n",
      "ggml_metal_init: simdgroup matrix mul. support = true\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "llama_kv_cache_init:        CPU KV buffer size =   248.00 MiB\n",
      "llama_kv_cache_init:      Metal KV buffer size =     8.00 MiB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_new_context_with_model:        CPU  output buffer size =     0.12 MiB\n",
      "llama_new_context_with_model:      Metal compute buffer size =   164.00 MiB\n",
      "llama_new_context_with_model:        CPU compute buffer size =   164.01 MiB\n",
      "llama_new_context_with_model: graph nodes  = 1030\n",
      "llama_new_context_with_model: graph splits = 499\n",
      "AVX = 0 | AVX_VNNI = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | AVX512_BF16 = 0 | FMA = 0 | NEON = 1 | SVE = 0 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n",
      "Model metadata: {'general.quantization_version': '2', 'tokenizer.ggml.unknown_token_id': '0', 'tokenizer.ggml.eos_token_id': '32000', 'tokenizer.ggml.bos_token_id': '1', 'tokenizer.ggml.model': 'llama', 'llama.attention.head_count_kv': '8', 'llama.context_length': '32768', 'llama.attention.head_count': '32', 'llama.rope.freq_base': '10000.000000', 'llama.rope.dimension_count': '128', 'general.file_type': '15', 'llama.feed_forward_length': '14336', 'llama.embedding_length': '4096', 'llama.block_count': '32', 'general.architecture': 'llama', 'llama.attention.layer_norm_rms_epsilon': '0.000010', 'general.name': 'open-orca_mistral-7b-openorca'}\n",
      "Using fallback chat format: llama-2\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.llms import LlamaCpp\n",
    "import os\n",
    "\n",
    "model_path = os.path.join(os.environ[\"HOME\"], \".cache/huggingface/hub/models--TheBloke--Mistral-7B-OpenOrca-GGUF/snapshots/fbd9cd059e5fa0bba72a0abe8aea7e127d7994cb/mistral-7b-openorca.Q4_K_M.gguf\")\n",
    "\n",
    "n_gpu_layers = 1  # Metal set to 1 is enough.\n",
    "n_batch = 512  # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.\n",
    "\n",
    "# Make sure the model path is correct for your system!\n",
    "model = LlamaCpp(\n",
    "    model_path=model_path,\n",
    "    n_gpu_layers=n_gpu_layers,\n",
    "    n_batch=n_batch,\n",
    "    n_ctx=2048,\n",
    "    f16_kv=True,  # MUST set to True, otherwise you will run into problem after a couple of calls\n",
    "    verbose=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dcb7b0a9-bec4-4159-94f4-8bc1fab523e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    4192.72 ms\n",
      "llama_print_timings:      sample time =      22.24 ms /   256 runs   (    0.09 ms per token, 11511.31 tokens per second)\n",
      "llama_print_timings: prompt eval time =    4192.62 ms /     6 tokens (  698.77 ms per token,     1.43 tokens per second)\n",
      "llama_print_timings:        eval time =   13631.74 ms /   255 runs   (   53.46 ms per token,    18.71 tokens per second)\n",
      "llama_print_timings:       total time =   17950.13 ms /   261 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'  Christmas is a Christian holiday that marks the birth of Jesus Christ, who is believed by Christians to be the son of God. The name “Christmas” comes from the combination of “Christ” (referring to Jesus) and “mass” (a short form of “Mass”, which is a religious service).\\n\\nChristmas is celebrated on December 25th each year, though some scholars argue that the actual date of birth of Jesus may have been earlier or later. The holiday has evolved over time, incorporating various customs and traditions from different cultures and countries.\\n\\nWhat are the main symbols of Christmas?\\n\\n1. The Tree: The tradition of the Christmas tree is thought to have originated in Germany during the 16th century. It was first recorded in 1530 when the church leader Martin Luther noticed the stars twinkling among the branches of a tree and brought the idea into his home as an example of the beauty and wonder of the birth of Jesus Christ.\\n\\n2. The Star: The star is symbolic of the Star of Bethlehem, which guided the Wise Men to the birthplace of Jesus in the biblical story.\\n\\n3. Santa Claus: Also known as Saint'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to batch ingest runs: LangSmithConnectionError('Connection error caused failure to POST https://api.smith.langchain.com/runs/batch  in LangSmith API. Please confirm your internet connection.. ConnectionError(MaxRetryError(\\'HTTPSConnectionPool(host=\\\\\\'api.smith.langchain.com\\\\\\', port=443): Max retries exceeded with url: /runs/batch (Caused by NameResolutionError(\"<urllib3.connection.HTTPSConnection object at 0x111634b80>: Failed to resolve \\\\\\'api.smith.langchain.com\\\\\\' ([Errno 8] nodename nor servname provided, or not known)\"))\\'))')\n",
      "Failed to batch ingest runs: LangSmithConnectionError('Connection error caused failure to POST https://api.smith.langchain.com/runs/batch  in LangSmith API. Please confirm your internet connection.. ConnectionError(MaxRetryError(\\'HTTPSConnectionPool(host=\\\\\\'api.smith.langchain.com\\\\\\', port=443): Max retries exceeded with url: /runs/batch (Caused by NameResolutionError(\"<urllib3.connection.HTTPSConnection object at 0x117b54640>: Failed to resolve \\\\\\'api.smith.langchain.com\\\\\\' ([Errno 8] nodename nor servname provided, or not known)\"))\\'))')\n"
     ]
    }
   ],
   "source": [
    "model.invoke(\"What is christmas?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3b100d32-c557-448d-b781-b711f1e902d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4192.72 ms\n",
      "llama_print_timings:      sample time =       0.27 ms /     3 runs   (    0.09 ms per token, 11320.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =     350.12 ms /     8 tokens (   43.77 ms per token,    22.85 tokens per second)\n",
      "llama_print_timings:        eval time =     151.15 ms /     2 runs   (   75.58 ms per token,    13.23 tokens per second)\n",
      "llama_print_timings:       total time =     503.07 ms /    10 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'.\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.messages import HumanMessage\n",
    "\n",
    "model.invoke([HumanMessage(content=\"Hi! I'm Bob\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "83a0a041-0b30-4c41-8e0e-a7fea3c38560",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4192.72 ms\n",
      "llama_print_timings:      sample time =      20.83 ms /   256 runs   (    0.08 ms per token, 12291.74 tokens per second)\n",
      "llama_print_timings: prompt eval time =    3762.14 ms /     4 tokens (  940.54 ms per token,     1.06 tokens per second)\n",
      "llama_print_timings:        eval time =   12971.76 ms /   255 runs   (   50.87 ms per token,    19.66 tokens per second)\n",
      "llama_print_timings:       total time =   16854.27 ms /   259 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\" A country in South Asia, it is the seventh-largest and second most populous country in the world. With a rich history and diverse culture, it has been an influential nation since ancient times. It is a land of spices, tea, Bollywood films, cricket, and many other fascinating aspects that make it a unique country.\\n\\nIndia has a long history that dates back thousands of years. Its roots can be traced back to the Indus Valley Civilization, which was one of the world's earliest civilizations. The Indian subcontinent has been home to various empires and dynasties over time, including the Maurya, Gupta, Mughal, and British Empires.\\n\\nIndia's cultural diversity is a result of its long history and interactions with different regions and peoples. There are numerous languages spoken in India, each with its own rich literature and traditions. Some of the most widely spoken languages include Hindi, Bengali, Telugu, Marathi, Tamil, Urdu, Gujarati, Malayalam, and Kannada.\\n\\nThe Indian subcontinent has a diverse climate that includes hot summers, mild winters, and torrential\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.invoke(\"What is India?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "34f77257-5c37-417b-9989-d7fe87f8e6fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n",
      "\n",
      "llama_print_timings:        load time =    4192.72 ms\n",
      "llama_print_timings:      sample time =      19.61 ms /   228 runs   (    0.09 ms per token, 11626.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     789.81 ms /    21 tokens (   37.61 ms per token,    26.59 tokens per second)\n",
      "llama_print_timings:        eval time =   11870.05 ms /   227 runs   (   52.29 ms per token,    19.12 tokens per second)\n",
      "llama_print_timings:       total time =   12767.74 ms /   248 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\" in 100 words or less.\\n\\nThe Tragedy of Macbeth is a classic play written by William Shakespeare, presenting a tale of ambition, power, and betrayal. The story centers on the titular character, Macbeth, a Scottish nobleman who receives prophecies from three witches. Influenced by these predictions, he becomes determined to ascend to the throne, leading him on a path of violence and deceit.\\n\\nMacbeth's wife, Lady Macbeth, is also consumed by her hunger for power and supports her husband in his pursuit of kingship. Together, they murder the existing king, Duncan, and frame their enemy, Banquo, for the crime. Their ascent to power, however, is met with growing resistance, chaos, and paranoia, ultimately culminating in Macbeth's tragic downfall.\\n\\nThroughout the play, Shakespeare explores themes of fate, moral responsibility, and the consequences of power, leaving audiences questioning the nature of ambition and whether it is worth the cost.\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.invoke(\"Summarise the play delimited by tripple quote '''The Tragedie of Macbeth''' by William Shakespeare\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d01527cd-5100-4b77-a2f6-3f7b74730701",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any, AsyncIterator, Dict, Iterator, List, Optional\n",
    "\n",
    "from langchain_core.callbacks import (\n",
    "    AsyncCallbackManagerForLLMRun,\n",
    "    CallbackManagerForLLMRun,\n",
    ")\n",
    "from langchain_core.language_models import BaseChatModel, SimpleChatModel\n",
    "from langchain_core.messages import AIMessageChunk, BaseMessage, HumanMessage\n",
    "from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult\n",
    "from langchain_core.runnables import run_in_executor\n",
    "\n",
    "\n",
    "class CustomChatModelAdvanced(BaseChatModel):\n",
    "    \"\"\"A custom chat model that echoes the first `n` characters of the input.\n",
    "\n",
    "    When contributing an implementation to LangChain, carefully document\n",
    "    the model including the initialization parameters, include\n",
    "    an example of how to initialize the model and include any relevant\n",
    "    links to the underlying models documentation or API.\n",
    "\n",
    "    Example:\n",
    "\n",
    "        .. code-block:: python\n",
    "\n",
    "            model = CustomChatModel(n=2)\n",
    "            result = model.invoke([HumanMessage(content=\"hello\")])\n",
    "            result = model.batch([[HumanMessage(content=\"hello\")],\n",
    "                                 [HumanMessage(content=\"world\")]])\n",
    "    \"\"\"\n",
    "\n",
    "    model_name: str\n",
    "    \"\"\"The name of the model\"\"\"\n",
    "    n: int\n",
    "    \"\"\"The number of characters from the last message of the prompt to be echoed.\"\"\"\n",
    "\n",
    "    def _generate(\n",
    "        self,\n",
    "        messages: List[BaseMessage],\n",
    "        stop: Optional[List[str]] = None,\n",
    "        run_manager: Optional[CallbackManagerForLLMRun] = None,\n",
    "        **kwargs: Any,\n",
    "    ) -> ChatResult:\n",
    "        \"\"\"Override the _generate method to implement the chat model logic.\n",
    "\n",
    "        This can be a call to an API, a call to a local model, or any other\n",
    "        implementation that generates a response to the input prompt.\n",
    "\n",
    "        Args:\n",
    "            messages: the prompt composed of a list of messages.\n",
    "            stop: a list of strings on which the model should stop generating.\n",
    "                  If generation stops due to a stop token, the stop token itself\n",
    "                  SHOULD BE INCLUDED as part of the output. This is not enforced\n",
    "                  across models right now, but it's a good practice to follow since\n",
    "                  it makes it much easier to parse the output of the model\n",
    "                  downstream and understand why generation stopped.\n",
    "            run_manager: A run manager with callbacks for the LLM.\n",
    "        \"\"\"\n",
    "        # Replace this with actual logic to generate a response from a list\n",
    "        # of messages.\n",
    "        last_message = messages[-1]\n",
    "        tokens = last_message.content[: self.n]\n",
    "        message = AIMessage(\n",
    "            content=tokens,\n",
    "            additional_kwargs={},  # Used to add additional payload (e.g., function calling request)\n",
    "            response_metadata={  # Use for response metadata\n",
    "                \"time_in_seconds\": 3,\n",
    "            },\n",
    "        )\n",
    "        ##\n",
    "\n",
    "        generation = ChatGeneration(message=message)\n",
    "        return ChatResult(generations=[generation])\n",
    "\n",
    "    def _stream(\n",
    "        self,\n",
    "        messages: List[BaseMessage],\n",
    "        stop: Optional[List[str]] = None,\n",
    "        run_manager: Optional[CallbackManagerForLLMRun] = None,\n",
    "        **kwargs: Any,\n",
    "    ) -> Iterator[ChatGenerationChunk]:\n",
    "        \"\"\"Stream the output of the model.\n",
    "\n",
    "        This method should be implemented if the model can generate output\n",
    "        in a streaming fashion. If the model does not support streaming,\n",
    "        do not implement it. In that case streaming requests will be automatically\n",
    "        handled by the _generate method.\n",
    "\n",
    "        Args:\n",
    "            messages: the prompt composed of a list of messages.\n",
    "            stop: a list of strings on which the model should stop generating.\n",
    "                  If generation stops due to a stop token, the stop token itself\n",
    "                  SHOULD BE INCLUDED as part of the output. This is not enforced\n",
    "                  across models right now, but it's a good practice to follow since\n",
    "                  it makes it much easier to parse the output of the model\n",
    "                  downstream and understand why generation stopped.\n",
    "            run_manager: A run manager with callbacks for the LLM.\n",
    "        \"\"\"\n",
    "        last_message = messages[-1]\n",
    "        tokens = last_message.content[: self.n]\n",
    "\n",
    "        for token in tokens:\n",
    "            chunk = ChatGenerationChunk(message=AIMessageChunk(content=token))\n",
    "\n",
    "            if run_manager:\n",
    "                # This is optional in newer versions of LangChain\n",
    "                # The on_llm_new_token will be called automatically\n",
    "                run_manager.on_llm_new_token(token, chunk=chunk)\n",
    "\n",
    "            yield chunk\n",
    "\n",
    "        # Let's add some other information (e.g., response metadata)\n",
    "        chunk = ChatGenerationChunk(\n",
    "            message=AIMessageChunk(content=\"\", response_metadata={\"time_in_sec\": 3})\n",
    "        )\n",
    "        if run_manager:\n",
    "            # This is optional in newer versions of LangChain\n",
    "            # The on_llm_new_token will be called automatically\n",
    "            run_manager.on_llm_new_token(token, chunk=chunk)\n",
    "        yield chunk\n",
    "\n",
    "    @property\n",
    "    def _llm_type(self) -> str:\n",
    "        \"\"\"Get the type of language model used by this chat model.\"\"\"\n",
    "        return \"echoing-chat-model-advanced\"\n",
    "\n",
    "    @property\n",
    "    def _identifying_params(self) -> Dict[str, Any]:\n",
    "        \"\"\"Return a dictionary of identifying parameters.\n",
    "\n",
    "        This information is used by the LangChain callback system, which\n",
    "        is used for tracing purposes make it possible to monitor LLMs.\n",
    "        \"\"\"\n",
    "        return {\n",
    "            # The model name allows users to specify custom token counting\n",
    "            # rules in LLM monitoring applications (e.g., in LangSmith users\n",
    "            # can provide per token pricing for their model and monitor\n",
    "            # costs for the given LLM.)\n",
    "            \"model_name\": self.model_name,\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c0d7a53d-0a0a-42e4-af6a-d5d753666356",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='Meo', response_metadata={'time_in_seconds': 3}, id='run-b4f5a1a8-c520-4585-8935-45995fb41707-0')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.messages import (\n",
    "    AIMessage,\n",
    "    BaseMessage,\n",
    "    FunctionMessage,\n",
    "    HumanMessage,\n",
    "    SystemMessage,\n",
    "    ToolMessage,\n",
    ")\n",
    "model = CustomChatModelAdvanced(n=3, model_name=\"my_custom_model\")\n",
    "\n",
    "model.invoke(\n",
    "    [\n",
    "        HumanMessage(content=\"hello!\"),\n",
    "        AIMessage(content=\"Hi there human!\"),\n",
    "        HumanMessage(content=\"Meow!\"),\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bc16716-9747-4848-a946-e4d205cc7c9e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}