kafka_producer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d13f7eb6-e0d8-4136-9843-92bd2e63332a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from kafka import KafkaProducer\n",
    "from time import sleep\n",
    "from json import dumps\n",
    "import json\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c4a24ff6-7802-4f88-8e47-086882af4177",
   "metadata": {},
   "outputs": [],
   "source": [
    "producer = KafkaProducer(bootstrap_servers=['54.160.104.110:9092'], \n",
    "                         value_serializer=lambda x: \n",
    "                         dumps(x).encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b88808a6-116a-4542-aaeb-f50cbcc34903",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 6\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m row \u001b[38;5;129;01min\u001b[39;00m reader_obj:\n\u001b[1;32m      5\u001b[0m     producer\u001b[38;5;241m.\u001b[39msend(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124maws_venkat_stream\u001b[39m\u001b[38;5;124m'\u001b[39m, value\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(row))\n\u001b[0;32m----> 6\u001b[0m     \u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "with open(\"Stockmarket.csv\") as file_obj:\n",
    "    reader_obj = csv.reader(file_obj)\n",
    "    headers = next(reader_obj) # skip the header row\n",
    "    for row in reader_obj:\n",
    "        producer.send('aws_venkat_stream', value=','.join(row))\n",
    "        sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "22403545-8bdc-4055-ae88-fa83049ef17a",
   "metadata": {},
   "outputs": [],
   "source": [
    "producer.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f34f9f4e-20e9-4a8b-82ad-4c508bb46d40",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}