Developing subdata access.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from icf.frame import Frame\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "from icf.pyicf import ICFFile\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from icf.frame import dispatch_serializer\n",
    "import struct\n",
    "import numpy as np\n",
    "from importlib import import_module\n",
    "import logging\n",
    "import sys\n",
    "class TopicFrame(Frame):\n",
    "    \n",
    "    def serialize(self) -> bytes:\n",
    "        \"\"\"Serializes the frame to a bytestream\n",
    "\n",
    "        Returns:\n",
    "            bytes: serialized frame\n",
    "        \"\"\"\n",
    "        data_stream = bytearray()\n",
    "        index = []\n",
    "        classes = []\n",
    "        pos = 0\n",
    "        for k, v in self.items():\n",
    "            v = dispatch_serializer(v)\n",
    "            _module = v.__class__.__module__\n",
    "            _module = _module if _module not in [__name__,\"icf.frame\"] else \"\"\n",
    "            classes.append(\"{},{},{}\\n\".format(k, v.__class__.__name__, _module))\n",
    "            d = v.serialize()\n",
    "            pos += len(d)\n",
    "            index.append(pos)\n",
    "            data_stream.extend(d)\n",
    "        n_obj = len(index)\n",
    "\n",
    "        classes = \"\".join(classes)\n",
    "        header = bytearray(struct.pack(\n",
    "            \"<2H{}I{}s\".format(n_obj, len(classes)),\n",
    "            len(classes),\n",
    "            n_obj,\n",
    "            *index,\n",
    "            classes.encode(),\n",
    "        ))\n",
    "        print(classes)\n",
    "        header.extend(data_stream)\n",
    "        return header\n",
    "    \n",
    "    @classmethod\n",
    "    def deserialize_header(cls,data):\n",
    "        ncls, nobj=struct.unpack(\"<2H\",data[:4])\n",
    "        print(ncls,nobj,ncls+4*nobj)\n",
    "        index = struct.unpack(\"<{}I{}s\".format(nobj, ncls), data[4:ncls+4*nobj+4])\n",
    "        classes = index[nobj:][0].decode()\n",
    "        index = list(index[:nobj])\n",
    "        print(classes)\n",
    "        print(index)\n",
    "        \n",
    "class FileStream:\n",
    "    def __init__(self, path, mode=\"\"):\n",
    "        self.buffer = bytearray()\n",
    "        self.size = len(self.buffer)\n",
    "        self.fp = 0\n",
    "        self.path = path\n",
    "        self.mode = mode\n",
    "        \n",
    "    def read(self, size=-1):\n",
    "        if size == -1:\n",
    "            size = self.size\n",
    "        self.fp += size\n",
    "        return self.buffer[self.fp-size:self.fp]\n",
    "    \n",
    "    def write(self, buffer):\n",
    "        self.buffer.extend(buffer)\n",
    "        self.size = len(self.buffer)\n",
    "        self.fp = self.size\n",
    "        return len(buffer)\n",
    "    def flush(self):\n",
    "        pass\n",
    "    def close(self):\n",
    "        pass\n",
    "    def tell(self):\n",
    "        return self.fp\n",
    "    \n",
    "    def seek(self,cookie, whence=0):\n",
    "        if whence == 0:\n",
    "            self.fp = cookie\n",
    "            return self.fp\n",
    "        if whence == 1:\n",
    "            self.fp += cookie\n",
    "            return self.fp\n",
    "        if whence == 2:\n",
    "            self.fp = self.size+cookie\n",
    "            return self.fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame = TopicFrame()\n",
    "frame.add(\"First\",1)\n",
    "frame.add(\"Second\", np.array([123,2,3,54,5,3,23,4,35,42,3,43,],dtype=np.uint8))\n",
    "print(frame.serialize())\n",
    "print(len(frame.serialize()))\n",
    "TopicFrame.deserialize_header(frame.serialize())\n",
    "frame = Frame()\n",
    "frame.add(\"First\",1)\n",
    "frame.add(\"Second\", [123,2,3,54,5,3,23,4,35,42,3,43,])\n",
    "print(frame.serialize())\n",
    "print(len(frame.serialize()))\n",
    "f = FileStream(frame.serialize())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = ICFFile(\"test\",custom_stream=FileStream(\"test\"))\n",
    "file.write(frame.serialize())\n",
    "frame[\"some array\"] = np.arange(10)\n",
    "file.write(frame.serialize())\n",
    "file.flush()\n",
    "file.close()\n",
    "file._file.buffer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filer= ICFFile(\"test.icf\",\"trunc\")\n",
    "\n",
    "filer.write(frame.serialize())\n",
    "\n",
    "filer.write(frame.serialize())\n",
    "filer.flush()\n",
    "#file._file.buffer\n",
    "filer.close()\n",
    "d =  open(\"test.icf\",\"rb\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(d))\n",
    "print(len(file._file.buffer))\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Frame.deserialize(file.read_at(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end = file._file.seek(0,2)\n",
    "file._file.seek(end-4)\n",
    "d = file._file.read(4)\n",
    "i = struct.unpack(\"<I\", d)[0]\n",
    "print(d,i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file._file.buffer.extend(file._file.buffer)\n",
    "file._file.size = len(file._file.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(file._file.buffer)\n",
    "\n",
    "file._file.seek(0)\n",
    "file2 = ICFFile(\"test\",custom_stream=file._file)\n",
    "print(file2)\n",
    "Frame.deserialize(file2.read_at(3))['some array']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f.read(10))\n",
    "print(f.read(10))\n",
    "print(f.tell())\n",
    "print(f.seek(0))\n",
    "print(f.read(10))\n",
    "print(f.seek(-8,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "startTime = datetime.now()\n",
    "writer = ICFFile(\"testing.icf\", compressor=None)  #'bz2')\n",
    "\n",
    "for i in range(90):\n",
    "\n",
    "    frame = Frame()\n",
    "    frame.add(\"r\",1)\n",
    "    frame.add(\"r2\", np.random.uniform(0,1012321))\n",
    "    #frame[\"a_list_of_lists\"] = [1, 3, 4, 5, [9, 4, 5], (93, 3.034)]\n",
    "    writer.write(frame.serialize())\n",
    "writer.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}