qie.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "创建文件夹\tsplit_wei\n",
      "目标路径：\tsplit_wei\n",
      "分割文件：\twei.csv\n",
      "分割大小：\t100\n",
      "wei_1_101.csv\t保存成功\n",
      "wei_101_201.csv\t保存成功\n",
      "wei_201_223.csv\t保存成功\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "res_file_path = Path(r\"wei.csv\")                 #待分割文件路径\n",
    "split_size = 100          #子文件行数最大值\n",
    "tar_dir = res_file_path.parent/(\"split_\"+res_file_path.name.split(\".\")[0])\n",
    "if not tar_dir.exists():\n",
    "    tar_dir.mkdir()\n",
    "    print(\"创建文件夹\\t\"+str(tar_dir))\n",
    "print(\"目标路径：\\t\"+str(tar_dir))\n",
    "print(\"分割文件：\\t\"+str(res_file_path))\n",
    "print(\"分割大小：\\t\"+\"{:,}\".format(split_size))\n",
    "tmp = pd.read_csv(res_file_path,nrows = 10)\n",
    "columns = tmp.columns.to_list()\n",
    "idx = 0\n",
    "while(len(tmp)>0):\n",
    "    start = 1+(idx*split_size)\n",
    "    tmp = pd.read_csv(res_file_path,\n",
    "                header = None,\n",
    "                names = columns,\n",
    "                skiprows = start,\n",
    "                nrows = split_size)\n",
    "    if len(tmp) <= 0:\n",
    "        break\n",
    "    file_name = res_file_path.name.split(\".\")[0]+\"_{}_{}\".format(start,start+len(tmp))+\".csv\"\n",
    "    file_path = tar_dir/file_name\n",
    "    tmp.to_csv(file_path,index=False)\n",
    "    idx+=1\n",
    "    print(file_name +\"\\t保存成功\")\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python3\n",
    "# -*- coding:utf-8 -*-\n",
    "# @FileName :Test.py\n",
    "# @Software PyCharm\n",
    " \n",
    "import os\n",
    "import pandas as pd\n",
    " \n",
    "# filename为文件路径，file_num为拆分后的文件行数\n",
    "# 根据是否有表头执行不同程序，默认有表头的\n",
    "def Data_split(filename,file_num,header=True):\n",
    "    if header:\n",
    "        # 设置每个文件需要有的行数,初始化为1000W\n",
    "        chunksize=500\n",
    "        data1=pd.read_table(filename,chunksize=chunksize,sep=',',encoding='gbk')\n",
    "        # print(data1)\n",
    "        # num表示总行数\n",
    "        num=0\n",
    "        for chunk in data1:\n",
    "            num+=len(chunk)\n",
    "        # print(num)\n",
    "        # chunksize表示每个文件需要分配到的行数\n",
    "        chunksize=round(num/file_num+1)\n",
    "        # print(chunksize)\n",
    "        # 分离文件名与扩展名os.path.split(filename)\n",
    "        head,tail=os.path.split(filename)\n",
    "        data2=pd.read_table(filename,chunksize=chunksize,sep=',',encoding='gbk')\n",
    "        i=0\n",
    "        for chunk in data2:\n",
    "            chunk.to_csv('{0}_{1}{2}'.format(head,i,tail),header=None,index=False)\n",
    "            print('保存第{0}个数据'.format(i))\n",
    "            i+=1\n",
    "    else:\n",
    "        # 获得每个文件需要的行数\n",
    "        chunksize=10000\n",
    "        data1=pd.read_table(filename,chunksize=chunksize,header=None,sep=',')\n",
    "        num=0\n",
    "        for chunk in data1:\n",
    "            num+=len(chunk)\n",
    "            chunksize=round(num/file_num+1)\n",
    " \n",
    "            head,tail=os.path.split(filename)\n",
    "            data2=pd.read_table(filename,chunksize=chunksize,header=None,sep=',')\n",
    "            i=0\n",
    "            for chunk in data2:\n",
    "                chunk.to_csv('{0}_{1}{2}'.format(head,i,tail),header=None,index=False)\n",
    "                print('保存第{0}个数据'.format(i))\n",
    "                i+=1\n",
    " \n",
    "filename='文件路径'\n",
    "#num为拆分为的文件个数\n",
    "Data_split(filename,num,header=True)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "57baa5815c940fdaff4d14510622de9616cae602444507ba5d0b6727c008cbd6"
  },
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}