Psoriasis_Paper_PubMed_scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6G0BlnDnyYup",
        "outputId": "2a594483-be98-4aec-90fb-a15585a84a24"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting biopython\n",
            "  Downloading biopython-1.84-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.10/dist-packages (3.1.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (2.32.3)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (4.12.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.2.2)\n",
            "Collecting pycountry\n",
            "  Downloading pycountry-24.6.1-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting country_converter\n",
            "  Downloading country_converter-1.2-py3-none-any.whl.metadata (24 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from biopython) (1.26.4)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.10/dist-packages (from openpyxl) (1.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests) (2024.8.30)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4) (2.6)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
            "Downloading biopython-1.84-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m31.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pycountry-24.6.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m63.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading country_converter-1.2-py3-none-any.whl (45 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pycountry, biopython, country_converter\n",
            "Successfully installed biopython-1.84 country_converter-1.2 pycountry-24.6.1\n"
          ]
        }
      ],
      "source": [
        "!pip install biopython openpyxl requests beautifulsoup4 pandas pycountry country_converter"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z1tT5lkEyc_3",
        "outputId": "e39cff12-5fff-4298-9d77-40dcd018c640"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "PubMed Searching"
      ],
      "metadata": {
        "id": "W8Mbh2FaKXE0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import time\n",
        "from datetime import datetime\n",
        "import pandas as pd\n",
        "from Bio import Entrez, Medline\n",
        "\n",
        "def search_pubmed(query, start_date, end_date, email):\n",
        "    Entrez.email = email\n",
        "    handle = Entrez.esearch(db=\"pubmed\",\n",
        "                            term=query,\n",
        "                            mindate=start_date,\n",
        "                            maxdate=end_date,\n",
        "                            retmax=100000)\n",
        "    record = Entrez.read(handle)\n",
        "    return record[\"IdList\"]\n",
        "\n",
        "def fetch_details(id_list):\n",
        "    ids = \",\".join(id_list)\n",
        "    handle = Entrez.efetch(db=\"pubmed\", id=ids, rettype=\"medline\", retmode=\"text\")\n",
        "    records = Medline.parse(handle)\n",
        "    return list(records)\n",
        "\n",
        "def extract_info(record):\n",
        "    pmid = record.get(\"PMID\", \"\")\n",
        "    title = record.get(\"TI\", \"\")\n",
        "    abstract = record.get(\"AB\", \"\")\n",
        "    date = record.get(\"DP\", \"\")  # Publication date\n",
        "    pmdate = record.get(\"EDAT\", \"\")  # Entrez Date (when added to PubMed)\n",
        "    authors = \", \".join(record.get(\"AU\", []))\n",
        "    affiliations = \"; \".join(record.get(\"AD\", []))\n",
        "    journal = record.get(\"TA\", \"\")  # TA is the journal title abbreviation\n",
        "\n",
        "    return {\n",
        "        \"PMID\": pmid,\n",
        "        \"Title\": title,\n",
        "        \"Abstract\": abstract,\n",
        "        \"Publication Date\": date,\n",
        "        \"PubMed Date\": pmdate,\n",
        "        \"Authors\": authors,\n",
        "        \"Affiliations\": affiliations,\n",
        "        \"Journal\": journal\n",
        "    }\n",
        "\n",
        "def main():\n",
        "    # Mount Google Drive\n",
        "    #drive.mount('/content/drive')\n",
        "\n",
        "    query = \"psoriasis\"\n",
        "    start_date = \"2022/01/01\"\n",
        "    end_date = \"2023/12/31\"\n",
        "    email = \"your_email@example.com\"  # 请替换为您的邮箱\n",
        "\n",
        "    print(\"searching PubMed...\")\n",
        "    id_list = search_pubmed(query, start_date, end_date, email)\n",
        "    print(f\"find {len(id_list)} papers\")\n",
        "\n",
        "    print(\"details...\")\n",
        "    records = fetch_details(id_list)\n",
        "\n",
        "    print(\"scraping...\")\n",
        "    data = [extract_info(record) for record in records]\n",
        "\n",
        "    df = pd.DataFrame(data)\n",
        "\n",
        "    # 重新排列列的顺序\n",
        "    columns_order = ['PMID', 'Publication Date', 'PubMed Date', 'Journal', 'Title', 'Abstract', 'Authors', 'Affiliations']\n",
        "    df = df[columns_order]\n",
        "\n",
        "    # 设置保存路径\n",
        "    save_path = \"/content\"\n",
        "    output_file = os.path.join(save_path, f\"psoriasis_papers_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx\")\n",
        "\n",
        "    df.to_excel(output_file, index=False)\n",
        "    print(f\"data has saved to {output_file}\")\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "zzgwRQ5v0ML1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "XETNT1LT5DL_"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}