001a_Download_über_API_der_DDB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e9dff5d2-6617-4de8-af36-fc2e337627bf",
   "metadata": {},
   "source": [
    "# Summer School: Digitale Methoden der Zeitungsanalyse\n",
    "\n",
    "*Hinweis:* Dieses Notebook enthält nur den Python-Code. Über „Run -> Run All Cells“ kann die Zeitung „[La otra Alemania](https://www.deutsche-digitale-bibliothek.de/newspaper/2149754-0)“ heruntergeladen werden."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac86f476-9955-4d6e-8c5d-bb9963ae7662",
   "metadata": {},
   "outputs": [],
   "source": [
    "# benötigt Python-Bibliotheken installieren\n",
    "%pip install -q pysolr pandas requests lxml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4126df25-428c-4efc-9609-1c04705fdd5d",
   "metadata": {},
   "source": [
    "## Suchindex `newspaper`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01056062-68e8-41b0-b034-2609ce6d3962",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pysolr\n",
    "\n",
    "# Solr-Endpunkt-URL\n",
    "solr_url = 'https://api.deutsche-digitale-bibliothek.de/2/search/index/newspaper'\n",
    "\n",
    "# Solr-Client initialisieren\n",
    "solr = pysolr.Solr(solr_url, timeout=10)\n",
    "\n",
    "# Suchparameter\n",
    "q = {\n",
    "    'q': 'location:\"buenos aires\" AND hasLoadedIssues:true',\n",
    "    'fl': 'id,title,location,frequency,progress',\n",
    "    'rows': 10  # Anzahl der zurückzugebenden Ergebnisse\n",
    "}\n",
    "\n",
    "# Suche ausführen\n",
    "results = solr.search(**q)\n",
    "\n",
    "# Ergebnisse ausgeben\n",
    "for result in results:\n",
    "    print(f\"ID: {result.get('id', 'N/A')}\")\n",
    "    print(f\"Title: {result.get('title', 'N/A')}\")\n",
    "    print(f\"Location: {result.get('location', 'N/A')}\")\n",
    "    print(f\"Frequency: {result.get('frequency', 'N/A')}\")\n",
    "    print(f\"Progress: {result.get('progress', 'N/A')}\")\n",
    "    print(\"-\" * 40)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4a32e08-5be4-4c20-a966-82946e0e8211",
   "metadata": {},
   "source": [
    "## Suchindex `newspaper-issues`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87a6e147-54a8-4f63-a82a-ac3fbb1fc7d0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "solr_url = 'https://api.deutsche-digitale-bibliothek.de/2/search/index/newspaper-issues'\n",
    "solr = pysolr.Solr(solr_url, always_commit=True, timeout=10)\n",
    "\n",
    "q = {\n",
    "    'q': 'zdb_id:2149754-0 AND type:issue',\n",
    "    'rows': 1000\n",
    "}\n",
    "\n",
    "response = solr.search(**q)\n",
    "    \n",
    "# Überführen der Ergebnisse in ein Pandas DataFrame\n",
    "df = pd.DataFrame(response.docs)\n",
    "\n",
    "# DataFrame anzeigen\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "191ef63c-864e-4739-803b-ea2f2d4c0a5e",
   "metadata": {},
   "source": [
    "### Erste Datenanalyse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e640fb9d-06cf-4a4a-938d-c81e1fa4bc5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sicherstellen, dass publication_date als Datumswerte formatiert sind\n",
    "df['publication_date'] = pd.to_datetime(df['publication_date'])\n",
    "\n",
    "# Frühestes und spätestes Datum ermitteln\n",
    "earliest_date = df['publication_date'].min()\n",
    "latest_date = df['publication_date'].max()\n",
    "\n",
    "# Ergebnisse anzeigen\n",
    "print(f\"Frühestes Veröffentlichungsdatum: {earliest_date}\")\n",
    "print(f\"Spätestes Veröffentlichungsdatum: {latest_date}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bd67e3f-bdfa-43ea-9664-3ba622f8b700",
   "metadata": {},
   "source": [
    "## Download der METS/MODS-Daten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf81231f-1290-4ef8-883c-edf40ed642c0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "\n",
    "# Verzeichnis für die XML-Dateien erstellen\n",
    "directory = 'La_Otra_Alemania'\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "\n",
    "# Durch jede Zeile des DataFrames iterieren\n",
    "for index, row in df.iterrows():\n",
    "    # Extrahiere den Wert der Spalte id\n",
    "    item_id = row['id']\n",
    "    \n",
    "    # Formatiere den Wert der Spalte publication_date im Format YYYY-MM-DD\n",
    "    publication_date = row['publication_date'].strftime('%Y-%m-%d')\n",
    "    \n",
    "    # Generiere die URL zur API-Abfrage\n",
    "    url = f'https://api.deutsche-digitale-bibliothek.de/2/items/{item_id}/source/record'\n",
    "    \n",
    "    # Setze die HTTP-Header\n",
    "    headers = {\n",
    "        'Accept': 'application/xml'\n",
    "    }\n",
    "    \n",
    "    # API-Anfrage senden\n",
    "    response = requests.get(url, headers=headers)\n",
    "    \n",
    "    if response.status_code == 200:\n",
    "        # Erstelle einen Dateipfad für die XML-Datei\n",
    "        file_path = os.path.join(directory, f'{publication_date}_{item_id}.xml')\n",
    "        \n",
    "        # Speichere die XML-Datei im erstellten Verzeichnis\n",
    "        with open(file_path, 'wb') as file:\n",
    "            file.write(response.content)\n",
    "        print(f'Datei gespeichert: {file_path}')\n",
    "    else:\n",
    "        print(f'Fehler beim Abrufen der Datei für ID {item_id}: {response.status_code}')\n",
    "\n",
    "print('Fertig!')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbb5694f-835d-4d75-84b2-fb9a66053ac8",
   "metadata": {},
   "source": [
    "## Download der Bild- und Volltextdaten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1251259d-2e7f-4019-9dc3-393d175749ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "\n",
    "# Definiere die Namensräume\n",
    "NAMESPACES = {\n",
    "    'mets': 'http://www.loc.gov/METS/',\n",
    "    'xlink': 'http://www.w3.org/1999/xlink'\n",
    "}\n",
    "\n",
    "# Verzeichnisse definieren\n",
    "xml_directory = 'La_Otra_Alemania'\n",
    "download_directory = 'La_Otra_Alemania'\n",
    "\n",
    "# Erstelle das Download-Verzeichnis, falls es nicht existiert\n",
    "if not os.path.exists(download_directory):\n",
    "    os.makedirs(download_directory)\n",
    "\n",
    "# Funktion, um URLs aus einer XML-Datei zu extrahieren und herunterzuladen\n",
    "def download_files_from_xml(xml_file_path, xpath_expr, subfolder, extension):\n",
    "    # Lade die XML-Datei ein\n",
    "    with open(xml_file_path, 'rb') as xml_file:\n",
    "        tree = etree.parse(xml_file)\n",
    "\n",
    "    # Extrahiere die URLs mit dem gegebenen XPath-Ausdruck\n",
    "    urls = tree.xpath(xpath_expr, namespaces=NAMESPACES)\n",
    "\n",
    "    # Erstelle das Unterverzeichnis, benannt nach der XML-Datei\n",
    "    xml_file_name = os.path.splitext(os.path.basename(xml_file_path))[0]\n",
    "    destination_dir = os.path.join(download_directory, xml_file_name, subfolder)\n",
    "    os.makedirs(destination_dir, exist_ok=True)\n",
    "\n",
    "    # Lade jede URL herunter und speichere die Datei\n",
    "    for i, url in enumerate(urls, start=1):\n",
    "        try:\n",
    "            response = requests.get(url)\n",
    "            response.raise_for_status()  # Überprüfe auf HTTP-Fehler\n",
    "            file_path = os.path.join(destination_dir, f'{i}.{extension}')\n",
    "            with open(file_path, 'wb') as output_file:\n",
    "                output_file.write(response.content)\n",
    "            print(f'Downloaded {url} to {file_path}')\n",
    "        except requests.RequestException as e:\n",
    "            print(f'Failed to download {url}: {e}')\n",
    "\n",
    "# Durchlaufe alle XML-Dateien im Verzeichnis\n",
    "for xml_file in os.listdir(xml_directory):\n",
    "    if xml_file.endswith('.xml'):\n",
    "        xml_file_path = os.path.join(xml_directory, xml_file)\n",
    "        # Lade und speichere Dateien mit dem ersten XPath-Ausdruck\n",
    "        download_files_from_xml(xml_file_path, '//mets:mets/mets:fileSec/mets:fileGrp[@USE=\"DEFAULT\"]/mets:file/mets:FLocat/@xlink:href', 'DEFAULT', 'jpeg')\n",
    "        # Lade und speichere Dateien mit dem zweiten XPath-Ausdruck\n",
    "        download_files_from_xml(xml_file_path, '//mets:mets/mets:fileSec/mets:fileGrp[@USE=\"DDB_FULLTEXT\"]/mets:file/mets:FLocat/@xlink:href', 'DDB_FULLTEXT', 'xml')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}