Merge pull request #8 from eyeseast/httpx

Use httpx and a shared client

.github/workflows/publish.yml

@@ -0,0 +1,58 @@
+name: Publish Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        pip install -e '.[test]'
+    - name: Run tests
+      run: |
+        pytest
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [test]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-publish-pip-
+    - name: Install dependencies
+      run: |
+        pip install setuptools wheel twine
+    - name: Publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*
+

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Test
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      name: Configure pip caching
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        pip install -e '.[test]'
+    - name: Run tests
+      run: |
+        pytest
+

feed_to_sqlite/cli.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import click
+import httpx
 
 from .ingest import ingest_feed
 
@@ -13,5 +14,6 @@
 )
 @click.argument("urls", nargs=-1)
 def cli(database, urls, table=None):
-    for url in urls:
-        ingest_feed(database, table_name=table, url=url)
+    with httpx.Client() as client:
+        for url in urls:
+            ingest_feed(database, table_name=table, url=url, client=client)

feed_to_sqlite/ingest.py

@@ -1,6 +1,6 @@
 import datetime
 import feedparser
-import requests
+import httpx
 from slugify import Slugify
 from sqlite_utils import Database
 
@@ -9,24 +9,31 @@
 FEEDS_TABLE = "feeds"
 
 
-def ingest_feed(db, *, url=None, feed_content="", table_name=None, normalize=None):
+def ingest_feed(
+    db, *, url=None, feed_content="", table_name=None, normalize=None, client=None
+):
     """
     `db` is a path or Database instance
-    
+
     if `table_name` is None, auto create from feed title
-    
+
     if `url` is given, fetch and parse
-    
+
     if `feed_content` is given, but not url, parse that
 
     `normalize` is a function that will be called on each feed item, useful for fixing links
     or doing additional work. It's signature is normalize(table, entry, feed_details).
+
+    `client` is an instance of `httpx.Client` to pool requests.
     """
     if not isinstance(db, Database):
         db = Database(db)
 
+    if client is None:
+        client = httpx.Client()
+
     if url:
-        r = requests.get(url)
+        r = client.get(url)
         r.raise_for_status()
         feed_content = r.text
 

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 import os
 
-VERSION = "0.3.0"
+VERSION = "0.4.0"
 
 README = os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md")
 
@@ -26,7 +26,7 @@ def get_long_description():
     """,
     install_requires=[
         "sqlite-utils>=2.22",
-        "requests",
+        "httpx",
         "feedparser",
         "awesome-slugify",
     ],