From 69aada2bc6ba7c5510fc57065ea3a861be0bdc55 Mon Sep 17 00:00:00 2001
From: h-vetinari <33685575+h-vetinari@users.noreply.github.com>
Date: Sat, 15 Dec 2018 19:04:23 +0100
Subject: [PATCH] CI for boto: fix errors; add coverage; add skip for
 uncatchable ResourceWarning (#23731)

---
 .travis.yml                 |  6 +++
 ci/deps/travis-36.yaml      |  3 +-
 ci/deps/travis-37.yaml      |  4 ++
 environment.yml             |  3 ++
 pandas/tests/io/conftest.py | 91 +++++++++++++++++++++----------------
 pandas/util/testing.py      | 16 +++++++
 requirements-dev.txt        |  3 ++
 7 files changed, 85 insertions(+), 41 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 03026647d6bb8c..e0ab770ac46ba9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -90,6 +90,12 @@ before_install:
   - uname -a
   - git --version
   - git tag
+  # Because travis runs on Google Cloud and has a /etc/boto.cfg,
+  # it breaks moto import, see:
+  # https://github.com/spulec/moto/issues/1771
+  # https://github.com/boto/boto/issues/3741
+  # This overrides travis and tells it to look nowhere.
+  - export BOTO_CONFIG=/dev/null
 
 install:
   - echo "install start"
diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml
index bfd69652730ed2..37cea281d48a61 100644
--- a/ci/deps/travis-36.yaml
+++ b/ci/deps/travis-36.yaml
@@ -4,6 +4,7 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
+  - botocore>=1.11
   - cython>=0.28.2
   - dask
   - fastparquet
@@ -35,10 +36,10 @@ dependencies:
   - pytest
   - pytest-xdist
   - pytest-cov
-  - moto
   - hypothesis>=3.58.0
   - pip:
     - brotlipy
     - coverage
+    - moto
     - pandas-datareader
     - python-dateutil
diff --git a/ci/deps/travis-37.yaml b/ci/deps/travis-37.yaml
index a297786f6b14d0..c503124d8cd260 100644
--- a/ci/deps/travis-37.yaml
+++ b/ci/deps/travis-37.yaml
@@ -5,6 +5,7 @@ channels:
   - c3i_test
 dependencies:
   - python=3.7
+  - botocore>=1.11
   - cython>=0.28.2
   - numpy
   - python-dateutil
@@ -14,3 +15,6 @@ dependencies:
   - pytest
   - pytest-xdist
   - hypothesis>=3.58.0
+  - s3fs
+  - pip:
+    - moto
diff --git a/environment.yml b/environment.yml
index a2342d37dd6a41..a7eb6608b60459 100644
--- a/environment.yml
+++ b/environment.yml
@@ -27,6 +27,8 @@ dependencies:
   # optional
   - beautifulsoup4>=4.2.1
   - blosc
+  - botocore>=1.11
+  - boto3
   - bottleneck>=1.2.0
   - fastparquet>=0.1.2
   - html5lib
@@ -42,6 +44,7 @@ dependencies:
   - pytables>=3.4.2
   - pytest-cov
   - pytest-xdist
+  - s3fs
   - scipy>=1.1
   - seaborn
   - sqlalchemy
diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py
index 928519d39aed3f..af6f7ac4ef5287 100644
--- a/pandas/tests/io/conftest.py
+++ b/pandas/tests/io/conftest.py
@@ -1,4 +1,10 @@
+from distutils.version import LooseVersion
+import os
+
 import pytest
+
+import pandas.util.testing as tm
+
 from pandas.io.parsers import read_csv
 
 
@@ -37,43 +43,48 @@ def s3_resource(tips_file, jsonl_file):
     """
     pytest.importorskip('s3fs')
     boto3 = pytest.importorskip('boto3')
-    # GH-24092. See if boto.plugin skips the test or fails.
-    try:
-        pytest.importorskip("boto.plugin")
-    except AttributeError:
-        raise pytest.skip("moto/moto error")
-    moto = pytest.importorskip('moto')
-
-    test_s3_files = [
-        ('tips.csv', tips_file),
-        ('tips.csv.gz', tips_file + '.gz'),
-        ('tips.csv.bz2', tips_file + '.bz2'),
-        ('items.jsonl', jsonl_file),
-    ]
-
-    def add_tips_files(bucket_name):
-        for s3_key, file_name in test_s3_files:
-            with open(file_name, 'rb') as f:
-                conn.Bucket(bucket_name).put_object(
-                    Key=s3_key,
-                    Body=f)
-
-    try:
-
-        s3 = moto.mock_s3()
-        s3.start()
-
-        # see gh-16135
-        bucket = 'pandas-test'
-        conn = boto3.resource("s3", region_name="us-east-1")
-
-        conn.create_bucket(Bucket=bucket)
-        add_tips_files(bucket)
-
-        conn.create_bucket(Bucket='cant_get_it', ACL='private')
-        add_tips_files('cant_get_it')
-        yield conn
-    except:  # noqa: flake8
-        pytest.skip("failure to use s3 resource")
-    finally:
-        s3.stop()
+    botocore = pytest.importorskip('botocore')
+
+    if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"):
+        # botocore leaks an uncatchable ResourceWarning before 1.11.0;
+        # see GH 23731 and https://github.com/boto/botocore/issues/1464
+        pytest.skip("botocore is leaking resources before 1.11.0")
+
+    with tm.ensure_safe_environment_variables():
+        # temporary workaround as moto fails for botocore >= 1.11 otherwise,
+        # see https://github.com/spulec/moto/issues/1924 & 1952
+        os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
+        os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
+
+        moto = pytest.importorskip('moto')
+
+        test_s3_files = [
+            ('tips.csv', tips_file),
+            ('tips.csv.gz', tips_file + '.gz'),
+            ('tips.csv.bz2', tips_file + '.bz2'),
+            ('items.jsonl', jsonl_file),
+        ]
+
+        def add_tips_files(bucket_name):
+            for s3_key, file_name in test_s3_files:
+                with open(file_name, 'rb') as f:
+                    conn.Bucket(bucket_name).put_object(
+                        Key=s3_key,
+                        Body=f)
+
+        try:
+            s3 = moto.mock_s3()
+            s3.start()
+
+            # see gh-16135
+            bucket = 'pandas-test'
+            conn = boto3.resource("s3", region_name="us-east-1")
+
+            conn.create_bucket(Bucket=bucket)
+            add_tips_files(bucket)
+
+            conn.create_bucket(Bucket='cant_get_it', ACL='private')
+            add_tips_files('cant_get_it')
+            yield conn
+        finally:
+            s3.stop()
diff --git a/pandas/util/testing.py b/pandas/util/testing.py
index 56a3cda1ba89fe..8a408f5613a018 100644
--- a/pandas/util/testing.py
+++ b/pandas/util/testing.py
@@ -782,6 +782,22 @@ def ensure_clean_dir():
             pass
 
 
+@contextmanager
+def ensure_safe_environment_variables():
+    """
+    Get a context manager to safely set environment variables
+
+    All changes will be undone on close, hence environment variables set
+    within this contextmanager will neither persist nor change global state.
+    """
+    saved_environ = dict(os.environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(saved_environ)
+
+
 # -----------------------------------------------------------------------------
 # Comparators
 
diff --git a/requirements-dev.txt b/requirements-dev.txt
index dfaf280d65f1cd..fe319f8be1386f 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -16,6 +16,8 @@ sphinxcontrib-spelling
 numpydoc
 beautifulsoup4>=4.2.1
 blosc
+botocore>=1.11
+boto3
 bottleneck>=1.2.0
 fastparquet>=0.1.2
 html5lib
@@ -31,6 +33,7 @@ pyarrow>=0.7.0
 tables>=3.4.2
 pytest-cov
 pytest-xdist
+s3fs
 scipy>=1.1
 seaborn
 sqlalchemy