OikotieApartment fix (#76)

* Update listing card xpath

* Add last page wait

* Print benchmark stats in case of xfail

finscraper/scrapy_spiders/oikotieapartment.py

@@ -173,6 +173,8 @@ def start_requests(self):
     def _get_last_page(self, driver):
         logger.debug('Getting last page...')
         last_page_xpath = '//span[contains(@ng-bind, "ctrl.totalPages")]'
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.XPATH, last_page_xpath)))
         last_page_element = driver.find_element(By.XPATH, last_page_xpath)
         last_page = int(last_page_element.text.split('/')[-1].strip())
         logger.debug(f'Last page found: {last_page}')
@@ -211,7 +213,7 @@ def _handle_pagination_page(self, request, spider, driver):
         driver.get(request.url)
 
         logger.debug('Scrolling pagination page to bottom...')
-        listings_xpath = '//div[contains(@class, "cards__card")]'
+        listings_xpath = '//div[contains(@class, "ot-card-v2__wrapper")]'
         driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
 
         logger.debug('Waiting for listings to be available...')

tests/test_spiders.py

@@ -112,9 +112,9 @@ def test_functionality(spider_cls, spider_params):
 @pytest.mark.xfail(reason='Benchmark')
 def test_benchmark_scrape_1_min(spider_cls, spider_params, capsys):
     df = spider_cls(**spider_params).scrape(0, 60).get()
-    assert len(df) >= 60
     with capsys.disabled():
         print(f'-- {len(df)} items')
+    assert len(df) >= 60
 
 
 @pytest.mark.parametrize('spider_cls, spider_params', other_cases,
@@ -124,6 +124,6 @@ def test_benchmark_scrape_100_items(spider_cls, spider_params, capsys):
     start = time.perf_counter()
     df = spider_cls(**spider_params).scrape(100).get()
     elapsed_time = int(time.perf_counter() - start)
-    assert len(df) >= 100
     with capsys.disabled():
         print(f'-- {elapsed_time} seconds ({len(df)} items)')
+    assert len(df) >= 100