add smartscraper lite

examples/anthropic/smart_scraper_lite_anthropic.py

@@ -0,0 +1,32 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))
+

examples/azure/smart_scraper_lite_azure.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-4o"
+    },
+    "verbose": True,
+    "headless": False
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/bedrock/smart_scraper_lite_bedrock.py

@@ -0,0 +1,26 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import json
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    }
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/deepseek/smart_scraper_lite_deepseek.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("DEEPSEEK_API_KEY"),
+        "model": "deepseek/deepseek-coder-33b-instruct",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/ernie/smart_scraper_lite_ernie.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ERNIE_API_KEY"),
+        "model": "ernie/ernie-bot-4",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/fireworks/smart_scraper_lite_fireworks.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("FIREWORKS_API_KEY"),
+        "model": "fireworks/llama-v2-70b-chat",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/google_genai/smart_scraper_lite_google_genai.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("GOOGLE_API_KEY"),
+        "model": "gemini-pro",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/google_vertexai/smart_scraper_lite_google_vertexai.py

@@ -0,0 +1,33 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "project": os.getenv("GOOGLE_CLOUD_PROJECT"),
+        "location": "us-central1",
+        "model": "text-bison@001",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))
+

examples/google_vertexai/smart_scraper_multi_lite_vertex.py

@@ -0,0 +1,47 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperMultiLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "project": os.getenv("GOOGLE_CLOUD_PROJECT"),
+        "location": "us-central1",
+        "model": "text-bison@001",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph(
+    prompt="Who is Marco Perini?",
+    source= [
+        "https://perinim.github.io/",
+        "https://perinim.github.io/cv/"
+    ],
+    config=graph_config
+)
+
+result = smart_scraper_multi_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/groq/smart_scraper_lite_groq.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("GROQ_API_KEY"),
+        "model": "mixtral-8x7b-32768",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/huggingfacehub/smart_scraper_lite_huggingfacehub.py

@@ -0,0 +1,31 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("HUGGINGFACEHUB_API_TOKEN"),
+        "model": "huggingfacehub/meta-llama/Llama-2-70b-chat-hf",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/local_models/smart_scraper_lite_ollama.py

@@ -0,0 +1,30 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+
+"""
+import json
+from scrapegraphai.graphs import SmartScraperLiteGraph
+from scrapegraphai.utils import prettify_exec_info
+
+graph_config = {
+    "llm": {
+        "model": "ollama/llama3.1",
+        "temperature": 0,
+        "format": "json",
+        "base_url": "http://localhost:11434",
+    },
+    "verbose": True,
+    "headless": False
+}
+
+smart_scraper_lite_graph = SmartScraperLiteGraph(
+    prompt="Who is Marco Perini?",
+    source="https://perinim.github.io/",
+    config=graph_config
+)
+
+result = smart_scraper_lite_graph.run()
+print(json.dumps(result, indent=4))
+
+graph_exec_info = smart_scraper_lite_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))