cerbervs/scrape
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v2.0.0

Browse Source 
* add valves

* add user valves
This commit is contained in:
Mark Bailey 2025-01-26 15:09:18 -05:00
parent 0873723991
commit de2d8be111
1 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
scrape.py
View File

@ -3,7 +3,7 @@ import websocket
import requests import requests
from urllib.parse import urlparse as parse_url from urllib.parse import urlparse as parse_url
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from pydantic import BaseModel, ConfigDict from pydantic import BaseModel, ConfigDict, Field
DEFAULT_REQUEST_LIMIT = 1 DEFAULT_REQUEST_LIMIT = 1
@ -143,15 +143,18 @@ class Tools:
return data return data
def reset(self):
self.visited = []
self.data = {}
self.netloc = None
self.scheme = None
def __init__(self, req_limit: int): def __init__(self, req_limit: int):
super().__init__(req_limit=req_limit) super().__init__(req_limit=req_limit)
self.reset()
class Valves(BaseModel):
request_limit: int = Field(
default=5, description="Request Limit"
)
pass
class UserValves(BaseModel):
single_request: bool = Field(default=False, description="Single Request")
pass
def __init__(self): def __init__(self):
""" """
@ -160,23 +163,23 @@ class Tools:
""" """
self.citation: bool = True self.citation: bool = True
self.valves = self.Valves()
self.user_valves = self.UserValves()
def scrape_recursively( def scrape_recursively(self, url: str) -> str:
self, url: str, request_limit: int = DEFAULT_REQUEST_LIMIT
) -> str:
""" """
Scrapes data from a web page using requests and BeautifulSoup. Scrapes data from a web page using requests and BeautifulSoup.
:params url: The URL of the web page to be scraped. :params url: The URL of the web page to be scraped.
:params request_limit: The number of requests to be made to scrape the website. (Optional, Default = 1)
""" """
if self.user_valves.single_request:
request_limit = 1
else:
request_limit = self.valves.request_limit
scraper = self.RecursiveScraper(request_limit) scraper = self.RecursiveScraper(request_limit)
data = scraper.scrape_website(url) data = scraper.scrape_website(url)
json_s = json.dumps(data) json_s = json.dumps(data)
# if scraper.visited != [] or scraper.data != {} or scraper.netloc is not None or scraper.scheme is not None:
# scraper.reset()
scraper = None scraper = None
return json_s return json_s