cerbervs/scrape
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v2.1.3

Browse Source 
* make text finding recursive
This commit is contained in:
Mark Bailey 2025-01-26 16:37:36 -05:00
parent 08e99b5b09
commit 9c54bd2481
1 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scrape.py
View File

@ -5,7 +5,7 @@ author_url: https://git.markbailey.dev/cerbervs
git_url: https://git.markbailey.dev/cerbervs/scrape.git
description: Scrapes web with option for recursive scraping.
requirements: websocket, requests, bs4, pydantic
version: 2.1.2
version: 2.1.3
licence: MIT
"""
@ -128,21 +128,20 @@ class Tools:
}
# Find all headings (h1 to h6)
heading_tag = ["h1", "h2", "h3", "h4", "h5", "h6"]
headings = soup.find_all(heading_tag)
headings = soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"])
if headings:
data["headings"].extend([tag.get_text() for tag in headings])
# Find all paragraphs
paragraphs = soup.find_all("p")
paragraphs = soup.find_all("p", recursive=True)
if paragraphs:
data["paragraphs"] = [p.get_text() for p in paragraphs]
divs = soup.find_all("div")
divs = soup.find_all("div", recursive=True)
if divs:
data["divs"] = [div.get_text() for div in divs]
lis = soup.find_all("li")
lis = soup.find_all("li", recursive=True)
if lis:
data["lis"] = [li.get_text() for li in lis]
@ -185,7 +184,7 @@ class Tools:
:params url: The URL of the web page to be scraped.
"""
if self.user_valves.single_request or self.valves.single_request:
if self.user_valves.single_request:
request_limit = 1
else:
request_limit = self.valves.request_limit
@ -198,4 +197,4 @@ class Tools:
if __name__ == "__main__":
tools = Tools()
print(tools.scrape_recursively("https://stackoverflow.com/questions/6486450/compute-list-difference"))
print(tools.scrape_recursively("https://en.wikipedia.org/wiki/Shamisen"))