初始化项目,由ModelHub XC社区提供模型
Model: chatpbc1/chatpbc-v33 Source: Original Platform
This commit is contained in:
36
chatpbc_webscraper.py
Normal file
36
chatpbc_webscraper.py
Normal file
@@ -0,0 +1,36 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
def scrape_url(url):
|
||||
"""
|
||||
Scrapes the content of a given URL for business analysis.
|
||||
"""
|
||||
try:
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
|
||||
# Remove script and style elements
|
||||
for script in soup(["script", "style"]):
|
||||
script.decompose()
|
||||
|
||||
# Get text
|
||||
text = soup.get_text()
|
||||
|
||||
# Break into lines and remove leading and trailing whitespace
|
||||
lines = (line.strip() for line in text.splitlines())
|
||||
# Break multi-headlines into a line each
|
||||
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
||||
# Drop blank lines
|
||||
text = '\n'.join(chunk for chunk in chunks if chunk)
|
||||
|
||||
return text[:5000] # Return first 5000 characters for context
|
||||
except Exception as e:
|
||||
return f"Error scraping URL: {str(e)}"
|
||||
|
||||
if __name__ == "__main__":
|
||||
url = "https://www.mik-tse.com"
|
||||
content = scrape_url(url)
|
||||
print(f"Scraped content from {url}:\n{content[:200]}...")
|
||||
Reference in New Issue
Block a user