37 lines
1.3 KiB
Python
37 lines
1.3 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
def scrape_url(url):
|
|
"""
|
|
Scrapes the content of a given URL for business analysis.
|
|
"""
|
|
try:
|
|
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
|
|
response = requests.get(url, headers=headers, timeout=10)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
|
|
# Remove script and style elements
|
|
for script in soup(["script", "style"]):
|
|
script.decompose()
|
|
|
|
# Get text
|
|
text = soup.get_text()
|
|
|
|
# Break into lines and remove leading and trailing whitespace
|
|
lines = (line.strip() for line in text.splitlines())
|
|
# Break multi-headlines into a line each
|
|
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
|
# Drop blank lines
|
|
text = '\n'.join(chunk for chunk in chunks if chunk)
|
|
|
|
return text[:5000] # Return first 5000 characters for context
|
|
except Exception as e:
|
|
return f"Error scraping URL: {str(e)}"
|
|
|
|
if __name__ == "__main__":
|
|
url = "https://www.mik-tse.com"
|
|
content = scrape_url(url)
|
|
print(f"Scraped content from {url}:\n{content[:200]}...")
|