import requests
from urllib.robotparser import RobotFileParser

def check_robots_txt(url):
    """Check if scraping is allowed by robots.txt"""
    rp = RobotFileParser()
    rp.set_url(url + "/robots.txt")
    rp.read()

    # Check if scraping is allowed for your user agent
    return rp.can_fetch("*", url)

import time
import random

class RateLimiter:
    def __init__(self, min_delay=1, max_delay=5):
        self.min_delay = min_delay
        self.max_delay = max_delay
        self.last_request = 0

    def wait(self):
        """Wait appropriate time between requests"""
        elapsed = time.time() - self.last_request
        delay = random.uniform(self.min_delay, self.max_delay)

        if elapsed < delay:
            time.sleep(delay - elapsed)

        self.last_request = time.time()

# Realistic user agents to avoid detection
USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15'
]

pip install requests beautifulsoup4 lxml selenium pandas

import requests
from bs4 import BeautifulSoup
import pandas as pd
from datetime import datetime

class BasicScraper:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        })

    def scrape_quotes(self, url="http://quotes.toscrape.com/"):
        """Scrape quotes from a test website"""
        response = self.session.get(url)

        if response.status_code != 200:
            print(f"Failed to fetch page: {response.status_code}")
            return []

        soup = BeautifulSoup(response.content, 'html.parser')
        quotes = []

        # Find all quote elements
        quote_elements = soup.find_all('div', class_='quote')

        for quote_elem in quote_elements:
            quote = {
                'text': quote_elem.find('span', class_='text').text.strip('"'),
                'author': quote_elem.find('small', class_='author').text,
                'tags': [tag.text for tag in quote_elem.find_all('a', class_='tag')],
                'scraped_at': datetime.now().isoformat()
            }
            quotes.append(quote)

        return quotes

    def save_to_csv(self, data, filename):
        """Save scraped data to CSV"""
        df = pd.DataFrame(data)
        df.to_csv(f"data/{filename}.csv", index=False)
        print(f"Saved {len(data)} records to {filename}.csv")

# Usage
if __name__ == "__main__":
    scraper = BasicScraper()
    quotes = scraper.scrape_quotes()
    scraper.save_to_csv(quotes, "quotes")

def scrape_all_pages(self, base_url):
    """Scrape data from all pages"""
    all_data = []
    page = 1

    while True:
        url = f"{base_url}/page/{page}/"
        response = self.session.get(url)

        if response.status_code != 200:
            break

        soup = BeautifulSoup(response.content, 'html.parser')

        # Check if page has content
        if not soup.find_all('div', class_='quote'):
            break

        # Scrape current page
        page_data = self.scrape_quotes_from_page(soup)
        all_data.extend(page_data)

        print(f"Scraped page {page}: {len(page_data)} items")
        page += 1

        # Be respectful - wait between requests
        time.sleep(2)

    return all_data

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

class SeleniumScraper:
    def __init__(self):
        self.options = Options()
        self.options.add_argument('--headless')  # Run in background
        self.options.add_argument('--no-sandbox')
        self.options.add_argument('--disable-dev-shm-usage')
        self.options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')

        self.driver = webdriver.Chrome(options=self.options)

    def scrape_dynamic_content(self, url):
        """Scrape content that loads with JavaScript"""
        self.driver.get(url)

        # Wait for dynamic content to load
        try:
            WebDriverWait(self.driver, 10).until(
                EC.presence_of_element_located((By.CLASS_NAME, "product-list"))
            )
        except:
            print("Timeout waiting for page to load")
            return []

        # Scroll to load more content
        self.scroll_to_bottom()

        # Extract data
        products = self.driver.find_elements(By.CLASS_NAME, "product-item")

        data = []
        for product in products:
            try:
                product_data = {
                    'name': product.find_element(By.CLASS_NAME, "product-name").text,
                    'price': product.find_element(By.CLASS_NAME, "product-price").text,
                    'rating': product.find_element(By.CLASS_NAME, "rating").get_attribute("data-rating"),
                    'url': product.find_element(By.TAG_NAME, "a").get_attribute("href")
                }
                data.append(product_data)
            except Exception as e:
                print(f"Error extracting product data: {e}")
                continue

        return data

    def scroll_to_bottom(self):
        """Scroll to bottom of page to load all content"""
        last_height = self.driver.execute_script("return document.body.scrollHeight")

        while True:
            # Scroll down
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

            # Wait for content to load
            time.sleep(2)

            # Check if we've reached the bottom
            new_height = self.driver.execute_script("return document.body.scrollHeight")

            if new_height == last_height:
                break

            last_height = new_height

    def handle_login(self, login_url, username, password):
        """Handle login for authenticated scraping"""
        self.driver.get(login_url)

        # Fill login form
        self.driver.find_element(By.ID, "username").send_keys(username)
        self.driver.find_element(By.ID, "password").send_keys(password)

        # Submit form
        self.driver.find_element(By.ID, "login-button").click()

        # Wait for login to complete
        WebDriverWait(self.driver, 10).until(
            EC.url_changes(login_url)
        )

    def close(self):
        """Close the browser"""
        self.driver.quit()

import scrapy
from scrapy.crawler import CrawlerProcess

class ProductSpider(scrapy.Spider):
    name = 'product_spider'
    allowed_domains = ['example.com']
    start_urls = ['https://example.com/products']

    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'DOWNLOAD_DELAY': 2,  # Be respectful
        'CONCURRENT_REQUESTS': 1,  # Limit concurrent requests
        'FEEDS': {
            'data/products.json': {'format': 'json'},
        }
    }

    def parse(self, response):
        """Parse product listing page"""
        # Extract product URLs
        product_urls = response.css('a.product-link::attr(href)').getall()

        # Follow each product URL
        for url in product_urls:
            yield response.follow(url, self.parse_product)

        # Follow pagination
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

    def parse_product(self, response):
        """Parse individual product page"""
        yield {
            'name': response.css('h1.product-title::text').get(),
            'price': response.css('span.price::text').get(),
            'description': response.css('div.description::text').get(),
            'sku': response.css('[data-sku]::attr(data-sku)').get(),
            'images': response.css('img.product-image::attr(src)').getall(),
            'url': response.url,
            'scraped_at': datetime.now().isoformat()
        }

# Run the spider
if __name__ == "__main__":
    process = CrawlerProcess()
    process.crawl(ProductSpider)
    process.start()

import scrapy
from scrapy.loader import ItemLoader
from itemloaders.processors import TakeFirst, MapCompose, Join
from urllib.parse import urljoin

class ProductItem(scrapy.Item):
    name = scrapy.Field()
    price = scrapy.Field()
    description = scrapy.Field()
    sku = scrapy.Field()
    images = scrapy.Field()
    url = scrapy.Field()
    scraped_at = scrapy.Field()

class ProductLoader(ItemLoader):
    default_item_class = ProductItem
    default_output_processor = TakeFirst()

    # Custom processors
    price_in = MapCompose(lambda x: x.strip(), lambda x: float(x.strip('$')))
    description_in = MapCompose(str.strip)
    description_out = Join(' ')
    images_out = lambda x: [urljoin('https://example.com', img) for img in x]

class AdvancedProductSpider(scrapy.Spider):
    name = 'advanced_product_spider'

    def parse_product(self, response):
        loader = ProductLoader(response=response)

        loader.add_css('name', 'h1.product-title::text')
        loader.add_css('price', 'span.price::text')
        loader.add_css('description', 'div.description p::text')
        loader.add_css('sku', '[data-sku]::attr(data-sku)')
        loader.add_css('images', 'img.product-image::attr(src)')
        loader.add_value('url', response.url)
        loader.add_value('scraped_at', datetime.now().isoformat())

        yield loader.load_item()

import random
import requests
from urllib.parse import urlparse

class ProxyManager:
    def __init__(self, proxy_list):
        self.proxy_list = proxy_list
        self.failed_proxies = set()

    def get_proxy(self):
        """Get a working proxy"""
        available_proxies = [p for p in self.proxy_list if p not in self.failed_proxies]

        if not available_proxies:
            raise Exception("No working proxies available")

        return random.choice(available_proxies)

    def test_proxy(self, proxy):
        """Test if proxy is working"""
        try:
            response = requests.get(
                'http://httpbin.org/ip',
                proxies={'http': proxy, 'https': proxy},
                timeout=5
            )
            return response.status_code == 200
        except:
            return False

    def mark_failed(self, proxy):
        """Mark proxy as failed"""
        self.failed_proxies.add(proxy)

    def rotate_proxy(self, current_proxy):
        """Get next proxy in rotation"""
        current_index = self.proxy_list.index(current_proxy)
        next_index = (current_index + 1) % len(self.proxy_list)
        return self.proxy_list[next_index]

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import random

class StealthBrowser:
    def __init__(self):
        self.user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
        ]

        self.viewports = [
            {'width': 1920, 'height': 1080},
            {'width': 1366, 'height': 768},
            {'width': 1536, 'height': 864}
        ]

    def create_stealth_driver(self):
        """Create a stealthy browser instance"""
        options = Options()

        # Basic stealth options
        options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-dev-shm-usage')
        options.add_argument('--disable-blink-features=AutomationControlled')
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option('useAutomationExtension', False)

        # Random user agent
        user_agent = random.choice(self.user_agents)
        options.add_argument(f'--user-agent={user_agent}')

        # Random viewport
        viewport = random.choice(self.viewports)

        driver = webdriver.Chrome(options=options)

        # Set viewport size
        driver.set_window_size(viewport['width'], viewport['height'])

        # Execute script to remove webdriver property
        driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")

        return driver

import pandas as pd
import re
from decimal import Decimal, InvalidOperation

class DataCleaner:
    def __init__(self):
        self.price_pattern = re.compile(r'[\d,]+\.?\d*')
        self.email_pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')

    def clean_dataframe(self, df):
        """Clean entire dataframe"""
        # Remove duplicates
        df = df.drop_duplicates()

        # Clean text columns
        text_columns = df.select_dtypes(include=['object']).columns
        for col in text_columns:
            df[col] = df[col].apply(self.clean_text)

        # Clean specific columns
        if 'price' in df.columns:
            df['price'] = df['price'].apply(self.clean_price)

        if 'email' in df.columns:
            df['email'] = df['email'].apply(self.clean_email)

        # Remove rows with missing critical data
        df = df.dropna(subset=['name', 'price'])

        return df

    def clean_text(self, text):
        """Clean text data"""
        if pd.isna(text):
            return text

        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', str(text)).strip()

        # Remove HTML tags
        text = re.sub(r'<[^>]+>', '', text)

        # Decode HTML entities
        import html
        text = html.unescape(text)

        return text

    def clean_price(self, price):
        """Clean and standardize price data"""
        if pd.isna(price):
            return price

        price_str = str(price)

        # Extract numeric value
        match = self.price_pattern.search(price_str)
        if match:
            try:
                return Decimal(match.group().replace(',', ''))
            except InvalidOperation:
                return None

        return None

    def clean_email(self, email):
        """Validate and clean email addresses"""
        if pd.isna(email):
            return email

        email_str = str(email).strip().lower()

        if self.email_pattern.match(email_str):
            return email_str

        return None

import sqlite3
from datetime import datetime

class ScrapingDatabase:
    def __init__(self, db_name='scraping.db'):
        self.conn = sqlite3.connect(db_name)
        self.create_tables()

    def create_tables(self):
        """Create database tables"""
        self.conn.execute('''
            CREATE TABLE IF NOT EXISTS scrapes (
                id INTEGER PRIMARY KEY,
                url TEXT,
                status TEXT,
                scraped_at TIMESTAMP,
                data TEXT
            )
        ''')

        self.conn.execute('''
            CREATE TABLE IF NOT EXISTS products (
                id INTEGER PRIMARY KEY,
                name TEXT,
                price DECIMAL,
                description TEXT,
                sku TEXT,
                url TEXT UNIQUE,
                scraped_at TIMESTAMP,
                updated_at TIMESTAMP
            )
        ''')

    def save_product(self, product_data):
        """Save or update product data"""
        now = datetime.now()

        # Check if product exists
        existing = self.conn.execute(
            'SELECT id FROM products WHERE url = ?',
            (product_data['url'],)
        ).fetchone()

        if existing:
            # Update existing product
            self.conn.execute('''
                UPDATE products
                SET name = ?, price = ?, description = ?, updated_at = ?
                WHERE url = ?
            ''', (
                product_data['name'],
                product_data['price'],
                product_data['description'],
                now,
                product_data['url']
            ))
        else:
            # Insert new product
            self.conn.execute('''
                INSERT INTO products (name, price, description, sku, url, scraped_at, updated_at)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            ''', (
                product_data['name'],
                product_data['price'],
                product_data['description'],
                product_data.get('sku'),
                product_data['url'],
                now,
                now
            ))

        self.conn.commit()

    def get_products_updated_since(self, since_date):
        """Get products updated since date"""
        cursor = self.conn.execute('''
            SELECT * FROM products
            WHERE updated_at > ?
            ORDER BY updated_at DESC
        ''', (since_date,))

        columns = [desc[0] for desc in cursor.description]
        return [dict(zip(columns, row)) for row in cursor.fetchall()]

    def close(self):
        """Close database connection"""
        self.conn.close()

from pymongo import MongoClient
from datetime import datetime

class MongoScrapingDB:
    def __init__(self, connection_string="mongodb://localhost:27017/"):
        self.client = MongoClient(connection_string)
        self.db = self.client['scraping_db']
        self.products = self.db['products']

    def save_product(self, product_data):
        """Save product data to MongoDB"""
        product_data['scraped_at'] = datetime.now()
        product_data['updated_at'] = datetime.now()

        # Upsert based on URL
        self.products.update_one(
            {'url': product_data['url']},
            {'$set': product_data},
            upsert=True
        )

    def get_products_by_category(self, category, limit=100):
        """Get products by category"""
        return list(self.products.find(
            {'category': category}
        ).limit(limit))

    def get_price_changes(self, url, days=30):
        """Get price change history"""
        from datetime import timedelta

        since_date = datetime.now() - timedelta(days=days)

        pipeline = [
            {'$match': {'url': url, 'updated_at': {'$gte': since_date}}},
            {'$sort': {'updated_at': 1}},
            {'$group': {
                '_id': None,
                'prices': {'$push': {'price': '$price', 'date': '$updated_at'}}
            }}
        ]

        result = list(self.products.aggregate(pipeline))
        return result[0]['prices'] if result else []

import time
import psutil
from collections import defaultdict

class ScrapingMonitor:
    def __init__(self):
        self.metrics = defaultdict(list)
        self.start_time = time.time()

    def record_request(self, url, response_time, status_code, success=True):
        """Record scraping request metrics"""
        self.metrics['requests'].append({
            'url': url,
            'response_time': response_time,
            'status_code': status_code,
            'success': success,
            'timestamp': time.time()
        })

    def record_error(self, url, error_type, error_message):
        """Record scraping errors"""
        self.metrics['errors'].append({
            'url': url,
            'error_type': error_type,
            'error_message': error_message,
            'timestamp': time.time()
        })

    def get_performance_stats(self):
        """Get performance statistics"""
        total_requests = len(self.metrics['requests'])
        successful_requests = len([r for r in self.metrics['requests'] if r['success']])
        total_errors = len(self.metrics['errors'])

        if total_requests > 0:
            success_rate = successful_requests / total_requests * 100
            avg_response_time = sum(r['response_time'] for r in self.metrics['requests']) / total_requests
        else:
            success_rate = 0
            avg_response_time = 0

        runtime = time.time() - self.start_time

        return {
            'total_requests': total_requests,
            'successful_requests': successful_requests,
            'success_rate': success_rate,
            'total_errors': total_errors,
            'average_response_time': avg_response_time,
            'runtime_seconds': runtime,
            'requests_per_second': total_requests / runtime if runtime > 0 else 0
        }

    def generate_report(self):
        """Generate scraping performance report"""
        stats = self.get_performance_stats()

        report = f"""
Scraping Performance Report
==========================
Total Runtime: {stats['runtime_seconds']:.2f} seconds
Total Requests: {stats['total_requests']}
Successful Requests: {stats['successful_requests']}
Success Rate: {stats['success_rate']:.1f}%
Average Response Time: {stats['average_response_time']:.2f} seconds
Requests per Second: {stats['requests_per_second']:.2f}
Total Errors: {stats['total_errors']}
"""

        # Error breakdown
        error_types = defaultdict(int)
        for error in self.metrics['errors']:
            error_types[error['error_type']] += 1

        if error_types:
            report += "\nError Breakdown:\n"
            for error_type, count in error_types.items():
                report += f"- {error_type}: {count}\n"

        return report

from celery import Celery
import requests
from bs4 import BeautifulSoup

app = Celery('scraping_tasks', broker='redis://localhost:6379/0')

@app.task
def scrape_url_task(url, parser_config):
    """Celery task for scraping a URL"""
    try:
        response = requests.get(url, headers={
            'User-Agent': 'ScrapingBot/1.0'
        }, timeout=30)

        if response.status_code == 200:
            soup = BeautifulSoup(response.content, 'html.parser')

            # Apply parser configuration
            data = {}
            for field, selector in parser_config.items():
                element = soup.select_one(selector)
                if element:
                    data[field] = element.text.strip()

            return {'url': url, 'data': data, 'status': 'success'}
        else:
            return {'url': url, 'status': 'error', 'error_code': response.status_code}

    except Exception as e:
        return {'url': url, 'status': 'error', 'error_message': str(e)}

# Usage
def scrape_multiple_urls(urls, parser_config):
    """Scrape multiple URLs asynchronously"""
    from celery import group

    # Create task group
    task_group = group(scrape_url_task.s(url, parser_config) for url in urls)

    # Execute tasks
    result = task_group.apply_async()

    # Get results
    return result.get(timeout=300)  # 5 minute timeout