import requests
from bs4 import BeautifulSoup
import json
import time
from typing import List, Dict, Any, Optional
import os
import cloudscraper
import random
from fake_useragent import UserAgent
from requests.adapters import HTTPAdapter
from urllib3.util import Retry

DIR_PATH = os.path.dirname(os.path.abspath(__file__))

class WebScraper:
    def __init__(self, base_url: str, output_dir: str = 'scraped_data'):
        """
        Initialize the web scraper.
        
        Args:
            base_url (str): The base URL of the website to scrape
            output_dir (str): Directory to save scraped data
        """
        self.base_url = base_url
        self.output_dir = output_dir
        self.user_agent = UserAgent()
        
        # Create output directory if it doesn't exist
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

    def get_data_from_url(self, url: str, max_retries: int = 3) -> Optional[BeautifulSoup]:
        """
        Fetch and parse a single page with retry mechanism and rotating proxies.
        
        Args:
            url (str): URL to fetch
            max_retries (int): Maximum number of retry attempts
            
        Returns:
            BeautifulSoup: Parsed HTML page_data
        """
        retry_count = 0
        while retry_count < max_retries:
            try:
                # Create a cloudscraper instance with random user agent
                scraper = cloudscraper.create_scraper(
                    browser={
                        'browser': 'chrome',
                        'platform': 'darwin',
                        'desktop': True
                    }
                )

                # Set random headers
                headers = {
                    'User-Agent': self.user_agent.random,
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                    'Accept-Language': 'en-US,en;q=0.5',
                    'Referer': self.base_url,
                    'DNT': '1',
                }

                # Make the request
                response = scraper.get(
                    url,
                    headers=headers,
                    timeout=30
                )
                response.raise_for_status()

                # Add random delay between requests
                time.sleep(random.uniform(2, 5))
                
                return BeautifulSoup(response.text, 'html.parser').body

            except requests.RequestException as e:
                print(f"Error fetching {url} (attempt {retry_count + 1}/{max_retries}): {e}")
                retry_count += 1
                time.sleep(random.uniform(5, 15))  # Exponential backoff

        return None

    def get_next_page_url(self, base_url: str, page_index: int) -> str:
        """
        Get the URL of the next page from the pagination element.
        
        Args:
            
            
        Returns:
            str: URL of the next page, or None if no next page exists
        """
        
        return f"{base_url}?startPage={page_index}"

    def scrape_with_pagination(self, max_pages: int = None, startPage: int = None, startItem: int = 0) -> List[Dict[str, Any]]:
        """
        Scrape multiple pages using pagination.
        
        Args:
            start_url (str): URL of the first page to scrape
            max_pages (int, optional): Maximum number of pages to scrape
            
        Returns:
            List[Dict[str, Any]]: List of scraped data dictionaries
        """
        if startPage:
            page_count = startPage
        current_url = self.get_next_page_url(self.base_url, page_count)
        
        while current_url:
            print(f"Scraping page {page_count}: {current_url}")
            soup = self.get_data_from_url(current_url)
            
            if not soup:
                break
            
            if startPage and page_count == startPage:
                # Get all list items on the current page
                list_items = soup.select('li.ng-image-challenge-browse_list-item')[startItem:]
            else:
                # Get all list items on the current page
                list_items = soup.select('li.ng-image-challenge-browse_list-item') 

            for item in list_items:
                # Find the link for each case
                link = item.select_one('a.issue-item_title-link')
                if link:
                    case_url = f"{self.base_url}{link.get('href')}"
                    print(f"Scraping case: {case_url}")
                
                # Get the case page content
                    case_soup = self.get_data_from_url(case_url)
                    if not case_soup:
                        continue
                # Extract data from current page
                page_data = {}

                page_data['Patient_Info'] = case_soup.select_one('div.image-challenge-qa_question').text.strip()
                answer_list = [element.text.strip() for element in case_soup.select('span.image-challenge-qa_label-text')]
                correct_list = [element.get('data-correct').strip() for element in case_soup.select('input[name="imageChallengeAnswer"]')]

                page_data['Answers'] = []
                for answer, correct in zip(answer_list, correct_list):
                    page_data['Answers'].append({
                        'text': answer,
                        'correct': correct
                    })
                page_data['Answer_Explanation'] = case_soup.select_one('div.image-challenge-qa_explanation-text').text.strip()
                page_data['Correct_Response_Percentage'] = case_soup.select_one('div.image-challenge-qa_progBar.correct').select_one('span.ng-progress-bar_label').text.strip()
                page_data['Total_Responses'] = case_soup.select_one('span.image-challenge-qa_totalResponses').text.strip()
                page_data['Image_URL'] = case_soup.select_one('img.image-challenge-qa_img').get('src')
                page_data['Date'] = case_soup.select_one('h2.image-challenge-qa_date').text.strip()
                    
                # Save results after each case
                self.save_result(page_data, "scraped.jsonl")

                # Random delay between requests
                time.sleep(random.uniform(5, 9))
            
            # Check if we've reached the maximum number of pages
            page_count += 1
            if max_pages and page_count > max_pages:
                break
                
            # Get next page URL
            current_url = self.get_next_page_url(self.base_url, page_count)
         
            # Random delay between pages
            time.sleep(random.uniform(3, 7))

    def save_result(self, result: Dict[str, Any], filename: str):
        """
        Save scraped results to a JSON file.
        
        Args:
            results (List[Dict[str, Any]]): Scraped data to save
            filename (str): Name of the output file
        """
        
        output_path = os.path.join(self.output_dir, filename)
        # Create directory if it doesn't exist
        if not os.path.exists(output_path):
            open(filename, 'w').close()
        with open(output_path, 'a', encoding='utf-8') as f:
            json.dump(result, f, ensure_ascii=False)
            f.write('\n')

# Example usage:
if __name__ == "__main__":
    # Initialize scraper
    scraper = WebScraper(base_url="https://www.nejm.org/image-challenge")
    
    # Perform scraping with pagination
    scraper.scrape_with_pagination(
        max_pages=20,
        startPage=6,
        startItem=0
    )