OSINT Methodology Framework

# OSINT Web Scraping Framework
import requests
from bs4 import BeautifulSoup
import time
import random
from urllib.parse import urljoin, urlparse
import json

class OSINTCollector:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        self.collected_data = []
        
    def collect_from_source(self, url, selectors):
        """
        Collect structured data from web sources
        """
        try:
            # Implement respectful scraping with delays
            time.sleep(random.uniform(1, 3))
            
            response = self.session.get(url, timeout=30)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Extract data based on CSS selectors
            data = {}
            for field, selector in selectors.items():
                elements = soup.select(selector)
                data[field] = [elem.get_text(strip=True) for elem in elements]
            
            # Add metadata
            data['source_url'] = url
            data['collection_timestamp'] = time.time()
            data['domain'] = urlparse(url).netloc
            
            self.collected_data.append(data)
            return data
            
        except Exception as e:
            print(f"Error collecting from {url}: {str(e)}")
            return None

# Usage Example
collector = OSINTCollector()
news_selectors = {
    'headlines': 'h1, h2.headline',
    'articles': '.article-content p',
    'authors': '.author-name',
    'timestamps': '.publish-date'
}

# Collect from multiple sources
sources = ['https://example-news.com/geopolitics']
for source in sources:
    collector.collect_from_source(source, news_selectors)