Web Scraping Ethics for Reddit: Legal Compliance and Best Practices

class EthicalDataCollector:
    """
    Framework for ethical Reddit data collection.
    Implements rate limiting, consent awareness, and data minimization.
    """

    def __init__(self, api_client, purpose: str):
        self.client = api_client
        self.purpose = purpose
        self.collection_log = []

        # Data minimization: only collect what's needed
        self.required_fields = self._define_required_fields()

    def _define_required_fields(self) -> set:
        """Define minimum required fields based on purpose."""
        # Don't collect more than necessary
        base_fields = {'id', 'title', 'selftext', 'created_utc', 'subreddit'}

        if self.purpose == 'sentiment_analysis':
            return base_fields | {'score'}
        elif self.purpose == 'topic_modeling':
            return base_fields
        else:
            return base_fields

    def should_collect(self, post) -> bool:
        """
        Check if post should be collected based on ethical criteria.
        """
        # Skip deleted or removed content
        if post.get('selftext') in ['[deleted]', '[removed]']:
            return False

        # Skip private subreddits (shouldn't be accessible anyway)
        if post.get('subreddit_type') == 'private':
            return False

        # Skip quarantined content unless explicitly authorized
        if post.get('quarantine'):
            return False

        # Check for opt-out signals (hypothetical)
        if 'do not scrape' in post.get('selftext', '').lower():
            return False

        return True

    def anonymize_data(self, post: dict) -> dict:
        """
        Remove or hash personally identifiable information.
        """
        import hashlib

        anonymized = {}

        for field in self.required_fields:
            if field in post:
                anonymized[field] = post[field]

        # Hash author instead of storing username
        if 'author' in post and post['author']:
            anonymized['author_hash'] = hashlib.sha256(
                post['author'].encode()
            ).hexdigest()[:16]

        # Remove direct URLs that might identify users
        if 'url' in anonymized:
            del anonymized['url']

        return anonymized

    def log_collection(self, post_id: str, reason: str):
        """Maintain audit log of what was collected and why."""
        from datetime import datetime

        self.collection_log.append({
            'post_id': post_id,
            'collected_at': datetime.utcnow().isoformat(),
            'purpose': self.purpose,
            'reason': reason
        })

    def collect(self, posts: list) -> list:
        """Collect posts with ethical filtering and anonymization."""
        collected = []

        for post in posts:
            if self.should_collect(post):
                anonymized = self.anonymize_data(post)
                collected.append(anonymized)
                self.log_collection(post['id'], 'met_criteria')
            else:
                self.log_collection(post['id'], 'filtered_out')

        return collected

import time
import logging
from datetime import datetime, timedelta

class CompliantAPIClient:
    """API client with built-in compliance features."""

    def __init__(self, client_id: str, client_secret: str, user_agent: str):
        self.client_id = client_id
        self.client_secret = client_secret
        self.user_agent = user_agent

        # Rate limiting
        self.requests_per_minute = 100
        self.request_times = []

        # Logging for audit
        self.logger = logging.getLogger('reddit_api')

    def _check_rate_limit(self):
        """Enforce rate limiting before requests."""
        now = datetime.now()
        minute_ago = now - timedelta(minutes=1)

        # Remove old timestamps
        self.request_times = [
            t for t in self.request_times
            if t > minute_ago
        ]

        # Check if at limit
        if len(self.request_times) >= self.requests_per_minute:
            # Wait until oldest request expires
            sleep_time = (self.request_times[0] - minute_ago).total_seconds()
            self.logger.info(f"Rate limit reached, sleeping {sleep_time:.2f}s")
            time.sleep(sleep_time + 0.1)
            self.request_times = self.request_times[1:]

        self.request_times.append(now)

    def _log_request(self, endpoint: str, params: dict):
        """Log API request for audit trail."""
        self.logger.info(f"API Request: {endpoint} | Params: {params}")

    def make_request(self, endpoint: str, params: dict = None):
        """Make rate-limited, logged API request."""
        self._check_rate_limit()
        self._log_request(endpoint, params)

        # Actual API call would go here
        # response = self._api_call(endpoint, params)
        # return response
        pass

    def respect_robots_txt(self) -> bool:
        """Check robots.txt (API access doesn't require this, but good practice)."""
        # Reddit's robots.txt allows API access
        # This is informational compliance
        return True