import requests import logging import os import time from datetime import timedelta from django.utils import timezone from apps.cpd.models import MembershipCache from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock logger = logging.getLogger(__name__) # Rate limiting: 60 requests per minute RATE_LIMIT = 60 RATE_WINDOW = 60 # seconds rate_limiter_lock = Lock() request_timestamps = [] class MembershipAPIService: """Service for interacting with ICTAZ Membership System API""" BASE_URL = os.getenv('MEMBERSHIP_API_BASE_URL') API_KEY = os.getenv('MEMBERSHIP_API_KEY') API_SECRET = os.getenv('MEMBERSHIP_API_SECRET') @classmethod def get_headers(cls): """Get API headers with authentication""" if not cls.BASE_URL: raise ValueError("Membership API base URL not configured. Set MEMBERSHIP_API_BASE_URL in environment variables.") if not cls.API_KEY or not cls.API_SECRET: raise ValueError("Membership API credentials not configured. Set MEMBERSHIP_API_KEY and MEMBERSHIP_API_SECRET in environment variables.") return { 'X-API-Key': cls.API_KEY, 'X-API-Secret': cls.API_SECRET, 'Content-Type': 'application/json' } @classmethod def wait_for_rate_limit(cls): """Rate limiter: ensures we don't exceed 60 requests per minute""" global request_timestamps with rate_limiter_lock: now = time.time() # Remove timestamps older than 60 seconds request_timestamps = [ts for ts in request_timestamps if now - ts < RATE_WINDOW] # If we've hit the limit, wait if len(request_timestamps) >= RATE_LIMIT: oldest = request_timestamps[0] wait_time = RATE_WINDOW - (now - oldest) if wait_time > 0: logger.info(f"Rate limit reached. Waiting {wait_time:.2f} seconds...") time.sleep(wait_time) # Clean up again after waiting now = time.time() request_timestamps = [ts for ts in request_timestamps if now - ts < RATE_WINDOW] # Record this request request_timestamps.append(time.time()) @classmethod def lookup_member(cls, national_id_type, national_id_no): """ Lookup member by national ID (NRC or Passport) with caching Args: national_id_type: "NRC" or "Passport" national_id_no: National ID number Returns: dict: Member data if found, None if not found or error """ # Check cache first (cache valid for 7 days) cache_expiry = timezone.now() - timedelta(days=7) try: cached = MembershipCache.objects.get( national_id_type=national_id_type, national_id_no=national_id_no, last_lookup__gte=cache_expiry ) logger.info(f"✓ Cache HIT for {national_id_type} {national_id_no}") if cached.found_in_system: return { 'membership_number': cached.membership_number, 'first_name': cached.first_name, 'middle_name': cached.middle_name, 'last_name': cached.last_name, 'email': cached.email, 'mobile': cached.mobile, } else: return None # Previously looked up and not found except MembershipCache.DoesNotExist: logger.info(f"✗ Cache MISS for {national_id_type} {national_id_no} - calling API") # Cache miss - call API url = f"{cls.BASE_URL}/members/individual/lookup" payload = { "nationalIdType": national_id_type, "nationalIdNo": national_id_no } headers = cls.get_headers() logger.info("=== MEMBERSHIP API CALL ===") logger.info(f"URL: {url}") logger.info(f"Payload: {payload}") logger.info(f"Headers: {headers}") # Apply rate limiting cls.wait_for_rate_limit() try: response = requests.post( url, json=payload, headers=headers, timeout=10, verify=False # Disable SSL verification for self-signed certs ) logger.info(f"Response Status: {response.status_code}") logger.info(f"Response Headers: {dict(response.headers)}") logger.info(f"Response Body: {response.text}") logger.info("=== END API CALL ===") logger.info("") response.raise_for_status() data = response.json() if data.get('success') and data.get('data'): member_data = data['data'] result = { 'membership_number': member_data.get('membershipNo', ''), 'first_name': member_data.get('firstName', ''), 'middle_name': member_data.get('middleName', ''), 'last_name': member_data.get('lastName', ''), 'email': member_data.get('email', ''), 'mobile': member_data.get('mobile', ''), } # Cache the successful result MembershipCache.objects.update_or_create( national_id_type=national_id_type, national_id_no=national_id_no, defaults={ 'found_in_system': True, 'membership_number': result['membership_number'], 'first_name': result['first_name'], 'middle_name': result['middle_name'], 'last_name': result['last_name'], 'email': result['email'], 'mobile': result['mobile'], } ) return result # Cache the "not found" result to avoid repeated lookups MembershipCache.objects.update_or_create( national_id_type=national_id_type, national_id_no=national_id_no, defaults={'found_in_system': False} ) return None except requests.exceptions.RequestException as e: logger.error(f"Error looking up member {national_id_type} {national_id_no}: {e}") # Cache the failure to avoid immediate retry MembershipCache.objects.update_or_create( national_id_type=national_id_type, national_id_no=national_id_no, defaults={'found_in_system': False} ) return None @classmethod def bulk_enrich_members(cls, registered_members, max_workers=10): """ Enrich multiple members in parallel with rate limiting Args: registered_members: List of RegisteredMember instances max_workers: Maximum number of parallel threads (default 10) Returns: list: List of enriched member data dicts """ enriched_results = [] # Separate members that need API lookups from those in cache members_to_lookup = [] for member in registered_members: if not member.document_number: # No document number, use attendance data only enriched_results.append((member, None)) continue doc_num = member.document_number.strip() national_id_type = "NRC" if '/' in doc_num else "Passport" # Check if in cache cache_expiry = timezone.now() - timedelta(days=7) try: cached = MembershipCache.objects.get( national_id_type=national_id_type, national_id_no=doc_num, last_lookup__gte=cache_expiry ) # Cache hit - use cached data if cached.found_in_system: member_data = { 'membership_number': cached.membership_number, 'first_name': cached.first_name, 'middle_name': cached.middle_name, 'last_name': cached.last_name, 'email': cached.email, 'mobile': cached.mobile, } enriched_results.append((member, member_data)) else: enriched_results.append((member, None)) logger.info(f"✓ Cache HIT for {national_id_type} {doc_num}") except MembershipCache.DoesNotExist: # Cache miss - need to lookup members_to_lookup.append((member, national_id_type, doc_num)) # Parallel API lookups for cache misses if members_to_lookup: logger.info(f"Performing {len(members_to_lookup)} parallel API lookups...") def lookup_single(item): member, national_id_type, doc_num = item member_data = cls.lookup_member(national_id_type, doc_num) return (member, member_data) with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {executor.submit(lookup_single, item): item for item in members_to_lookup} for future in as_completed(futures): try: result = future.result() enriched_results.append(result) except Exception as e: member, _, _ = futures[future] logger.error(f"Error enriching member {member.registration_code}: {e}") enriched_results.append((member, None)) # Build final enriched data list in original order member_to_data = {m.id: data for m, data in enriched_results} final_results = [] for member in registered_members: member_data = member_to_data.get(member.id) enriched = cls._build_enriched_dict(member, member_data) final_results.append(enriched) return final_results @classmethod def _build_enriched_dict(cls, registered_member, member_data): """Helper to build enriched data dict from member and optional API data""" if member_data: # Use membership system data doc_num = registered_member.document_number.strip() if registered_member.document_number else '' national_id_type = "NRC" if '/' in doc_num else "Passport" if doc_num else '' return { 'membership_number': member_data.get('membership_number', ''), 'national_id_type': national_id_type, 'national_id_number': doc_num, 'first_name': member_data.get('first_name', ''), 'middle_name': member_data.get('middle_name', ''), 'last_name': member_data.get('last_name', ''), 'email': member_data.get('email', ''), 'mobile': member_data.get('mobile', ''), 'source': 'membership_system' } else: # Fallback to attendance list data full_name_parts = registered_member.full_name.split(' ', 2) first_name = full_name_parts[0] if len(full_name_parts) > 0 else '' middle_name = full_name_parts[1] if len(full_name_parts) > 2 else '' last_name = full_name_parts[-1] if len(full_name_parts) > 1 else '' doc_num = registered_member.document_number or '' national_id_type = "NRC" if '/' in doc_num else "Passport" if doc_num else '' return { 'membership_number': registered_member.membership_number or '', 'national_id_type': national_id_type, 'national_id_number': doc_num, 'first_name': first_name, 'middle_name': middle_name, 'last_name': last_name, 'email': registered_member.email or '', 'mobile': registered_member.phone_number or '', 'source': 'attendance_list' } @classmethod def enrich_member_data(cls, registered_member): """ Enrich registered member data with membership system data Args: registered_member: RegisteredMember instance Returns: dict: Enriched member data """ # Try to lookup member in membership system member_data = None if registered_member.document_number: # Determine ID type from document number format doc_num = registered_member.document_number.strip() # NRC format: ######/##/# (e.g., 535628/61/1) # Passport: alphanumeric (e.g., MWZ015434) if '/' in doc_num: national_id_type = "NRC" else: national_id_type = "Passport" member_data = cls.lookup_member(national_id_type, doc_num) # Build enriched data if member_data: # Use membership system data (already in correct format from lookup_member) # Determine ID type from document number doc_num = registered_member.document_number.strip() if registered_member.document_number else '' national_id_type = "NRC" if '/' in doc_num else "Passport" if doc_num else '' enriched = { 'membership_number': member_data.get('membership_number', ''), 'national_id_type': national_id_type, 'national_id_number': doc_num, 'first_name': member_data.get('first_name', ''), 'middle_name': member_data.get('middle_name', ''), 'last_name': member_data.get('last_name', ''), 'email': member_data.get('email', ''), 'mobile': member_data.get('mobile', ''), 'source': 'membership_system' } logger.info(f"Enriched data for {registered_member.registration_code}: Mobile={enriched['mobile']}") return enriched else: # Fallback to attendance list data full_name_parts = registered_member.full_name.split(' ', 2) first_name = full_name_parts[0] if len(full_name_parts) > 0 else '' middle_name = full_name_parts[1] if len(full_name_parts) > 2 else '' last_name = full_name_parts[-1] if len(full_name_parts) > 1 else '' # Determine ID type doc_num = registered_member.document_number or '' national_id_type = "NRC" if '/' in doc_num else "Passport" if doc_num else '' return { 'membership_number': registered_member.membership_number or '', 'national_id_type': national_id_type, 'national_id_number': doc_num, 'first_name': first_name, 'middle_name': middle_name, 'last_name': last_name, 'email': registered_member.email or '', 'mobile': '', 'source': 'attendance_list' }