Technical Documentation

# Import required libraries
from bs4 import BeautifulSoup
import requests
import pandas as pd

def scrape_phd_data(url):
    """Scrape PhD candidate data from university websites"""
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # Example extraction pattern
    candidates = []
    for profile in soup.select('.faculty-listing'):
        name = profile.find('h3').text.strip()
        department = profile.find('div', class_='dept').text
        candidates.append({'name': name, 'department': department})
    
    return pd.DataFrame(candidates)
            

# Gender detection function
def detect_gender(name):
    from gender_guesser.detector import Detector
    d = Detector()
    first_name = name.split()[0]
    return d.get_gender(first_name)
            

# Import and merge university datasets
def merge_university_data():
    # Load individual university datasets
    harvard_df = pd.read_csv('data/harvard_candidates.csv')
    stanford_df = pd.read_csv('data/stanford_candidates.csv') 
    yale_df = pd.read_csv('data/yale_candidates.csv')
    
    # Add university column to each dataset
    harvard_df['university'] = 'Harvard University'
    stanford_df['university'] = 'Stanford University'
    yale_df['university'] = 'Yale University'
    
    # Merge all dataframes
    merged_df = pd.concat([
        harvard_df,
        stanford_df, 
        yale_df
    ], ignore_index=True)
    
    # Standardize column names
    merged_df.columns = merged_df.columns.str.lower()
    
    # Add gender classification
    merged_df['gender'] = merged_df['name'].apply(detect_gender)
    
    return merged_df