Session 1: Data Acquisition

import openml
import pandas as pd

# OpenML ID for the classic Titanic dataset
dataset = openml.datasets.get_dataset(40945)

# Extract features and target
# categorical_indicator helps identify which columns to treat as categories
X, y, categorical_indicator, attribute_names = dataset.get_data(
    dataset_format="dataframe", 
    target=dataset.default_target_attribute
)

print(f"Dataset: {dataset.name} | Rows: {X.shape[0]}, Cols: {X.shape[1]}")
X.head(3)

import pandas as pd
import requests
from io import StringIO

# Wikipedia started blocking requests without a user-agent header, so we need to set it to mimic a browser
headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "Accept-Language": "en-US,en;q=0.9",
}
response = requests.get("https://en.wikipedia.org/wiki/Berlin_population_statistics", headers=headers, timeout=20)
response.raise_for_status()

# Parse HTML text
tables = pd.read_html(StringIO(response.text))
print(f"Found {len(tables)} tables. Showing table 0 ({tables[0].shape[0]} rows, {tables[0].shape[1]} columns):")
tables[0].head(3)

import requests
from bs4 import BeautifulSoup

url = "https://books.toscrape.com"
response = requests.get(url, timeout=20)
response.raise_for_status()

# Build the soup object parsing the HTML
soup = BeautifulSoup(response.text, "html.parser")

print(f"Title: {soup.title.get_text(strip=True)}")

# Extract product information
product_pods = soup.select("article.product_pod")

single_product = product_pods[0] # Select first product
print(single_product.prettify())

# Extract title information
title_element = single_product.select_one("h3 a")
title_element

text_title = title_element.get_text() # Get content within the <a> tag
attr_title = title_element.get("title") # Get value of an attribute

print(f"Title from get_text(): {text_title}")
print(f"Title from title attribute: {attr_title}")

# Get description from subpage
href = title_element.get("href")
response = requests.get(f"{url}/{href}", timeout=20)
response.raise_for_status()

subsoup = BeautifulSoup(response.text, "html.parser")
description = subsoup.select_one("div[id='product_description'] + p").get_text(strip=True)
description

import requests
from bs4 import BeautifulSoup
import pandas as pd
from IPython.display import display

url = "https://books.toscrape.com"
response = requests.get(url, timeout=20)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")

def extract_books_single_page(soup: BeautifulSoup, url: str):
    # Retrieve all books
    product_pods = soup.select("article.product_pod")

    # Dictionary in which we fill our data
    scraped_data = {"Title": [], "Price": [], "Availability": [], "Rating": [], "Description": []}

    # Loop over the products to extract
    for product in product_pods:
        # Info directly on main page
        title = product.select_one("h3 a").get("title")
        price = product.select_one("div.product_price p.price_color").get_text()[1:]
        stock = product.select_one("div.product_price p.instock.availability").get_text().strip()
        rating = product.select_one("p").get("class")[1]

        # Info on subpage
        href = product.select_one("h3 a").get("href")
        response = requests.get(url=f"{url}/{href}", timeout=3)
        response.raise_for_status()
        subsoup = BeautifulSoup(response.text, "html.parser")
        description = subsoup.select_one("article.product_page > p").get_text() # Sometime you might need ">" to enforce direct child selection
        
        scraped_data["Title"].append(title)
        scraped_data["Price"].append(price)
        scraped_data["Availability"].append(stock)
        scraped_data["Rating"].append(rating)
        scraped_data["Description"].append(description)
    return scraped_data

scraped_data = pd.DataFrame(extract_books_single_page(soup, url))

display(scraped_data.head())

scraped_data.to_csv("../downloads/books.csv", index=False)

# Code Example 1: Computing Inter-Annotator Agreement

import numpy as np
from sklearn.metrics import cohen_kappa_score
from statsmodels.stats.inter_rater import fleiss_kappa, aggregate_raters

# Simulated annotations: 50 items, 3 annotators, 4 classes
np.random.seed(42)
n_items, n_annotators, n_classes = 50, 3, 4

# Generate ground truth and noisy labels
true_labels = np.random.randint(0, n_classes, n_items)
annotations = np.array([
    np.where(np.random.random(n_items) > 0.15, true_labels, 
             np.random.randint(0, n_classes, n_items))
    for _ in range(n_annotators)
]).T  # Shape: (50 items, 3 annotators)

# Cohen's Kappa (pairwise)
kappa_01 = cohen_kappa_score(annotations[:, 0], annotations[:, 1])
kappa_02 = cohen_kappa_score(annotations[:, 0], annotations[:, 2])
kappa_12 = cohen_kappa_score(annotations[:, 1], annotations[:, 2])

print("Inter-Annotator Agreement")
print("=" * 45)
print(f"Cohen's Kappa (A0 vs A1): {kappa_01:.3f}")
print(f"Cohen's Kappa (A0 vs A2): {kappa_02:.3f}")
print(f"Cohen's Kappa (A1 vs A2): {kappa_12:.3f}")
print(f"Mean Kappa (pairwise)   : {np.mean([kappa_01, kappa_02, kappa_12]):.3f}")

# Fleiss' Kappa (all annotators)
agg_table, _ = aggregate_raters(annotations)
fleiss = fleiss_kappa(agg_table)
print(f"Fleiss' Kappa (3 raters): {fleiss:.3f}")

# Code Example 2: Majority Voting with Tie-Breaking

from scipy.stats import mode

def majority_vote(annotations):
    """
    annotations: (n_items, n_annotators) array of integer labels
    Returns: (n_items,) array of aggregated labels
    """
    result = mode(annotations, axis=1, keepdims=False)
    return result.mode

# Apply majority voting
mv_labels = majority_vote(annotations)

# Evaluate against ground truth
mv_accuracy = (mv_labels == true_labels).mean()
individual_accuracies = [
    (annotations[:, j] == true_labels).mean() 
    for j in range(n_annotators)
]

print("Majority Voting Results")
print("=" * 45)
for j, acc in enumerate(individual_accuracies):
    print(f"Annotator {j} accuracy     : {acc:.3f}")
print("-" * 45)
print(f"Majority vote accuracy     : {mv_accuracy:.3f}")
print(f"Improvement over best solo : {mv_accuracy - max(individual_accuracies):.3f}")

# Show first 10 items
print("\nFirst 10 items (True | A0 A1 A2 | MV):")
for i in range(10):
    ann_str = ' '.join(map(str, annotations[i]))
    print(f"  {i:2d}: {true_labels[i]} | {ann_str} | {mv_labels[i]}")

# Code Example 3: Dawid-Skene Implementation (Simplified EM)

def dawid_skene(annotations, n_classes, max_iter=20, tol=1e-4):
    """
    annotations: (n_items, n_annotators)
    Returns: aggregated labels, annotator error rates
    """
    n_items, n_ann = annotations.shape
    
    # Initialize with majority vote
    labels = majority_vote(annotations)
    
    # Confusion matrices: (n_annotators, n_classes, n_classes)
    confusion = np.zeros((n_ann, n_classes, n_classes))
    
    for iteration in range(max_iter):
        old_labels = labels.copy()
        
        # M-step: estimate confusion matrices
        for j in range(n_ann):
            for k in range(n_classes):
                mask = (labels == k)
                if mask.sum() > 0:
                    for l in range(n_classes):
                        confusion[j, k, l] = ((annotations[mask, j] == l).sum() + 1) / (mask.sum() + n_classes)
        
        # E-step: estimate labels from weighted votes
        for i in range(n_items):
            log_probs = np.zeros(n_classes)
            for k in range(n_classes):
                for j in range(n_ann):
                    log_probs[k] += np.log(confusion[j, k, annotations[i, j]] + 1e-10)
            labels[i] = np.argmax(log_probs)
        
        # Check convergence
        if np.all(labels == old_labels):
            break
    
    # Compute error rates (1 - accuracy on diagonal)
    error_rates = [1 - confusion[j].diagonal().mean() for j in range(n_ann)]
    
    return labels, error_rates, confusion

# Code Example 4: Comparing Aggregation Methods

# Run Dawid-Skene
ds_labels, error_rates, confusion_matrices = dawid_skene(annotations, n_classes)
ds_accuracy = (ds_labels == true_labels).mean()

print("Dawid-Skene Results")
print("=" * 45)
for j, err in enumerate(error_rates):
    print(f"Annotator {j} error rate   : {err:.3f}")
print("-" * 45)
print(f"Dawid-Skene accuracy       : {ds_accuracy:.3f}")
print(f"Majority vote accuracy     : {mv_accuracy:.3f}")
print(f"Best individual accuracy   : {max(individual_accuracies):.3f}")
print("-" * 45)
print(f"DS improvement over MV     : {ds_accuracy - mv_accuracy:.3f}")

# Show confusion matrix for worst annotator
worst_ann = np.argmax(error_rates)
print(f"\nConfusion Matrix for Annotator {worst_ann} (worst):")
print(confusion_matrices[worst_ann].round(2))

# Count disagreements
disagree = (mv_labels != ds_labels).sum()
print(f"\nMV vs DS disagreements: {disagree}/{n_items} items ({disagree/n_items*100:.1f}%)")

# Code Example 1: Datasheet as a Structured Object

datasheet = {
    "dataset_name": "FacialExpressions-5K",
    "version": "1.0",
    "motivation": {
        "purpose": "Expression recognition for assistive technology",
        "funded_by": "EU Horizon Grant #12345"
    },
    "composition": {
        "instances": 5000,
        "type": "Facial images (224x224 JPEG)",
        "labels": ["happy", "sad", "neutral", "surprised", "angry"],
        "sensitive_data": "Contains biometric data; consent obtained"
    },
    "uses": {
        "intended": ["Affective computing research"],
        "prohibited": ["Surveillance", "Hiring decisions", "Law enforcement"]
    },
    "distribution": {
        "license": "CC BY-NC 4.0",
        "url": "https://example.org/dataset"
    }
}

import json
print(json.dumps(datasheet, indent=2))