{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "84e79595",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import re\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "\n",
    "epsilon = 0.2\n",
    "delta = 0.1\n",
    "total_clauses = 900\n",
    "PAC = int ((1/epsilon) * (math.log((math.pow(2,total_clauses) / delta), 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "28b6133b-6cff-4c4c-9d2f-bfed64055df4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_rules(model_type):\n",
    "    rules = []\n",
    "    for file in os.listdir(model_type):\n",
    "        if file.endswith('.json'):\n",
    "            with open(f'{model_type}/{file}', 'r') as f:\n",
    "                rules.append(json.load(f))\n",
    "    return rules\n",
    "\n",
    "roberta_base = load_rules(\"roberta-base\")\n",
    "roberta_large = load_rules(\"roberta-large\")\n",
    "mistral7b_sampling = load_rules(\"Mistral-7B-Instruct-v0.2-Sampling\")\n",
    "mistral7b = load_rules(\"Mistral-7B-Instruct-v0.2\")\n",
    "mistral24b = load_rules(\"Mistral-Small-24B-Instruct-2501\")\n",
    "deepseek = load_rules(\"DeepSeek-R1-Distill-Llama-8B\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8346c68a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 142588 characters]",
      "text/plain": [
       "<Figure size 1250x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_log_queries(models):\n",
    "\n",
    "    plt.figure(figsize=(12.5,4))\n",
    "\n",
    "    max_len = max([len(m[i]['sample_numbers']) for (m, _) in models for i in range(len(m))])\n",
    "    model_counts = []\n",
    "    for i in range(len(models)):\n",
    "        data, title = models[i]\n",
    "        model_counts.append((title, round(np.mean([sum(d['sample_numbers']) for d in data]),2)))\n",
    "        avg = average_counts(data, 'sample_numbers')\n",
    "        plt.plot(np.log2(avg), label=title)\n",
    "    \n",
    "    plt.axhline(y=np.log2(PAC), linewidth=2, linestyle='dashed')\n",
    "    plt.annotate(\"PAC threshold\", xy=(15, np.log2(PAC)), xytext=(200, 5), textcoords=\"offset points\")\n",
    "    plt.axvline(x=200, linewidth=2, linestyle='dashed')\n",
    "    plt.annotate(\"Original iteration limit\", xy=(200, 13), xytext=(5, -100), textcoords=\"offset points\")\n",
    "    plt.ylabel(\"Log$_2$(#queries)\")\n",
    "    plt.xlabel(\"Iteration\")\n",
    "    plt.ylim((0, 13))\n",
    "    plt.xlim((0, max_len))\n",
    "    plt.title(\"Number of queries at each iteration - Sampling method\")\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.show()\n",
    "\n",
    "def average_counts(data, column):\n",
    "    max_len = max(len(d[column]) for d in data)\n",
    "    averages = []\n",
    "    for i in range(max_len):\n",
    "        current_counts = []\n",
    "        for d in data:\n",
    "            if i < len(d[column]):\n",
    "                current_counts.append(d[column][i])\n",
    "        if current_counts:\n",
    "            avg = sum(current_counts) / len(current_counts)\n",
    "            averages.append(avg)\n",
    "    return averages\n",
    "\n",
    "plot_log_queries([(roberta_base, \"Roberta-base\"), (roberta_large, \"Roberta-large\"), (mistral7b_sampling, \"Mistral7B-sampling\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b8a551fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def common_rules(rules):\n",
    "    item_counts = Counter()\n",
    "    length = len(rules)\n",
    "    for sublist in rules:\n",
    "        unique_items = set(sublist)\n",
    "        item_counts.update(unique_items) \n",
    "    return sorted([(item, round(count/length, 2)) for item, count in item_counts.items()], key=lambda x: x[1], reverse=True)\n",
    "\n",
    "def confidence(occupation, gender, continent, time_idx, consequent=[]):\n",
    "    with open('../data/known_countries.csv', 'r') as f:\n",
    "        countries = pd.read_csv(f)    \n",
    "\n",
    "    if 'occupation' in consequent or occupation == '':\n",
    "        occs_ = []\n",
    "        for job in os.listdir('../data/extracted_occupations/'):\n",
    "            with open(f'../data/extracted_occupations/{job}', 'r') as f:\n",
    "                occ_ = pd.read_csv(f)\n",
    "            occ_['occupation'] = job.replace('.csv', '')\n",
    "            occs_.append(occ_)\n",
    "        occ = pd.concat(occs_)\n",
    "    else:\n",
    "        with open(f'../data/extracted_occupations/{occupation}.csv', 'r') as f:\n",
    "            occ = pd.read_csv(f)\n",
    "            occ['occupation'] = occupation\n",
    "    if occupation == '':\n",
    "        yes_occupation = True\n",
    "    else:\n",
    "        yes_occupation = (occ['occupation'] == occupation)\n",
    "    \n",
    "    if continent == 'Australia':\n",
    "        continent = 'Oceania'\n",
    "    if continent == '':\n",
    "        continent_countries = countries['nationality'].tolist()\n",
    "    elif continent == 'Eurasia':\n",
    "        continent_countries = countries[countries['continent'].isin(['Europe', 'Asia'])]['nationality'].tolist()\n",
    "    elif continent == 'Americas':\n",
    "        continent_countries = countries[countries['continent'].isin(['North America', 'South America'])]['nationality'].tolist()\n",
    "    else:\n",
    "        continent_countries = countries[countries['continent'] == continent]['nationality'].tolist()\n",
    "\n",
    "    time_periods = [(occ['birth'] < 1875), (occ['birth'] >= 1875) & (occ['birth'] < 1925), (occ['birth'] >= 1925) & (occ['birth'] < 1951), (occ['birth'] >= 1951) & (occ['birth'] < 1970), (occ['birth'] >= 1970), True]\n",
    "    if time_idx == '':\n",
    "        time_idx = 5\n",
    "    \n",
    "    if gender == '' and consequent == []:\n",
    "        yes = occ[time_periods[time_idx] & (occ['nationality'].isin(continent_countries)) & yes_occupation]\n",
    "        no = len(occ)\n",
    "        conf = 0.00 if no == 0 else len(yes)/no\n",
    "        return (conf, len(yes), no)\n",
    "    \n",
    "    no_continent = True if 'continent' in consequent else (occ['nationality'].isin(continent_countries))    \n",
    "    no_time      = True if 'time' in consequent else time_periods[time_idx]\n",
    "    no_gender    = True if 'gender' in consequent else (occ['gender']==gender)\n",
    "\n",
    "    yes = occ[(occ['gender']==gender) & time_periods[time_idx] & (occ['nationality'].isin(continent_countries)) & yes_occupation]\n",
    "    no = occ[no_continent & no_time & no_gender]\n",
    "\n",
    "    conf = 0.00 if len(no) == 0 else len(yes)/len(no)\n",
    "    return (conf, len(yes), len(no))\n",
    "    \n",
    "def confidence_of_rule(rule, consequent):\n",
    "    rule = re.escape(rule)\n",
    "    gender_pattern = r'\\b(?:man|woman)\\b'\n",
    "    gender_match = re.search(gender_pattern, rule, re.IGNORECASE)\n",
    "    if not gender_match:\n",
    "        gender = ''\n",
    "    else:\n",
    "        if '→' in rule:\n",
    "            gender = 'female' if re.search(r'\\bwoman\\b', rule, re.IGNORECASE) else 'male'\n",
    "        else:\n",
    "            gender = 'male' if re.search(r'\\bwoman\\b', rule, re.IGNORECASE) else 'female'\n",
    "            consequent.append('gender')\n",
    "    \n",
    "    occupations = ['fashion_designer', 'nurse', 'dancer', 'priest', 'footballer', 'banker', 'singer', 'lawyer', 'mathematician', 'diplomat']\n",
    "    occupation = ''\n",
    "    for o in occupations:\n",
    "        o_pattern = r'\\b' + o + r'\\b'\n",
    "        if re.search(o_pattern, rule):\n",
    "            occupation = o\n",
    "            break\n",
    "    \n",
    "    if occupation == '' and gender == '':\n",
    "        return (-1, 0.00, 0.00)\n",
    "\n",
    "    continents = ['Africa', 'Americas', 'Asia', 'Europe', 'North_America', 'Oceania', 'South_America', 'Australia', 'Eurasia']\n",
    "    continent = ''\n",
    "    for c in continents:\n",
    "        c_pattern = r'\\b' + c + r'\\b'\n",
    "        if re.search(c_pattern, rule):\n",
    "            continent = c.replace('_', ' ')\n",
    "            break\n",
    "    \n",
    "    time_periods = ['before_the_year_1875', 'between_the_years_1875_and_1925', 'between_the_years_1925_and_1951', 'between_the_years_1951_and_1970', 'after_the_year_1970']\n",
    "    time_period = ''\n",
    "    for i, t in enumerate(time_periods):\n",
    "        t_pattern = r'\\b' + t + r'\\b'\n",
    "        if re.search(t_pattern, rule):\n",
    "            time_period = i\n",
    "            break\n",
    "    \n",
    "    return confidence(occupation, gender, continent, time_period, consequent=consequent)\n",
    "\n",
    "def rules_confidence(rules, name=\"\", n=-1):\n",
    "    rule_table = []\n",
    "    conf = -np.inf\n",
    "    for (rule, rate) in rules:\n",
    "        cons = []\n",
    "        if '→' in rule:\n",
    "            consequent = rule[rule.index('→')+1:]\n",
    "            if 'man' in consequent or 'woman' in consequent:\n",
    "                cons.append('gender')\n",
    "            for cont in ['Africa', 'Americas', 'Asia', 'Europe', 'North_America', 'Oceania', 'South_America', 'Australia', 'Eurasia']:\n",
    "                if cont in consequent:\n",
    "                    cons.append('continent')\n",
    "            for time in ['before_the_year_1875', 'between_the_years_1875_and_1925', 'between_the_years_1925_and_1951', 'between_the_years_1951_and_1970', 'after_the_year_1970']:\n",
    "                if time in consequent:\n",
    "                    cons.append('time')\n",
    "            for occ in ['fashion_designer', 'nurse', 'dancer', 'priest', 'footballer', 'banker', 'singer', 'lawyer', 'mathematician', 'diplomat']:\n",
    "                if occ in consequent:\n",
    "                    cons.append('occupation')\n",
    "        conf, yes, no = confidence_of_rule(rule, consequent=cons)\n",
    "        rule_table.append((rule, round(float(rate), 2), conf))\n",
    "    rule_table = sorted([(rule, rate, conf) for (rule, rate, conf) in rule_table], key=lambda x:(x[2], x[1]), reverse=True)\n",
    "    if n > 0:\n",
    "        rule_table = rule_table[:n]\n",
    "    delete_indicies = []\n",
    "    \n",
    "    for i, rule in enumerate(rule_table):\n",
    "        if rule[2] < 0:\n",
    "            delete_indicies.append(i)\n",
    "    rule_table_ = [it for i, it in enumerate(rule_table) if i not in delete_indicies]\n",
    "    del_table = [it for i, it in enumerate(rule_table) if i in delete_indicies]\n",
    "\n",
    "\n",
    "    df = pd.DataFrame(rule_table_, columns=['Rule', 'Rate', 'Confidence'])\n",
    "    latex_string= df.to_latex(index=False, escape=False, float_format=\"%.2f\")\n",
    "    latex_string = latex_string.replace('¬', '$\\\\neg$')\n",
    "    latex_string = latex_string.replace('∧', '$\\\\land$')\n",
    "    latex_string = latex_string.replace('_the_years_', ' ')\n",
    "    latex_string = latex_string.replace('_the_year_', ' ')\n",
    "    latex_string = latex_string.replace('_and_', ' and ')\n",
    "    latex_string = latex_string.replace('fashion_designer', 'fashion designer')\n",
    "    latex_string = latex_string.replace('North_America', 'North America')\n",
    "    latex_string = latex_string.replace('South_America', 'South America')\n",
    "    latex_string = latex_string.replace('\\\\begin{tabular}{lrr}', '')\n",
    "    latex_string = latex_string.replace('\\\\toprule', '')\n",
    "    latex_string = latex_string.replace('\\\\midrule', '')\n",
    "    latex_string = latex_string.replace('\\\\bottomrule', '')\n",
    "    latex_string = latex_string.replace('Rule & Rate & Confidence \\\\\\\\', '')\n",
    "    latex_string = latex_string.replace('\\\\end{tabular}', '')\n",
    "    return rule_table_, del_table, latex_string\n",
    "\n",
    "\n",
    "roberta_base_extracted_rules_table, roberta_base_deleted_rules, roberta_base_latex_str                   = rules_confidence(common_rules([it['rules'] for it in roberta_base]), name=\"Roberta-base\")\n",
    "roberta_large_extracted_rules_table, roberta_large_deleted_rules, roberta_large_latex_str                = rules_confidence(common_rules([it['rules'] for it in roberta_large]), name=\"Roberta-large\")\n",
    "mistral7b_sampling_extracted_rules_table, mistral7b_sampling_deleted_rules, mistral7b_sampling_latex_str = rules_confidence(common_rules([it['rules'] for it in mistral7b_sampling]), name=\"Mistral(7B) - Sampling\")\n",
    "mistral7b_extracted_rules_table, mistral7b_deleted_rules, mistral7b_latex_str                            = rules_confidence(common_rules([it['rules'] for it in mistral7b]), name=\"Mistral(7B)\")\n",
    "mistral24b_extracted_rules_table, mistral24b_deleted_rules, mistral24b_latex_str                         = rules_confidence(common_rules([it['rules'] for it in mistral24b]), name=\"Mistral(24B)\")\n",
    "deepseek_extracted_rules_table, deepseek_deleted_rules, deepseek_latex_str                               = rules_confidence(common_rules([it['rules'] for it in deepseek]), name=\"Deepseek\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9c65cc21-eef1-4a46-b9a0-109a6f7d6c19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllll}\n",
      "\\toprule\n",
      "Model & Size($\\mathcal{H}$) & $|\\cap|$ & E($|\\cap|$) & Conf & #Queries & Size($\\mathcal{Q}$) \\\\\n",
      "\\midrule\n",
      "roberta-base & 94.0$\\pm$0.0 & 94.0$\\pm$0.0 & 9.82$\\pm$0.0 & 0.48$\\pm$0.32 & 14204.8$\\pm$1923.46 & 91.0$\\pm$0.0 \\\\\n",
      "roberta-large & 110.9$\\pm$0.3 & 110.8$\\pm$0.4 & 13.67$\\pm$0.05 & 0.48$\\pm$0.32 & 12640.5$\\pm$1467.07 & 63.9$\\pm$0.3 \\\\\n",
      "Mistral-7B-Instruct-v0.2 & 55.9$\\pm$0.7 & 54.42$\\pm$2.07 & 3.47$\\pm$0.06 & 0.48$\\pm$0.32 & 18012.4$\\pm$2240.84 & 28.5$\\pm$1.2 \\\\\n",
      "Mistral-7B-Instruct-v0.2 & 2.5$\\pm$2.18 & 1.9$\\pm$0.3 & 0.01$\\pm$0.01 & 0.48$\\pm$0.32 & 4.8$\\pm$3.84 & 0.1$\\pm$0.44 \\\\\n",
      "Mistral-Small-24B-Instruct-2501 & 15.75$\\pm$6.66 & 3.42$\\pm$1.76 & 0.27$\\pm$0.16 & 0.48$\\pm$0.32 & 8.3$\\pm$6.02 & 0.0$\\pm$0.0 \\\\\n",
      "DeepSeek-R1-Distill-Llama-8B & 1.85$\\pm$2.17 & 0.14$\\pm$0.36 & 0.0$\\pm$0.01 & 0.48$\\pm$0.32 & 3.2$\\pm$4.59 & 0.05$\\pm$0.22 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def rule_stats(rules, possible):\n",
    "    intersections = []\n",
    "    expecteds = []\n",
    "    rule_lengths = [len(r['rules']) for r in rules]\n",
    "    for i in range(len(rules)-1):\n",
    "        for ii in range(i+1, len(rules)):\n",
    "            in_len = len(set(rules[i]['rules']).intersection(set(rules[ii]['rules'])))\n",
    "            intersection = (in_len, f'{in_len} / ({len(rules[i][\"rules\"])}, {len(rules[ii][\"rules\"])})')\n",
    "            expected = len(rules[i]['rules'])*len(rules[ii]['rules'])/possible\n",
    "\n",
    "            intersections.append(intersection[0])\n",
    "            expecteds.append(expected)\n",
    "    intersection_mean = np.mean(intersections) if len(intersections) > 0 else 0\n",
    "    intersection_stdev = np.std(intersections) if len(intersections) > 0 else 0\n",
    "    expected_mean = np.mean(expecteds) if len(expecteds) > 0 else 0\n",
    "    expected_stdev = np.std(expecteds) if len(expecteds) > 0 else 0\n",
    "    rule_length_mean = np.mean(rule_lengths) if len(rule_lengths) > 0 else 0\n",
    "    rule_length_stdev = np.std(rule_lengths) if len(rule_lengths) > 0 else 0\n",
    "    return (f'{round(rule_length_mean, 2)}$\\pm${round(rule_length_stdev,2)}',f'{round(intersection_mean, 2)}$\\pm${round(intersection_stdev,2)}', f'{round(expected_mean,2)}$\\pm${round(expected_stdev,2)}')\n",
    "\n",
    "def print_stats_latex(models):\n",
    "\n",
    "    model_stats = []\n",
    "    for model in models:\n",
    "        model_name = model[0]['model name']\n",
    "        if 'features' in model[0].keys():\n",
    "            features = model[0]['features']\n",
    "            possible_rules = 1\n",
    "            for f in features:\n",
    "                possible_rules *= len(features[f])\n",
    "        else:\n",
    "            possible_rules = total_clauses\n",
    "        \n",
    "        (rule_len, intersection, expected) = rule_stats(model, possible_rules)\n",
    "        q_rules = [len(d['q_rules']) for d in model]\n",
    "        q_rule_str = f'{round(np.mean(q_rules),2)}$\\pm${round(np.std(q_rules),2)}'\n",
    "\n",
    "        num_queries = [sum(d['sample_numbers']) for d in model]\n",
    "        num_queries_str = f'{round(np.mean(num_queries),2)}$\\pm${round(np.std(num_queries),2)}'\n",
    "\n",
    "        table, _, _ = rules_confidence(common_rules([it['rules'] for it in roberta_base]))\n",
    "        confidence = [conf for (_,_,conf) in table]\n",
    "        confidence_str = f'{round(np.mean(confidence),2)}$\\pm${round(np.std(confidence),2)}'\n",
    "\n",
    "        model_stats.append((model_name, rule_len, intersection, expected, confidence_str, num_queries_str, q_rule_str))\n",
    "    \n",
    "    df = pd.DataFrame(model_stats, columns=['Model', 'Size($\\mathcal{H}$)', '$|\\cap|$', 'E($|\\cap|$)', 'Conf','#Queries', 'Size($\\mathcal{Q}$)'])\n",
    "    print(df.to_latex(index=False))\n",
    "\n",
    "\n",
    "print_stats_latex([roberta_base, roberta_large, mistral7b_sampling, mistral7b, mistral24b, deepseek])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e4033a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "$\\neg$(Australia $\\land$ before 1875 $\\land$ lawyer $\\land$ woman) & 1.00 & 1.00 \\\\\n",
      "$\\neg$(priest $\\land$ woman) & 1.00 & 0.98 \\\\\n",
      "$\\neg$(before 1875 $\\land$ mathematician $\\land$ woman) & 1.00 & 0.98 \\\\\n",
      "$\\neg$(between 1875 and 1925 $\\land$ diplomat $\\land$ woman) & 1.00 & 0.98 \\\\\n",
      "$\\neg$(banker $\\land$ woman) & 1.00 & 0.96 \\\\\n",
      "$\\neg$(Asia $\\land$ mathematician $\\land$ woman) & 1.00 & 0.94 \\\\\n",
      "$\\neg$(Asia $\\land$ diplomat $\\land$ woman) & 1.00 & 0.93 \\\\\n",
      "$\\neg$(between 1925 and 1951 $\\land$ diplomat $\\land$ woman) & 1.00 & 0.92 \\\\\n",
      "$\\neg$(man $\\land$ nurse) & 1.00 & 0.91 \\\\\n",
      "$\\neg$(Europe $\\land$ mathematician $\\land$ woman) & 1.00 & 0.91 \\\\\n",
      "$\\neg$(between 1875 and 1925 $\\land$ mathematician $\\land$ woman) & 1.00 & 0.90 \\\\\n",
      "$\\neg$(Oceania $\\land$ diplomat $\\land$ woman) & 1.00 & 0.89 \\\\\n",
      "$\\neg$(Americas $\\land$ between 1925 and 1951 $\\land$ mathematician $\\land$ woman) & 1.00 & 0.89 \\\\\n",
      "(Oceania $\\land$ before 1875 $\\land$ woman) → nurse & 1.00 & 0.88 \\\\\n",
      "$\\neg$(Australia $\\land$ between 1925 and 1951 $\\land$ lawyer $\\land$ woman) & 1.00 & 0.87 \\\\\n",
      "$\\neg$(Americas $\\land$ before 1875 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.86 \\\\\n",
      "$\\neg$(footballer $\\land$ woman) & 1.00 & 0.84 \\\\\n",
      "$\\neg$(Australia $\\land$ between 1875 and 1925 $\\land$ lawyer $\\land$ woman) & 1.00 & 0.81 \\\\\n",
      "$\\neg$(Oceania $\\land$ mathematician $\\land$ woman) & 1.00 & 0.81 \\\\\n",
      "$\\neg$(between 1951 and 1970 $\\land$ diplomat $\\land$ woman) & 1.00 & 0.80 \\\\\n",
      "$\\neg$(Australia $\\land$ before 1875 $\\land$ man $\\land$ singer) & 1.00 & 0.80 \\\\\n",
      "$\\neg$(Asia $\\land$ before 1875 $\\land$ dancer $\\land$ man) & 1.00 & 0.78 \\\\\n",
      "$\\neg$(Americas $\\land$ between 1875 and 1925 $\\land$ dancer $\\land$ man) & 1.00 & 0.74 \\\\\n",
      "$\\neg$(Africa $\\land$ before 1875 $\\land$ singer $\\land$ woman) & 1.00 & 0.73 \\\\\n",
      "$\\neg$(Europe $\\land$ between 1875 and 1925 $\\land$ dancer $\\land$ man) & 1.00 & 0.72 \\\\\n",
      "$\\neg$(Australia $\\land$ fashion designer $\\land$ man) & 1.00 & 0.72 \\\\\n",
      "$\\neg$(Asia $\\land$ before 1875 $\\land$ singer $\\land$ woman) & 1.00 & 0.69 \\\\\n",
      "(between 1951 and 1970 $\\land$ singer $\\land$ woman) → Eurasia & 1.00 & 0.69 \\\\\n",
      "$\\neg$(Americas $\\land$ between 1875 and 1925 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.66 \\\\\n",
      "$\\neg$(Australia $\\land$ dancer $\\land$ man) & 1.00 & 0.65 \\\\\n",
      "(between 1925 and 1951 $\\land$ singer $\\land$ woman) → Eurasia & 1.00 & 0.65 \\\\\n",
      "$\\neg$(Americas $\\land$ between 1925 and 1951 $\\land$ dancer $\\land$ man) & 1.00 & 0.64 \\\\\n",
      "(between 1875 and 1925 $\\land$ singer $\\land$ woman) → Eurasia & 1.00 & 0.64 \\\\\n",
      "$\\neg$(Africa $\\land$ after 1970 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.63 \\\\\n",
      "$\\neg$(Australia $\\land$ between 1951 and 1970 $\\land$ lawyer $\\land$ woman) & 1.00 & 0.63 \\\\\n",
      "$\\neg$(Asia $\\land$ between 1875 and 1925 $\\land$ dancer $\\land$ man) & 1.00 & 0.61 \\\\\n",
      "$\\neg$(Africa $\\land$ dancer $\\land$ man) & 1.00 & 0.60 \\\\\n",
      "$\\neg$(Eurasia $\\land$ dancer $\\land$ man) & 1.00 & 0.59 \\\\\n",
      "$\\neg$(Europe $\\land$ after 1970 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.59 \\\\\n",
      "$\\neg$(Americas $\\land$ before 1875 $\\land$ dancer $\\land$ man) & 1.00 & 0.58 \\\\\n",
      "$\\neg$(Americas $\\land$ between 1925 and 1951 $\\land$ fashion designer $\\land$ woman) & 1.00 & 0.57 \\\\\n",
      "$\\neg$(Australia $\\land$ after 1970 $\\land$ man $\\land$ singer) & 1.00 & 0.57 \\\\\n",
      "$\\neg$(Africa $\\land$ after 1970 $\\land$ singer $\\land$ woman) & 1.00 & 0.56 \\\\\n",
      "(Oceania $\\land$ between 1875 and 1925 $\\land$ woman) → nurse & 1.00 & 0.56 \\\\\n",
      "$\\neg$(Australia $\\land$ after 1970 $\\land$ lawyer $\\land$ man) & 1.00 & 0.56 \\\\\n",
      "$\\neg$(Asia $\\land$ after 1970 $\\land$ dancer $\\land$ man) & 1.00 & 0.56 \\\\\n",
      "(Asia $\\land$ fashion designer $\\land$ woman) → after 1970 & 1.00 & 0.56 \\\\\n",
      "$\\neg$(Europe $\\land$ before 1875 $\\land$ dancer $\\land$ man) & 1.00 & 0.56 \\\\\n",
      "$\\neg$(Eurasia $\\land$ fashion designer $\\land$ man) & 1.00 & 0.55 \\\\\n",
      "$\\neg$(Asia $\\land$ after 1970 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.54 \\\\\n",
      "$\\neg$(Africa $\\land$ between 1951 and 1970 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.53 \\\\\n",
      "$\\neg$(Europe $\\land$ before 1875 $\\land$ singer $\\land$ woman) & 1.00 & 0.52 \\\\\n",
      "$\\neg$(Africa $\\land$ between 1875 and 1925 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.50 \\\\\n",
      "$\\neg$(Africa $\\land$ between 1925 and 1951 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.50 \\\\\n",
      "$\\neg$(Americas $\\land$ before 1875 $\\land$ singer $\\land$ woman) & 1.00 & 0.49 \\\\\n",
      "$\\neg$(Eurasia $\\land$ man $\\land$ singer) & 1.00 & 0.48 \\\\\n",
      "$\\neg$(Australia $\\land$ after 1970 $\\land$ diplomat $\\land$ man) & 1.00 & 0.47 \\\\\n",
      "$\\neg$(Europe $\\land$ after 1970 $\\land$ singer $\\land$ woman) & 1.00 & 0.47 \\\\\n",
      "$\\neg$(Asia $\\land$ after 1970 $\\land$ singer $\\land$ woman) & 1.00 & 0.43 \\\\\n",
      "$\\neg$(Europe $\\land$ after 1970 $\\land$ dancer $\\land$ woman) & 1.00 & 0.40 \\\\\n",
      "$\\neg$(Europe $\\land$ between 1875 and 1925 $\\land$ fashion designer $\\land$ woman) & 1.00 & 0.39 \\\\\n",
      "$\\neg$(Europe $\\land$ before 1875 $\\land$ fashion designer $\\land$ woman) & 1.00 & 0.36 \\\\\n",
      "(South America $\\land$ before 1875 $\\land$ woman) → nurse & 1.00 & 0.35 \\\\\n",
      "(North America $\\land$ before 1875 $\\land$ woman) → nurse & 1.00 & 0.34 \\\\\n",
      "(Africa $\\land$ mathematician $\\land$ woman) → between 1925 and 1951 & 1.00 & 0.27 \\\\\n",
      "$\\neg$(Australia $\\land$ between 1951 and 1970 $\\land$ man $\\land$ mathematician) & 1.00 & 0.23 \\\\\n",
      "(Oceania $\\land$ between 1925 and 1951 $\\land$ woman) → nurse & 1.00 & 0.22 \\\\\n",
      "$\\neg$(Australia $\\land$ after 1970 $\\land$ man $\\land$ mathematician) & 1.00 & 0.21 \\\\\n",
      "$\\neg$(Australia $\\land$ between 1925 and 1951 $\\land$ man $\\land$ mathematician) & 1.00 & 0.21 \\\\\n",
      "$\\neg$(Africa $\\land$ between 1925 and 1951 $\\land$ man $\\land$ mathematician) & 1.00 & 0.16 \\\\\n",
      "$\\neg$(Eurasia $\\land$ between 1951 and 1970 $\\land$ man $\\land$ mathematician) & 1.00 & 0.14 \\\\\n",
      "(North America $\\land$ between 1875 and 1925 $\\land$ woman) → nurse & 1.00 & 0.13 \\\\\n",
      "(Oceania $\\land$ between 1951 and 1970 $\\land$ woman) → nurse & 1.00 & 0.08 \\\\\n",
      "$\\neg$(Eurasia $\\land$ between 1925 and 1951 $\\land$ man $\\land$ mathematician) & 1.00 & 0.08 \\\\\n",
      "(North America $\\land$ between 1925 and 1951 $\\land$ woman) → nurse & 1.00 & 0.06 \\\\\n",
      "(South America $\\land$ between 1875 and 1925 $\\land$ woman) → nurse & 1.00 & 0.06 \\\\\n",
      "(Europe $\\land$ between 1925 and 1951 $\\land$ woman) → nurse & 1.00 & 0.05 \\\\\n",
      "(South America $\\land$ between 1951 and 1970 $\\land$ woman) → nurse & 1.00 & 0.04 \\\\\n",
      "(South America $\\land$ between 1925 and 1951 $\\land$ woman) → nurse & 1.00 & 0.03 \\\\\n",
      "(Europe $\\land$ between 1951 and 1970 $\\land$ woman) → nurse & 1.00 & 0.03 \\\\\n",
      "(Americas $\\land$ between 1951 and 1970 $\\land$ woman) → nurse & 1.00 & 0.03 \\\\\n",
      "(North America $\\land$ between 1951 and 1970 $\\land$ woman) → nurse & 1.00 & 0.03 \\\\\n",
      "(Asia $\\land$ between 1925 and 1951 $\\land$ woman) → nurse & 1.00 & 0.02 \\\\\n",
      "(Asia $\\land$ between 1951 and 1970 $\\land$ woman) → nurse & 1.00 & 0.02 \\\\\n",
      "(after 1970 $\\land$ diplomat $\\land$ woman) → Australia & 1.00 & 0.02 \\\\\n",
      "(lawyer $\\land$ woman) → Australia & 1.00 & 0.02 \\\\\n",
      "(South America $\\land$ after 1970 $\\land$ woman) → nurse & 1.00 & 0.01 \\\\\n",
      "(Americas $\\land$ after 1970 $\\land$ woman) → nurse & 1.00 & 0.01 \\\\\n",
      "(after 1970 $\\land$ mathematician $\\land$ woman) → Australia & 1.00 & 0.01 \\\\\n",
      "(North America $\\land$ after 1970 $\\land$ woman) → nurse & 1.00 & 0.01 \\\\\n",
      "(Oceania $\\land$ after 1970 $\\land$ woman) → nurse & 1.00 & 0.00 \\\\\n",
      "$\\neg$(Africa $\\land$ before 1875 $\\land$ fashion designer $\\land$ man) & 1.00 & 0.00 \\\\\n",
      "$\\neg$(Australia $\\land$ before 1875 $\\land$ diplomat $\\land$ man) & 1.00 & 0.00 \\\\\n",
      "(before 1875 $\\land$ diplomat $\\land$ woman) → Australia & 1.00 & 0.00 \\\\\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Prints the tables of all the rules that can be found in the appendix\n",
    "\n",
    "print(roberta_base_latex_str)\n",
    "#print(roberta_large_latex_str)\n",
    "#print(mistral7b_sampling_latex_str)\n",
    "#print(mistral7b_latex_str)\n",
    "#print(mistral24b_latex_str)\n",
    "#print(deepseek_latex_str)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
