{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wZebm5H_yAXD"
      },
      "source": [
        "# **Library Imports**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "hfswus2JcdP7"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import random\n",
        "import math\n",
        "import statistics\n",
        "import operator\n",
        "from typing import Counter\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.linear_model import BayesianRidge\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "from scipy.stats import wasserstein_distance as wass_d"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "F-wW78T7vjE9"
      },
      "outputs": [],
      "source": [
        "class histogram():\n",
        "  def __init__(self, input):\n",
        "    if not input.keys():\n",
        "      print('Not a valid input!')\n",
        "      return -1\n",
        "    self.histogram = input\n",
        "\n",
        "  def normalize(self):\n",
        "    nhist = self.histogram\n",
        "    n = sum(self.histogram.values())\n",
        "    for key in self.histogram.keys():\n",
        "      nhist[key] = self.histogram[key]/n\n",
        "    self.histogram = nhist\n",
        "\n",
        "  def plot(self):\n",
        "    if len(list(self.histogram.keys())) == 0:\n",
        "      print(\"No Histogram!\")\n",
        "      return 0\n",
        "    nhist = normalize(self.histogram)\n",
        "    print(sum(nhist.values()))\n",
        "    plt.bar(nhist.keys(),nhist.values(), width = 1 , color = 'b')\n",
        "    plt.show()\n",
        "\n",
        "  def wasserstein(self, other):\n",
        "    return wass_d(list(self.histogram.keys()),list(other.histogram.keys()),list(self.histogram.values()),list(other.histogram.values()))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U7U-Df5GRjOg"
      },
      "source": [
        "# **Misra Gries**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "0iS7Ml5oRiP-"
      },
      "outputs": [],
      "source": [
        "class MisraGries(dict):\n",
        "    #k: counters\n",
        "    #n: number of elments seen\n",
        "\n",
        "\n",
        "    def __init__(self, k=2, n=0):\n",
        "        super().__init__()\n",
        "        self.k = k\n",
        "        self.n = n\n",
        "\n",
        "    def _decrement(self):\n",
        "        \"\"\"Decrement all elements and remove zeros.\"\"\"\n",
        "        while len(self) > self.k:\n",
        "            for element in list(self):\n",
        "                self[element] -= 1\n",
        "                if self[element] <= 0:\n",
        "                    del self[element]\n",
        "\n",
        "    def record(self, element):\n",
        "        \"\"\"Record a stream element.\"\"\"\n",
        "        self.n += 1\n",
        "        if element in self:\n",
        "            self[element] += 1\n",
        "        elif len(self) <= self.k:\n",
        "            self[element] = 1\n",
        "        else:\n",
        "            self._decrement()\n",
        "\n",
        "    def __add__(self, other):\n",
        "        \"\"\"Merge two together to get a new one.\"\"\"\n",
        "        new = self.__class__(k=self.k, n=self.n + other.n)\n",
        "        new += other\n",
        "        new += self\n",
        "        return new\n",
        "\n",
        "    def __iadd__(self, other):\n",
        "        \"\"\"Merge two together to get a new one.\"\"\"\n",
        "        self.n += other.n\n",
        "        for element, count in other.items():\n",
        "            self.setdefault(element, 0)\n",
        "            self[element] += count\n",
        "        self._decrement()\n",
        "        return self\n",
        "\n",
        "    def scan(self, stream):\n",
        "        \"\"\"Scan an entire array>\"\"\"\n",
        "        for element in stream:\n",
        "            self.record(element)\n",
        "        return self\n",
        "\n",
        "    def output(self):\n",
        "        \"\"\"Every item which occurs more than n/k times is guaranteed to appear in the output array.\"\"\"\n",
        "        nk = self.n/self.k\n",
        "        return [key for key, count in self.items() if count > nk]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1AMUmUESr6Tq"
      },
      "source": [
        "# Mergable Misra Gries [[Paper](https://arxiv.org/abs/1705.07001)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "VrXlHYxIsPEi"
      },
      "outputs": [],
      "source": [
        "class MergableMisraGries(dict):\n",
        "    #k: counters\n",
        "    #n: number of elments seen\n",
        "\n",
        "\n",
        "    def __init__(self, k=2, kstar = -1, n=0):\n",
        "        super().__init__()\n",
        "        self.k = k\n",
        "        if kstar == -1:\n",
        "          self.kstar =  k/2\n",
        "        if kstar != k/2 and kstar != -1:\n",
        "          print(\"Warning: Choosing other threshold than median!\")\n",
        "        self.kstar = kstar\n",
        "        self.n = n\n",
        "\n",
        "    def _decrement(self):\n",
        "        \"\"\"Decrement all elements and remove zeros.\"\"\"\n",
        "        c = statistics.median(list(self.values()))\n",
        "        for element in list(self):\n",
        "                self[element] -= c\n",
        "                if self[element] <= 0:\n",
        "                    del self[element]\n",
        "        return c\n",
        "\n",
        "    def record(self, element):\n",
        "        \"\"\"Record a stream element.\"\"\"\n",
        "        self.n += 1\n",
        "        # print(element)\n",
        "        if element[0] in self:\n",
        "            self[element[0]] += element[1]\n",
        "        elif len(self) < self.k:\n",
        "            self[element[0]] = element[1]\n",
        "        else:\n",
        "            c = self._decrement()\n",
        "            #  print(c)\n",
        "            #  print(self.keys())\n",
        "            if element[1] > c:\n",
        "                self[element[0]] = element[1]\n",
        "\n",
        "\n",
        "    def merge(self, other):\n",
        "        \"\"\"Merge two together to get a new one.\"\"\"\n",
        "        new = self\n",
        "        for element in list(other.keys()):\n",
        "          new.record([element,other[element]])\n",
        "        return new\n",
        "\n",
        "    def __add__(self, other):\n",
        "        return self.merge(other)\n",
        "\n",
        "    def __iadd__(self, other):\n",
        "        \"\"\"Merge two together to get a new one.\"\"\"\n",
        "        self.n += other.n\n",
        "        for element, count in other.items():\n",
        "            self.setdefault(element, 0)\n",
        "            self[element] += count\n",
        "        self._decrement()\n",
        "        return self\n",
        "\n",
        "    def scanunweighted(self, stream):\n",
        "        \"\"\"Scan an entire array>\"\"\"\n",
        "        for element in stream:\n",
        "            self.record([element,1])\n",
        "        return self\n",
        "\n",
        "    def scanweighted(self, stream):\n",
        "        \"\"\"Scan an entire array>\"\"\"\n",
        "        for element in stream:\n",
        "            self.record(element)\n",
        "        return self\n",
        "\n",
        "    def output(self):\n",
        "        \"\"\"Every item which occurs more than n/k times is guaranteed to appear in the output array.\"\"\"\n",
        "        nk = self.n / self.k\n",
        "        return [key for key, count in self.items() if count > nk]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D54zhl4nHKEK"
      },
      "source": [
        "# **ACS Income Pull**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 263,
      "metadata": {
        "id": "tvx8AsOYJIML"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AGEP</th>\n",
              "      <th>COW</th>\n",
              "      <th>SCHL</th>\n",
              "      <th>MAR</th>\n",
              "      <th>OCCP</th>\n",
              "      <th>POBP</th>\n",
              "      <th>RELP</th>\n",
              "      <th>WKHP</th>\n",
              "      <th>SEX</th>\n",
              "      <th>RAC1P</th>\n",
              "      <th>Income</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>18.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>4720.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1600.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>53.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>3605.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>10000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>41.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>7330.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>24000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>18.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>2722.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>180.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>21.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>3870.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>29000.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   AGEP  COW  SCHL  MAR    OCCP  POBP  RELP  WKHP  SEX  RAC1P   Income\n",
              "0  18.0  1.0  18.0  5.0  4720.0  13.0  17.0  21.0  2.0    2.0   1600.0\n",
              "1  53.0  5.0  17.0  5.0  3605.0  18.0  16.0  40.0  1.0    1.0  10000.0\n",
              "2  41.0  1.0  16.0  5.0  7330.0   1.0  17.0  40.0  1.0    1.0  24000.0\n",
              "3  18.0  6.0  18.0  5.0  2722.0   1.0  17.0   2.0  2.0    1.0    180.0\n",
              "4  21.0  5.0  19.0  5.0  3870.0  12.0  17.0  50.0  1.0    1.0  29000.0"
            ]
          },
          "execution_count": 263,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from fairlearn.datasets import fetch_acs_income\n",
        "\n",
        "data = fetch_acs_income()\n",
        "data1 = pd.DataFrame(data= np.c_[data['data'], data['target']],\n",
        "                     columns= data['feature_names'][:10] + ['Income'])\n",
        "data1.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 264,
      "metadata": {
        "id": "VBqTcVAZKLew"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1664500, 11)\n"
          ]
        }
      ],
      "source": [
        "print(data1.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IcojpCa8zm0g"
      },
      "source": [
        "# Trials"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ChsHWDNbsqbh",
        "outputId": "5f6fba63-ff61-43b0-92e3-0b04a5bb2681"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[6, 20, 8, 6, 18, 10, 6, 4, 0, 18]\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "rng = np.random.default_rng()\n",
        "\n",
        "def discretize(n, disc =2):\n",
        "  m = []\n",
        "  for element in n:\n",
        "    q = int(element/disc)\n",
        "    m.append(q*disc)\n",
        "  return m\n",
        "\n",
        "\n",
        "n1 = [int(x) for x in rng.exponential(10,10000)]\n",
        "n2 = [int(x) for x in rng.exponential(50,10000)]\n",
        "\n",
        "n1 = discretize(n1)\n",
        "n2 = discretize(n2)\n",
        "print(n1[:10])\n",
        "\n",
        "mg1 = MisraGries(k=30)\n",
        "mg2 = MisraGries(k=150)\n",
        "\n",
        "mg1.scan(n1)\n",
        "mg2.scan(n2)\n",
        "\n",
        "mmg1 = MergableMisraGries(k = 60, kstar = 30)\n",
        "mmg2 = MergableMisraGries(k = 300, kstar = 150)\n",
        "\n",
        "mmg1.scanunweighted(n1)\n",
        "mmg2.scanunweighted(n2)\n",
        "\n",
        "mg3 = mg1+mg2\n",
        "mmg3 = mmg1+mmg2\n",
        "\n",
        "# print(mg1 + mg2)\n",
        "# mg1 += mg2\n",
        "# print(mg1.n)\n",
        "# print(mg1.output())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 898
        },
        "id": "Hd9Vtkrwy6z8",
        "outputId": "de91f64c-79f4-45d2-9e95-4d2b3fcdf421"
      },
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 15596 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "dict_keys([100, 40, 16, 18, 122, 80, 8, 90, 46, 4, 64, 12, 10, 56, 104, 26, 206, 92, 6, 62, 182, 96, 132, 76, 86, 108, 72, 34, 94, 36, 0, 24, 88, 112, 2, 144, 38, 52, 54, 14, 20, 58, 32, 22, 48, 74, 42, 28, 116, 168, 30, 162, 82, 78, 110, 50, 106, 138, 170, 84, 166, 98, 66, 158, 204, 154, 148, 140, 208, 68, 156, 60, 274, 44, 252, 232, 124, 70, 142, 152, 186, 260, 118, 218, 134, 120, 224, 126, 184, 262, 230, 150, 226, 176, 164, 102, 192, 130, 210, 146, 272, 190, 334, 234, 220, 136, 172, 128, 114, 178, 296, 160, 264, 216, 308, 276, 238, 196, 188, 316, 200, 202, 282, 280, 306, 194, 222, 244, 240, 228, 266, 268, 278, 212, 406, 452, 292, 294, 286, 214, 242, 246, 328, 180, 248, 288, 486, 174, 330, 198, 320, 236, 258, 298, 386, 254, 314, 396, 426, 250, 378, 284, 256, 290, 318, 408, 360])\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<BarContainer object of 167 artists>"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 17232 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from collections import Counter as counter\n",
        "\n",
        "c = counter(n1)\n",
        "plt.bar(c.keys(), c.values(), width = 1, color='b')\n",
        "plt.show()\n",
        "c = counter(n2)\n",
        "print(c.keys())\n",
        "plt.bar(c.keys(), c.values(), width = 1, color='b')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QwnaBPgRUJz-",
        "outputId": "c3f2b1a0-c234-4af1-94c4-e37a83674ca5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{6: 0.09985632183908046, 20: 0.025348932676518884, 8: 0.08579638752052546, 18: 0.02904351395730706, 10: 0.06845238095238096, 4: 0.12140804597701149, 0: 0.19098932676518884, 2: 0.14480706075533661, 16: 0.03992200328407225, 36: 0.005336617405582923, 14: 0.03981937602627258, 22: 0.02062807881773399, 24: 0.012520525451559935, 12: 0.05582922824302135, 26: 0.012623152709359606, 32: 0.007594417077175698, 38: 0.0051313628899835794, 40: 0.002976190476190476, 30: 0.0083128078817734, 28: 0.010057471264367816, 34: 0.00451559934318555, 42: 0.0028735632183908046, 44: 0.0031814449917898194, 46: 0.0009236453201970443, 48: 0.0008210180623973727, 52: 0.0007183908045977011, 60: 0.00010262725779967159, 70: 0.00010262725779967159, 50: 0.0003078817733990148}\n"
          ]
        }
      ],
      "source": [
        "def normalize(hist):\n",
        "  nhist = hist\n",
        "  n = sum(hist.values())\n",
        "  for key in hist.keys():\n",
        "    nhist[key] = hist[key]/n\n",
        "  n = sum(nhist.values())\n",
        "  return nhist\n",
        "\n",
        "print(normalize(mg1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 878
        },
        "id": "NXSLeHVrU4mk",
        "outputId": "17c4d8c2-7066-44ac-a6ad-ed022e401d86"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.9999999999999998\n"
          ]
        },
        {
          "data": {
            "image/png": "[encoded data, 19476 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<BarContainer object of 130 artists>"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 20888 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "nmg1 = normalize(mg1)\n",
        "print(sum(nmg1.values()))\n",
        "plt.bar(nmg1.keys(),nmg1.values(), width = 1 , color = 'b')\n",
        "plt.show()\n",
        "nmg2 = normalize(mg2)\n",
        "plt.bar(nmg2.keys(),nmg2.values(), width = 1 , color = 'b')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 878
        },
        "id": "gbxD4oO80j4j",
        "outputId": "685f79c2-5b3f-4da0-f3a9-23c35b3962d1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.9999999999999996\n"
          ]
        },
        {
          "data": {
            "image/png": "[encoded data, 16792 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<BarContainer object of 167 artists>"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 20040 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "nmmg1 = normalize(mmg1)\n",
        "print(sum(nmmg1.values()))\n",
        "plt.bar(nmmg1.keys(),nmmg1.values(), width = 1 , color = 'b')\n",
        "plt.show()\n",
        "nmmg2 = normalize(mmg2)\n",
        "plt.bar(nmmg2.keys(),nmmg2.values(), width = 1 , color = 'b')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 860
        },
        "id": "kCQs9PGdrf12",
        "outputId": "cd61474c-6827-4f79-ee01-edc7495b6678"
      },
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 17372 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<BarContainer object of 167 artists>"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 20040 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "n1 = normalize(counter(n1))\n",
        "plt.bar(n1.keys(),n1.values(), width = 1 , color = 'b')\n",
        "plt.show()\n",
        "n2 = normalize(counter(n2))\n",
        "plt.bar(n2.keys(),n2.values(), width = 1 , color = 'b')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "ccfNFplKtGeL",
        "outputId": "3dab6551-e20e-46fd-df09-3e7e3449bf92"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<BarContainer object of 44 artists>"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 12356 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def makequantile(f1):\n",
        "  keys1 = list(f1.keys())\n",
        "  keys1.sort()\n",
        "  f1 = {i: f1[i] for i in keys1}\n",
        "  cdf = 0\n",
        "  for key in f1.keys():\n",
        "    cdf = cdf + f1[key]\n",
        "    f1[key] = cdf\n",
        "  return f1\n",
        "\n",
        "q1 = makequantile(n1)\n",
        "plt.bar(q1.keys(),q1.values(), width = 1, color = 'b')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "icu2ZdKY22Yh"
      },
      "outputs": [],
      "source": [
        "# def hist(f_1,f_2):\n",
        "#   f_1 = normalize(f_1)\n",
        "#   f_2 = normalize(f_2)\n",
        "#   keys = list(f_1.keys())+list(f_2.keys())\n",
        "#   keys.sort()\n",
        "#   for key in keys:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "D8mXjvnm7_EN"
      },
      "outputs": [],
      "source": [
        "from typing import Counter\n",
        "class stream():\n",
        "  def __init__(self, n = 100000):\n",
        "    self.stream = []\n",
        "    self.n = n\n",
        "\n",
        "  def generatenormal(self, mu = 0, sigma = 1):\n",
        "    self.stream = np.random.normal(mu, sigma, self.n)\n",
        "\n",
        "  def discretize(self, disc = 0.05):\n",
        "    m = []\n",
        "    for element in self.stream:\n",
        "      q = math.floor(element/disc)\n",
        "      m.append(q*disc+disc/2)\n",
        "    self.stream = m\n",
        "\n",
        "  def histogram(self):\n",
        "    self.hist = counter(self.stream)\n",
        "    return self.hist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uc81EhOQEzVo",
        "outputId": "331d87c3-52db-404d-d68f-5e78e1a9cd60"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.01432126843429422\n"
          ]
        }
      ],
      "source": [
        "str1 = stream()\n",
        "str1.generatenormal()\n",
        "str1.discretize()\n",
        "str1.histogram()\n",
        "\n",
        "\n",
        "mg = MisraGries(k = 100)\n",
        "mmg = MergableMisraGries(k = 150, kstar = 75)\n",
        "\n",
        "mg.scan(str1.stream)\n",
        "mmg.scanunweighted(str1.stream)\n",
        "\n",
        "histbase = histogram(str1.hist)\n",
        "histbase.normalize()\n",
        "histmg = histogram(mg)\n",
        "histmmg = histogram(mmg)\n",
        "\n",
        "\n",
        "\n",
        "# histbase.plot()\n",
        "# histmg.plot()\n",
        "# histmmg.plot()\n",
        "\n",
        "print(histmg.wasserstein(histmmg))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Sr63O73VpJ4_",
        "outputId": "338b9063-395d-4bdd-f063-f8256440c6a6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.0\n"
          ]
        },
        {
          "data": {
            "image/png": "[encoded data, 16112 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.0000000000000004\n"
          ]
        },
        {
          "data": {
            "image/png": "[encoded data, 15924 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.0\n"
          ]
        },
        {
          "data": {
            "image/png": "[encoded data, 16112 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.0000000000000007\n"
          ]
        },
        {
          "data": {
            "image/png": "[encoded data, 16344 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.0067425873784034435\n",
            "0.09745791860698501\n"
          ]
        }
      ],
      "source": [
        "str1 = stream()\n",
        "str1.generatenormal()\n",
        "str1.discretize()\n",
        "str1.histogram()\n",
        "\n",
        "mmg1 = MergableMisraGries(k = 150)\n",
        "mmg2 = MergableMisraGries(k = 150)\n",
        "mmg = MergableMisraGries(k = 150)\n",
        "\n",
        "mmg1.scanunweighted(str1.stream[:50000])\n",
        "mmg2.scanunweighted(str1.stream[50000:])\n",
        "mmg.scanunweighted(str1.stream)\n",
        "\n",
        "mmgmerged = mmg1.merge(mmg2)\n",
        "\n",
        "mmg1hist = histogram(mmg1)\n",
        "mmg2hist = histogram(mmg2)\n",
        "mmgmergedhist = histogram(mmgmerged)\n",
        "mmghist = histogram(mmg)\n",
        "\n",
        "mmg1hist.plot()\n",
        "mmg2hist.plot()\n",
        "mmgmergedhist.plot()\n",
        "mmghist.plot()\n",
        "\n",
        "basehist = histogram(str1.hist)\n",
        "basehist.normalize()\n",
        "print(mmghist.wasserstein(mmgmergedhist))\n",
        "print(mmghist.wasserstein(basehist))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 843
        },
        "id": "eRMx_74dyhTV",
        "outputId": "e10493ed-d665-44b5-9bdd-3dd4b7e35e63"
      },
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 18660 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "[encoded data, 23720 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "str1 = stream(100000)\n",
        "str1.generatenormal(10,10)\n",
        "str1.discretize()\n",
        "str1.histogram()\n",
        "basehist = histogram(str1.hist)\n",
        "basehist.normalize()\n",
        "\n",
        "mmgtracker = {}\n",
        "for n in range(50,2501,50):\n",
        "  mmg = MergableMisraGries(k = n)\n",
        "  mmg.scanunweighted(str1.stream)\n",
        "  mmghist = histogram(mmg)\n",
        "  dist = mmghist.wasserstein(basehist)\n",
        "  mmgtracker[n] = dist\n",
        "\n",
        "# print(mmgtracker)\n",
        "plt.plot(mmgtracker.keys(),mmgtracker.values(), color = 'b')\n",
        "plt.yscale('log')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(mmgtracker.keys(),mmgtracker.values(), color = 'b')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 843
        },
        "id": "r0xCjcAL2gm6",
        "outputId": "fc4cd499-5a22-47a8-905c-0376065c86fb"
      },
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 18660 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "[encoded data, 23720 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.plot(mmgtracker.keys(),mmgtracker.values(), color = 'b')\n",
        "plt.yscale('log')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(mmgtracker.keys(),mmgtracker.values(), color = 'b')\n",
        "# plt.yscale('log')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# ACS INCOME Analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 265,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(866735, 11)\n",
            "(797765, 11)\n"
          ]
        }
      ],
      "source": [
        "ACS_male = data1.loc[data1['SEX'] == 1]\n",
        "print(ACS_male.shape)\n",
        "ACS_female = data1.loc[data1['SEX'] == 2]\n",
        "print(ACS_female.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 266,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<Axes: xlabel='Income', ylabel='Density'>"
            ]
          },
          "execution_count": 266,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 29560 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "sns.set_theme()\n",
        "sns.kdeplot(data = data1, x = 'Income', hue = 'SEX')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 267,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "23157.60993367511\n"
          ]
        }
      ],
      "source": [
        "print(wass_d(ACS_female['Income'],ACS_male['Income']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "23157.60993367511\n"
          ]
        }
      ],
      "source": [
        "ACS_male_disc = discretize(ACS_male['Income'],1000)\n",
        "ACS_female_disc = discretize(ACS_female['Income'],1000)\n",
        "print(wass_d(ACS_male_disc,ACS_female_disc))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 269,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "18342.632810439336\n"
          ]
        }
      ],
      "source": [
        "MG_Male = MergableMisraGries(k = 5000)\n",
        "MG_Male.scanunweighted(ACS_male_disc)\n",
        "MG_Female = MergableMisraGries(k = 5000)\n",
        "MG_Female.scanunweighted(ACS_female_disc)\n",
        "\n",
        "hist_MG_Male = histogram(MG_Male)\n",
        "hist_MG_Female = histogram(MG_Female)\n",
        "print(hist_MG_Male.wasserstein(hist_MG_Female))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 270,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Buckets = 2000, Distance = 16035.943692742832\n",
            "Buckets = 4000, Distance = 17022.311403183\n",
            "Buckets = 6000, Distance = 19719.537018779785\n",
            "Buckets = 8000, Distance = 20077.413543251925\n",
            "Buckets = 10000, Distance = 21078.50904122288\n",
            "Buckets = 12000, Distance = 21060.30906091839\n",
            "Buckets = 14000, Distance = 20567.458666988594\n",
            "Buckets = 16000, Distance = 23157.609933675107\n",
            "Buckets = 18000, Distance = 23157.609933675107\n",
            "Buckets = 20000, Distance = 23157.609933675107\n",
            "Buckets = 22000, Distance = 23157.609933675107\n",
            "Buckets = 24000, Distance = 23157.609933675107\n",
            "Buckets = 26000, Distance = 23157.609933675107\n",
            "Buckets = 28000, Distance = 23157.609933675107\n",
            "Buckets = 30000, Distance = 23157.609933675107\n"
          ]
        }
      ],
      "source": [
        "for i in range(2000,30001,2000):\n",
        "    dist = 0\n",
        "    runs = 1\n",
        "    for _ in range(runs):\n",
        "        random.shuffle(ACS_female_disc)\n",
        "        random.shuffle(ACS_male_disc)\n",
        "        MG_Male = MergableMisraGries(k = i)\n",
        "        MG_Male.scanunweighted(ACS_male_disc)\n",
        "        MG_Female = MergableMisraGries(k = i)\n",
        "        MG_Female.scanunweighted(ACS_female_disc)\n",
        "        hist_MG_Male = histogram(MG_Male)\n",
        "        hist_MG_Female = histogram(MG_Female)\n",
        "        # print(hist_MG_Male.wasserstein(hist_MG_Female))\n",
        "        dist += hist_MG_Male.wasserstein(hist_MG_Female)\n",
        "    print(\"Buckets = \" + str(i) + \", Distance = \" + str(dist/runs))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Buckets = 1000, Distance = 15045.216042817152\n",
        "Buckets = 2000, Distance = 15510.089093567334\n",
        "Buckets = 3000, Distance = 16366.997977371017\n",
        "Buckets = 4000, Distance = 16739.17814568069\n",
        "Buckets = 5000, Distance = 19765.52828182616\n",
        "Buckets = 6000, Distance = 23157.817472739458"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 271,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Income\n",
            "30000.0     21852\n",
            "40000.0     19223\n",
            "20000.0     19086\n",
            "50000.0     17643\n",
            "25000.0     15455\n",
            "            ...  \n",
            "128060.0        1\n",
            "94550.0         1\n",
            "189210.0        1\n",
            "95950.0         1\n",
            "15770.0         1\n",
            "Name: count, Length: 11883, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print(ACS_female['Income'].value_counts())\n",
        "Income = ACS_female['Income'].value_counts()\n",
        "Income.to_csv('Income_Distinct.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# ACS Income Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 272,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(416125, 10)\n",
            "(416125, 1)\n"
          ]
        }
      ],
      "source": [
        "#Train Test Split\n",
        "\n",
        "data1 = data1.sample(frac=1)\n",
        "\n",
        "# Select ratio\n",
        "ratio = 0.75\n",
        "\n",
        "total_rows = data1.shape[0]\n",
        "train_size = int(total_rows*ratio)\n",
        "\n",
        "# #Standardize\n",
        "# data1['Income'] =(data1['Income'] - data1['Income'].mean() ) / data1['Income'].std()\n",
        "\n",
        "# Split data into test and train\n",
        "train = data1[0:train_size]\n",
        "test = data1[train_size:]\n",
        "\n",
        "x_train = train.iloc[:,:10]\n",
        "y_train = train.iloc[:, 10:12]\n",
        "x_train = x_train.to_numpy()\n",
        "y_train = y_train.to_numpy()\n",
        "x_test = test.iloc[:,:10]\n",
        "y_test = test.iloc[:, 10:12]\n",
        "x_test = x_test.to_numpy()\n",
        "y_test = y_test.to_numpy()\n",
        "print(x_test.shape)\n",
        "print(y_test.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 273,
      "metadata": {},
      "outputs": [],
      "source": [
        "ls = LinearRegression(fit_intercept=False)\n",
        "ls.fit(x_train, y_train)\n",
        "\n",
        "y_pred = ls.predict(x_test)\n",
        "\n",
        "y_pred_male = y_pred[x_test[:, -2] == 1][:,0]\n",
        "y_pred_female = y_pred[x_test[:, -2] == 2][:,0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 274,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "25133.418767424333\n"
          ]
        }
      ],
      "source": [
        "print(wass_d(y_pred_male,y_pred_female))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 287,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2190\n",
            "25126.849154800933\n"
          ]
        }
      ],
      "source": [
        "ACS_pred_male_disc = discretize(y_pred_male,100)\n",
        "ACS_pred_female_disc = discretize(y_pred_female,100)\n",
        "distance = wass_d(ACS_pred_male_disc,ACS_pred_female_disc)\n",
        "print(len(set(ACS_pred_male_disc)))\n",
        "print(distance)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 276,
      "metadata": {},
      "outputs": [],
      "source": [
        "Male_Disc_base = Counter(ACS_pred_male_disc)\n",
        "Male_Disc_Hist = histogram(Male_Disc_base)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 278,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Buckets = 1000,  Computed Distance = 1707.0467946028689, Relative Error = 0.0679194030378785\n",
            "Buckets = 2000,  Computed Distance = 1484.364838273775, Relative Error = 0.05905940834470574\n",
            "Buckets = 3000,  Computed Distance = 1535.7761315268044, Relative Error = 0.061104943568573704\n",
            "Buckets = 4000,  Computed Distance = 1367.9069178576954, Relative Error = 0.05442581982287906\n",
            "Buckets = 5000,  Computed Distance = 894.0595323262652, Relative Error = 0.03557253960929157\n",
            "Buckets = 6000,  Computed Distance = 1654.6210856227672, Relative Error = 0.06583350658265306\n",
            "Buckets = 7000,  Computed Distance = 496.60491651403123, Relative Error = 0.019758749193020984\n",
            "Buckets = 8000,  Computed Distance = 1663.3941484160725, Relative Error = 0.06618256625085844\n",
            "Buckets = 9000,  Computed Distance = 1372.7875828545393, Relative Error = 0.05462001007827309\n",
            "Buckets = 10000,  Computed Distance = 502.84108070251375, Relative Error = 0.020006871593805237\n",
            "Buckets = 11000,  Computed Distance = 509.7057351617898, Relative Error = 0.02028000015384791\n",
            "Buckets = 12000,  Computed Distance = 1627.807785333591, Relative Error = 0.06476666801977705\n",
            "Buckets = 13000,  Computed Distance = 436.44398713579074, Relative Error = 0.017365086393329156\n",
            "Buckets = 14000,  Computed Distance = 529.982042159752, Relative Error = 0.021086747028900468\n",
            "Buckets = 15000,  Computed Distance = 497.7864323542799, Relative Error = 0.01980575894741272\n"
          ]
        }
      ],
      "source": [
        "for i in range(1000,15001,1000):\n",
        "    dist = 0\n",
        "    runs = 30\n",
        "    for _ in range(runs):\n",
        "        random.shuffle(ACS_pred_female_disc)\n",
        "        random.shuffle(ACS_pred_male_disc)\n",
        "        MG_Male = MergableMisraGries(k = i)\n",
        "        MG_Male.scanunweighted(ACS_pred_male_disc)\n",
        "        # MG_Female = MergableMisraGries(k = i)\n",
        "        # MG_Female.scanunweighted(ACS_pred_female_disc)\n",
        "        hist_MG_Male = histogram(MG_Male)\n",
        "        # hist_MG_Female = histogram(MG_Female)\n",
        "        # print(hist_MG_Male.wasserstein(Male_Disc_Hist))\n",
        "        # print(hist_MG_Male.wasserstein(hist_MG_Female))\n",
        "        # dist += hist_MG_Male.wasserstein(hist_MG_Female)\n",
        "        dist += hist_MG_Male.wasserstein(Male_Disc_Hist)\n",
        "    print(\"Buckets = \" + str(i) + \",  Computed Distance = \" + str(dist/runs) + \", Relative Error = \" + str(dist/(runs*distance)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Buckets = 200,  Computed Distance = 24293.31476289477, Relative Error = 0.961024788838336\n",
        "Buckets = 400,  Computed Distance = 25013.354244940576, Relative Error = 0.9895089952112451\n",
        "Buckets = 600,  Computed Distance = 24603.233089404974, Relative Error = 0.9732849187217437\n",
        "Buckets = 800,  Computed Distance = 24200.213858057356, Relative Error = 0.9573417888737247\n",
        "Buckets = 1000,  Computed Distance = 24430.228515371582, Relative Error = 0.966440991252346\n",
        "Buckets = 1200,  Computed Distance = 24935.53401663188, Relative Error = 0.9864304870205084\n",
        "Buckets = 1400,  Computed Distance = 25448.22239108431, Relative Error = 1.0067120435559949\n",
        "Buckets = 1600,  Computed Distance = 25008.946216646953, Relative Error = 0.9893346170129047\n",
        "Buckets = 1800,  Computed Distance = 24964.025132601982, Relative Error = 0.9875575735863467\n",
        "Buckets = 2000,  Computed Distance = 25045.881852516166, Relative Error = 0.9907957622706953\n",
        "Buckets = 2200,  Computed Distance = 25542.364116187822, Relative Error = 1.010436217567302\n",
        "Buckets = 2400,  Computed Distance = 25387.902344956958, Relative Error = 1.004325829070321\n",
        "Buckets = 2600,  Computed Distance = 25197.224135378878, Relative Error = 0.9967827462146319\n",
        "Buckets = 2800,  Computed Distance = 25171.74049457176, Relative Error = 0.9957746330458643\n",
        "Buckets = 3000,  Computed Distance = 25522.170049691736, Relative Error = 1.0096373558771785\n",
        "Buckets = 3200,  Computed Distance = 25134.024770340788, Relative Error = 0.9942826280943492\n",
        "Buckets = 3400,  Computed Distance = 24979.604992449167, Relative Error = 0.9881739008214689\n",
        "Buckets = 3600,  Computed Distance = 25939.432561486246, Relative Error = 1.0261439389104645\n",
        "Buckets = 3800,  Computed Distance = 25508.925818497322, Relative Error = 1.009113424309537\n",
        "Buckets = 4000,  Computed Distance = 25242.803542570735, Relative Error = 0.9985858323969604\n",
        "Buckets = 4200,  Computed Distance = 25279.566351327678, Relative Error = 1.0000401407475321\n",
        "Buckets = 4400,  Computed Distance = 25528.427597803344, Relative Error = 1.0098848996525462\n",
        "Buckets = 4600,  Computed Distance = 25455.540109524896, Relative Error = 1.0070015268516168\n",
        "Buckets = 4800,  Computed Distance = 24949.637922723752, Relative Error = 0.9869884266638229\n",
        "Buckets = 5000,  Computed Distance = 25149.358546044154, Relative Error = 0.9948892204306049"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Buckets = 1000,  Computed Distance = 2283.652495964721, Relative Error = 0.09033953081885325\n",
        "Buckets = 2000,  Computed Distance = 1879.8845166861377, Relative Error = 0.07436678108037151\n",
        "Buckets = 3000,  Computed Distance = 1805.9091226744213, Relative Error = 0.07144037156799292\n",
        "Buckets = 4000,  Computed Distance = 1876.572271750493, Relative Error = 0.07423575122623564\n",
        "Buckets = 5000,  Computed Distance = 1160.6102772792228, Relative Error = 0.04591284711584417\n",
        "Buckets = 6000,  Computed Distance = 1877.412866324711, Relative Error = 0.07426900449904222\n",
        "Buckets = 7000,  Computed Distance = 569.4062212166426, Relative Error = 0.02252527079358327\n",
        "Buckets = 8000,  Computed Distance = 1811.5505799843006, Relative Error = 0.07166354326658088\n",
        "Buckets = 9000,  Computed Distance = 1579.5298801557594, Relative Error = 0.0624849833938286\n",
        "Buckets = 10000,  Computed Distance = 490.0564628110932, Relative Error = 0.019386255572303543\n",
        "Buckets = 11000,  Computed Distance = 465.4247752217706, Relative Error = 0.018411845015518746\n",
        "Buckets = 12000,  Computed Distance = 1462.2928051052343, Relative Error = 0.057847175157525584\n",
        "Buckets = 13000,  Computed Distance = 444.33426134194127, Relative Error = 0.017577520558536338\n",
        "Buckets = 14000,  Computed Distance = 548.8599640716552, Relative Error = 0.021712476713117206\n",
        "Buckets = 15000,  Computed Distance = 572.8321407713621, Relative Error = 0.0226607975279456"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 288,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "23743.511107282764\n"
          ]
        }
      ],
      "source": [
        "random.shuffle(ACS_pred_male_disc)\n",
        "random.shuffle(ACS_pred_female_disc)\n",
        "MG_Male = MergableMisraGries(k = 1000)\n",
        "MG_Male.scanunweighted(ACS_pred_male_disc)\n",
        "MG_Female = MergableMisraGries(k = 1000)\n",
        "MG_Female.scanunweighted(ACS_pred_female_disc)\n",
        "\n",
        "hist_MG_Male = histogram(MG_Male)\n",
        "hist_MG_Female = histogram(MG_Female)\n",
        "print(hist_MG_Male.wasserstein(hist_MG_Female))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 289,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "976\n",
            "2276\n"
          ]
        }
      ],
      "source": [
        "MG_Male_data = []\n",
        "MG_Male_Keys = list(MG_Male.keys())\n",
        "MG_Male_Values = list(MG_Male.values())\n",
        "for i in range(len(MG_Male_Keys)):\n",
        "    for _ in range(int(MG_Male_Values[i])):\n",
        "        MG_Male_data.append(MG_Male_Keys[i])\n",
        "print(len(MG_Male_Keys))\n",
        "print(len(MG_Male_data))\n",
        "# MG_male_df = pd.DataFrame('Income' : MG_Male.keys(), 'Count' : MG_Male.values())\n",
        "# plt.bar(MG_Male.keys(), MG_Male.values(), width = 1000, color='b')\n",
        "# fig, axs = plt.subplots(1, 2)\n",
        "# _ =plt.hist(MG_Male_data,bins = 50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 290,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 19400 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "_ =plt.hist(ACS_pred_male_disc,bins = 50, density=True)\n",
        "_ = plt.hist(MG_Male_data,bins = 50, density=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 286,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<BarContainer object of 167 artists>"
            ]
          },
          "execution_count": 286,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "[encoded data, 18828 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# c = counter(ACS_pred_male_disc)\n",
        "plt.clf()\n",
        "plt.bar(c.keys(), c.values(), width = 1000, color='b')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "U7U-Df5GRjOg"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
