confidence_interval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\": \"kLr4fYcqcSpbuI95brIH3vnnYCquzzSxHPU6XGQCIkQRGJwhg0StNbj1eegrHs12\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\": \"xIGPmVtaOm+z0BqfSOMn4lOR6ciex448GIKG4eE61LsAvmGj48XcMQZtKcE/UXZe\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\": \"Dc9u1wF/0zApGIWoBbH77iWEHtdmkuYWG839Uzmv8y8yBLXebjO9ZnERsde5Ln/P\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\": \"cT9JaBz7GiRXdENrJLZNSC6eMNF3nh3fa5fTF51Svp+ukxPdwcU5kGXGPBgDCa2j\"};\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      if (url in hashes) {\n",
       "        element.crossOrigin = \"anonymous\";\n",
       "        element.integrity = \"sha384-\" + hashes[url];\n",
       "      }\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\": \"kLr4fYcqcSpbuI95brIH3vnnYCquzzSxHPU6XGQCIkQRGJwhg0StNbj1eegrHs12\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\": \"xIGPmVtaOm+z0BqfSOMn4lOR6ciex448GIKG4eE61LsAvmGj48XcMQZtKcE/UXZe\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\": \"Dc9u1wF/0zApGIWoBbH77iWEHtdmkuYWG839Uzmv8y8yBLXebjO9ZnERsde5Ln/P\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\": \"cT9JaBz7GiRXdENrJLZNSC6eMNF3nh3fa5fTF51Svp+ukxPdwcU5kGXGPBgDCa2j\"};\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      if (url in hashes) {\n        element.crossOrigin = \"anonymous\";\n        element.integrity = \"sha384-\" + hashes[url];\n      }\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from bokeh.io import show\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.io import output_notebook\n",
    "from bokeh.layouts import gridplot\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confidence Intervals and Two-Tail Tests\n",
    "### goals:\n",
    "* Understand the confidence intervals\n",
    "* understand a two-tail test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"Please skip to below. This just copies code from the previous notebook\"\n",
    "#NOTE!! Do not use seed for results. Just using this for instructional purposes\n",
    "random.seed(13)\n",
    "def resample(l):\n",
    "    final = []\n",
    "    for i in range(len(l)):\n",
    "        final.append(random.choice(l))\n",
    "    return final\n",
    "\n",
    "def repeat_resample(sample_a, sample_b, num_iter = 1000):\n",
    "    difference_in_means = []#keep track of the difference in heights for each experiment\n",
    "    for i in range(num_iter):\n",
    "        resample_a = resample(sample_a)\n",
    "        resample_b = resample(sample_b)\n",
    "        difference = np.mean(resample_a) - np.mean(resample_b)\n",
    "        difference_in_means.append(difference)\n",
    "    return difference_in_means\n",
    "\n",
    "def make_grid(sample_a, sample_b):\n",
    "    resample_a = resample(sample_a)\n",
    "    resample_b = resample(sample_b)\n",
    "    p1= make_bar(labels = [x for x in range(len(resample_a))], nums = sorted(resample_a), \n",
    "               title = \"Resample A\", y_range = (120, 180))\n",
    "    p2 = make_bar(labels = [x for x in range(len(resample_b))], nums = sorted(resample_b), \n",
    "               title = \"Resample B\", y_range = (120, 180))\n",
    "    grid = gridplot([p1, p2], ncols = 2)\n",
    "    print('the mean of resample 1 is {r1} and the mean of resample2 is {r2}'.format(\n",
    "        r1 = np.mean(resample_a), r2 = np.mean(resample_b)))\n",
    "    return grid\n",
    "\n",
    "def hist(l):\n",
    "    hist, edges = np.histogram(l, density=True)\n",
    "    p = figure()\n",
    "    p.quad(top = hist, bottom=0, left=edges[:-1], right=edges[1:], alpha = .4)\n",
    "    return p\n",
    "\n",
    "def get_classroom_willow_a():\n",
    "    return [146.1,152.7, 146.3, 142.0, 151.8,\n",
    "     151.4, 145.1, 153.5, 151.2, 143.5, 158.2,\n",
    "     150.6, 143.5, 151.6, 149.9, 154.2, 142.4,\n",
    "     154.6, 154.1, 152.8, 152.4, 155.9, 152.9,\n",
    "     149.9, 145.0]\n",
    "\n",
    "def get_classroom_willow_b():\n",
    "    return [142.9, 150.9, 154.0, 146.4, 142.4,\n",
    "     148.7, 151.4, 154.5, 142.9, 142.5, 152.5,\n",
    "     151.4, 156.7, 153.9, 148.5, 147.6, 161.9,\n",
    "     147.6, 145.1, 143.3, 149.5, 147.3,\n",
    "     148.7, 150.4]\n",
    "\n",
    "WILLOW_A = get_classroom_willow_a()\n",
    "WILLOW_B = get_classroom_willow_b()\n",
    "\n",
    "def get_classroom_birch_a():\n",
    "    return [141.8, 150.2, 147.6, 146.6, 153.8,\n",
    "     149.3, 147.6, 158.0, 146.5, 142.7, 142.1,\n",
    "     146.4, 152.3, 153.3, 154.7, 158.3, 157.6,\n",
    "     152.3, 155.8, 152.4, 146.4, 153.3, 149.5,\n",
    "     148.2, 159.3]\n",
    "\n",
    "def get_classroom_birch_b():\n",
    "    return [169.6, 163.3,  177.6, 164.5,\n",
    "     169.5, 168.9, 168.4, 168.7, 163.3, 163.4,\n",
    "     165.0, 164.0, 169.9, 173.6, 161.3, 168.5,\n",
    "     160.8, 162.3, 164.6, 166.3, 163.6, 152.6,\n",
    "     172.8, 164.4]\n",
    "\n",
    "BIRCH_A = get_classroom_birch_a()\n",
    "BIRCH_B = get_classroom_birch_b()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confidence Interval\n",
    "Let us revisit the two classrooms from the Willow School. We went to the city and found a sample of kids with freckles and a sample of kids with no freckes. We recoreded the means of each group and found the difference between them. We repeated the experiment 1,000 times and recorded the result each time. This new set of numbers is a new sample. The histogram looks like this:\n",
    "\n",
    "![title](data/hist.jpeg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that some results occur a lot, those in the middle. For example, we see that there is a difference of 0 about 25% of the time. On the other hand, some results do not occurr that often. -2 occurs less than 2.5% of the time, and 4 occurrs less than 2.5% of the time. That means that 95% of the time, the differences lie between -2 and 4. If we conduct this experiment again and again, 95% of the time we would find that the difference would be no less than -2, and no greater than 4. This is the 95% confidence interval. We think that the population mean lies between -2 and 4. \n",
    "\n",
    "It appears the mean is approxtimately 1. We can calculate the mean and confidence interval more precisely and directly. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the mean of the resamples is 0.8203034999999992\n",
      "the lower 2.5% is -1.7721666666666378\n",
      "the upper 97.5% is 3.259370833333325\n"
     ]
    }
   ],
   "source": [
    "RESAMPLES_WILLOW = repeat_resample(WILLOW_A, WILLOW_B)\n",
    "print(\"the mean of the resamples is {m}\".format(m = np.mean(RESAMPLES_WILLOW)))\n",
    "print(\"the lower 2.5% is {l}\".format(l = np.percentile(RESAMPLES_WILLOW, 2.5)))\n",
    "print(\"the upper 97.5% is {l}\".format(l = np.percentile(RESAMPLES_WILLOW, 97.5)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The mean is approxtimately .8, and the 95% confidence interval lies between -1.7 and 3.3. Remember, the reason we need statistics is because we have samples and we are trying to estimate the true population. In this case, we can say that the  difference between the freckles and non freckles is .8. But it could be as little as -1.7, or as great as 3.3. We can't know for sure.\n",
    "\n",
    "If you tell your friend that students with freckles are taller than .8cm. But your friend answers that the true difference might be as low as -1.7. So we can't really be sure there is a difference. In other words, we do not reject the null hypothesis. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two Tail Test\n",
    "Previously, we tested only if one sample was greater than the other. Now we can perform a different type of test: is one of the samples different than the other sample, either greater or less? \n",
    "### Formal Procedure. \n",
    "\n",
    "In order to determine if a result is significant, we do the following steps:\n",
    "1. Determine your \"test statistic,\" what you are testing. Usually this will be 0. \n",
    "2. Resample to get a sample of means\n",
    "3. Determine the confidence interval.\n",
    "4. If the test statistic lies outside of the confidence interval, (what is know as the \"reject regions\") reject the null hypothesis. Otherwise, do not reject the null hypothesis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we want to determine if the two samples are different, so our test statistic is 0\n",
      "The 95% interval lies between -1.7721666666666378 and 3.259370833333325\n",
      "0 lies within the confidence interval. We will not reject the null hypothesis.\n"
     ]
    }
   ],
   "source": [
    "print('we want to determine if the two samples are different, so our test statistic is 0')\n",
    "#get the reasmpled means\n",
    "# (just repeating the step from above, for clarity)\n",
    "resamples_of_the_means = repeat_resample(WILLOW_A, WILLOW_B)\n",
    "lower = np.percentile(RESAMPLES_WILLOW, 2.5)\n",
    "upper = np.percentile(RESAMPLES_WILLOW, 97.5)\n",
    "print('The 95% interval lies between {l} and {u}'.format(\n",
    "    l = lower, u = upper))\n",
    "lies_inside =   lower < 0 < upper\n",
    "if lies_inside:\n",
    "    print('0 lies within the confidence interval. We will not reject the null hypothesis.')\n",
    "else:\n",
    "    print('0 does not lie within the confidence interval. We will  reject the null hypothesis.')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's perform the same procedure with the Birch School."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"e2f811ff-1977-4613-a1a6-34c21eb630e6\" data-root-id=\"1002\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"63232cd0-70b5-499f-8717-f253883fec94\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1011\"}],\"center\":[{\"id\":\"1014\"},{\"id\":\"1018\"}],\"left\":[{\"id\":\"1015\"}],\"renderers\":[{\"id\":\"1036\"}],\"title\":{\"id\":\"1038\"},\"toolbar\":{\"id\":\"1026\"},\"x_range\":{\"id\":\"1003\"},\"x_scale\":{\"id\":\"1007\"},\"y_range\":{\"id\":\"1005\"},\"y_scale\":{\"id\":\"1009\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1023\",\"type\":\"ResetTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1033\"},\"glyph\":{\"id\":\"1034\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1035\"},\"selection_glyph\":null,\"view\":{\"id\":\"1037\"}},\"id\":\"1036\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1016\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1024\",\"type\":\"HelpTool\"},{\"attributes\":{\"axis\":{\"id\":\"1015\"},\"dimension\":1,\"ticker\":null},\"id\":\"1018\",\"type\":\"Grid\"},{\"attributes\":{\"bottom\":{\"value\":0},\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"left\":{\"field\":\"left\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"right\":{\"field\":\"right\"},\"top\":{\"field\":\"top\"}},\"id\":\"1035\",\"type\":\"Quad\"},{\"attributes\":{\"formatter\":{\"id\":\"1043\"},\"ticker\":{\"id\":\"1012\"}},\"id\":\"1011\",\"type\":\"LinearAxis\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1019\"},{\"id\":\"1020\"},{\"id\":\"1021\"},{\"id\":\"1022\"},{\"id\":\"1023\"},{\"id\":\"1024\"}]},\"id\":\"1026\",\"type\":\"Toolbar\"},{\"attributes\":{\"axis\":{\"id\":\"1011\"},\"ticker\":null},\"id\":\"1014\",\"type\":\"Grid\"},{\"attributes\":{\"bottom\":{\"value\":0},\"fill_alpha\":{\"value\":0.4},\"fill_color\":{\"value\":\"#1f77b4\"},\"left\":{\"field\":\"left\"},\"line_alpha\":{\"value\":0.4},\"line_color\":{\"value\":\"#1f77b4\"},\"right\":{\"field\":\"right\"},\"top\":{\"field\":\"top\"}},\"id\":\"1034\",\"type\":\"Quad\"},{\"attributes\":{},\"id\":\"1041\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"formatter\":{\"id\":\"1041\"},\"ticker\":{\"id\":\"1016\"}},\"id\":\"1015\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1005\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"BasicTicker\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1038\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1007\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1025\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"source\":{\"id\":\"1033\"}},\"id\":\"1037\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1019\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1044\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1020\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1022\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"LinearScale\"},{\"attributes\":{\"data\":{\"left\":{\"__ndarray__\":\"INv5fmqcJUDDI94nNIYnQGZswtD9bylACrWmecdZK0Ct/YoikUMtQFBGb8taLS9AescpOpKLMEDL65sOd4AxQB0QDuNbdTJAbjSAt0BqM0A=\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[10]},\"right\":{\"__ndarray__\":\"wyPeJzSGJ0BmbMLQ/W8pQAq1pnnHWStArf2KIpFDLUBQRm/LWi0vQHrHKTqSizBAy+ubDneAMUAdEA7jW3UyQG40gLdAajNAwFjyiyVfNEA=\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[10]},\"top\":{\"__ndarray__\":\"V3CP4IQgYT99Mc4Cv56YP2FMsximaLU/0OZ4SesTxj/H/29lpTHRP6wTTMPBudA/9dCu8aHfxD/8muTF00urP1p5hpJ8DpA/TnCP4IQgUT8=\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[10]}},\"selected\":{\"id\":\"1045\"},\"selection_policy\":{\"id\":\"1044\"}},\"id\":\"1033\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1045\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1003\",\"type\":\"DataRange1d\"},{\"attributes\":{\"overlay\":{\"id\":\"1025\"}},\"id\":\"1021\",\"type\":\"BoxZoomTool\"}],\"root_ids\":[\"1002\"]},\"title\":\"Bokeh Application\",\"version\":\"2.1.1\"}};\n",
       "  var render_items = [{\"docid\":\"63232cd0-70b5-499f-8717-f253883fec94\",\"root_ids\":[\"1002\"],\"roots\":{\"1002\":\"e2f811ff-1977-4613-a1a6-34c21eb630e6\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else {\n",
       "        attempts++;\n",
       "        if (attempts > 100) {\n",
       "          clearInterval(timer);\n",
       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        }\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1002"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "RESAMPLES_BIRCH = repeat_resample(BIRCH_B, BIRCH_A)\n",
    "show(hist(RESAMPLES_BIRCH))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It appears that the average difference is 16cm. The lowest in the 95% interval is 12cm, and the highest is 18cm. You tell your friend that the 17-year olds are taller than the 12-year olds by 16cm. Even with randomness, the lowest is at least 12cm. 0 does not lie within the 95% interval, so it is unlikely the population contains 0. So we can be certain the difference is real and not just due to randomness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 95% interval lies between 12.75349166666668 and 18.192141666666675\n",
      "0 does not lie within the confidence interval. We will reject the null hypothesis.\n"
     ]
    }
   ],
   "source": [
    "mean_birch = np.mean(RESAMPLES_BIRCH)\n",
    "lower_birch = np.percentile(RESAMPLES_BIRCH, 2.5)\n",
    "upper_birch = np.percentile(RESAMPLES_BIRCH, 97.5)\n",
    "print('The 95% interval lies between {l} and {u}'.format(\n",
    "    l = lower_birch, u = upper_birch))\n",
    "lies_inside =   lower_birch < 0 < upper_birch\n",
    "if lies_inside:\n",
    "    print('0 lies within the confidence interval. We will not reject the null hypothesis.')\n",
    "else:\n",
    "    print('0 does not lie within the confidence interval. We will reject the null hypothesis.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# is the class of 17-year olds taller than the class of 12-year olds by 12cm? (Is 12cm significant?)\n",
    "# test statistic is 12\n",
    "lies_inside =   lower_birch < 12 < upper_birch\n",
    "lies_inside\n",
    "# is significant. 12 is not certain, so not likely to be in the population\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# is 14 cm significant? \n",
    "# test statistic is 14\n",
    "lies_inside =   lower_birch < 14 < upper_birch\n",
    "lies_inside\n",
    "# not significant. 14 is somewhat certain, and could lie within the population"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## p value for confidence intervals\n",
    "The last section we calculated a p value. A p value is implied when we use confidence intervals. If we use a 95% confidence interval, our p vaue is <= .05. In other words, if the reject region lies in the 5% region, that is the same as saying \"the probability of the null hypothesis being true is .05.\" \n",
    "\n",
    "If we were to use a 99% confidence interval, our p value would be .01."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary\n",
    "\n",
    "* We resample to estimate the population\n",
    "* The differences between two samples can be quantified by a confidence interval. A 95% confidence interval means there is a 95% probability that the true (population) mean lies within that interval. \n",
    "* If the test statistic lies outside the confidence interval, it is population contains the test statistic. For example, if 0 lies in the reject region, then the true difference between the samples is not 0 (and we reject the null hypothesis). \n",
    "* The p value is equal to 1 - confidence interval. For a 95% confidence interval, the p vaue = 1 - .95 = .05\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}