12_pitfalls.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), December 12, 2021**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Possible pitfalls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Know what is copied when creating a `DataFrame`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>3 rows × 5 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th></tr><tr><th></th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th></tr></thead><tbody><tr><th>1</th><td>0.345332</td><td>0.88169</td><td>0.636903</td><td>0.275687</td><td>0.441932</td></tr><tr><th>2</th><td>0.942451</td><td>0.243809</td><td>0.860925</td><td>0.710149</td><td>0.0612701</td></tr><tr><th>3</th><td>0.937008</td><td>0.67613</td><td>0.391914</td><td>0.549464</td><td>0.178747</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5\\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 0.345332 & 0.88169 & 0.636903 & 0.275687 & 0.441932 \\\\\n",
       "\t2 & 0.942451 & 0.243809 & 0.860925 & 0.710149 & 0.0612701 \\\\\n",
       "\t3 & 0.937008 & 0.67613 & 0.391914 & 0.549464 & 0.178747 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×5 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64   \u001b[0m\n",
       "─────┼───────────────────────────────────────────────────\n",
       "   1 │ 0.345332  0.88169   0.636903  0.275687  0.441932\n",
       "   2 │ 0.942451  0.243809  0.860925  0.710149  0.0612701\n",
       "   3 │ 0.937008  0.67613   0.391914  0.549464  0.178747"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(rand(3, 5), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = copy(x)\n",
    "x === y # not the same object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = DataFrame(x)\n",
    "x === y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "any(x[!, i] === y[!, i] for i in ncol(x)) # the columns are also not the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = DataFrame(x, copycols=false)\n",
    "x === y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all(x[!, i] === y[!, i] for i in ncol(x)) # the columns are the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>3 rows × 2 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x     \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     2      2\n",
       "   3 │     3      3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = 1:3; y = [1, 2, 3]; df = DataFrame(x=x,y=y) # the same when creating data frames using kwarg syntax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y === df.y # different object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(UnitRange{Int64}, Vector{Int64})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "typeof(x), typeof(df.x) # range is converted to a vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y === df[:, :y] # slicing rows always creates a copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can avoid copying by using `copycols=false` keyword argument in functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>3 rows × 2 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x     \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     2      2\n",
       "   3 │     3      3"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = DataFrame(x=x,y=y, copycols=false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y === df.y # now it is the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "select(df, :y)[!, 1] === y # not the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "select(df, :y, copycols=false)[!, 1] === y # the same"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do not modify the parent of `GroupedDataFrame` or `view`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 2 groups based on key: id</b></p><p><i>First Group (3 rows): id = 1</i></p><div class=\"data-frame\"><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>1</td><td>3</td></tr><tr><th>3</th><td>1</td><td>5</td></tr></tbody></table></div><p>&vellip;</p><p><i>Last Group (3 rows): id = 2</i></p><div class=\"data-frame\"><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>2</td></tr><tr><th>2</th><td>2</td><td>4</td></tr><tr><th>3</th><td>2</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "\n",
       "First Group (3 rows): id = 1\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 1 & 3 \\\\\n",
       "\t3 & 1 & 5 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (3 rows): id = 2\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 2 \\\\\n",
       "\t2 & 2 & 4 \\\\\n",
       "\t3 & 2 & 6 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "First Group (3 rows): id = 1\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     1      3\n",
       "   3 │     1      5\n",
       "⋮\n",
       "Last Group (3 rows): id = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      2\n",
       "   2 │     2      4\n",
       "   3 │     2      6"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(id=repeat([1,2], outer=3), x=1:6)\n",
    "g = groupby(x, :id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 2 groups based on key: id</b></p><p><i>First Group (3 rows): id = 2</i></p><div class=\"data-frame\"><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>1</td></tr><tr><th>2</th><td>2</td><td>3</td></tr><tr><th>3</th><td>1</td><td>5</td></tr></tbody></table></div><p>&vellip;</p><p><i>Last Group (3 rows): id = 2</i></p><div class=\"data-frame\"><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>2</td></tr><tr><th>2</th><td>2</td><td>4</td></tr><tr><th>3</th><td>2</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "\n",
       "First Group (3 rows): id = 2\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 1 \\\\\n",
       "\t2 & 2 & 3 \\\\\n",
       "\t3 & 1 & 5 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (3 rows): id = 2\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 2 \\\\\n",
       "\t2 & 2 & 4 \\\\\n",
       "\t3 & 2 & 6 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "First Group (3 rows): id = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      1\n",
       "   2 │     2      3\n",
       "   3 │     1      5\n",
       "⋮\n",
       "Last Group (3 rows): id = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      2\n",
       "   2 │     2      4\n",
       "   3 │     2      6"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[1:3, 1] = [2,2,2]\n",
    "g # well - it is wrong now, g is only a view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>2 rows × 2 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>5</td></tr><tr><th>2</th><td>2</td><td>6</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 5 \\\\\n",
       "\t2 & 2 & 6 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×2 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      5\n",
       "   2 │     2      6"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = view(x, 5:6, :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>2 rows × 2 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 1 \\\\\n",
       "\t2 & 2 & 2 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      1\n",
       "   2 │     2      2"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delete!(x, 3:6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "BoundsError",
     "evalue": "BoundsError: attempt to access 2-element Vector{Int64} at index [5:6]",
     "output_type": "error",
     "traceback": [
      "BoundsError: attempt to access 2-element Vector{Int64} at index [5:6]",
      "",
      "Stacktrace:",
      "  [1] throw_boundserror(A::Vector{Int64}, I::Tuple{UnitRange{Int64}})",
      "    @ Base .\\abstractarray.jl:691",
      "  [2] checkbounds",
      "    @ .\\abstractarray.jl:656 [inlined]",
      "  [3] view(A::Vector{Int64}, I::UnitRange{Int64})",
      "    @ Base .\\subarray.jl:177",
      "  [4] view",
      "    @ C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\subdataframe\\subdataframe.jl:129 [inlined]",
      "  [5] getindex",
      "    @ C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\subdataframe\\subdataframe.jl:166 [inlined]",
      "  [6] getindex",
      "    @ C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\iteration.jl:200 [inlined]",
      "  [7] iterate(itr::DataFrames.DataFrameColumns{SubDataFrame{DataFrame, DataFrames.Index, UnitRange{Int64}}}, i::Int64) (repeats 2 times)",
      "    @ DataFrames C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\iteration.jl:198",
      "  [8] _show(io::IOContext{IOBuffer}, df::SubDataFrame{DataFrame, DataFrames.Index, UnitRange{Int64}}; allrows::Bool, allcols::Bool, rowlabel::Symbol, summary::Bool, eltypes::Bool, rowid::Nothing, truncate::Int64, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})",
      "    @ DataFrames C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\show.jl:172",
      "  [9] #show#735",
      "    @ C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\show.jl:353 [inlined]",
      " [10] show(io::IOContext{IOBuffer}, df::SubDataFrame{DataFrame, DataFrames.Index, UnitRange{Int64}})",
      "    @ DataFrames C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\show.jl:353",
      " [11] #show#750",
      "    @ C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\io.jl:138 [inlined]",
      " [12] show",
      "    @ C:\\Users\\bogum\\.julia\\packages\\DataFrames\\ORSVA\\src\\abstractdataframe\\io.jl:138 [inlined]",
      " [13] limitstringmime(mime::MIME{Symbol(\"text/plain\")}, x::SubDataFrame{DataFrame, DataFrames.Index, UnitRange{Int64}})",
      "    @ IJulia C:\\Users\\bogum\\.julia\\packages\\IJulia\\e8kqU\\src\\inline.jl:43",
      " [14] display_mimestring",
      "    @ C:\\Users\\bogum\\.julia\\packages\\IJulia\\e8kqU\\src\\display.jl:71 [inlined]",
      " [15] display_dict(x::SubDataFrame{DataFrame, DataFrames.Index, UnitRange{Int64}})",
      "    @ IJulia C:\\Users\\bogum\\.julia\\packages\\IJulia\\e8kqU\\src\\display.jl:102",
      " [16] #invokelatest#2",
      "    @ .\\essentials.jl:716 [inlined]",
      " [17] invokelatest",
      "    @ .\\essentials.jl:714 [inlined]",
      " [18] execute_request(socket::ZMQ.Socket, msg::IJulia.Msg)",
      "    @ IJulia C:\\Users\\bogum\\.julia\\packages\\IJulia\\e8kqU\\src\\execute_request.jl:112",
      " [19] #invokelatest#2",
      "    @ .\\essentials.jl:716 [inlined]",
      " [20] invokelatest",
      "    @ .\\essentials.jl:714 [inlined]",
      " [21] eventloop(socket::ZMQ.Socket)",
      "    @ IJulia C:\\Users\\bogum\\.julia\\packages\\IJulia\\e8kqU\\src\\eventloop.jl:8",
      " [22] (::IJulia.var\"#15#18\")()",
      "    @ IJulia .\\task.jl:423"
     ]
    }
   ],
   "source": [
    "s # error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single column selection for `DataFrame` creates aliases with `!` and `getproperty` syntax and copies with `:`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>3 rows × 5 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>a</th><th>b</th><th>c</th><th>d</th><th>e</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td><td>2</td><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td><td>3</td><td>3</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccc}\n",
       "\t& a & b & c & d & e\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 & 2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 & 3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×5 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m a     \u001b[0m\u001b[1m b     \u001b[0m\u001b[1m c     \u001b[0m\u001b[1m d     \u001b[0m\u001b[1m e     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────────────────────────\n",
       "   1 │     1      1      1      1      1\n",
       "   2 │     2      2      2      2      2\n",
       "   3 │     3      3      3      3      3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>3 rows × 5 columns</p><table class=\"data-frame\"><thead><tr><th></th><th>a</th><th>b</th><th>c</th><th>d</th><th>e</th></tr><tr><th></th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th><th title=\"Int64\">Int64</th></tr></thead><tbody><tr><th>1</th><td>100</td><td>100</td><td>1</td><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td><td>2</td><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td><td>3</td><td>3</td><td>3</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccc}\n",
       "\t& a & b & c & d & e\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 100 & 100 & 1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 & 2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 & 3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×5 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m a     \u001b[0m\u001b[1m b     \u001b[0m\u001b[1m c     \u001b[0m\u001b[1m d     \u001b[0m\u001b[1m e     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────────────────────────\n",
       "   1 │   100    100      1      1      1\n",
       "   2 │     2      2      2      2      2\n",
       "   3 │     3      3      3      3      3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = DataFrame(a=1:3)\n",
    "x.b = x[!, 1] # alias\n",
    "x.c = x[:, 1] # copy\n",
    "x.d = x[!, 1][:] # copy\n",
    "x.e = copy(x[!, 1]) # explicit copy\n",
    "display(x)\n",
    "x[1,1] = 100\n",
    "display(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### When iterating rows of a data frame use `eachrow` to avoid compilation cost (wide tables), but `Tables.namedtupleiterator` for fast execution (tall tables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "this table is wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>2 rows × 900 columns (omitted printing of 887 columns)</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th><th>x10</th><th>x11</th><th>x12</th><th>x13</th></tr><tr><th></th><th title=\"Bool\">Bool</th><th title=\"Int64\">Int64</th><th title=\"Char\">Char</th><th title=\"Bool\">Bool</th><th title=\"Float64\">Float64</th><th title=\"Char\">Char</th><th title=\"Char\">Char</th><th title=\"Char\">Char</th><th title=\"Bool\">Bool</th><th title=\"Bool\">Bool</th><th title=\"Char\">Char</th><th title=\"Float64\">Float64</th><th title=\"Bool\">Bool</th></tr></thead><tbody><tr><th>1</th><td>0</td><td>1</td><td>a</td><td>0</td><td>1.0</td><td>a</td><td>a</td><td>a</td><td>0</td><td>0</td><td>a</td><td>1.0</td><td>0</td></tr><tr><th>2</th><td>1</td><td>2</td><td>b</td><td>1</td><td>2.0</td><td>b</td><td>b</td><td>b</td><td>1</td><td>1</td><td>b</td><td>2.0</td><td>1</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & x10 & x11 & x12 & x13 & \\\\\n",
       "\t\\hline\n",
       "\t& Bool & Int64 & Char & Bool & Float64 & Char & Char & Char & Bool & Bool & Char & Float64 & Bool & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0 & 1 & a & 0 & 1.0 & a & a & a & 0 & 0 & a & 1.0 & 0 & $\\dots$ \\\\\n",
       "\t2 & 1 & 2 & b & 1 & 2.0 & b & b & b & 1 & 1 & b & 2.0 & 1 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×900 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1    \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3   \u001b[0m\u001b[1m x4    \u001b[0m\u001b[1m x5      \u001b[0m\u001b[1m x6   \u001b[0m\u001b[1m x7   \u001b[0m\u001b[1m x8   \u001b[0m\u001b[1m x9    \u001b[0m\u001b[1m x10   \u001b[0m\u001b[1m x11\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Bool  \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Bool  \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Bool  \u001b[0m\u001b[90m Bool  \u001b[0m\u001b[90m Cha\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ false      1  a     false      1.0  a     a     a     false  false  a   ⋯\n",
       "   2 │  true      2  b      true      2.0  b     b     b      true   true  b\n",
       "\u001b[36m                                                             890 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = DataFrame([rand([1:2, 'a':'b', false:true, 1.0:2.0]) for i in 1:900], :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.083117 seconds (86.08 k allocations: 4.741 MiB, 99.93% compilation time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2-element Vector{DataFrameRow}:\n",
       " \u001b[1mDataFrameRow\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1    \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3   \u001b[0m\u001b[1m x4    \u001b[0m\u001b[1m x5      \u001b[0m\u001b[1m x6   \u001b[0m\u001b[1m x7   \u001b[0m\u001b[1m x8   \u001b[0m\u001b[1m x9    \u001b[0m\u001b[1m x10   \u001b[0m\u001b[1m x11\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Bool  \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Bool  \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Bool  \u001b[0m\u001b[90m Bool  \u001b[0m\u001b[90m Cha\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ false      1  a     false      1.0  a     a     a     false  false  a   ⋯\n",
       "\u001b[36m                                                             890 columns omitted\u001b[0m\n",
       " \u001b[1mDataFrameRow\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1   \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3   \u001b[0m\u001b[1m x4   \u001b[0m\u001b[1m x5      \u001b[0m\u001b[1m x6   \u001b[0m\u001b[1m x7   \u001b[0m\u001b[1m x8   \u001b[0m\u001b[1m x9   \u001b[0m\u001b[1m x10  \u001b[0m\u001b[1m x11  \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Bool \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Bool \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Bool \u001b[0m\u001b[90m Bool \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   2 │ true      2  b     true      2.0  b     b     b     true  true  b       ⋯\n",
       "\u001b[36m                                                             889 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time collect(eachrow(df1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 11.948307 seconds (2.66 M allocations: 167.463 MiB, 0.68% gc time, 99.89% compilation time)\n"
     ]
    }
   ],
   "source": [
    "@time collect(Tables.namedtupleiterator(df1));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "as you can see the time to compile `Tables.namedtupleiterator` is very large in this case, and it would get much worse if the table was wider (that is why we include this tip in pitfalls notebook)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the table below is tall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"data-frame\"><p>1,000,000 rows × 10 columns (omitted printing of 2 columns)</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th></tr><tr><th></th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th><th title=\"Float64\">Float64</th></tr></thead><tbody><tr><th>1</th><td>0.812598</td><td>0.199008</td><td>0.489529</td><td>0.173243</td><td>0.174659</td><td>0.629508</td><td>0.261476</td><td>0.384585</td></tr><tr><th>2</th><td>0.00280933</td><td>0.0697785</td><td>0.721708</td><td>0.985117</td><td>0.748635</td><td>0.747097</td><td>0.95652</td><td>0.234513</td></tr><tr><th>3</th><td>0.674185</td><td>0.642761</td><td>0.277017</td><td>0.576259</td><td>0.641213</td><td>0.905591</td><td>0.880482</td><td>0.558141</td></tr><tr><th>4</th><td>0.637642</td><td>0.0178559</td><td>0.938028</td><td>0.309252</td><td>0.446349</td><td>0.577177</td><td>0.966324</td><td>0.75673</td></tr><tr><th>5</th><td>0.847842</td><td>0.0951855</td><td>0.185581</td><td>0.437136</td><td>0.0211549</td><td>0.90453</td><td>0.382964</td><td>0.450601</td></tr><tr><th>6</th><td>0.85145</td><td>0.388947</td><td>0.265416</td><td>0.628984</td><td>0.433217</td><td>0.806872</td><td>0.39362</td><td>0.866561</td></tr><tr><th>7</th><td>0.808643</td><td>0.328728</td><td>0.753462</td><td>0.404849</td><td>0.107149</td><td>0.285132</td><td>0.82107</td><td>0.0583062</td></tr><tr><th>8</th><td>0.513336</td><td>0.771539</td><td>0.344971</td><td>0.71855</td><td>0.266392</td><td>0.745457</td><td>0.370414</td><td>0.996824</td></tr><tr><th>9</th><td>0.235142</td><td>0.7607</td><td>0.243749</td><td>0.386569</td><td>0.0957563</td><td>0.789125</td><td>0.390211</td><td>0.28802</td></tr><tr><th>10</th><td>0.970881</td><td>0.00220123</td><td>0.305367</td><td>0.34369</td><td>0.6262</td><td>0.251802</td><td>0.0153362</td><td>0.533094</td></tr><tr><th>11</th><td>0.800898</td><td>0.455806</td><td>0.19927</td><td>0.572369</td><td>0.196033</td><td>0.660683</td><td>0.775653</td><td>0.680364</td></tr><tr><th>12</th><td>0.696595</td><td>0.960139</td><td>0.242386</td><td>0.0847333</td><td>0.427189</td><td>0.53297</td><td>0.785826</td><td>0.599125</td></tr><tr><th>13</th><td>0.932493</td><td>0.889251</td><td>0.79769</td><td>0.591297</td><td>0.664426</td><td>0.414793</td><td>0.29345</td><td>0.704092</td></tr><tr><th>14</th><td>0.791435</td><td>0.146096</td><td>0.213213</td><td>0.0977305</td><td>0.81124</td><td>0.390915</td><td>0.885303</td><td>0.19021</td></tr><tr><th>15</th><td>0.206303</td><td>0.672786</td><td>0.184465</td><td>0.481344</td><td>0.995939</td><td>0.810043</td><td>0.480981</td><td>0.345796</td></tr><tr><th>16</th><td>0.535187</td><td>0.125527</td><td>0.267538</td><td>0.825189</td><td>0.33463</td><td>0.619829</td><td>0.563728</td><td>0.483909</td></tr><tr><th>17</th><td>0.232218</td><td>0.380849</td><td>0.58704</td><td>0.258559</td><td>0.527797</td><td>0.487358</td><td>0.244215</td><td>0.306474</td></tr><tr><th>18</th><td>0.100788</td><td>0.789073</td><td>0.875051</td><td>0.612904</td><td>0.306586</td><td>0.786476</td><td>0.0499812</td><td>0.97055</td></tr><tr><th>19</th><td>0.567395</td><td>0.474193</td><td>0.457527</td><td>0.326257</td><td>0.359375</td><td>0.705074</td><td>0.895929</td><td>0.501663</td></tr><tr><th>20</th><td>0.446746</td><td>0.232416</td><td>0.064617</td><td>0.376457</td><td>0.493576</td><td>0.4788</td><td>0.0746602</td><td>0.737677</td></tr><tr><th>21</th><td>0.0909145</td><td>0.428829</td><td>0.447152</td><td>0.188665</td><td>0.602939</td><td>0.709546</td><td>0.610028</td><td>0.0622533</td></tr><tr><th>22</th><td>0.956029</td><td>0.672475</td><td>0.274408</td><td>0.193462</td><td>0.766002</td><td>0.818443</td><td>0.51466</td><td>0.293607</td></tr><tr><th>23</th><td>0.34909</td><td>0.565187</td><td>0.270686</td><td>0.213333</td><td>0.403518</td><td>0.0299558</td><td>0.995217</td><td>0.251452</td></tr><tr><th>24</th><td>0.11957</td><td>0.0879578</td><td>0.721854</td><td>0.186702</td><td>0.250901</td><td>0.0832353</td><td>0.475087</td><td>0.471285</td></tr><tr><th>25</th><td>0.0656886</td><td>0.586737</td><td>0.126349</td><td>0.895501</td><td>0.47768</td><td>0.748157</td><td>0.418267</td><td>0.250066</td></tr><tr><th>26</th><td>0.077623</td><td>0.650055</td><td>0.106039</td><td>0.535195</td><td>0.436161</td><td>0.967486</td><td>0.811753</td><td>0.408328</td></tr><tr><th>27</th><td>0.16542</td><td>0.359966</td><td>0.959966</td><td>0.573472</td><td>0.534036</td><td>0.026867</td><td>0.168101</td><td>0.590052</td></tr><tr><th>28</th><td>0.246313</td><td>0.600301</td><td>0.511784</td><td>0.872385</td><td>0.386095</td><td>0.549019</td><td>0.965729</td><td>0.226105</td></tr><tr><th>29</th><td>0.516978</td><td>0.0858493</td><td>0.809388</td><td>0.21923</td><td>0.32178</td><td>0.202469</td><td>0.690369</td><td>0.667672</td></tr><tr><th>30</th><td>0.0455554</td><td>0.198724</td><td>0.73012</td><td>0.638181</td><td>0.950534</td><td>0.552568</td><td>0.715544</td><td>0.0543341</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.812598 & 0.199008 & 0.489529 & 0.173243 & 0.174659 & 0.629508 & 0.261476 & 0.384585 & $\\dots$ \\\\\n",
       "\t2 & 0.00280933 & 0.0697785 & 0.721708 & 0.985117 & 0.748635 & 0.747097 & 0.95652 & 0.234513 & $\\dots$ \\\\\n",
       "\t3 & 0.674185 & 0.642761 & 0.277017 & 0.576259 & 0.641213 & 0.905591 & 0.880482 & 0.558141 & $\\dots$ \\\\\n",
       "\t4 & 0.637642 & 0.0178559 & 0.938028 & 0.309252 & 0.446349 & 0.577177 & 0.966324 & 0.75673 & $\\dots$ \\\\\n",
       "\t5 & 0.847842 & 0.0951855 & 0.185581 & 0.437136 & 0.0211549 & 0.90453 & 0.382964 & 0.450601 & $\\dots$ \\\\\n",
       "\t6 & 0.85145 & 0.388947 & 0.265416 & 0.628984 & 0.433217 & 0.806872 & 0.39362 & 0.866561 & $\\dots$ \\\\\n",
       "\t7 & 0.808643 & 0.328728 & 0.753462 & 0.404849 & 0.107149 & 0.285132 & 0.82107 & 0.0583062 & $\\dots$ \\\\\n",
       "\t8 & 0.513336 & 0.771539 & 0.344971 & 0.71855 & 0.266392 & 0.745457 & 0.370414 & 0.996824 & $\\dots$ \\\\\n",
       "\t9 & 0.235142 & 0.7607 & 0.243749 & 0.386569 & 0.0957563 & 0.789125 & 0.390211 & 0.28802 & $\\dots$ \\\\\n",
       "\t10 & 0.970881 & 0.00220123 & 0.305367 & 0.34369 & 0.6262 & 0.251802 & 0.0153362 & 0.533094 & $\\dots$ \\\\\n",
       "\t11 & 0.800898 & 0.455806 & 0.19927 & 0.572369 & 0.196033 & 0.660683 & 0.775653 & 0.680364 & $\\dots$ \\\\\n",
       "\t12 & 0.696595 & 0.960139 & 0.242386 & 0.0847333 & 0.427189 & 0.53297 & 0.785826 & 0.599125 & $\\dots$ \\\\\n",
       "\t13 & 0.932493 & 0.889251 & 0.79769 & 0.591297 & 0.664426 & 0.414793 & 0.29345 & 0.704092 & $\\dots$ \\\\\n",
       "\t14 & 0.791435 & 0.146096 & 0.213213 & 0.0977305 & 0.81124 & 0.390915 & 0.885303 & 0.19021 & $\\dots$ \\\\\n",
       "\t15 & 0.206303 & 0.672786 & 0.184465 & 0.481344 & 0.995939 & 0.810043 & 0.480981 & 0.345796 & $\\dots$ \\\\\n",
       "\t16 & 0.535187 & 0.125527 & 0.267538 & 0.825189 & 0.33463 & 0.619829 & 0.563728 & 0.483909 & $\\dots$ \\\\\n",
       "\t17 & 0.232218 & 0.380849 & 0.58704 & 0.258559 & 0.527797 & 0.487358 & 0.244215 & 0.306474 & $\\dots$ \\\\\n",
       "\t18 & 0.100788 & 0.789073 & 0.875051 & 0.612904 & 0.306586 & 0.786476 & 0.0499812 & 0.97055 & $\\dots$ \\\\\n",
       "\t19 & 0.567395 & 0.474193 & 0.457527 & 0.326257 & 0.359375 & 0.705074 & 0.895929 & 0.501663 & $\\dots$ \\\\\n",
       "\t20 & 0.446746 & 0.232416 & 0.064617 & 0.376457 & 0.493576 & 0.4788 & 0.0746602 & 0.737677 & $\\dots$ \\\\\n",
       "\t21 & 0.0909145 & 0.428829 & 0.447152 & 0.188665 & 0.602939 & 0.709546 & 0.610028 & 0.0622533 & $\\dots$ \\\\\n",
       "\t22 & 0.956029 & 0.672475 & 0.274408 & 0.193462 & 0.766002 & 0.818443 & 0.51466 & 0.293607 & $\\dots$ \\\\\n",
       "\t23 & 0.34909 & 0.565187 & 0.270686 & 0.213333 & 0.403518 & 0.0299558 & 0.995217 & 0.251452 & $\\dots$ \\\\\n",
       "\t24 & 0.11957 & 0.0879578 & 0.721854 & 0.186702 & 0.250901 & 0.0832353 & 0.475087 & 0.471285 & $\\dots$ \\\\\n",
       "\t25 & 0.0656886 & 0.586737 & 0.126349 & 0.895501 & 0.47768 & 0.748157 & 0.418267 & 0.250066 & $\\dots$ \\\\\n",
       "\t26 & 0.077623 & 0.650055 & 0.106039 & 0.535195 & 0.436161 & 0.967486 & 0.811753 & 0.408328 & $\\dots$ \\\\\n",
       "\t27 & 0.16542 & 0.359966 & 0.959966 & 0.573472 & 0.534036 & 0.026867 & 0.168101 & 0.590052 & $\\dots$ \\\\\n",
       "\t28 & 0.246313 & 0.600301 & 0.511784 & 0.872385 & 0.386095 & 0.549019 & 0.965729 & 0.226105 & $\\dots$ \\\\\n",
       "\t29 & 0.516978 & 0.0858493 & 0.809388 & 0.21923 & 0.32178 & 0.202469 & 0.690369 & 0.667672 & $\\dots$ \\\\\n",
       "\t30 & 0.0455554 & 0.198724 & 0.73012 & 0.638181 & 0.950534 & 0.552568 & 0.715544 & 0.0543341 & $\\dots$ \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ &  \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1000000×10 DataFrame\u001b[0m\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x1         \u001b[0m\u001b[1m x2         \u001b[0m\u001b[1m x3        \u001b[0m\u001b[1m x4        \u001b[0m\u001b[1m x5        \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m \u001b[0m ⋯\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Float64    \u001b[0m\u001b[90m Float64    \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m \u001b[0m ⋯\n",
       "─────────┼──────────────────────────────────────────────────────────────────────\n",
       "       1 │ 0.812598    0.199008    0.489529   0.173243   0.174659   0.629508   ⋯\n",
       "       2 │ 0.00280933  0.0697785   0.721708   0.985117   0.748635   0.747097\n",
       "       3 │ 0.674185    0.642761    0.277017   0.576259   0.641213   0.905591\n",
       "       4 │ 0.637642    0.0178559   0.938028   0.309252   0.446349   0.577177\n",
       "       5 │ 0.847842    0.0951855   0.185581   0.437136   0.0211549  0.90453    ⋯\n",
       "       6 │ 0.85145     0.388947    0.265416   0.628984   0.433217   0.806872\n",
       "       7 │ 0.808643    0.328728    0.753462   0.404849   0.107149   0.285132\n",
       "       8 │ 0.513336    0.771539    0.344971   0.71855    0.266392   0.745457\n",
       "       9 │ 0.235142    0.7607      0.243749   0.386569   0.0957563  0.789125   ⋯\n",
       "      10 │ 0.970881    0.00220123  0.305367   0.34369    0.6262     0.251802\n",
       "      11 │ 0.800898    0.455806    0.19927    0.572369   0.196033   0.660683\n",
       "    ⋮    │     ⋮           ⋮           ⋮          ⋮          ⋮         ⋮       ⋱\n",
       "  999991 │ 0.0410003   0.462709    0.677799   0.963837   0.163653   0.813361\n",
       "  999992 │ 0.530049    0.41385     0.736268   0.300554   0.793102   0.866098   ⋯\n",
       "  999993 │ 0.270374    0.105039    0.132824   0.850973   0.0863429  0.845847\n",
       "  999994 │ 0.0546815   0.708984    0.46075    0.943581   0.719482   0.886945\n",
       "  999995 │ 0.1872      0.898951    0.458422   0.790131   0.630058   0.492997\n",
       "  999996 │ 0.574525    0.991815    0.705654   0.308657   0.92873    0.195947   ⋯\n",
       "  999997 │ 0.315078    0.118292    0.24476    0.693066   0.742942   0.293202\n",
       "  999998 │ 0.749306    0.149496    0.404374   0.7858     0.778498   0.307536\n",
       "  999999 │ 0.515942    0.259932    0.765135   0.550848   0.975477   0.26296\n",
       " 1000000 │ 0.0898478   0.623094    0.567737   0.718526   0.706433   0.35637    ⋯\n",
       "\u001b[36m                                               4 columns and 999979 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = DataFrame(rand(10^6, 10), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  3.892058 seconds (60.19 M allocations: 1.061 GiB, 8.04% gc time, 3.45% compilation time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Vector{Float64}:\n",
       " 3.5415480933137844\n",
       " 5.205413796084415\n",
       " 5.538985173625073\n",
       " 5.679680368162268\n",
       " 4.819966279265384\n",
       " 6.039599576343484\n",
       " 4.559158997855922\n",
       " 5.506623184812048\n",
       " 4.357268924654652\n",
       " 3.9055762037646815\n",
       " 5.468423085862514\n",
       " 5.135525917068438\n",
       " 6.650543085096766\n",
       " ⋮\n",
       " 4.808328609279509\n",
       " 4.135386777246283\n",
       " 5.411053708396329\n",
       " 6.091779993930741\n",
       " 4.527597648004583\n",
       " 6.795736552178264\n",
       " 5.536045157550963\n",
       " 4.279163510515187\n",
       " 3.9958529869139956\n",
       " 5.377263758627355\n",
       " 5.361734558567568\n",
       " 4.74749740730375"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, eachrow(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  3.526568 seconds (59.99 M allocations: 1.050 GiB, 2.39% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Vector{Float64}:\n",
       " 3.5415480933137844\n",
       " 5.205413796084415\n",
       " 5.538985173625073\n",
       " 5.679680368162268\n",
       " 4.819966279265384\n",
       " 6.039599576343484\n",
       " 4.559158997855922\n",
       " 5.506623184812048\n",
       " 4.357268924654652\n",
       " 3.9055762037646815\n",
       " 5.468423085862514\n",
       " 5.135525917068438\n",
       " 6.650543085096766\n",
       " ⋮\n",
       " 4.808328609279509\n",
       " 4.135386777246283\n",
       " 5.411053708396329\n",
       " 6.091779993930741\n",
       " 4.527597648004583\n",
       " 6.795736552178264\n",
       " 5.536045157550963\n",
       " 4.279163510515187\n",
       " 3.9958529869139956\n",
       " 5.377263758627355\n",
       " 5.361734558567568\n",
       " 4.74749740730375"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, eachrow(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.275496 seconds (500.67 k allocations: 35.538 MiB, 93.38% compilation time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Vector{Float64}:\n",
       " 3.5415480933137844\n",
       " 5.205413796084415\n",
       " 5.538985173625073\n",
       " 5.679680368162268\n",
       " 4.819966279265384\n",
       " 6.039599576343484\n",
       " 4.559158997855922\n",
       " 5.506623184812048\n",
       " 4.357268924654652\n",
       " 3.9055762037646815\n",
       " 5.468423085862514\n",
       " 5.135525917068438\n",
       " 6.650543085096766\n",
       " ⋮\n",
       " 4.808328609279509\n",
       " 4.135386777246283\n",
       " 5.411053708396329\n",
       " 6.091779993930741\n",
       " 4.527597648004583\n",
       " 6.795736552178264\n",
       " 5.536045157550963\n",
       " 4.279163510515187\n",
       " 3.9958529869139956\n",
       " 5.377263758627355\n",
       " 5.361734558567568\n",
       " 4.74749740730375"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, Tables.namedtupleiterator(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.017857 seconds (13 allocations: 7.630 MiB)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Vector{Float64}:\n",
       " 3.5415480933137844\n",
       " 5.205413796084415\n",
       " 5.538985173625073\n",
       " 5.679680368162268\n",
       " 4.819966279265384\n",
       " 6.039599576343484\n",
       " 4.559158997855922\n",
       " 5.506623184812048\n",
       " 4.357268924654652\n",
       " 3.9055762037646815\n",
       " 5.468423085862514\n",
       " 5.135525917068438\n",
       " 6.650543085096766\n",
       " ⋮\n",
       " 4.808328609279509\n",
       " 4.135386777246283\n",
       " 5.411053708396329\n",
       " 6.091779993930741\n",
       " 4.527597648004583\n",
       " 6.795736552178264\n",
       " 5.536045157550963\n",
       " 4.279163510515187\n",
       " 3.9958529869139956\n",
       " 5.377263758627355\n",
       " 5.361734558567568\n",
       " 4.74749740730375"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, Tables.namedtupleiterator(df2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "as you can see - this time it is much faster to iterate a type stable container"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "still you might want to use the `select` syntax, which is optimized for such reductions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1.123701 seconds (2.16 M allocations: 134.329 MiB, 2.20% gc time, 98.66% compilation time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Vector{Float64}:\n",
       " 3.5415480933137844\n",
       " 5.205413796084415\n",
       " 5.538985173625073\n",
       " 5.679680368162268\n",
       " 4.819966279265384\n",
       " 6.039599576343484\n",
       " 4.559158997855922\n",
       " 5.506623184812048\n",
       " 4.357268924654652\n",
       " 3.9055762037646815\n",
       " 5.468423085862514\n",
       " 5.135525917068438\n",
       " 6.650543085096766\n",
       " ⋮\n",
       " 4.808328609279509\n",
       " 4.135386777246283\n",
       " 5.411053708396329\n",
       " 6.091779993930741\n",
       " 4.527597648004583\n",
       " 6.795736552178264\n",
       " 5.536045157550963\n",
       " 4.279163510515187\n",
       " 3.9958529869139956\n",
       " 5.377263758627355\n",
       " 5.361734558567568\n",
       " 4.74749740730375"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time select(df2, AsTable(:) => ByRow(sum) => \"sum\").sum # this includes compilation time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.013828 seconds (151 allocations: 7.637 MiB)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Vector{Float64}:\n",
       " 3.5415480933137844\n",
       " 5.205413796084415\n",
       " 5.538985173625073\n",
       " 5.679680368162268\n",
       " 4.819966279265384\n",
       " 6.039599576343484\n",
       " 4.559158997855922\n",
       " 5.506623184812048\n",
       " 4.357268924654652\n",
       " 3.9055762037646815\n",
       " 5.468423085862514\n",
       " 5.135525917068438\n",
       " 6.650543085096766\n",
       " ⋮\n",
       " 4.808328609279509\n",
       " 4.135386777246283\n",
       " 5.411053708396329\n",
       " 6.091779993930741\n",
       " 4.527597648004583\n",
       " 6.795736552178264\n",
       " 5.536045157550963\n",
       " 4.279163510515187\n",
       " 3.9958529869139956\n",
       " 5.377263758627355\n",
       " 5.361734558567568\n",
       " 4.74749740730375"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time select(df2, AsTable(:) => ByRow(sum) => \"sum\").sum"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.7.0",
   "language": "julia",
   "name": "julia-1.7"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}