{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 4 new time-saving tricks in pandas ([video](https://www.youtube.com/watch?v=-NbY7E9hKxk&list=PL5-da3qGB5ICCsgW1MxlZ0Hq8LL5U3u9y&index=32))\n", "\n", "- [My pandas video series (30 videos)](http://www.dataschool.io/easier-data-analysis-with-pandas/)\n", "- [GitHub repository](https://github.com/justmarkham/pandas-videos)\n", "- [pandas release notes](http://pandas.pydata.org/pandas-docs/stable/whatsnew.html)" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'0.22.0'" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "pd.__version__" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1. Create a datetime column from a DataFrame\n", "\n", "*New in 0.18.1*" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
monthdayyearhour
01225201710
1115201811
\n", "
" ], "text/plain": [ " month day year hour\n", "0 12 25 2017 10\n", "1 1 15 2018 11" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# create an example DataFrame\n", "df = pd.DataFrame([[12, 25, 2017, 10], [1, 15, 2018, 11]],\n", " columns=['month', 'day', 'year', 'hour'])\n", "df" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 2017-12-25 10:00:00\n", "1 2018-01-15 11:00:00\n", "dtype: datetime64[ns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# new: create a datetime column from the entire DataFrame\n", "pd.to_datetime(df)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 2017-12-25\n", "1 2018-01-15\n", "dtype: datetime64[ns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# new: create a datetime column from a subset of columns\n", "pd.to_datetime(df[['month', 'day', 'year']])" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
monthdayyearhour
2017-12-251225201710
2018-01-15115201811
\n", "
" ], "text/plain": [ " month day year hour\n", "2017-12-25 12 25 2017 10\n", "2018-01-15 1 15 2018 11" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# overwrite the index\n", "df.index = pd.to_datetime(df[['month', 'day', 'year']])\n", "df" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- [More information](http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#assembling-datetimes)\n", "- [Video: How do I work with dates and times in pandas?](https://www.youtube.com/watch?v=yCgJGsg0Xa4&list=PL5-da3qGB5ICCsgW1MxlZ0Hq8LL5U3u9y&index=25)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2. Create a category column during file reading\n", "\n", "*New in 0.19.0*" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countrybeer_servingsspirit_servingswine_servingstotal_litres_of_pure_alcoholcontinent
0Afghanistan0000.0Asia
1Albania89132544.9Europe
2Algeria250140.7Africa
3Andorra24513831212.4Europe
4Angola21757455.9Africa
\n", "
" ], "text/plain": [ " country beer_servings spirit_servings wine_servings \\\n", "0 Afghanistan 0 0 0 \n", "1 Albania 89 132 54 \n", "2 Algeria 25 0 14 \n", "3 Andorra 245 138 312 \n", "4 Angola 217 57 45 \n", "\n", " total_litres_of_pure_alcohol continent \n", "0 0.0 Asia \n", "1 4.9 Europe \n", "2 0.7 Africa \n", "3 12.4 Europe \n", "4 5.9 Africa " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# read the drinks dataset into a DataFrame\n", "drinks = pd.read_csv('http://bit.ly/drinksbycountry')\n", "drinks.head()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "country object\n", "beer_servings int64\n", "spirit_servings int64\n", "wine_servings int64\n", "total_litres_of_pure_alcohol float64\n", "continent object\n", "dtype: object" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# data types are automatically detected\n", "drinks.dtypes" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "country object\n", "beer_servings int64\n", "spirit_servings int64\n", "wine_servings int64\n", "total_litres_of_pure_alcohol float64\n", "continent category\n", "dtype: object" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# old way to create a category (after file reading)\n", "drinks['continent'] = drinks.continent.astype('category')\n", "drinks.dtypes" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "country object\n", "beer_servings int64\n", "spirit_servings int64\n", "wine_servings int64\n", "total_litres_of_pure_alcohol float64\n", "continent category\n", "dtype: object" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# new way to create a category (during file reading)\n", "drinks = pd.read_csv('http://bit.ly/drinksbycountry', dtype={'continent':'category'})\n", "drinks.dtypes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- [More information](http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#read-csv-supports-parsing-categorical-directly)\n", "- [Video: How do I make my pandas DataFrame smaller and faster?](https://www.youtube.com/watch?v=wDYDYGyN_cw&list=PL5-da3qGB5ICCsgW1MxlZ0Hq8LL5U3u9y&index=21)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 3. Convert the data type of multiple columns at once\n", "\n", "*New in 0.19.0*" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "country object\n", "beer_servings int64\n", "spirit_servings int64\n", "wine_servings int64\n", "total_litres_of_pure_alcohol float64\n", "continent object\n", "dtype: object" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# read the drinks dataset into a DataFrame\n", "drinks = pd.read_csv('http://bit.ly/drinksbycountry')\n", "drinks.dtypes" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "country object\n", "beer_servings float64\n", "spirit_servings float64\n", "wine_servings int64\n", "total_litres_of_pure_alcohol float64\n", "continent object\n", "dtype: object" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# old way to convert data types (one at a time)\n", "drinks['beer_servings'] = drinks.beer_servings.astype('float')\n", "drinks['spirit_servings'] = drinks.spirit_servings.astype('float')\n", "drinks.dtypes" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "country object\n", "beer_servings float64\n", "spirit_servings float64\n", "wine_servings int64\n", "total_litres_of_pure_alcohol float64\n", "continent object\n", "dtype: object" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# new way to convert data types (all at once)\n", "drinks = pd.read_csv('http://bit.ly/drinksbycountry')\n", "drinks = drinks.astype({'beer_servings':'float', 'spirit_servings':'float'})\n", "drinks.dtypes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- [More information](http://pandas.pydata.org/pandas-docs/stable/basics.html#astype)\n", "- [Video: How do I change the data type of a pandas Series?](https://www.youtube.com/watch?v=V0AWyzVMf54&list=PL5-da3qGB5ICCsgW1MxlZ0Hq8LL5U3u9y&index=13)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 4. Apply multiple aggregations on a Series or DataFrame\n", "\n", "*New in 0.20.0*" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "continent\n", "Africa 61.471698\n", "Asia 37.045455\n", "Europe 193.777778\n", "North America 145.434783\n", "Oceania 89.687500\n", "South America 175.083333\n", "Name: beer_servings, dtype: float64" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# example of a single aggregation function after a groupby\n", "drinks.groupby('continent').beer_servings.mean()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
meanminmax
continent
Africa61.4716980.0376.0
Asia37.0454550.0247.0
Europe193.7777780.0361.0
North America145.4347831.0285.0
Oceania89.6875000.0306.0
South America175.08333393.0333.0
\n", "
" ], "text/plain": [ " mean min max\n", "continent \n", "Africa 61.471698 0.0 376.0\n", "Asia 37.045455 0.0 247.0\n", "Europe 193.777778 0.0 361.0\n", "North America 145.434783 1.0 285.0\n", "Oceania 89.687500 0.0 306.0\n", "South America 175.083333 93.0 333.0" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# multiple aggregation functions can be applied simultaneously\n", "drinks.groupby('continent').beer_servings.agg(['mean', 'min', 'max'])" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "mean 106.160622\n", "min 0.000000\n", "max 376.000000\n", "Name: beer_servings, dtype: float64" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# new: apply the same aggregations to a Series\n", "drinks.beer_servings.agg(['mean', 'min', 'max'])" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countrybeer_servingsspirit_servingswine_servingstotal_litres_of_pure_alcoholcontinent
maxZimbabwe376.000000438.000000370.00000014.400000South America
meanNaN106.16062280.99481949.4507774.717098NaN
minAfghanistan0.0000000.0000000.0000000.000000Africa
\n", "
" ], "text/plain": [ " country beer_servings spirit_servings wine_servings \\\n", "max Zimbabwe 376.000000 438.000000 370.000000 \n", "mean NaN 106.160622 80.994819 49.450777 \n", "min Afghanistan 0.000000 0.000000 0.000000 \n", "\n", " total_litres_of_pure_alcohol continent \n", "max 14.400000 South America \n", "mean 4.717098 NaN \n", "min 0.000000 Africa " ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# new: apply the same aggregations to a DataFrame\n", "drinks.agg(['mean', 'min', 'max'])" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
beer_servingsspirit_servingswine_servingstotal_litres_of_pure_alcohol
count193.000000193.000000193.000000193.000000
mean106.16062280.99481949.4507774.717098
std101.14310388.28431279.6975983.773298
min0.0000000.0000000.0000000.000000
25%20.0000004.0000001.0000001.300000
50%76.00000056.0000008.0000004.200000
75%188.000000128.00000059.0000007.200000
max376.000000438.000000370.00000014.400000
\n", "
" ], "text/plain": [ " beer_servings spirit_servings wine_servings \\\n", "count 193.000000 193.000000 193.000000 \n", "mean 106.160622 80.994819 49.450777 \n", "std 101.143103 88.284312 79.697598 \n", "min 0.000000 0.000000 0.000000 \n", "25% 20.000000 4.000000 1.000000 \n", "50% 76.000000 56.000000 8.000000 \n", "75% 188.000000 128.000000 59.000000 \n", "max 376.000000 438.000000 370.000000 \n", "\n", " total_litres_of_pure_alcohol \n", "count 193.000000 \n", "mean 4.717098 \n", "std 3.773298 \n", "min 0.000000 \n", "25% 1.300000 \n", "50% 4.200000 \n", "75% 7.200000 \n", "max 14.400000 " ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# DataFrame describe method provides similar functionality but is less flexible\n", "drinks.describe()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- [More information](http://pandas.pydata.org/pandas-docs/stable/basics.html#basics-aggregate)\n", "- [Video: When should I use a \"groupby\" in pandas?](https://www.youtube.com/watch?v=qy0fDqoMJx8&list=PL5-da3qGB5ICCsgW1MxlZ0Hq8LL5U3u9y&index=14)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Bonus: Download the official pandas cheat sheet\n", "\n", "*New in 0.19.2*\n", "\n", "[Cheat sheet (PDF)](https://github.com/pandas-dev/pandas/blob/master/doc/cheatsheet/Pandas_Cheat_Sheet.pdf)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.4" } }, "nbformat": 4, "nbformat_minor": 2 }