{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "___\n", "\n", " \n", "___\n", "
*Copyright Pierian Data 2017*
\n", "
*For more information, visit us at www.pieriandata.com*
" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Time Shifting\n", "\n", "Sometimes you may need to shift all your data up or down along the time series index, in fact, a lot of pandas built-in methods do this under the hood. This isn't something we won't do often in the course, but its definitely good to know about this anyways!" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df = pd.read_csv('time_data/walmart_stock.csv',index_col='Date')\n", "df.index = pd.to_datetime(df.index)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2012-01-0359.97000161.06000159.86999960.3300021266880052.619235
2012-01-0460.20999960.34999859.47000159.709999959330052.078475
2012-01-0559.34999859.61999958.36999959.4199981276820051.825539
2012-01-0659.41999859.45000158.86999959.000000806940051.459220
2012-01-0959.02999959.54999958.91999859.180000667930051.616215
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2012-01-03 59.970001 61.060001 59.869999 60.330002 12668800 52.619235\n", "2012-01-04 60.209999 60.349998 59.470001 59.709999 9593300 52.078475\n", "2012-01-05 59.349998 59.619999 58.369999 59.419998 12768200 51.825539\n", "2012-01-06 59.419998 59.450001 58.869999 59.000000 8069400 51.459220\n", "2012-01-09 59.029999 59.549999 58.919998 59.180000 6679300 51.616215" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2016-12-2369.43000069.75000069.36000169.540001480390069.032411
2016-12-2769.30000369.82000069.25000069.699997443570069.191240
2016-12-2869.94000270.00000069.26000269.309998487570068.804087
2016-12-2969.20999969.51999769.12000369.260002429840068.754456
2016-12-3069.12000369.43000068.83000269.120003688950068.615479
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2016-12-23 69.430000 69.750000 69.360001 69.540001 4803900 69.032411\n", "2016-12-27 69.300003 69.820000 69.250000 69.699997 4435700 69.191240\n", "2016-12-28 69.940002 70.000000 69.260002 69.309998 4875700 68.804087\n", "2016-12-29 69.209999 69.519997 69.120003 69.260002 4298400 68.754456\n", "2016-12-30 69.120003 69.430000 68.830002 69.120003 6889500 68.615479" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.tail()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## .shift() forward" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2012-01-03NaNNaNNaNNaNNaNNaN
2012-01-0459.97000161.06000159.86999960.33000212668800.052.619235
2012-01-0560.20999960.34999859.47000159.7099999593300.052.078475
2012-01-0659.34999859.61999958.36999959.41999812768200.051.825539
2012-01-0959.41999859.45000158.86999959.0000008069400.051.459220
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2012-01-03 NaN NaN NaN NaN NaN NaN\n", "2012-01-04 59.970001 61.060001 59.869999 60.330002 12668800.0 52.619235\n", "2012-01-05 60.209999 60.349998 59.470001 59.709999 9593300.0 52.078475\n", "2012-01-06 59.349998 59.619999 58.369999 59.419998 12768200.0 51.825539\n", "2012-01-09 59.419998 59.450001 58.869999 59.000000 8069400.0 51.459220" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shift(1).head()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2016-12-2371.23999871.23999869.20999969.58999612106800.069.082042
2016-12-2769.43000069.75000069.36000169.5400014803900.069.032411
2016-12-2869.30000369.82000069.25000069.6999974435700.069.191240
2016-12-2969.94000270.00000069.26000269.3099984875700.068.804087
2016-12-3069.20999969.51999769.12000369.2600024298400.068.754456
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2016-12-23 71.239998 71.239998 69.209999 69.589996 12106800.0 69.082042\n", "2016-12-27 69.430000 69.750000 69.360001 69.540001 4803900.0 69.032411\n", "2016-12-28 69.300003 69.820000 69.250000 69.699997 4435700.0 69.191240\n", "2016-12-29 69.940002 70.000000 69.260002 69.309998 4875700.0 68.804087\n", "2016-12-30 69.209999 69.519997 69.120003 69.260002 4298400.0 68.754456" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# You will lose that last piece of data that no longer has an index!\n", "df.shift(1).tail()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## shift() backwards" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2012-01-0360.20999960.34999859.47000159.7099999593300.052.078475
2012-01-0459.34999859.61999958.36999959.41999812768200.051.825539
2012-01-0559.41999859.45000158.86999959.0000008069400.051.459220
2012-01-0659.02999959.54999958.91999859.1800006679300.051.616215
2012-01-0959.43000059.70999958.98000059.0400016907300.051.494109
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2012-01-03 60.209999 60.349998 59.470001 59.709999 9593300.0 52.078475\n", "2012-01-04 59.349998 59.619999 58.369999 59.419998 12768200.0 51.825539\n", "2012-01-05 59.419998 59.450001 58.869999 59.000000 8069400.0 51.459220\n", "2012-01-06 59.029999 59.549999 58.919998 59.180000 6679300.0 51.616215\n", "2012-01-09 59.430000 59.709999 58.980000 59.040001 6907300.0 51.494109" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shift(-1).head()" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2016-12-2369.30000369.82000069.25000069.6999974435700.069.191240
2016-12-2769.94000270.00000069.26000269.3099984875700.068.804087
2016-12-2869.20999969.51999769.12000369.2600024298400.068.754456
2016-12-2969.12000369.43000068.83000269.1200036889500.068.615479
2016-12-30NaNNaNNaNNaNNaNNaN
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2016-12-23 69.300003 69.820000 69.250000 69.699997 4435700.0 69.191240\n", "2016-12-27 69.940002 70.000000 69.260002 69.309998 4875700.0 68.804087\n", "2016-12-28 69.209999 69.519997 69.120003 69.260002 4298400.0 68.754456\n", "2016-12-29 69.120003 69.430000 68.830002 69.120003 6889500.0 68.615479\n", "2016-12-30 NaN NaN NaN NaN NaN NaN" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shift(-1).tail()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Shifting based off Time String Code \n", "### Using tshift()" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
OpenHighLowCloseVolumeAdj Close
Date
2012-01-3159.97000161.06000159.86999960.3300021266880052.619235
2012-01-3160.20999960.34999859.47000159.709999959330052.078475
2012-01-3159.34999859.61999958.36999959.4199981276820051.825539
2012-01-3159.41999859.45000158.86999959.000000806940051.459220
2012-01-3159.02999959.54999958.91999859.180000667930051.616215
\n", "
" ], "text/plain": [ " Open High Low Close Volume Adj Close\n", "Date \n", "2012-01-31 59.970001 61.060001 59.869999 60.330002 12668800 52.619235\n", "2012-01-31 60.209999 60.349998 59.470001 59.709999 9593300 52.078475\n", "2012-01-31 59.349998 59.619999 58.369999 59.419998 12768200 51.825539\n", "2012-01-31 59.419998 59.450001 58.869999 59.000000 8069400 51.459220\n", "2012-01-31 59.029999 59.549999 58.919998 59.180000 6679300 51.616215" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Shift everything forward one month\n", "df.tshift(periods=1,freq='M').head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "That is it for now!" ] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python [conda root]", "language": "python", "name": "conda-root-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.3" } }, "nbformat": 4, "nbformat_minor": 0 }