Skip to content

Instantly share code, notes, and snippets.

@alexlauforgithub

alexlauforgithub/-lag.ipynb

Created January 22, 2021 08:33
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alexlauforgithub/69a6626d670feb50c93aa6cc8f4296c5 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alexlauforgithub/69a6626d670feb50c93aa6cc8f4296c5 to your computer and use it in GitHub Desktop.
Download ZIP
極速lag.ipynb
Display the source blob
Display the rendered blob
Raw
-lag.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "極速lag.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyOxZQFkke5AX9kP2AzpBsAm",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/alexlauforgithub/69a6626d670feb50c93aa6cc8f4296c5/-lag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "vzMBmn8p4Gdl"
},
"source": [
"import numpy as np\n",
"from numpy.lib.stride_tricks import as_strided as strided \n",
"import pandas as pd"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "tizNCQja8jge",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"outputId": "4d0aa885-43bb-4590-b769-70ed32fb4f11"
},
"source": [
"x = np.array(range(0,20))\n",
"print(x)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"text": [
"[ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19]\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "7rZSH6VR_0Yq"
},
"source": [
"a = x\n",
"W = 5"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "-2zEP_QQFEK6"
},
"source": [
"def dododo(a, W):\n",
" a_ext = np.concatenate(( np.full(W-1,np.nan) ,a))\n",
" n = a_ext.strides[0]\n",
" res = np.fliplr(strided(a_ext[W-1:], shape=(a.size,W), strides=(n,-n)))\n",
" return(res)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "0TCxwxrE_lKb"
},
"source": [
"a = np.array(range(1,21))\n",
"b = np.array(range(21,41))\n",
"c = np.random.random_sample((20,))\n",
"d = np.repeat(['A', 'B'], 10)\n",
"df = pd.DataFrame({'val1':a, 'val2':b, 'val3':c, 'grp': d})"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "hziG0Zpyf8j0"
},
"source": [
"x = df.groupby(['grp'], group_keys=False).apply(lambda x: dododo(x['val1'], 5))\n",
"y = df.groupby(['grp'], group_keys=False).apply(lambda x: dododo(x['val2'], 5))"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "GmwDVv4GRuwB",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 639
},
"outputId": "bf698241-6172-4cd9-ba3c-dafce7074ff6"
},
"source": [
"df"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>val1</th>\n",
" <th>val2</th>\n",
" <th>val3</th>\n",
" <th>grp</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>21</td>\n",
" <td>0.645247</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>22</td>\n",
" <td>0.049844</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>23</td>\n",
" <td>0.192304</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>24</td>\n",
" <td>0.772429</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>25</td>\n",
" <td>0.100788</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>6</td>\n",
" <td>26</td>\n",
" <td>0.867648</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>7</td>\n",
" <td>27</td>\n",
" <td>0.966026</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>8</td>\n",
" <td>28</td>\n",
" <td>0.986281</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>9</td>\n",
" <td>29</td>\n",
" <td>0.986891</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>10</td>\n",
" <td>30</td>\n",
" <td>0.788358</td>\n",
" <td>A</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>11</td>\n",
" <td>31</td>\n",
" <td>0.028804</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>12</td>\n",
" <td>32</td>\n",
" <td>0.827435</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>13</td>\n",
" <td>33</td>\n",
" <td>0.951224</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>14</td>\n",
" <td>34</td>\n",
" <td>0.606368</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>15</td>\n",
" <td>35</td>\n",
" <td>0.755293</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>16</td>\n",
" <td>36</td>\n",
" <td>0.596170</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>17</td>\n",
" <td>37</td>\n",
" <td>0.533669</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>18</td>\n",
" <td>38</td>\n",
" <td>0.260281</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>19</td>\n",
" <td>39</td>\n",
" <td>0.909233</td>\n",
" <td>B</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>20</td>\n",
" <td>40</td>\n",
" <td>0.877136</td>\n",
" <td>B</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" val1 val2 val3 grp\n",
"0 1 21 0.645247 A\n",
"1 2 22 0.049844 A\n",
"2 3 23 0.192304 A\n",
"3 4 24 0.772429 A\n",
"4 5 25 0.100788 A\n",
"5 6 26 0.867648 A\n",
"6 7 27 0.966026 A\n",
"7 8 28 0.986281 A\n",
"8 9 29 0.986891 A\n",
"9 10 30 0.788358 A\n",
"10 11 31 0.028804 B\n",
"11 12 32 0.827435 B\n",
"12 13 33 0.951224 B\n",
"13 14 34 0.606368 B\n",
"14 15 35 0.755293 B\n",
"15 16 36 0.596170 B\n",
"16 17 37 0.533669 B\n",
"17 18 38 0.260281 B\n",
"18 19 39 0.909233 B\n",
"19 20 40 0.877136 B"
]
},
"metadata": {
"tags": []
},
"execution_count": 7
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "hWFu3Ja7UBxk",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 84
},
"outputId": "500d9748-ddf1-4444-8db0-e365e09ac836"
},
"source": [
"x"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"grp\n",
"A [[nan, nan, nan, nan, 1.0], [nan, nan, nan, 1....\n",
"B [[nan, nan, nan, nan, 11.0], [nan, nan, nan, 1...\n",
"dtype: object"
]
},
"metadata": {
"tags": []
},
"execution_count": 8
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "drpVkIsGRznN",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 357
},
"outputId": "f1737f01-335f-4508-b242-b50c8a7bf7e6"
},
"source": [
"x2 = np.vstack(x.to_list())\n",
"x2"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array([[nan, nan, nan, nan, 1.],\n",
" [nan, nan, nan, 1., 2.],\n",
" [nan, nan, 1., 2., 3.],\n",
" [nan, 1., 2., 3., 4.],\n",
" [ 1., 2., 3., 4., 5.],\n",
" [ 2., 3., 4., 5., 6.],\n",
" [ 3., 4., 5., 6., 7.],\n",
" [ 4., 5., 6., 7., 8.],\n",
" [ 5., 6., 7., 8., 9.],\n",
" [ 6., 7., 8., 9., 10.],\n",
" [nan, nan, nan, nan, 11.],\n",
" [nan, nan, nan, 11., 12.],\n",
" [nan, nan, 11., 12., 13.],\n",
" [nan, 11., 12., 13., 14.],\n",
" [11., 12., 13., 14., 15.],\n",
" [12., 13., 14., 15., 16.],\n",
" [13., 14., 15., 16., 17.],\n",
" [14., 15., 16., 17., 18.],\n",
" [15., 16., 17., 18., 19.],\n",
" [16., 17., 18., 19., 20.]])"
]
},
"metadata": {
"tags": []
},
"execution_count": 9
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "mM8OyfRMjKC0"
},
"source": [
"x2 = np.vstack(x.to_list())"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "DyiVnTnuhtv1"
},
"source": [
"y2 = np.vstack(y.to_list())"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "fw4uXHzciXTc"
},
"source": [
"z = np.hstack([x2, y2])"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "pDbIuIk9pYOK",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 357
},
"outputId": "8b690678-ddc4-4cd3-e648-da05abf37c26"
},
"source": [
"z"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array([[nan, nan, nan, nan, 1., nan, nan, nan, nan, 21.],\n",
" [nan, nan, nan, 1., 2., nan, nan, nan, 21., 22.],\n",
" [nan, nan, 1., 2., 3., nan, nan, 21., 22., 23.],\n",
" [nan, 1., 2., 3., 4., nan, 21., 22., 23., 24.],\n",
" [ 1., 2., 3., 4., 5., 21., 22., 23., 24., 25.],\n",
" [ 2., 3., 4., 5., 6., 22., 23., 24., 25., 26.],\n",
" [ 3., 4., 5., 6., 7., 23., 24., 25., 26., 27.],\n",
" [ 4., 5., 6., 7., 8., 24., 25., 26., 27., 28.],\n",
" [ 5., 6., 7., 8., 9., 25., 26., 27., 28., 29.],\n",
" [ 6., 7., 8., 9., 10., 26., 27., 28., 29., 30.],\n",
" [nan, nan, nan, nan, 11., nan, nan, nan, nan, 31.],\n",
" [nan, nan, nan, 11., 12., nan, nan, nan, 31., 32.],\n",
" [nan, nan, 11., 12., 13., nan, nan, 31., 32., 33.],\n",
" [nan, 11., 12., 13., 14., nan, 31., 32., 33., 34.],\n",
" [11., 12., 13., 14., 15., 31., 32., 33., 34., 35.],\n",
" [12., 13., 14., 15., 16., 32., 33., 34., 35., 36.],\n",
" [13., 14., 15., 16., 17., 33., 34., 35., 36., 37.],\n",
" [14., 15., 16., 17., 18., 34., 35., 36., 37., 38.],\n",
" [15., 16., 17., 18., 19., 35., 36., 37., 38., 39.],\n",
" [16., 17., 18., 19., 20., 36., 37., 38., 39., 40.]])"
]
},
"metadata": {
"tags": []
},
"execution_count": 78
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "jPChw6WTjXjQ",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"outputId": "168cb1c2-cb26-4e9b-dbc2-cc54dadd8d14"
},
"source": [
"z.reshape((-1,5,2), order='F')"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array([[[nan, nan],\n",
" [nan, nan],\n",
" [nan, nan],\n",
" [nan, nan],\n",
" [ 1., 21.]],\n",
"\n",
" [[nan, nan],\n",
" [nan, nan],\n",
" [nan, nan],\n",
" [ 1., 21.],\n",
" [ 2., 22.]],\n",
"\n",
" [[nan, nan],\n",
" [nan, nan],\n",
" [ 1., 21.],\n",
" [ 2., 22.],\n",
" [ 3., 23.]],\n",
"\n",
" [[nan, nan],\n",
" [ 1., 21.],\n",
" [ 2., 22.],\n",
" [ 3., 23.],\n",
" [ 4., 24.]],\n",
"\n",
" [[ 1., 21.],\n",
" [ 2., 22.],\n",
" [ 3., 23.],\n",
" [ 4., 24.],\n",
" [ 5., 25.]],\n",
"\n",
" [[ 2., 22.],\n",
" [ 3., 23.],\n",
" [ 4., 24.],\n",
" [ 5., 25.],\n",
" [ 6., 26.]],\n",
"\n",
" [[ 3., 23.],\n",
" [ 4., 24.],\n",
" [ 5., 25.],\n",
" [ 6., 26.],\n",
" [ 7., 27.]],\n",
"\n",
" [[ 4., 24.],\n",
" [ 5., 25.],\n",
" [ 6., 26.],\n",
" [ 7., 27.],\n",
" [ 8., 28.]],\n",
"\n",
" [[ 5., 25.],\n",
" [ 6., 26.],\n",
" [ 7., 27.],\n",
" [ 8., 28.],\n",
" [ 9., 29.]],\n",
"\n",
" [[ 6., 26.],\n",
" [ 7., 27.],\n",
" [ 8., 28.],\n",
" [ 9., 29.],\n",
" [10., 30.]],\n",
"\n",
" [[nan, nan],\n",
" [nan, nan],\n",
" [nan, nan],\n",
" [nan, nan],\n",
" [11., 31.]],\n",
"\n",
" [[nan, nan],\n",
" [nan, nan],\n",
" [nan, nan],\n",
" [11., 31.],\n",
" [12., 32.]],\n",
"\n",
" [[nan, nan],\n",
" [nan, nan],\n",
" [11., 31.],\n",
" [12., 32.],\n",
" [13., 33.]],\n",
"\n",
" [[nan, nan],\n",
" [11., 31.],\n",
" [12., 32.],\n",
" [13., 33.],\n",
" [14., 34.]],\n",
"\n",
" [[11., 31.],\n",
" [12., 32.],\n",
" [13., 33.],\n",
" [14., 34.],\n",
" [15., 35.]],\n",
"\n",
" [[12., 32.],\n",
" [13., 33.],\n",
" [14., 34.],\n",
" [15., 35.],\n",
" [16., 36.]],\n",
"\n",
" [[13., 33.],\n",
" [14., 34.],\n",
" [15., 35.],\n",
" [16., 36.],\n",
" [17., 37.]],\n",
"\n",
" [[14., 34.],\n",
" [15., 35.],\n",
" [16., 36.],\n",
" [17., 37.],\n",
" [18., 38.]],\n",
"\n",
" [[15., 35.],\n",
" [16., 36.],\n",
" [17., 37.],\n",
" [18., 38.],\n",
" [19., 39.]],\n",
"\n",
" [[16., 36.],\n",
" [17., 37.],\n",
" [18., 38.],\n",
" [19., 39.],\n",
" [20., 40.]]])"
]
},
"metadata": {
"tags": []
},
"execution_count": 68
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "fX0zmKDApWko"
},
"source": [
""
],
"execution_count": null,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment