Skip to content

Instantly share code, notes, and snippets.

@alonsosilvaallende

alonsosilvaallende/notebook.ipynb

Last active December 11, 2025 12:58
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alonsosilvaallende/a49a5f0e795d84ce2405fda01dc9f52f to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alonsosilvaallende/a49a5f0e795d84ce2405fda01dc9f52f to your computer and use it in GitHub Desktop.
Download ZIP
llguidance_issue
Display the source blob
Display the rendered blob
Raw
notebook.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "86ee02f5-c4d5-41f2-abf2-4bc455c0d414",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:31:51.046504Z",
"iopub.status.busy": "2025-12-11T12:31:51.046049Z",
"iopub.status.idle": "2025-12-11T12:31:51.054910Z",
"shell.execute_reply": "2025-12-11T12:31:51.053135Z",
"shell.execute_reply.started": "2025-12-11T12:31:51.046456Z"
}
},
"outputs": [],
"source": [
"def get_valid_tokens(interpreter):\n",
" \"\"\"Print valid tokens from the interpreter\"\"\"\n",
" a = interpreter.compute_mask()\n",
" valid_tokens = [i for i in range(len(a[0])) if a[0][i]!=0]\n",
" for valid_token in valid_tokens:\n",
" print(f\"\\x1b[33mtoken id: {valid_token},\\x1b[0m \\x1b[36mtoken: '{tokenizer.decode(valid_token)}'\\x1b[0m\")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ffcaa6e4-310c-4d6a-abb4-2cc5422aa454",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:31:51.056512Z",
"iopub.status.busy": "2025-12-11T12:31:51.056055Z",
"iopub.status.idle": "2025-12-11T12:31:55.972761Z",
"shell.execute_reply": "2025-12-11T12:31:55.970824Z",
"shell.execute_reply.started": "2025-12-11T12:31:51.056465Z"
}
},
"outputs": [],
"source": [
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
"\n",
"model_id = \"Qwen/Qwen2.5-0.5B-Instruct\"\n",
"tokenizer = AutoTokenizer.from_pretrained(model_id)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "65483836-d9a8-4278-84e8-2352274f9c35",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:31:55.974688Z",
"iopub.status.busy": "2025-12-11T12:31:55.973952Z",
"iopub.status.idle": "2025-12-11T12:31:55.996343Z",
"shell.execute_reply": "2025-12-11T12:31:55.994485Z",
"shell.execute_reply.started": "2025-12-11T12:31:55.974642Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"start: month\n",
"month: \"April\" | \"August\"\n",
"\n"
]
}
],
"source": [
"import llguidance\n",
"\n",
"grammar_def = \"\"\"\n",
"start: month\n",
"month: \"April\" | \"August\"\n",
"\"\"\"\n",
"\n",
"grammar = llguidance.LLMatcher.grammar_from_lark(grammar_def)\n",
"\n",
"print(grammar)"
]
},
{
"cell_type": "markdown",
"id": "287a7535-17ce-46fd-a11b-9f459db7d05a",
"metadata": {},
"source": [
"## Precomputing all the possible paths"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d0516457-77f8-4592-8db6-fa39c39a5192",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:31:55.997633Z",
"iopub.status.busy": "2025-12-11T12:31:55.997298Z",
"iopub.status.idle": "2025-12-11T12:31:56.812474Z",
"shell.execute_reply": "2025-12-11T12:31:56.811512Z",
"shell.execute_reply.started": "2025-12-11T12:31:55.997603Z"
}
},
"outputs": [
{
"data": {
"image/svg+xml": [
"<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1349pt\" height=\"628pt\" viewBox=\"0.00 0.00 1349.00 628.00\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 624)\">\n",
"<title>%3</title>\n",
"<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-624 1345,-624 1345,4 -4,4\"/>\n",
"<text text-anchor=\"middle\" x=\"126.5\" y=\"-604.8\" font-family=\"Times,serif\" font-size=\"14.00\">Allowed Paths</text>\n",
"<text text-anchor=\"middle\" x=\"126.5\" y=\"-589.8\" font-family=\"Times,serif\" font-size=\"14.00\">Regular expression: April|August</text>\n",
"<!-- 0 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>0</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"109\" cy=\"-206\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"109\" y=\"-202.3\" font-family=\"Times,serif\" font-size=\"14.00\">0</text>\n",
"</g>\n",
"<!-- 1 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>1</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"734.5\" cy=\"-453\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"734.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\">1</text>\n",
"</g>\n",
"<!-- 0&#45;&gt;1 -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>0-&gt;1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M111.52,-224.1C116.95,-275.39 140.39,-421.35 234,-474 400.06,-567.4 478.94,-550.81 662,-498 680.5,-492.66 698.82,-480.97 712.29,-470.84\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"714.68,-473.42 720.39,-464.49 710.36,-467.91 714.68,-473.42\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"350,-561 350,-582 402,-582 402,-561 350,-561\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"350,-561 350,-582 402,-582 402,-561 350,-561\"/>\n",
"<text text-anchor=\"start\" x=\"369\" y=\"-567.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"402,-561 402,-582 448,-582 448,-561 402,-561\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"402,-561 402,-582 448,-582 448,-561 402,-561\"/>\n",
"<text text-anchor=\"start\" x=\"405\" y=\"-567.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"350,-540 350,-561 402,-561 402,-540 350,-540\"/>\n",
"<text text-anchor=\"start\" x=\"353\" y=\"-546.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">21872</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"402,-540 402,-561 448,-561 448,-540 402,-540\"/>\n",
"<text text-anchor=\"start\" x=\"411\" y=\"-546.8\" font-family=\"Times,serif\" font-size=\"14.00\">Aug</text>\n",
"</g>\n",
"<!-- 4 -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>4</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"506\" cy=\"-134\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"506\" y=\"-130.3\" font-family=\"Times,serif\" font-size=\"14.00\">4</text>\n",
"</g>\n",
"<!-- 0&#45;&gt;4 -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>0-&gt;4</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M126.3,-200.11C164.67,-186.56 263.95,-153.32 350,-140 394.28,-133.15 446.48,-132.67 477.59,-133.17\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"477.6,-136.67 487.67,-133.38 477.75,-129.67 477.6,-136.67\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"234,-187 234,-208 286,-208 286,-187 234,-187\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"234,-187 234,-208 286,-208 286,-187 234,-187\"/>\n",
"<text text-anchor=\"start\" x=\"253\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"286,-187 286,-208 332,-208 332,-187 286,-187\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"286,-187 286,-208 332,-208 332,-187 286,-187\"/>\n",
"<text text-anchor=\"start\" x=\"289\" y=\"-193.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"234,-166 234,-187 286,-187 286,-166 234,-166\"/>\n",
"<text text-anchor=\"start\" x=\"237\" y=\"-172.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">10611</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"286,-166 286,-187 332,-187 332,-166 286,-166\"/>\n",
"<text text-anchor=\"start\" x=\"299.5\" y=\"-172.8\" font-family=\"Times,serif\" font-size=\"14.00\">Ap</text>\n",
"</g>\n",
"<!-- 5 -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>5</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"734.5\" cy=\"-137\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"734.5\" y=\"-133.3\" font-family=\"Times,serif\" font-size=\"14.00\">5</text>\n",
"</g>\n",
"<!-- 0&#45;&gt;5 -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>0-&gt;5</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M120.39,-191.86C126.77,-183.98 135.52,-174.48 145,-168 225.76,-112.84 253.88,-106.09 350,-88 483.44,-62.89 645.46,-108.06 707.57,-128.12\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"706.73,-131.53 717.32,-131.33 708.91,-124.88 706.73,-131.53\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"350,-109 350,-130 402,-130 402,-109 350,-109\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"350,-109 350,-130 402,-130 402,-109 350,-109\"/>\n",
"<text text-anchor=\"start\" x=\"369\" y=\"-115.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"402,-109 402,-130 448,-130 448,-109 402,-109\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"402,-109 402,-130 448,-130 448,-109 402,-109\"/>\n",
"<text text-anchor=\"start\" x=\"405\" y=\"-115.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"350,-88 350,-109 402,-109 402,-88 350,-88\"/>\n",
"<text text-anchor=\"start\" x=\"353\" y=\"-94.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">20722</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"402,-88 402,-109 448,-109 448,-88 402,-88\"/>\n",
"<text text-anchor=\"start\" x=\"412\" y=\"-94.8\" font-family=\"Times,serif\" font-size=\"14.00\">Apr</text>\n",
"</g>\n",
"<!-- 7 -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>7</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"506\" cy=\"-450\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"506\" y=\"-446.3\" font-family=\"Times,serif\" font-size=\"14.00\">7</text>\n",
"</g>\n",
"<!-- 0&#45;&gt;7 -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>0-&gt;7</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M113.26,-223.72C117.65,-243.73 127.09,-276.43 145,-299 213.12,-384.84 246.67,-399.5 350,-436 392.62,-451.06 445.72,-452.41 477.41,-451.55\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"477.81,-455.03 487.67,-451.17 477.55,-448.04 477.81,-455.03\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"234,-449 234,-470 286,-470 286,-449 234,-449\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"234,-449 234,-470 286,-470 286,-449 234,-449\"/>\n",
"<text text-anchor=\"start\" x=\"253\" y=\"-455.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"286,-449 286,-470 332,-470 332,-449 286,-449\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"286,-449 286,-470 332,-470 332,-449 286,-449\"/>\n",
"<text text-anchor=\"start\" x=\"289\" y=\"-455.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"234,-428 234,-449 286,-449 286,-428 234,-428\"/>\n",
"<text text-anchor=\"start\" x=\"237\" y=\"-434.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">65332</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"286,-428 286,-449 332,-449 332,-428 286,-428\"/>\n",
"<text text-anchor=\"start\" x=\"299.5\" y=\"-434.8\" font-family=\"Times,serif\" font-size=\"14.00\">Au</text>\n",
"</g>\n",
"<!-- 8 -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>8</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"283\" cy=\"-286\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"283\" y=\"-282.3\" font-family=\"Times,serif\" font-size=\"14.00\">8</text>\n",
"</g>\n",
"<!-- 0&#45;&gt;8 -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>0-&gt;8</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M125.57,-213.24C155.35,-227.09 220.63,-257.46 256.74,-274.25\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"255.55,-277.56 266.09,-278.6 258.5,-271.21 255.55,-277.56\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"145.5,-274 145.5,-295 170.5,-295 170.5,-274 145.5,-274\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"145.5,-274 145.5,-295 170.5,-295 170.5,-274 145.5,-274\"/>\n",
"<text text-anchor=\"start\" x=\"151\" y=\"-280.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"170.5,-274 170.5,-295 216.5,-295 216.5,-274 170.5,-274\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"170.5,-274 170.5,-295 216.5,-295 216.5,-274 170.5,-274\"/>\n",
"<text text-anchor=\"start\" x=\"173.5\" y=\"-280.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"145.5,-253 145.5,-274 170.5,-274 170.5,-253 145.5,-253\"/>\n",
"<text text-anchor=\"start\" x=\"148.5\" y=\"-259.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">32</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"170.5,-253 170.5,-274 216.5,-274 216.5,-253 170.5,-253\"/>\n",
"<text text-anchor=\"start\" x=\"188.5\" y=\"-259.8\" font-family=\"Times,serif\" font-size=\"14.00\">A</text>\n",
"</g>\n",
"<!-- 9 -->\n",
"<g id=\"node10\" class=\"node\">\n",
"<title>9</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-222\" rx=\"18\" ry=\"18\"/>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"1319\" cy=\"-222\" rx=\"22\" ry=\"22\"/>\n",
"<text text-anchor=\"middle\" x=\"1319\" y=\"-218.3\" font-family=\"Times,serif\" font-size=\"14.00\">9</text>\n",
"</g>\n",
"<!-- 0&#45;&gt;9 -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>0-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M115.07,-188.79C130.84,-139.28 183.93,0 282,0 282,0 282,0 1151,0 1224.3,0 1237.38,-43.66 1279,-104 1297.01,-130.11 1307.2,-165.14 1312.6,-190.24\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1309.23,-191.22 1314.64,-200.33 1316.09,-189.84 1309.23,-191.22\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"680.5,-41.5 680.5,-62.5 732.5,-62.5 732.5,-41.5 680.5,-41.5\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"680.5,-41.5 680.5,-62.5 732.5,-62.5 732.5,-41.5 680.5,-41.5\"/>\n",
"<text text-anchor=\"start\" x=\"699.5\" y=\"-48.3\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"732.5,-41.5 732.5,-62.5 789.5,-62.5 789.5,-41.5 732.5,-41.5\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"732.5,-41.5 732.5,-62.5 789.5,-62.5 789.5,-41.5 732.5,-41.5\"/>\n",
"<text text-anchor=\"start\" x=\"741\" y=\"-48.3\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"680.5,-20.5 680.5,-41.5 732.5,-41.5 732.5,-20.5 680.5,-20.5\"/>\n",
"<text text-anchor=\"start\" x=\"683.5\" y=\"-27.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">31459</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"732.5,-20.5 732.5,-41.5 789.5,-41.5 789.5,-20.5 732.5,-20.5\"/>\n",
"<text text-anchor=\"start\" x=\"735.5\" y=\"-27.3\" font-family=\"Times,serif\" font-size=\"14.00\">August</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"680.5,0.5 680.5,-20.5 732.5,-20.5 732.5,0.5 680.5,0.5\"/>\n",
"<text text-anchor=\"start\" x=\"683.5\" y=\"-6.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">28046</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"732.5,0.5 732.5,-20.5 789.5,-20.5 789.5,0.5 732.5,0.5\"/>\n",
"<text text-anchor=\"start\" x=\"743.5\" y=\"-6.3\" font-family=\"Times,serif\" font-size=\"14.00\">April</text>\n",
"</g>\n",
"<!-- 2 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>2</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"1150\" cy=\"-378\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"1150\" y=\"-374.3\" font-family=\"Times,serif\" font-size=\"14.00\">2</text>\n",
"</g>\n",
"<!-- 1&#45;&gt;2 -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>1-&gt;2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M752.59,-454.64C805.3,-459.06 969.19,-467.78 1092,-420 1105.56,-414.72 1118.76,-405.35 1129.04,-396.79\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1131.37,-399.41 1136.61,-390.2 1126.77,-394.13 1131.37,-399.41\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"905,-477 905,-498 939,-498 939,-477 905,-477\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"905,-477 905,-498 939,-498 939,-477 905,-477\"/>\n",
"<text text-anchor=\"start\" x=\"915\" y=\"-483.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"939,-477 939,-498 985,-498 985,-477 939,-477\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"939,-477 939,-498 985,-498 985,-477 939,-477\"/>\n",
"<text text-anchor=\"start\" x=\"942\" y=\"-483.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"905,-456 905,-477 939,-477 939,-456 905,-456\"/>\n",
"<text text-anchor=\"start\" x=\"908\" y=\"-462.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">355</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"939,-456 939,-477 985,-477 985,-456 939,-456\"/>\n",
"<text text-anchor=\"start\" x=\"953.5\" y=\"-462.8\" font-family=\"Times,serif\" font-size=\"14.00\">us</text>\n",
"</g>\n",
"<!-- 3 -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>3</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"945\" cy=\"-367\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"945\" y=\"-363.3\" font-family=\"Times,serif\" font-size=\"14.00\">3</text>\n",
"</g>\n",
"<!-- 1&#45;&gt;3 -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>1-&gt;3</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M750.73,-445.01C761.83,-438.86 776.98,-429.77 789,-420 798.03,-412.66 796.95,-406.86 807,-401 825.51,-390.21 882.68,-378.29 917.04,-371.82\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"917.71,-375.26 926.9,-370 916.43,-368.38 917.71,-375.26\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"807.5,-422 807.5,-443 832.5,-443 832.5,-422 807.5,-422\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"807.5,-422 807.5,-443 832.5,-443 832.5,-422 807.5,-422\"/>\n",
"<text text-anchor=\"start\" x=\"813\" y=\"-428.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"832.5,-422 832.5,-443 878.5,-443 878.5,-422 832.5,-422\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"832.5,-422 832.5,-443 878.5,-443 878.5,-422 832.5,-422\"/>\n",
"<text text-anchor=\"start\" x=\"835.5\" y=\"-428.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"807.5,-401 807.5,-422 832.5,-422 832.5,-401 807.5,-401\"/>\n",
"<text text-anchor=\"start\" x=\"810.5\" y=\"-407.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">84</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"832.5,-401 832.5,-422 878.5,-422 878.5,-401 832.5,-401\"/>\n",
"<text text-anchor=\"start\" x=\"850.5\" y=\"-407.8\" font-family=\"Times,serif\" font-size=\"14.00\">u</text>\n",
"</g>\n",
"<!-- 1&#45;&gt;9 -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>1-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M751.54,-459.64C780.01,-471.04 841.54,-493.87 896,-502 1069.67,-527.92 1162.38,-536.28 1279,-405 1298.57,-382.98 1309.98,-300.3 1314.98,-253.97\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1318.48,-254.16 1316.03,-243.85 1311.51,-253.44 1318.48,-254.16\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1012,-538 1012,-559 1046,-559 1046,-538 1012,-538\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1012,-538 1012,-559 1046,-559 1046,-538 1012,-538\"/>\n",
"<text text-anchor=\"start\" x=\"1022\" y=\"-544.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1046,-538 1046,-559 1092,-559 1092,-538 1046,-538\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1046,-538 1046,-559 1092,-559 1092,-538 1046,-538\"/>\n",
"<text text-anchor=\"start\" x=\"1049\" y=\"-544.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1012,-517 1012,-538 1046,-538 1046,-517 1012,-517\"/>\n",
"<text text-anchor=\"start\" x=\"1015\" y=\"-523.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">590</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1046,-517 1046,-538 1092,-538 1092,-517 1046,-517\"/>\n",
"<text text-anchor=\"start\" x=\"1057.5\" y=\"-523.8\" font-family=\"Times,serif\" font-size=\"14.00\">ust</text>\n",
"</g>\n",
"<!-- 2&#45;&gt;9 -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>2-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1167.81,-373.69C1194.7,-366.01 1247.53,-347.5 1279,-314 1294.98,-296.99 1304.93,-272.44 1310.78,-252.99\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1314.16,-253.91 1313.48,-243.34 1307.42,-252.03 1314.16,-253.91\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1208.5,-380 1208.5,-401 1233.5,-401 1233.5,-380 1208.5,-380\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1208.5,-380 1208.5,-401 1233.5,-401 1233.5,-380 1208.5,-380\"/>\n",
"<text text-anchor=\"start\" x=\"1214\" y=\"-386.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1233.5,-380 1233.5,-401 1279.5,-401 1279.5,-380 1233.5,-380\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1233.5,-380 1233.5,-401 1279.5,-401 1279.5,-380 1233.5,-380\"/>\n",
"<text text-anchor=\"start\" x=\"1236.5\" y=\"-386.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1208.5,-359 1208.5,-380 1233.5,-380 1233.5,-359 1208.5,-359\"/>\n",
"<text text-anchor=\"start\" x=\"1211.5\" y=\"-365.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">83</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1233.5,-359 1233.5,-380 1279.5,-380 1279.5,-359 1233.5,-359\"/>\n",
"<text text-anchor=\"start\" x=\"1253\" y=\"-365.8\" font-family=\"Times,serif\" font-size=\"14.00\">t</text>\n",
"</g>\n",
"<!-- 3&#45;&gt;2 -->\n",
"<g id=\"edge12\" class=\"edge\">\n",
"<title>3-&gt;2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M963.31,-367.94C998.68,-369.85 1079.47,-374.23 1121.88,-376.53\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1121.85,-380.03 1132.02,-377.08 1122.23,-373.04 1121.85,-380.03\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1017,-395 1017,-416 1042,-416 1042,-395 1017,-395\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1017,-395 1017,-416 1042,-416 1042,-395 1017,-395\"/>\n",
"<text text-anchor=\"start\" x=\"1022.5\" y=\"-401.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1042,-395 1042,-416 1088,-416 1088,-395 1042,-395\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1042,-395 1042,-416 1088,-416 1088,-395 1042,-395\"/>\n",
"<text text-anchor=\"start\" x=\"1045\" y=\"-401.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1017,-374 1017,-395 1042,-395 1042,-374 1017,-374\"/>\n",
"<text text-anchor=\"start\" x=\"1020\" y=\"-380.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">82</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1042,-374 1042,-395 1088,-395 1088,-374 1042,-374\"/>\n",
"<text text-anchor=\"start\" x=\"1061\" y=\"-380.8\" font-family=\"Times,serif\" font-size=\"14.00\">s</text>\n",
"</g>\n",
"<!-- 3&#45;&gt;9 -->\n",
"<g id=\"edge13\" class=\"edge\">\n",
"<title>3-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M961.98,-360.77C1019.1,-338.51 1213.65,-262.68 1288.4,-233.54\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1290.09,-236.64 1298.13,-229.74 1287.54,-230.12 1290.09,-236.64\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1110,-322 1110,-343 1144,-343 1144,-322 1110,-322\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1110,-322 1110,-343 1144,-343 1144,-322 1110,-322\"/>\n",
"<text text-anchor=\"start\" x=\"1120\" y=\"-328.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1144,-322 1144,-343 1190,-343 1190,-322 1144,-322\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1144,-322 1144,-343 1190,-343 1190,-322 1144,-322\"/>\n",
"<text text-anchor=\"start\" x=\"1147\" y=\"-328.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1110,-301 1110,-322 1144,-322 1144,-301 1110,-301\"/>\n",
"<text text-anchor=\"start\" x=\"1113\" y=\"-307.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">267</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1144,-301 1144,-322 1190,-322 1190,-301 1144,-301\"/>\n",
"<text text-anchor=\"start\" x=\"1160\" y=\"-307.8\" font-family=\"Times,serif\" font-size=\"14.00\">st</text>\n",
"</g>\n",
"<!-- 4&#45;&gt;5 -->\n",
"<g id=\"edge14\" class=\"edge\">\n",
"<title>4-&gt;5</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M524.2,-133.99C552.81,-134 611.95,-134.14 662,-135 676.55,-135.25 692.76,-135.69 706.07,-136.09\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"706.35,-139.61 716.46,-136.42 706.57,-132.61 706.35,-139.61\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"578,-156 578,-177 603,-177 603,-156 578,-156\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"578,-156 578,-177 603,-177 603,-156 578,-156\"/>\n",
"<text text-anchor=\"start\" x=\"583.5\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"603,-156 603,-177 649,-177 649,-156 603,-156\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"603,-156 603,-177 649,-177 649,-156 603,-156\"/>\n",
"<text text-anchor=\"start\" x=\"606\" y=\"-162.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"578,-135 578,-156 603,-156 603,-135 578,-135\"/>\n",
"<text text-anchor=\"start\" x=\"581\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">81</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"603,-135 603,-156 649,-156 649,-135 603,-135\"/>\n",
"<text text-anchor=\"start\" x=\"622.5\" y=\"-141.8\" font-family=\"Times,serif\" font-size=\"14.00\">r</text>\n",
"</g>\n",
"<!-- 6 -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>6</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" cx=\"945\" cy=\"-224\" rx=\"18\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"945\" y=\"-220.3\" font-family=\"Times,serif\" font-size=\"14.00\">6</text>\n",
"</g>\n",
"<!-- 4&#45;&gt;6 -->\n",
"<g id=\"edge15\" class=\"edge\">\n",
"<title>4-&gt;6</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M519.36,-146.42C530.16,-156.45 546.77,-170.02 564,-177 657.39,-214.85 690.72,-182.74 789,-205 797.22,-206.86 798.75,-209.26 807,-211 844.56,-218.93 888.94,-222.03 916.78,-223.24\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"916.81,-226.74 926.94,-223.62 917.08,-219.74 916.81,-226.74\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"694.5,-226 694.5,-247 728.5,-247 728.5,-226 694.5,-226\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"694.5,-226 694.5,-247 728.5,-247 728.5,-226 694.5,-226\"/>\n",
"<text text-anchor=\"start\" x=\"704.5\" y=\"-232.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"728.5,-226 728.5,-247 774.5,-247 774.5,-226 728.5,-226\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"728.5,-226 728.5,-247 774.5,-247 774.5,-226 728.5,-226\"/>\n",
"<text text-anchor=\"start\" x=\"731.5\" y=\"-232.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"694.5,-205 694.5,-226 728.5,-226 728.5,-205 694.5,-205\"/>\n",
"<text text-anchor=\"start\" x=\"697.5\" y=\"-211.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">461</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"728.5,-205 728.5,-226 774.5,-226 774.5,-205 728.5,-205\"/>\n",
"<text text-anchor=\"start\" x=\"745.5\" y=\"-211.8\" font-family=\"Times,serif\" font-size=\"14.00\">ri</text>\n",
"</g>\n",
"<!-- 4&#45;&gt;9 -->\n",
"<g id=\"edge16\" class=\"edge\">\n",
"<title>4-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M523.28,-128.73C560.23,-117.25 653.45,-91 733.5,-91 733.5,-91 733.5,-91 1151,-91 1217.69,-91 1273.56,-156.48 1300.57,-194.85\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1297.96,-197.24 1306.5,-203.51 1303.73,-193.28 1297.96,-197.24\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"896,-112 896,-133 948,-133 948,-112 896,-112\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"896,-112 896,-133 948,-133 948,-112 896,-112\"/>\n",
"<text text-anchor=\"start\" x=\"915\" y=\"-118.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"948,-112 948,-133 994,-133 994,-112 948,-112\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"948,-112 948,-133 994,-133 994,-112 948,-112\"/>\n",
"<text text-anchor=\"start\" x=\"951\" y=\"-118.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"896,-91 896,-112 948,-112 948,-91 896,-91\"/>\n",
"<text text-anchor=\"start\" x=\"899\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">30560</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"948,-91 948,-112 994,-112 994,-91 948,-91\"/>\n",
"<text text-anchor=\"start\" x=\"963\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">ril</text>\n",
"</g>\n",
"<!-- 5&#45;&gt;6 -->\n",
"<g id=\"edge17\" class=\"edge\">\n",
"<title>5-&gt;6</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M752.59,-138.23C780.24,-140.79 835.92,-148.27 878,-169 895.56,-177.65 912.46,-191.89 924.77,-203.71\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"922.59,-206.48 932.15,-211.05 927.52,-201.52 922.59,-206.48\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"807.5,-190 807.5,-211 832.5,-211 832.5,-190 807.5,-190\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"807.5,-190 807.5,-211 832.5,-211 832.5,-190 807.5,-190\"/>\n",
"<text text-anchor=\"start\" x=\"813\" y=\"-196.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"832.5,-190 832.5,-211 878.5,-211 878.5,-190 832.5,-190\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"832.5,-190 832.5,-211 878.5,-211 878.5,-190 832.5,-190\"/>\n",
"<text text-anchor=\"start\" x=\"835.5\" y=\"-196.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"807.5,-169 807.5,-190 832.5,-190 832.5,-169 807.5,-169\"/>\n",
"<text text-anchor=\"start\" x=\"810.5\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">72</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"832.5,-169 832.5,-190 878.5,-190 878.5,-169 832.5,-169\"/>\n",
"<text text-anchor=\"start\" x=\"853\" y=\"-175.8\" font-family=\"Times,serif\" font-size=\"14.00\">i</text>\n",
"</g>\n",
"<!-- 5&#45;&gt;9 -->\n",
"<g id=\"edge18\" class=\"edge\">\n",
"<title>5-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M752.63,-138.19C779.42,-140.1 832.83,-144.15 878,-149 973.37,-159.25 997.35,-161.4 1092,-177 1162.2,-188.57 1243.93,-205.78 1287.29,-215.21\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1286.69,-218.66 1297.21,-217.38 1288.19,-211.82 1286.69,-218.66\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1012,-198 1012,-219 1046,-219 1046,-198 1012,-198\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1012,-198 1012,-219 1046,-219 1046,-198 1012,-198\"/>\n",
"<text text-anchor=\"start\" x=\"1022\" y=\"-204.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1046,-198 1046,-219 1092,-219 1092,-198 1046,-198\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1046,-198 1046,-219 1092,-219 1092,-198 1046,-198\"/>\n",
"<text text-anchor=\"start\" x=\"1049\" y=\"-204.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1012,-177 1012,-198 1046,-198 1046,-177 1012,-177\"/>\n",
"<text text-anchor=\"start\" x=\"1015\" y=\"-183.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">321</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1046,-177 1046,-198 1092,-198 1092,-177 1046,-177\"/>\n",
"<text text-anchor=\"start\" x=\"1064\" y=\"-183.8\" font-family=\"Times,serif\" font-size=\"14.00\">il</text>\n",
"</g>\n",
"<!-- 6&#45;&gt;9 -->\n",
"<g id=\"edge19\" class=\"edge\">\n",
"<title>6-&gt;9</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M963.44,-223.91C1021.9,-223.59 1211.45,-222.57 1286.75,-222.17\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1287.02,-225.67 1297,-222.11 1286.98,-218.67 1287.02,-225.67\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1115,-244 1115,-265 1140,-265 1140,-244 1115,-244\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1115,-244 1115,-265 1140,-265 1140,-244 1115,-244\"/>\n",
"<text text-anchor=\"start\" x=\"1120.5\" y=\"-250.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"1140,-244 1140,-265 1186,-265 1186,-244 1140,-244\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1140,-244 1140,-265 1186,-265 1186,-244 1140,-244\"/>\n",
"<text text-anchor=\"start\" x=\"1143\" y=\"-250.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1115,-223 1115,-244 1140,-244 1140,-223 1115,-223\"/>\n",
"<text text-anchor=\"start\" x=\"1118\" y=\"-229.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">75</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"1140,-223 1140,-244 1186,-244 1186,-223 1140,-223\"/>\n",
"<text text-anchor=\"start\" x=\"1160.5\" y=\"-229.8\" font-family=\"Times,serif\" font-size=\"14.00\">l</text>\n",
"</g>\n",
"<!-- 7&#45;&gt;1 -->\n",
"<g id=\"edge20\" class=\"edge\">\n",
"<title>7-&gt;1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M524.02,-450.23C562.97,-450.74 658.82,-452.01 706.15,-452.64\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"706.23,-456.14 716.28,-452.77 706.33,-449.14 706.23,-456.14\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"578,-473 578,-494 603,-494 603,-473 578,-473\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"578,-473 578,-494 603,-494 603,-473 578,-473\"/>\n",
"<text text-anchor=\"start\" x=\"583.5\" y=\"-479.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"603,-473 603,-494 649,-494 649,-473 603,-473\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"603,-473 603,-494 649,-494 649,-473 603,-473\"/>\n",
"<text text-anchor=\"start\" x=\"606\" y=\"-479.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"578,-452 578,-473 603,-473 603,-452 578,-452\"/>\n",
"<text text-anchor=\"start\" x=\"581\" y=\"-458.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">70</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"603,-452 603,-473 649,-473 649,-452 603,-452\"/>\n",
"<text text-anchor=\"start\" x=\"621\" y=\"-458.8\" font-family=\"Times,serif\" font-size=\"14.00\">g</text>\n",
"</g>\n",
"<!-- 7&#45;&gt;3 -->\n",
"<g id=\"edge21\" class=\"edge\">\n",
"<title>7-&gt;3</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M522.72,-442.15C552.53,-427.69 619.69,-397.13 680,-384 764.72,-365.55 868.07,-365.07 916.49,-366.09\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"916.68,-369.59 926.77,-366.34 916.86,-362.59 916.68,-369.59\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"690.5,-405 690.5,-426 733.5,-426 733.5,-405 690.5,-405\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"690.5,-405 690.5,-426 733.5,-426 733.5,-405 690.5,-405\"/>\n",
"<text text-anchor=\"start\" x=\"705\" y=\"-411.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"733.5,-405 733.5,-426 779.5,-426 779.5,-405 733.5,-405\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"733.5,-405 733.5,-426 779.5,-426 779.5,-405 733.5,-405\"/>\n",
"<text text-anchor=\"start\" x=\"736.5\" y=\"-411.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"690.5,-384 690.5,-405 733.5,-405 733.5,-384 690.5,-384\"/>\n",
"<text text-anchor=\"start\" x=\"693.5\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">8717</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"733.5,-384 733.5,-405 779.5,-405 779.5,-384 733.5,-384\"/>\n",
"<text text-anchor=\"start\" x=\"747\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\">gu</text>\n",
"</g>\n",
"<!-- 8&#45;&gt;1 -->\n",
"<g id=\"edge22\" class=\"edge\">\n",
"<title>8-&gt;1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M295.87,-298.78C308.19,-311.24 328.63,-329.78 350,-340 430.29,-378.39 460.63,-355.86 546,-381 577.65,-390.32 664.98,-425.21 708.37,-442.77\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"707.07,-446.02 717.66,-446.54 709.71,-439.54 707.07,-446.02\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"466,-402 466,-423 500,-423 500,-402 466,-402\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"466,-402 466,-423 500,-423 500,-402 466,-402\"/>\n",
"<text text-anchor=\"start\" x=\"476\" y=\"-408.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"500,-402 500,-423 546,-423 546,-402 500,-402\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"500,-402 500,-423 546,-423 546,-402 500,-402\"/>\n",
"<text text-anchor=\"start\" x=\"503\" y=\"-408.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"466,-381 466,-402 500,-402 500,-381 466,-381\"/>\n",
"<text text-anchor=\"start\" x=\"469\" y=\"-387.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">768</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"500,-381 500,-402 546,-402 546,-381 500,-381\"/>\n",
"<text text-anchor=\"start\" x=\"513.5\" y=\"-387.8\" font-family=\"Times,serif\" font-size=\"14.00\">ug</text>\n",
"</g>\n",
"<!-- 8&#45;&gt;3 -->\n",
"<g id=\"edge23\" class=\"edge\">\n",
"<title>8-&gt;3</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M300.4,-290.93C313.66,-294.75 332.89,-299.89 350,-303 406.36,-313.24 807.78,-353.47 916.76,-364.3\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"916.63,-367.8 926.93,-365.31 917.32,-360.84 916.63,-367.8\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"564,-359 564,-380 616,-380 616,-359 564,-359\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"564,-359 564,-380 616,-380 616,-359 564,-359\"/>\n",
"<text text-anchor=\"start\" x=\"583\" y=\"-365.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"616,-359 616,-380 662,-380 662,-359 616,-359\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"616,-359 616,-380 662,-380 662,-359 616,-359\"/>\n",
"<text text-anchor=\"start\" x=\"619\" y=\"-365.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"564,-338 564,-359 616,-359 616,-338 564,-338\"/>\n",
"<text text-anchor=\"start\" x=\"567\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">29785</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"616,-338 616,-359 662,-359 662,-338 616,-338\"/>\n",
"<text text-anchor=\"start\" x=\"625\" y=\"-344.8\" font-family=\"Times,serif\" font-size=\"14.00\">ugu</text>\n",
"</g>\n",
"<!-- 8&#45;&gt;4 -->\n",
"<g id=\"edge24\" class=\"edge\">\n",
"<title>8-&gt;4</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M287.88,-268.49C295.02,-240.67 313.01,-186.81 350,-160 369.94,-145.54 438.84,-138.55 477.73,-135.68\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"478.1,-139.16 487.83,-134.98 477.62,-132.18 478.1,-139.16\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"364,-181 364,-202 389,-202 389,-181 364,-181\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"364,-181 364,-202 389,-202 389,-181 364,-181\"/>\n",
"<text text-anchor=\"start\" x=\"369.5\" y=\"-187.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"389,-181 389,-202 435,-202 435,-181 389,-181\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"389,-181 389,-202 435,-202 435,-181 389,-181\"/>\n",
"<text text-anchor=\"start\" x=\"392\" y=\"-187.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"364,-160 364,-181 389,-181 389,-160 364,-160\"/>\n",
"<text text-anchor=\"start\" x=\"367\" y=\"-166.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">79</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"389,-160 389,-181 435,-181 435,-160 389,-160\"/>\n",
"<text text-anchor=\"start\" x=\"407\" y=\"-166.8\" font-family=\"Times,serif\" font-size=\"14.00\">p</text>\n",
"</g>\n",
"<!-- 8&#45;&gt;5 -->\n",
"<g id=\"edge25\" class=\"edge\">\n",
"<title>8-&gt;5</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M300.47,-280.61C332.3,-270.26 404.54,-247.13 466,-230 552.5,-205.9 579.24,-215.83 662,-181 679.52,-173.63 697.75,-162.34 711.42,-153.05\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"713.48,-155.88 719.68,-147.29 709.48,-150.14 713.48,-155.88\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"466,-251 466,-272 500,-272 500,-251 466,-251\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"466,-251 466,-272 500,-272 500,-251 466,-251\"/>\n",
"<text text-anchor=\"start\" x=\"476\" y=\"-257.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"500,-251 500,-272 546,-272 546,-251 500,-251\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"500,-251 500,-272 546,-272 546,-251 500,-251\"/>\n",
"<text text-anchor=\"start\" x=\"503\" y=\"-257.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"466,-230 466,-251 500,-251 500,-230 466,-230\"/>\n",
"<text text-anchor=\"start\" x=\"469\" y=\"-236.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">649</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"500,-230 500,-251 546,-251 546,-230 500,-230\"/>\n",
"<text text-anchor=\"start\" x=\"515\" y=\"-236.8\" font-family=\"Times,serif\" font-size=\"14.00\">pr</text>\n",
"</g>\n",
"<!-- 8&#45;&gt;6 -->\n",
"<g id=\"edge26\" class=\"edge\">\n",
"<title>8-&gt;6</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M300.99,-287.68C343.35,-291.33 456.7,-297.7 546,-272 554.79,-269.47 555.22,-264.59 564,-262 660.13,-233.65 689.35,-257.7 789,-247 834,-242.17 885.98,-233.9 916.86,-228.71\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"917.58,-232.13 926.86,-227.01 916.41,-225.23 917.58,-232.13\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"564,-283 564,-304 616,-304 616,-283 564,-283\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"564,-283 564,-304 616,-304 616,-283 564,-283\"/>\n",
"<text text-anchor=\"start\" x=\"583\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"616,-283 616,-304 662,-304 662,-283 616,-283\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"616,-283 616,-304 662,-304 662,-283 616,-283\"/>\n",
"<text text-anchor=\"start\" x=\"619\" y=\"-289.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"564,-262 564,-283 616,-283 616,-262 564,-262\"/>\n",
"<text text-anchor=\"start\" x=\"567\" y=\"-268.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">68608</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"616,-262 616,-283 662,-283 662,-262 616,-262\"/>\n",
"<text text-anchor=\"start\" x=\"628.5\" y=\"-268.8\" font-family=\"Times,serif\" font-size=\"14.00\">pri</text>\n",
"</g>\n",
"<!-- 8&#45;&gt;7 -->\n",
"<g id=\"edge27\" class=\"edge\">\n",
"<title>8-&gt;7</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M291.92,-301.84C302.46,-321.24 322.95,-353.64 350,-371 387.65,-395.16 410.55,-369.52 448,-394 459.16,-401.29 456.78,-408.37 466,-418 471.13,-423.36 477.13,-428.67 482.86,-433.38\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"480.94,-436.32 490.95,-439.79 485.29,-430.84 480.94,-436.32\"/>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"364,-415 364,-436 389,-436 389,-415 364,-415\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"364,-415 364,-436 389,-436 389,-415 364,-415\"/>\n",
"<text text-anchor=\"start\" x=\"369.5\" y=\"-421.8\" font-family=\"Times,serif\" font-size=\"14.00\">id</text>\n",
"<polygon fill=\"#ffebcd\" stroke=\"transparent\" points=\"389,-415 389,-436 435,-436 435,-415 389,-415\"/>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"389,-415 389,-436 435,-436 435,-415 389,-415\"/>\n",
"<text text-anchor=\"start\" x=\"392\" y=\"-421.8\" font-family=\"Times,serif\" font-size=\"14.00\">token</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"364,-394 364,-415 389,-415 389,-394 364,-394\"/>\n",
"<text text-anchor=\"start\" x=\"367\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#00b4d8\">84</text>\n",
"<polygon fill=\"none\" stroke=\"black\" points=\"389,-394 389,-415 435,-415 435,-394 389,-394\"/>\n",
"<text text-anchor=\"start\" x=\"407\" y=\"-400.8\" font-family=\"Times,serif\" font-size=\"14.00\">u</text>\n",
"</g>\n",
"<g id=\"node11\" class=\"node\">\n",
"<title/>\n",
"</g>\n",
"<!-- &#45;&gt;0 -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>-&gt;0</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M54.19,-206C62.65,-206 72.05,-206 80.6,-206\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"80.83,-209.5 90.83,-206 80.83,-202.5 80.83,-209.5\"/>\n",
"</g>\n",
"</g>\n",
"</svg>"
],
"text/plain": [
"<IPython.core.display.SVG object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from litelines.transformers import SchemaProcessor\n",
"\n",
"processor = SchemaProcessor(response_format=r\"April|August\", tokenizer=tokenizer)\n",
"processor.show_graph()"
]
},
{
"cell_type": "markdown",
"id": "99202a73-1998-4895-bcd9-9178694616db",
"metadata": {},
"source": [
"## Let's go to state 4\n",
"\n",
"I expect to see the following valid tokens at state $4$:\n",
"- token id: $81$, token: 'r'\n",
"- token id: $461$, token: 'ri'\n",
"- token id: $30560$, token: 'ril' "
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "30015928-6b30-4391-8d8f-9cc3eccd5554",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:31:56.813141Z",
"iopub.status.busy": "2025-12-11T12:31:56.812967Z",
"iopub.status.idle": "2025-12-11T12:31:58.464449Z",
"shell.execute_reply": "2025-12-11T12:31:58.463108Z",
"shell.execute_reply.started": "2025-12-11T12:31:56.813125Z"
}
},
"outputs": [],
"source": [
"import llguidance.hf\n",
"\n",
"ll_tok = llguidance.hf.from_tokenizer(tokenizer)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "b2bd019d-da93-4ae0-a0fd-bc6dfce22dfe",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:32:02.439144Z",
"iopub.status.busy": "2025-12-11T12:32:02.438597Z",
"iopub.status.idle": "2025-12-11T12:32:02.448380Z",
"shell.execute_reply": "2025-12-11T12:32:02.446641Z",
"shell.execute_reply.started": "2025-12-11T12:32:02.439097Z"
}
},
"outputs": [],
"source": [
"interpreter = llguidance.LLInterpreter(\n",
" ll_tok, grammar, enable_ff_tokens=False, enable_backtrack=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "3b1a6610-5187-4bbe-848a-a43e584a72f5",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:32:03.335135Z",
"iopub.status.busy": "2025-12-11T12:32:03.334623Z",
"iopub.status.idle": "2025-12-11T12:32:03.346390Z",
"shell.execute_reply": "2025-12-11T12:32:03.344570Z",
"shell.execute_reply.started": "2025-12-11T12:32:03.335094Z"
}
},
"outputs": [],
"source": [
"prompt = tokenizer.encode(\"The fourth month of the year is\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "2813585f-56d6-4c7a-9780-c66588044261",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:32:14.457812Z",
"iopub.status.busy": "2025-12-11T12:32:14.457302Z",
"iopub.status.idle": "2025-12-11T12:32:14.468949Z",
"shell.execute_reply": "2025-12-11T12:32:14.467058Z",
"shell.execute_reply.started": "2025-12-11T12:32:14.457769Z"
}
},
"outputs": [],
"source": [
"result = interpreter.process_prompt(prompt)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "e4e6a5cc-d529-421d-afe6-1e25400f5c8e",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:32:18.400295Z",
"iopub.status.busy": "2025-12-11T12:32:18.399786Z",
"iopub.status.idle": "2025-12-11T12:32:18.424255Z",
"shell.execute_reply": "2025-12-11T12:32:18.422489Z",
"shell.execute_reply.started": "2025-12-11T12:32:18.400253Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mtoken id: 32,\u001b[0m \u001b[36mtoken: 'A'\u001b[0m\n",
"\u001b[33mtoken id: 10611,\u001b[0m \u001b[36mtoken: 'Ap'\u001b[0m\n",
"\u001b[33mtoken id: 20722,\u001b[0m \u001b[36mtoken: 'Apr'\u001b[0m\n",
"\u001b[33mtoken id: 21872,\u001b[0m \u001b[36mtoken: 'Aug'\u001b[0m\n",
"\u001b[33mtoken id: 28046,\u001b[0m \u001b[36mtoken: 'April'\u001b[0m\n",
"\u001b[33mtoken id: 31459,\u001b[0m \u001b[36mtoken: 'August'\u001b[0m\n",
"\u001b[33mtoken id: 65332,\u001b[0m \u001b[36mtoken: 'Au'\u001b[0m\n"
]
}
],
"source": [
"get_valid_tokens(interpreter)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "a4670cd6-4dad-477f-ae95-c350c969ed2c",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:32:49.473264Z",
"iopub.status.busy": "2025-12-11T12:32:49.472762Z",
"iopub.status.idle": "2025-12-11T12:32:49.481664Z",
"shell.execute_reply": "2025-12-11T12:32:49.479799Z",
"shell.execute_reply.started": "2025-12-11T12:32:49.473221Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(0, [32])"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"interpreter.commit_token(32)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "0de71cd6-626c-4f07-aa8f-247a586f2109",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:32:59.169050Z",
"iopub.status.busy": "2025-12-11T12:32:59.168570Z",
"iopub.status.idle": "2025-12-11T12:32:59.193850Z",
"shell.execute_reply": "2025-12-11T12:32:59.191706Z",
"shell.execute_reply.started": "2025-12-11T12:32:59.169008Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mtoken id: 79,\u001b[0m \u001b[36mtoken: 'p'\u001b[0m\n",
"\u001b[33mtoken id: 84,\u001b[0m \u001b[36mtoken: 'u'\u001b[0m\n",
"\u001b[33mtoken id: 649,\u001b[0m \u001b[36mtoken: 'pr'\u001b[0m\n",
"\u001b[33mtoken id: 768,\u001b[0m \u001b[36mtoken: 'ug'\u001b[0m\n",
"\u001b[33mtoken id: 29785,\u001b[0m \u001b[36mtoken: 'ugu'\u001b[0m\n",
"\u001b[33mtoken id: 68608,\u001b[0m \u001b[36mtoken: 'pri'\u001b[0m\n"
]
}
],
"source": [
"get_valid_tokens(interpreter)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "76bf24d6-4c59-472d-8296-fc8fec8ac405",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:33:03.728768Z",
"iopub.status.busy": "2025-12-11T12:33:03.728299Z",
"iopub.status.idle": "2025-12-11T12:33:03.736694Z",
"shell.execute_reply": "2025-12-11T12:33:03.734802Z",
"shell.execute_reply.started": "2025-12-11T12:33:03.728723Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(0, [79])"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"interpreter.commit_token(79)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "a72dc2c1-c7fa-4a1e-83b0-5388d521a1d6",
"metadata": {
"execution": {
"iopub.execute_input": "2025-12-11T12:33:07.251072Z",
"iopub.status.busy": "2025-12-11T12:33:07.250574Z",
"iopub.status.idle": "2025-12-11T12:33:07.275321Z",
"shell.execute_reply": "2025-12-11T12:33:07.273810Z",
"shell.execute_reply.started": "2025-12-11T12:33:07.251030Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mtoken id: 30560,\u001b[0m \u001b[36mtoken: 'ril'\u001b[0m\n"
]
}
],
"source": [
"get_valid_tokens(interpreter)"
]
},
{
"cell_type": "markdown",
"id": "64bfc210-3dae-474a-b2f8-73e9231bdb9e",
"metadata": {},
"source": [
"I only see token id: $30560$, token: 'ril', instead of the three valid tokens at state $4$.\n",
"\n",
"## Questions:\n",
"\n",
"- Why is that?\n",
"- Is it possible to obtain all the valid tokens?"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "467047bf-25d0-4941-aef1-74b5fc44c93e",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment