Use compress json to write raw functions

Author: Ho-Hsiang Wu

notebooks/1 - Preprocess Data.ipynb

@@ -267,7 +267,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Output each set to train/valid/test.function/docstrings/lineage files"
+    "## Output each set to train/valid/test.function/docstrings/lineage files\n",
+    "Original functions are also written to compressed json files. (Raw functions contain `,`, `\\t`, `\\n`, etc., it is less error-prone using json format)"
    ]
   },
   {
@@ -280,7 +281,7 @@
    "source": [
     "def write_to(df, filename):\n",
     "    df.function_tokens.to_csv('{}.function'.format(filename), index=False)\n",
-    "    df.original_function.to_csv('{}.original_function'.format(filename), index=False)\n",
+    "    df.original_function.to_json('{}_original_function.json.gz'.format(filename), orient='values', compression='gzip')\n",
     "    if filename != 'without_docstrings':\n",
     "        df.docstring_tokens.to_csv('{}.docstring'.format(filename), index=False)\n",
     "    df.url.to_csv('{}.lineage'.format(filename), index=False)"