Skip to content

Commit d5f60ab

Browse files
JakeRadMSFTJakeRadMSFT
committed
Update all Notebooks to download data when it's needed.
1 parent ab5aa15 commit d5f60ab

14 files changed

+788
-774
lines changed

‎machine-learning/02-Data Preparation and Feature Engineering.ipynb‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,57 @@
8686
],
8787
"outputs": []
8888
},
89+
{
90+
"cell_type": "markdown",
91+
"metadata": {},
92+
"source": [
93+
"## Download or Locate Data\n",
94+
"The following code tries to locate the data file in a few known locations or it will download it from the known GitHub location."
95+
]
96+
},
97+
{
98+
"cell_type": "code",
99+
"execution_count": 1,
100+
"metadata": {
101+
"dotnet_interactive": {
102+
"language": "csharp"
103+
}
104+
},
105+
"source": [
106+
"using System;\n",
107+
"using System.IO;\n",
108+
"using System.Net;\n",
109+
"\n",
110+
"string EnsureDataSetDownloaded(string fileName)\n",
111+
"{\n",
112+
"\n",
113+
"\t// This is the path if the repo has been checked out.\n",
114+
"\tvar filePath = Path.Combine(Directory.GetCurrentDirectory(),\"data\", fileName);\n",
115+
"\n",
116+
"\tif (!File.Exists(filePath))\n",
117+
"\t{\n",
118+
"\t\t// This is the path if the file has already been downloaded.\n",
119+
"\t\tfilePath = Path.Combine(Directory.GetCurrentDirectory(), fileName);\n",
120+
"\t}\n",
121+
"\n",
122+
"\tif (!File.Exists(filePath))\n",
123+
"\t{\n",
124+
"\t\tusing (var client = new WebClient())\n",
125+
"\t\t{\n",
126+
"\t\t\tclient.DownloadFile($\"https://raw.githubusercontent.com/dotnet/csharp-notebooks/main/machine-learning/data/{fileName}\", filePath);\n",
127+
"\t\t}\n",
128+
"\t\tConsole.WriteLine($\"Downloaded {fileName} to : {filePath}\");\n",
129+
"\t}\n",
130+
"\telse\n",
131+
"\t{\n",
132+
"\t\tConsole.WriteLine($\"{fileName} found here: {filePath}\");\n",
133+
"\t}\n",
134+
"\n",
135+
"\treturn filePath;\n",
136+
"}"
137+
],
138+
"outputs": []
139+
},
89140
{
90141
"cell_type": "markdown",
91142
"metadata": {},
@@ -188,12 +239,14 @@
188239
}
189240
},
190241
"source": [
242+
"var trainDataPath = EnsureDataSetDownloaded(\"taxi-fare.csv\");\n",
243+
"\n",
191244
"// Create TextLoader based on the Model Input type. \n",
192245
"TextLoader textLoader = mlContext.Data.CreateTextLoader<ModelInput>(separatorChar: ',', hasHeader: true);\n",
193246
"\n",
194247
"// Load the data into an IDataView. Load() method can support multiple files. \n",
195248
"// Files must they have the same separator character, header, column names, etc. \n",
196-
"IDataView data = textLoader.Load(\"data/taxi-fare.csv\");\n",
249+
"IDataView data = textLoader.Load(trainDataPath);\n",
197250
"\n",
198251
"data.Preview(1); "
199252
],

‎machine-learning/03-Training and AutoML.ipynb‎

Lines changed: 117 additions & 150 deletions
Large diffs are not rendered by default.

‎machine-learning/04-Model Evaluation.ipynb‎

Lines changed: 45 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,8 @@
138138
"cell_type": "markdown",
139139
"metadata": {},
140140
"source": [
141-
"### Load your data\n",
142-
"\n",
143-
"Use the `#!value` and `#!share` magic commands to fetch the data from GitHub, store it in the `taxi_data` variable and load it into a `DataFrame` "
141+
"### Download or Locate Data\n",
142+
"The following code tries to locate the data file in a few known locations or it will download it from the known GitHub location."
144143
]
145144
},
146145
{
@@ -152,22 +151,48 @@
152151
}
153152
},
154153
"source": [
155-
"#!value --name taxi_data --from-url https://github.com/dotnet/csharp-notebooks/raw/main/machine-learning/data/taxi-fare.csv"
154+
"using System;\n",
155+
"using System.IO;\n",
156+
"using System.Net;\n",
157+
"\n",
158+
"string EnsureDataSetDownloaded(string fileName)\n",
159+
"{\n",
160+
"\n",
161+
"\t// This is the path if the repo has been checked out.\n",
162+
"\tvar filePath = Path.Combine(Directory.GetCurrentDirectory(),\"data\", fileName);\n",
163+
"\n",
164+
"\tif (!File.Exists(filePath))\n",
165+
"\t{\n",
166+
"\t\t// This is the path if the file has already been downloaded.\n",
167+
"\t\tfilePath = Path.Combine(Directory.GetCurrentDirectory(), fileName);\n",
168+
"\t}\n",
169+
"\n",
170+
"\tif (!File.Exists(filePath))\n",
171+
"\t{\n",
172+
"\t\tusing (var client = new WebClient())\n",
173+
"\t\t{\n",
174+
"\t\t\tclient.DownloadFile($\"https://raw.githubusercontent.com/dotnet/csharp-notebooks/main/machine-learning/data/{fileName}\", filePath);\n",
175+
"\t\t}\n",
176+
"\t\tConsole.WriteLine($\"Downloaded {fileName} to : {filePath}\");\n",
177+
"\t}\n",
178+
"\telse\n",
179+
"\t{\n",
180+
"\t\tConsole.WriteLine($\"{fileName} found here: {filePath}\");\n",
181+
"\t}\n",
182+
"\n",
183+
"\treturn filePath;\n",
184+
"}"
156185
],
157-
"outputs": []
158-
},
159-
{
160-
"cell_type": "code",
161-
"execution_count": 1,
162-
"metadata": {
163-
"dotnet_interactive": {
164-
"language": "csharp"
186+
"outputs": [
187+
{
188+
"output_type": "execute_result",
189+
"data": {
190+
"text/plain": "Train Data Path: C:\\dev\\csharp-notebooks\\machine-learning\\data\\taxi-fare.csv\r\n"
191+
},
192+
"execution_count": 1,
193+
"metadata": {}
165194
}
166-
},
167-
"source": [
168-
"#!share taxi_data --from value"
169-
],
170-
"outputs": []
195+
]
171196
},
172197
{
173198
"cell_type": "code",
@@ -178,7 +203,8 @@
178203
}
179204
},
180205
"source": [
181-
"var df = DataFrame.LoadCsvFromString(taxi_data);"
206+
"var trainDataPath = EnsureDataSetDownloaded(\"taxi-fare.csv\");\n",
207+
"var df = DataFrame.LoadCsv(trainDataPath);"
182208
],
183209
"outputs": []
184210
},
@@ -204,7 +230,7 @@
204230
{
205231
"output_type": "execute_result",
206232
"data": {
207-
"text/html": "<table id=\"table_637928803371228110\"><thead><tr><th><i>index</i></th><th>vendor_id</th><th>rate_code</th><th>passenger_count</th><th>trip_time_in_secs</th><th>trip_distance</th><th>payment_type</th><th>fare_amount</th></tr></thead><tbody><tr><td><i><div class=\"dni-plaintext\">0</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1271</div></td><td><div class=\"dni-plaintext\">3.8</div></td><td>CRD</td><td><div class=\"dni-plaintext\">17.5</div></td></tr><tr><td><i><div class=\"dni-plaintext\">1</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">474</div></td><td><div class=\"dni-plaintext\">1.5</div></td><td>CRD</td><td><div class=\"dni-plaintext\">8</div></td></tr><tr><td><i><div class=\"dni-plaintext\">2</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">637</div></td><td><div class=\"dni-plaintext\">1.4</div></td><td>CRD</td><td><div class=\"dni-plaintext\">8.5</div></td></tr><tr><td><i><div class=\"dni-plaintext\">3</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">181</div></td><td><div class=\"dni-plaintext\">0.6</div></td><td>CSH</td><td><div class=\"dni-plaintext\">4.5</div></td></tr><tr><td><i><div class=\"dni-plaintext\">4</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">661</div></td><td><div class=\"dni-plaintext\">1.1</div></td><td>CRD</td><td><div class=\"dni-plaintext\">8.5</div></td></tr></tbody></table>"
233+
"text/html": "<table id=\"table_637934937843853168\"><thead><tr><th><i>index</i></th><th>vendor_id</th><th>rate_code</th><th>passenger_count</th><th>trip_time_in_secs</th><th>trip_distance</th><th>payment_type</th><th>fare_amount</th></tr></thead><tbody><tr><td><i><div class=\"dni-plaintext\">0</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1271</div></td><td><div class=\"dni-plaintext\">3.8</div></td><td>CRD</td><td><div class=\"dni-plaintext\">17.5</div></td></tr><tr><td><i><div class=\"dni-plaintext\">1</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">474</div></td><td><div class=\"dni-plaintext\">1.5</div></td><td>CRD</td><td><div class=\"dni-plaintext\">8</div></td></tr><tr><td><i><div class=\"dni-plaintext\">2</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">637</div></td><td><div class=\"dni-plaintext\">1.4</div></td><td>CRD</td><td><div class=\"dni-plaintext\">8.5</div></td></tr><tr><td><i><div class=\"dni-plaintext\">3</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">181</div></td><td><div class=\"dni-plaintext\">0.6</div></td><td>CSH</td><td><div class=\"dni-plaintext\">4.5</div></td></tr><tr><td><i><div class=\"dni-plaintext\">4</div></i></td><td>CMT</td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">1</div></td><td><div class=\"dni-plaintext\">661</div></td><td><div class=\"dni-plaintext\">1.1</div></td><td>CRD</td><td><div class=\"dni-plaintext\">8.5</div></td></tr></tbody></table>"
208234
},
209235
"execution_count": 1,
210236
"metadata": {}

‎machine-learning/E2E-Classification with Iris Dataset.ipynb‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
{
6161
"output_type": "execute_result",
6262
"data": {
63-
"text/markdown": "Loading extensions from `Microsoft.Data.Analysis.Interactive.dll`"
63+
"text/markdown": "Loading extensions from `Microsoft.ML.AutoML.Interactive.dll`"
6464
},
6565
"execution_count": 1,
6666
"metadata": {}
@@ -76,7 +76,7 @@
7676
{
7777
"output_type": "execute_result",
7878
"data": {
79-
"text/markdown": "Loading extensions from `Microsoft.ML.AutoML.Interactive.dll`"
79+
"text/markdown": "Loading extensions from `Microsoft.Data.Analysis.Interactive.dll`"
8080
},
8181
"execution_count": 1,
8282
"metadata": {}
@@ -157,6 +157,49 @@
157157
],
158158
"outputs": []
159159
},
160+
{
161+
"cell_type": "code",
162+
"execution_count": 1,
163+
"metadata": {
164+
"dotnet_interactive": {
165+
"language": "csharp"
166+
}
167+
},
168+
"source": [
169+
"using System;\n",
170+
"using System.IO;\n",
171+
"using System.Net;\n",
172+
"\n",
173+
"string EnsureDataSetDownloaded(string fileName)\n",
174+
"{\n",
175+
"\n",
176+
"\t// This is the path if the repo has been checked out.\n",
177+
"\tvar filePath = Path.Combine(Directory.GetCurrentDirectory(),\"data\", fileName);\n",
178+
"\n",
179+
"\tif (!File.Exists(filePath))\n",
180+
"\t{\n",
181+
"\t\t// This is the path if the file has already been downloaded.\n",
182+
"\t\tfilePath = Path.Combine(Directory.GetCurrentDirectory(), fileName);\n",
183+
"\t}\n",
184+
"\n",
185+
"\tif (!File.Exists(filePath))\n",
186+
"\t{\n",
187+
"\t\tusing (var client = new WebClient())\n",
188+
"\t\t{\n",
189+
"\t\t\tclient.DownloadFile($\"https://raw.githubusercontent.com/dotnet/csharp-notebooks/main/machine-learning/data/{fileName}\", filePath);\n",
190+
"\t\t}\n",
191+
"\t\tConsole.WriteLine($\"Downloaded {fileName} to : {filePath}\");\n",
192+
"\t}\n",
193+
"\telse\n",
194+
"\t{\n",
195+
"\t\tConsole.WriteLine($\"{fileName} found here: {filePath}\");\n",
196+
"\t}\n",
197+
"\n",
198+
"\treturn filePath;\n",
199+
"}"
200+
],
201+
"outputs": []
202+
},
160203
{
161204
"cell_type": "markdown",
162205
"metadata": {},
@@ -177,8 +220,8 @@
177220
"var mlContext = new MLContext();\n",
178221
"\n",
179222
"// Define path to training data\n",
180-
"string trainValidateDataPath = @\".\\data\\iris-train.tsv\";\n",
181-
"string testDataPath = @\".\\data\\iris-test.tsv\";\n",
223+
"string trainValidateDataPath = EnsureDataSetDownloaded(\"iris-train.tsv\");\n",
224+
"string testDataPath = EnsureDataSetDownloaded(\"iris-test.tsv\");\n",
182225
"\n",
183226
"// Load data from a text file to an IDataView (a flexible, efficient way of describing tabular data)\n",
184227
"IDataView trainValidateData = mlContext.Data.LoadFromTextFile<ModelInput>(\n",
@@ -200,6 +243,14 @@
200243
""
201244
],
202245
"outputs": [
246+
{
247+
"output_type": "execute_result",
248+
"data": {
249+
"text/plain": "iris-test.tsv found here: C:\\dev\\csharp-notebooks\\machine-learning\\data\\iris-test.tsv\r\n"
250+
},
251+
"execution_count": 1,
252+
"metadata": {}
253+
},
203254
{
204255
"output_type": "execute_result",
205256
"data": {

‎machine-learning/E2E-Forecasting using Regression with Luna Dataset.ipynb‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,50 @@
118118
}
119119
},
120120
"source": [
121-
"var dataPath = @\"./data/Luna.csv\";\n",
121+
"using System;\n",
122+
"using System.IO;\n",
123+
"using System.Net;\n",
124+
"\n",
125+
"string EnsureDataSetDownloaded(string fileName)\n",
126+
"{\n",
127+
"\n",
128+
"\t// This is the path if the repo has been checked out.\n",
129+
"\tvar filePath = Path.Combine(Directory.GetCurrentDirectory(),\"data\", fileName);\n",
130+
"\n",
131+
"\tif (!File.Exists(filePath))\n",
132+
"\t{\n",
133+
"\t\t// This is the path if the file has already been downloaded.\n",
134+
"\t\tfilePath = Path.Combine(Directory.GetCurrentDirectory(), fileName);\n",
135+
"\t}\n",
136+
"\n",
137+
"\tif (!File.Exists(filePath))\n",
138+
"\t{\n",
139+
"\t\tusing (var client = new WebClient())\n",
140+
"\t\t{\n",
141+
"\t\t\tclient.DownloadFile($\"https://raw.githubusercontent.com/dotnet/csharp-notebooks/main/machine-learning/data/{fileName}\", filePath);\n",
142+
"\t\t}\n",
143+
"\t\tConsole.WriteLine($\"Downloaded {fileName} to : {filePath}\");\n",
144+
"\t}\n",
145+
"\telse\n",
146+
"\t{\n",
147+
"\t\tConsole.WriteLine($\"{fileName} found here: {filePath}\");\n",
148+
"\t}\n",
149+
"\n",
150+
"\treturn filePath;\n",
151+
"}"
152+
],
153+
"outputs": []
154+
},
155+
{
156+
"cell_type": "code",
157+
"execution_count": 1,
158+
"metadata": {
159+
"dotnet_interactive": {
160+
"language": "csharp"
161+
}
162+
},
163+
"source": [
164+
"var dataPath = EnsureDataSetDownloaded(@\"Luna.csv\");\n",
122165
"\n",
123166
"// use previous 2 weeks data\n",
124167
"var df = DataFrame.LoadCsv(dataPath);\n",

‎machine-learning/E2E-Forecasting using SSA with Luna Dataset.ipynb‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,50 @@
125125
}
126126
},
127127
"source": [
128-
"var dataPath = @\"./data/Luna.csv\";\n",
128+
"using System;\n",
129+
"using System.IO;\n",
130+
"using System.Net;\n",
131+
"\n",
132+
"string EnsureDataSetDownloaded(string fileName)\n",
133+
"{\n",
134+
"\n",
135+
"\t// This is the path if the repo has been checked out.\n",
136+
"\tvar filePath = Path.Combine(Directory.GetCurrentDirectory(),\"data\", fileName);\n",
137+
"\n",
138+
"\tif (!File.Exists(filePath))\n",
139+
"\t{\n",
140+
"\t\t// This is the path if the file has already been downloaded.\n",
141+
"\t\tfilePath = Path.Combine(Directory.GetCurrentDirectory(), fileName);\n",
142+
"\t}\n",
143+
"\n",
144+
"\tif (!File.Exists(filePath))\n",
145+
"\t{\n",
146+
"\t\tusing (var client = new WebClient())\n",
147+
"\t\t{\n",
148+
"\t\t\tclient.DownloadFile($\"https://raw.githubusercontent.com/dotnet/csharp-notebooks/main/machine-learning/data/{fileName}\", filePath);\n",
149+
"\t\t}\n",
150+
"\t\tConsole.WriteLine($\"Downloaded {fileName} to : {filePath}\");\n",
151+
"\t}\n",
152+
"\telse\n",
153+
"\t{\n",
154+
"\t\tConsole.WriteLine($\"{fileName} found here: {filePath}\");\n",
155+
"\t}\n",
156+
"\n",
157+
"\treturn filePath;\n",
158+
"}"
159+
],
160+
"outputs": []
161+
},
162+
{
163+
"cell_type": "code",
164+
"execution_count": 1,
165+
"metadata": {
166+
"dotnet_interactive": {
167+
"language": "csharp"
168+
}
169+
},
170+
"source": [
171+
"var dataPath = EnsureDataSetDownloaded(@\"Luna.csv\");\n",
129172
"var df = DataFrame.LoadCsv(dataPath);\n",
130173
"var loads = df[\"load\"].Cast<float?>();"
131174
],

0 commit comments

Comments
 (0)