Skip to content

Commit 14b9536

Browse files
jalauzon-msftjalauzon-msft
authored
[Storage] Fix Datalake rename operations when new name contains ? (Azure#28561)
1 parent 4ccde7f commit 14b9536

17 files changed

+1109
-46
lines changed

‎.vscode/cspell.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -736,7 +736,8 @@
736736
"cpksubdirectory",
737737
"thead",
738738
"tbody",
739-
"racwdlmeop"
739+
"racwdlmeop",
740+
"newfs"
740741
]
741742
},
742743
{

‎sdk/storage/azure-storage-file-datalake/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
- Added ability to perform leasing actions on file append and flush. See new keyword `lease_action` for details.
88
- Added support for `AsyncIterable` as data type for async file upload.
99

10+
### Bugs Fixed
11+
- Fixed an issue where `rename_file` and `rename_directory` would not work correctly if the new file/directory name
12+
contained a `?` character.
13+
1014
### Other Changes
1115
- Removed `msrest` dependency.
1216
- Added `typing-extensions>=4.0.1` as a dependency.

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -375,17 +375,7 @@ def rename_directory(self, new_name, **kwargs):
375375
:dedent: 4
376376
:caption: Rename the source directory.
377377
"""
378-
new_name = new_name.strip('/')
379-
new_file_system = new_name.split('/')[0]
380-
new_path_and_token = new_name[len(new_file_system):].strip('/').split('?')
381-
new_path = new_path_and_token[0]
382-
try:
383-
new_dir_sas = new_path_and_token[1] or self._query_str.strip('?')
384-
except IndexError:
385-
if not self._raw_credential and new_file_system != self.file_system_name:
386-
raise ValueError("please provide the sas token for the new file")
387-
if not self._raw_credential and new_file_system == self.file_system_name:
388-
new_dir_sas = self._query_str.strip('?')
378+
new_file_system, new_path, new_dir_sas = self._parse_rename_path(new_name)
389379

390380
new_directory_client = DataLakeDirectoryClient(
391381
"{}://{}".format(self.scheme, self.primary_hostname), new_file_system, directory_name=new_path,

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -831,17 +831,7 @@ def rename_file(self, new_name, **kwargs):
831831
:dedent: 4
832832
:caption: Rename the source file.
833833
"""
834-
new_name = new_name.strip('/')
835-
new_file_system = new_name.split('/')[0]
836-
new_path_and_token = new_name[len(new_file_system):].strip('/').split('?')
837-
new_path = new_path_and_token[0]
838-
try:
839-
new_file_sas = new_path_and_token[1] or self._query_str.strip('?')
840-
except IndexError:
841-
if not self._raw_credential and new_file_system != self.file_system_name:
842-
raise ValueError("please provide the sas token for the new file")
843-
if not self._raw_credential and new_file_system == self.file_system_name:
844-
new_file_sas = self._query_str.strip('?')
834+
new_file_system, new_path, new_file_sas = self._parse_rename_path(new_name)
845835

846836
new_file_client = DataLakeFileClient(
847837
"{}://{}".format(self.scheme, self.primary_hostname), new_file_system, file_path=new_path,

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@
44
# Licensed under the MIT License. See License.txt in the project root for
55
# license information.
66
# --------------------------------------------------------------------------
7+
import re
78
from datetime import datetime
89
from typing import ( # pylint: disable=unused-import
9-
Any, Dict, Optional, Union,
10+
Any, Dict, Optional, Tuple, Union,
1011
TYPE_CHECKING
1112
)
1213
from urllib.parse import urlparse, quote
@@ -737,6 +738,30 @@ def _set_access_control_internal(self, options, progress_hook, max_batches=None)
737738
error.continuation_token = last_continuation_token
738739
raise error
739740

741+
def _parse_rename_path(self, new_name: str) -> Tuple[str, str, Optional[str]]:
742+
new_name = new_name.strip('/')
743+
new_file_system = new_name.split('/')[0]
744+
new_path = new_name[len(new_file_system):].strip('/')
745+
746+
new_sas = None
747+
sas_split = new_path.split('?')
748+
# If there is a ?, there could be a SAS token
749+
if len(sas_split) > 0:
750+
# Check last element for SAS by looking for sv= and sig=
751+
potential_sas = sas_split[-1]
752+
if re.search(r'sv=\d{4}-\d{2}-\d{2}', potential_sas) and 'sig=' in potential_sas:
753+
new_sas = potential_sas
754+
# Remove SAS from new path
755+
new_path = new_path[:-(len(new_sas) + 1)]
756+
757+
if not new_sas:
758+
if not self._raw_credential and new_file_system != self.file_system_name:
759+
raise ValueError("please provide the sas token for the new file")
760+
if not self._raw_credential and new_file_system == self.file_system_name:
761+
new_sas = self._query_str.strip('?')
762+
763+
return new_file_system, new_path, new_sas
764+
740765
def _rename_path_options(self, # pylint: disable=no-self-use
741766
rename_source, # type: str
742767
content_settings=None, # type: Optional[ContentSettings]

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -343,17 +343,7 @@ async def rename_directory(self, new_name, # type: str
343343
:dedent: 4
344344
:caption: Rename the source directory.
345345
"""
346-
new_name = new_name.strip('/')
347-
new_file_system = new_name.split('/')[0]
348-
new_path_and_token = new_name[len(new_file_system):].strip('/').split('?')
349-
new_path = new_path_and_token[0]
350-
try:
351-
new_dir_sas = new_path_and_token[1] or self._query_str.strip('?')
352-
except IndexError:
353-
if not self._raw_credential and new_file_system != self.file_system_name:
354-
raise ValueError("please provide the sas token for the new directory")
355-
if not self._raw_credential and new_file_system == self.file_system_name:
356-
new_dir_sas = self._query_str.strip('?')
346+
new_file_system, new_path, new_dir_sas = self._parse_rename_path(new_name)
357347

358348
new_directory_client = DataLakeDirectoryClient(
359349
"{}://{}".format(self.scheme, self.primary_hostname), new_file_system, directory_name=new_path,

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -675,17 +675,7 @@ async def rename_file(self, new_name, **kwargs):
675675
:dedent: 4
676676
:caption: Rename the source file.
677677
"""
678-
new_name = new_name.strip('/')
679-
new_file_system = new_name.split('/')[0]
680-
new_path_and_token = new_name[len(new_file_system):].strip('/').split('?')
681-
new_path = new_path_and_token[0]
682-
try:
683-
new_file_sas = new_path_and_token[1] or self._query_str.strip('?')
684-
except IndexError:
685-
if not self._raw_credential and new_file_system != self.file_system_name:
686-
raise ValueError("please provide the sas token for the new file")
687-
if not self._raw_credential and new_file_system == self.file_system_name:
688-
new_file_sas = self._query_str.strip('?')
678+
new_file_system, new_path, new_file_sas = self._parse_rename_path(new_name)
689679

690680
new_file_client = DataLakeFileClient(
691681
"{}://{}".format(self.scheme, self.primary_hostname), new_file_system, file_path=new_path,

‎sdk/storage/azure-storage-file-datalake/tests/recordings/test_directory.pyTestDirectorytest_rename_directory_special_chars.json‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
{
2+
"Entries": [
3+
{
4+
"RequestUri": "https://storagename.blob.core.windows.net/filesystem6e3c2f71?restype=container\u0026timeout=5",
5+
"RequestMethod": "PUT",
6+
"RequestHeaders": {
7+
"Accept": "application/xml",
8+
"Accept-Encoding": "gzip, deflate",
9+
"Connection": "keep-alive",
10+
"Content-Length": "0",
11+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
12+
"x-ms-date": "Tue, 31 Jan 2023 23:57:44 GMT",
13+
"x-ms-version": "2021-12-02"
14+
},
15+
"RequestBody": null,
16+
"StatusCode": 201,
17+
"ResponseHeaders": {
18+
"Content-Length": "0",
19+
"Date": "Tue, 31 Jan 2023 23:57:44 GMT",
20+
"ETag": "\u00220x8DB03E6F27FAE14\u0022",
21+
"Last-Modified": "Tue, 31 Jan 2023 23:57:44 GMT",
22+
"Server": [
23+
"Windows-Azure-Blob/1.0",
24+
"Microsoft-HTTPAPI/2.0"
25+
],
26+
"x-ms-version": "2021-12-02"
27+
},
28+
"ResponseBody": null
29+
},
30+
{
31+
"RequestUri": "https://storagename.dfs.core.windows.net/filesystem6e3c2f71/olddir?resource=directory",
32+
"RequestMethod": "PUT",
33+
"RequestHeaders": {
34+
"Accept": "application/json",
35+
"Accept-Encoding": "gzip, deflate",
36+
"Connection": "keep-alive",
37+
"Content-Length": "0",
38+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
39+
"x-ms-date": "Tue, 31 Jan 2023 23:57:45 GMT",
40+
"x-ms-version": "2021-12-02"
41+
},
42+
"RequestBody": null,
43+
"StatusCode": 201,
44+
"ResponseHeaders": {
45+
"Content-Length": "0",
46+
"Date": "Tue, 31 Jan 2023 23:57:45 GMT",
47+
"ETag": "\u00220x8DB03E6F2DE38DD\u0022",
48+
"Last-Modified": "Tue, 31 Jan 2023 23:57:45 GMT",
49+
"Server": [
50+
"Windows-Azure-HDFS/1.0",
51+
"Microsoft-HTTPAPI/2.0"
52+
],
53+
"x-ms-request-server-encrypted": "true",
54+
"x-ms-version": "2021-12-02"
55+
},
56+
"ResponseBody": null
57+
},
58+
{
59+
"RequestUri": "https://storagename.dfs.core.windows.net/filesystem6e3c2f71/%3F%21%40%23%24%25%5E%26%2A.%3Ftest?mode=legacy",
60+
"RequestMethod": "PUT",
61+
"RequestHeaders": {
62+
"Accept": "application/json",
63+
"Accept-Encoding": "gzip, deflate",
64+
"Connection": "keep-alive",
65+
"Content-Length": "0",
66+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
67+
"x-ms-date": "Tue, 31 Jan 2023 23:57:45 GMT",
68+
"x-ms-rename-source": "/filesystem6e3c2f71/olddir",
69+
"x-ms-source-lease-id": "",
70+
"x-ms-version": "2021-12-02"
71+
},
72+
"RequestBody": null,
73+
"StatusCode": 201,
74+
"ResponseHeaders": {
75+
"Content-Length": "0",
76+
"Date": "Tue, 31 Jan 2023 23:57:45 GMT",
77+
"ETag": "\u00220x8DB03E6F2DE38DD\u0022",
78+
"Last-Modified": "Tue, 31 Jan 2023 23:57:45 GMT",
79+
"Server": [
80+
"Windows-Azure-HDFS/1.0",
81+
"Microsoft-HTTPAPI/2.0"
82+
],
83+
"x-ms-version": "2021-12-02"
84+
},
85+
"ResponseBody": null
86+
},
87+
{
88+
"RequestUri": "https://storagename.blob.core.windows.net/filesystem6e3c2f71/%3F%21%40%23%24%25%5E%26%2A.%3Ftest",
89+
"RequestMethod": "HEAD",
90+
"RequestHeaders": {
91+
"Accept": "application/xml",
92+
"Accept-Encoding": "gzip, deflate",
93+
"Connection": "keep-alive",
94+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
95+
"x-ms-date": "Tue, 31 Jan 2023 23:57:45 GMT",
96+
"x-ms-version": "2021-12-02"
97+
},
98+
"RequestBody": null,
99+
"StatusCode": 200,
100+
"ResponseHeaders": {
101+
"Accept-Ranges": "bytes",
102+
"Content-Length": "0",
103+
"Content-Type": "application/octet-stream",
104+
"Date": "Tue, 31 Jan 2023 23:57:45 GMT",
105+
"ETag": "\u00220x8DB03E6F2DE38DD\u0022",
106+
"Last-Modified": "Tue, 31 Jan 2023 23:57:45 GMT",
107+
"Server": [
108+
"Windows-Azure-Blob/1.0",
109+
"Microsoft-HTTPAPI/2.0"
110+
],
111+
"x-ms-access-tier": "Hot",
112+
"x-ms-access-tier-inferred": "true",
113+
"x-ms-blob-type": "BlockBlob",
114+
"x-ms-creation-time": "Tue, 31 Jan 2023 23:57:45 GMT",
115+
"x-ms-group": "$superuser",
116+
"x-ms-lease-state": "available",
117+
"x-ms-lease-status": "unlocked",
118+
"x-ms-meta-hdi_isfolder": "true",
119+
"x-ms-owner": "$superuser",
120+
"x-ms-permissions": "rwxr-x---",
121+
"x-ms-resource-type": "directory",
122+
"x-ms-server-encrypted": "true",
123+
"x-ms-version": "2021-12-02"
124+
},
125+
"ResponseBody": null
126+
}
127+
],
128+
"Variables": {}
129+
}

‎sdk/storage/azure-storage-file-datalake/tests/recordings/test_directory_async.pyTestDirectoryAsynctest_rename_directory_special_chars.json‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
{
2+
"Entries": [
3+
{
4+
"RequestUri": "https://storagename.blob.core.windows.net/filesystema0d033ec?restype=container\u0026timeout=5",
5+
"RequestMethod": "PUT",
6+
"RequestHeaders": {
7+
"Accept": "application/xml",
8+
"Accept-Encoding": "gzip, deflate",
9+
"Content-Length": "0",
10+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
11+
"x-ms-date": "Wed, 01 Feb 2023 00:00:44 GMT",
12+
"x-ms-version": "2021-12-02"
13+
},
14+
"RequestBody": null,
15+
"StatusCode": 201,
16+
"ResponseHeaders": {
17+
"Content-Length": "0",
18+
"Date": "Wed, 01 Feb 2023 00:00:44 GMT",
19+
"ETag": "\u00220x8DB03E75DE541D8\u0022",
20+
"Last-Modified": "Wed, 01 Feb 2023 00:00:45 GMT",
21+
"Server": [
22+
"Windows-Azure-Blob/1.0",
23+
"Microsoft-HTTPAPI/2.0"
24+
],
25+
"x-ms-version": "2021-12-02"
26+
},
27+
"ResponseBody": null
28+
},
29+
{
30+
"RequestUri": "https://storagename.dfs.core.windows.net/filesystema0d033ec/olddir?resource=directory",
31+
"RequestMethod": "PUT",
32+
"RequestHeaders": {
33+
"Accept": "application/json",
34+
"Accept-Encoding": "gzip, deflate",
35+
"Content-Length": "0",
36+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
37+
"x-ms-date": "Wed, 01 Feb 2023 00:00:45 GMT",
38+
"x-ms-version": "2021-12-02"
39+
},
40+
"RequestBody": null,
41+
"StatusCode": 201,
42+
"ResponseHeaders": {
43+
"Content-Length": "0",
44+
"Date": "Wed, 01 Feb 2023 00:00:45 GMT",
45+
"ETag": "\u00220x8DB03E75E5BC906\u0022",
46+
"Last-Modified": "Wed, 01 Feb 2023 00:00:45 GMT",
47+
"Server": [
48+
"Windows-Azure-HDFS/1.0",
49+
"Microsoft-HTTPAPI/2.0"
50+
],
51+
"x-ms-request-server-encrypted": "true",
52+
"x-ms-version": "2021-12-02"
53+
},
54+
"ResponseBody": null
55+
},
56+
{
57+
"RequestUri": "https://storagename.dfs.core.windows.net/filesystema0d033ec/%3F!@%23$%25%5E\u0026*.%3Ftest?mode=legacy",
58+
"RequestMethod": "PUT",
59+
"RequestHeaders": {
60+
"Accept": "application/json",
61+
"Accept-Encoding": "gzip, deflate",
62+
"Content-Length": "0",
63+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
64+
"x-ms-date": "Wed, 01 Feb 2023 00:00:45 GMT",
65+
"x-ms-rename-source": "/filesystema0d033ec/olddir",
66+
"x-ms-source-lease-id": "",
67+
"x-ms-version": "2021-12-02"
68+
},
69+
"RequestBody": null,
70+
"StatusCode": 201,
71+
"ResponseHeaders": {
72+
"Content-Length": "0",
73+
"Date": "Wed, 01 Feb 2023 00:00:45 GMT",
74+
"ETag": "\u00220x8DB03E75E5BC906\u0022",
75+
"Last-Modified": "Wed, 01 Feb 2023 00:00:45 GMT",
76+
"Server": [
77+
"Windows-Azure-HDFS/1.0",
78+
"Microsoft-HTTPAPI/2.0"
79+
],
80+
"x-ms-version": "2021-12-02"
81+
},
82+
"ResponseBody": null
83+
},
84+
{
85+
"RequestUri": "https://storagename.blob.core.windows.net/filesystema0d033ec/%3F!@%23$%25%5E\u0026*.%3Ftest",
86+
"RequestMethod": "HEAD",
87+
"RequestHeaders": {
88+
"Accept": "application/xml",
89+
"Accept-Encoding": "gzip, deflate",
90+
"User-Agent": "azsdk-python-storage-dfs/12.10.0b1 Python/3.10.6 (Windows-10-10.0.22621-SP0)",
91+
"x-ms-date": "Wed, 01 Feb 2023 00:00:46 GMT",
92+
"x-ms-version": "2021-12-02"
93+
},
94+
"RequestBody": null,
95+
"StatusCode": 200,
96+
"ResponseHeaders": {
97+
"Accept-Ranges": "bytes",
98+
"Content-Length": "0",
99+
"Content-Type": "application/octet-stream",
100+
"Date": "Wed, 01 Feb 2023 00:00:45 GMT",
101+
"ETag": "\u00220x8DB03E75E5BC906\u0022",
102+
"Last-Modified": "Wed, 01 Feb 2023 00:00:45 GMT",
103+
"Server": [
104+
"Windows-Azure-Blob/1.0",
105+
"Microsoft-HTTPAPI/2.0"
106+
],
107+
"x-ms-access-tier": "Hot",
108+
"x-ms-access-tier-inferred": "true",
109+
"x-ms-blob-type": "BlockBlob",
110+
"x-ms-creation-time": "Wed, 01 Feb 2023 00:00:45 GMT",
111+
"x-ms-group": "$superuser",
112+
"x-ms-lease-state": "available",
113+
"x-ms-lease-status": "unlocked",
114+
"x-ms-meta-hdi_isfolder": "true",
115+
"x-ms-owner": "$superuser",
116+
"x-ms-permissions": "rwxr-x---",
117+
"x-ms-resource-type": "directory",
118+
"x-ms-server-encrypted": "true",
119+
"x-ms-version": "2021-12-02"
120+
},
121+
"ResponseBody": null
122+
}
123+
],
124+
"Variables": {}
125+
}

0 commit comments

Comments
 (0)