Skip to content

Commit 8a54588

Browse files
tasherif-msfttasherif-msft
authored
Added GA Changelog date + More Tests (Azure#21742)
* Added GA Changelog date * Added GA Changelog date * added more tests * OAuth test * fixed test * fixed test * added env vars * lint * added more tests * changelog * changelog * changelog
1 parent 303452e commit 8a54588

File tree

8 files changed

+362
-12
lines changed

8 files changed

+362
-12
lines changed

‎sdk/storage/azdatalake/CHANGELOG.md‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
11
# Release History
22

3-
## 0.2.0 (Unreleased)
4-
5-
### Features Added
6-
7-
### Breaking Changes
3+
## 1.0.0 (2023-10-18)
84

95
### Bugs Fixed
106
* Fixed an issue where customers could not capture the raw HTTP response of directory and file GetProperties operations.
117
* Fixed an issue where file/directory renames with source/destination SAS tokens fail with authorization failures.
128

13-
### Other Changes
14-
159
## 0.1.0-beta.1 (2023-08-16)
1610

1711
### Features Added

‎sdk/storage/azdatalake/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "go",
44
"TagPrefix": "go/storage/azdatalake",
5-
"Tag": "go/storage/azdatalake_93ff4d0fca"
5+
"Tag": "go/storage/azdatalake_7c0fb050cb"
66
}

‎sdk/storage/azdatalake/ci.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,9 @@ stages:
2525
- template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml
2626
parameters:
2727
ServiceDirectory: 'storage/azdatalake'
28-
RunLiveTests: true
28+
RunLiveTests: true
29+
EnvVars:
30+
AZURE_CLIENT_ID: $(AZDATALAKE_CLIENT_ID)
31+
AZURE_TENANT_ID: $(AZDATALAKE_TENANT_ID)
32+
AZURE_CLIENT_SECRET: $(AZDATALAKE_CLIENT_SECRET)
33+
AZURE_SUBSCRIPTION_ID: $(AZDATALAKE_SUBSCRIPTION_ID)

‎sdk/storage/azdatalake/directory/client_test.go‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2539,4 +2539,33 @@ func (s *RecordedTestSuite) TestDirGetPropertiesResponseCapture() {
25392539
_require.Equal("directory", respFromCtxService.Header.Get("x-ms-resource-type"))
25402540
}
25412541

2542-
// TODO: more tests for acls
2542+
func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() {
2543+
_require := require.New(s.T())
2544+
testName := s.T().Name()
2545+
2546+
filesystemName := testcommon.GenerateFileSystemName(testName)
2547+
fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
2548+
_require.NoError(err)
2549+
defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
2550+
2551+
_, err = fsClient.Create(context.Background(), nil)
2552+
_require.NoError(err)
2553+
2554+
cred, err := testcommon.GetGenericTokenCredential()
2555+
_require.NoError(err)
2556+
2557+
accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake)
2558+
_require.Greater(len(accountName), 0)
2559+
2560+
dirName := testcommon.GenerateDirName(testName)
2561+
dirURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + dirName
2562+
2563+
dirClient, err := directory.NewClient(dirURL, cred, nil)
2564+
_require.NoError(err)
2565+
2566+
_, err = dirClient.Create(context.Background(), nil)
2567+
_require.NoError(err)
2568+
2569+
_, err = dirClient.GetProperties(context.Background(), nil)
2570+
_require.NoError(err)
2571+
}

‎sdk/storage/azdatalake/file/client_test.go‎

Lines changed: 281 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2572,6 +2572,87 @@ func (s *RecordedTestSuite) TestSmallFileUploadFile() {
25722572
_require.EqualValues(downloadedContentMD5, contentMD5)
25732573
}
25742574

2575+
func (s *RecordedTestSuite) TestSmallFileUploadFileWithAccessConditionsAndHTTPHeaders() {
2576+
_require := require.New(s.T())
2577+
testName := s.T().Name()
2578+
2579+
filesystemName := testcommon.GenerateFileSystemName(testName)
2580+
fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
2581+
_require.NoError(err)
2582+
defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
2583+
2584+
_, err = fsClient.Create(context.Background(), nil)
2585+
_require.NoError(err)
2586+
2587+
var fileSize int64 = 10 * 1024
2588+
fileName := testcommon.GenerateFileName(testName)
2589+
fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil)
2590+
_require.NoError(err)
2591+
2592+
createFileOpts := &file.CreateOptions{
2593+
ProposedLeaseID: proposedLeaseIDs[0],
2594+
LeaseDuration: to.Ptr(int64(15)),
2595+
}
2596+
2597+
resp, err := fClient.Create(context.Background(), createFileOpts)
2598+
_require.NoError(err)
2599+
_require.NotNil(resp)
2600+
2601+
// create local file
2602+
_, content := testcommon.GenerateData(int(fileSize))
2603+
_require.NoError(err)
2604+
err = os.WriteFile("testFile", content, 0644)
2605+
_require.NoError(err)
2606+
2607+
defer func() {
2608+
err = os.Remove("testFile")
2609+
_require.NoError(err)
2610+
}()
2611+
2612+
fh, err := os.Open("testFile")
2613+
_require.NoError(err)
2614+
2615+
defer func(fh *os.File) {
2616+
err := fh.Close()
2617+
_require.NoError(err)
2618+
}(fh)
2619+
2620+
hash := md5.New()
2621+
_, err = io.Copy(hash, fh)
2622+
_require.NoError(err)
2623+
contentMD5 := hash.Sum(nil)
2624+
2625+
err = fClient.UploadFile(context.Background(), fh, &file.UploadFileOptions{
2626+
Concurrency: 5,
2627+
ChunkSize: 2 * 1024,
2628+
AccessConditions: &file.AccessConditions{LeaseAccessConditions: &file.LeaseAccessConditions{
2629+
LeaseID: proposedLeaseIDs[0],
2630+
}},
2631+
HTTPHeaders: &testcommon.BasicHeaders,
2632+
})
2633+
_require.NoError(err)
2634+
2635+
gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{
2636+
AccessConditions: &file.AccessConditions{LeaseAccessConditions: &file.LeaseAccessConditions{
2637+
LeaseID: proposedLeaseIDs[0],
2638+
},
2639+
}})
2640+
_require.NoError(err)
2641+
_require.Equal(*gResp2.ContentLength, fileSize)
2642+
2643+
dResp, err := fClient.DownloadStream(context.Background(), nil)
2644+
_require.NoError(err)
2645+
2646+
data, err := io.ReadAll(dResp.Body)
2647+
_require.NoError(err)
2648+
2649+
downloadedMD5Value := md5.Sum(data)
2650+
downloadedContentMD5 := downloadedMD5Value[:]
2651+
2652+
_require.EqualValues(downloadedContentMD5, contentMD5)
2653+
validatePropertiesSet(_require, fClient, *testcommon.BasicHeaders.ContentDisposition)
2654+
}
2655+
25752656
func (s *RecordedTestSuite) TestTinyFileUploadFile() {
25762657
_require := require.New(s.T())
25772658
testName := s.T().Name()
@@ -2732,6 +2813,176 @@ func (s *RecordedTestSuite) TestFileUploadSmallBuffer() {
27322813
_require.EqualValues(downloadedContentMD5, contentMD5)
27332814
}
27342815

2816+
func (s *RecordedTestSuite) TestFileUploadSmallBufferWithAccessConditions() {
2817+
_require := require.New(s.T())
2818+
testName := s.T().Name()
2819+
2820+
filesystemName := testcommon.GenerateFileSystemName(testName)
2821+
fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
2822+
_require.NoError(err)
2823+
defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
2824+
2825+
createFileOpts := &file.CreateOptions{
2826+
ProposedLeaseID: proposedLeaseIDs[0],
2827+
LeaseDuration: to.Ptr(int64(15)),
2828+
}
2829+
_, err = fsClient.Create(context.Background(), nil)
2830+
_require.NoError(err)
2831+
2832+
var fileSize int64 = 10 * 1024
2833+
fileName := testcommon.GenerateFileName(testName)
2834+
fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil)
2835+
_require.NoError(err)
2836+
2837+
resp, err := fClient.Create(context.Background(), createFileOpts)
2838+
_require.NoError(err)
2839+
_require.NotNil(resp)
2840+
2841+
_, content := testcommon.GenerateData(int(fileSize))
2842+
md5Value := md5.Sum(content)
2843+
contentMD5 := md5Value[:]
2844+
2845+
err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{
2846+
Concurrency: 5,
2847+
ChunkSize: 2 * 1024,
2848+
})
2849+
_require.NotNil(err)
2850+
2851+
err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{
2852+
Concurrency: 5,
2853+
ChunkSize: 2 * 1024,
2854+
AccessConditions: &file.AccessConditions{LeaseAccessConditions: &file.LeaseAccessConditions{
2855+
LeaseID: proposedLeaseIDs[0],
2856+
}},
2857+
})
2858+
_require.Nil(err)
2859+
2860+
gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{
2861+
AccessConditions: &file.AccessConditions{LeaseAccessConditions: &file.LeaseAccessConditions{
2862+
LeaseID: proposedLeaseIDs[0],
2863+
},
2864+
}})
2865+
_require.Nil(err)
2866+
_require.Equal(*gResp2.ContentLength, fileSize)
2867+
2868+
dResp, err := fClient.DownloadStream(context.Background(), nil)
2869+
_require.NoError(err)
2870+
2871+
data, err := io.ReadAll(dResp.Body)
2872+
_require.NoError(err)
2873+
2874+
downloadedMD5Value := md5.Sum(data)
2875+
downloadedContentMD5 := downloadedMD5Value[:]
2876+
2877+
_require.EqualValues(downloadedContentMD5, contentMD5)
2878+
}
2879+
2880+
func (s *RecordedTestSuite) TestFileUploadSmallBufferWithHTTPHeaders() {
2881+
_require := require.New(s.T())
2882+
testName := s.T().Name()
2883+
2884+
filesystemName := testcommon.GenerateFileSystemName(testName)
2885+
fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
2886+
_require.NoError(err)
2887+
defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
2888+
2889+
_, err = fsClient.Create(context.Background(), nil)
2890+
_require.NoError(err)
2891+
2892+
var fileSize int64 = 10 * 1024
2893+
fileName := testcommon.GenerateFileName(testName)
2894+
fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil)
2895+
_require.NoError(err)
2896+
2897+
resp, err := fClient.Create(context.Background(), nil)
2898+
_require.NoError(err)
2899+
_require.NotNil(resp)
2900+
2901+
_, content := testcommon.GenerateData(int(fileSize))
2902+
md5Value := md5.Sum(content)
2903+
contentMD5 := md5Value[:]
2904+
2905+
err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{
2906+
Concurrency: 5,
2907+
ChunkSize: 2 * 1024,
2908+
HTTPHeaders: &testcommon.BasicHeaders,
2909+
})
2910+
2911+
_require.NoError(err)
2912+
gResp2, err := fClient.GetProperties(context.Background(), nil)
2913+
_require.NoError(err)
2914+
_require.Equal(*gResp2.ContentLength, fileSize)
2915+
2916+
dResp, err := fClient.DownloadStream(context.Background(), nil)
2917+
_require.NoError(err)
2918+
2919+
data, err := io.ReadAll(dResp.Body)
2920+
_require.NoError(err)
2921+
2922+
downloadedMD5Value := md5.Sum(data)
2923+
downloadedContentMD5 := downloadedMD5Value[:]
2924+
2925+
_require.EqualValues(downloadedContentMD5, contentMD5)
2926+
validatePropertiesSet(_require, fClient, *testcommon.BasicHeaders.ContentDisposition)
2927+
}
2928+
2929+
func (s *RecordedTestSuite) TestDownloadDataContentMD5() {
2930+
_require := require.New(s.T())
2931+
testName := s.T().Name()
2932+
2933+
filesystemName := testcommon.GenerateFileSystemName(testName)
2934+
fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
2935+
_require.NoError(err)
2936+
defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
2937+
2938+
createFileOpts := &file.CreateOptions{
2939+
ProposedLeaseID: proposedLeaseIDs[0],
2940+
LeaseDuration: to.Ptr(int64(15)),
2941+
}
2942+
2943+
_, err = fsClient.Create(context.Background(), nil)
2944+
_require.NoError(err)
2945+
2946+
var fileSize int64 = 10 * 1024
2947+
fileName := testcommon.GenerateFileName(testName)
2948+
fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil)
2949+
_require.NoError(err)
2950+
resp, err := fClient.Create(context.Background(), createFileOpts)
2951+
_require.NoError(err)
2952+
_require.NotNil(resp)
2953+
2954+
_, content := testcommon.GenerateData(int(fileSize))
2955+
2956+
err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{
2957+
Concurrency: 5,
2958+
ChunkSize: 2 * 1024,
2959+
HTTPHeaders: &testcommon.BasicHeaders,
2960+
AccessConditions: &file.AccessConditions{LeaseAccessConditions: &file.LeaseAccessConditions{
2961+
LeaseID: proposedLeaseIDs[0],
2962+
},
2963+
}})
2964+
2965+
_require.NoError(err)
2966+
gResp2, err := fClient.GetProperties(context.Background(), nil)
2967+
_require.NoError(err)
2968+
_require.Equal(*gResp2.ContentLength, fileSize)
2969+
2970+
options := file.DownloadStreamOptions{
2971+
Range: &file.HTTPRange{
2972+
Count: 3,
2973+
Offset: 10,
2974+
},
2975+
RangeGetContentMD5: to.Ptr(true),
2976+
AccessConditions: &file.AccessConditions{LeaseAccessConditions: &file.LeaseAccessConditions{
2977+
LeaseID: proposedLeaseIDs[0],
2978+
},
2979+
}}
2980+
resp1, err := fClient.DownloadStream(context.Background(), &options)
2981+
_require.Nil(err)
2982+
mdf := md5.Sum(content[10:13])
2983+
_require.Equal(resp1.ContentMD5, mdf[:])
2984+
}
2985+
27352986
func (s *RecordedTestSuite) TestFileAppendAndFlushData() {
27362987
_require := require.New(s.T())
27372988
testName := s.T().Name()
@@ -4003,4 +4254,33 @@ func (s *RecordedTestSuite) TestFileGetPropertiesResponseCapture() {
40034254
_require.Equal("file", respFromCtxDir.Header.Get("x-ms-resource-type"))
40044255
}
40054256

4006-
// TODO tests all uploads/downloads with other opts
4257+
func (s *UnrecordedTestSuite) TestFileCreateDeleteUsingOAuth() {
4258+
_require := require.New(s.T())
4259+
testName := s.T().Name()
4260+
4261+
filesystemName := testcommon.GenerateFileSystemName(testName)
4262+
fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
4263+
_require.NoError(err)
4264+
defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
4265+
4266+
_, err = fsClient.Create(context.Background(), nil)
4267+
_require.NoError(err)
4268+
4269+
cred, err := testcommon.GetGenericTokenCredential()
4270+
_require.NoError(err)
4271+
4272+
accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake)
4273+
_require.Greater(len(accountName), 0)
4274+
4275+
fileName := testcommon.GenerateFileName(testName)
4276+
fileURL := "https://" + accountName + ".dfs.core.windows.net/" + filesystemName + "/" + fileName
4277+
4278+
fClient, err := file.NewClient(fileURL, cred, nil)
4279+
_require.NoError(err)
4280+
4281+
_, err = fClient.Create(context.Background(), nil)
4282+
_require.NoError(err)
4283+
4284+
_, err = fClient.GetProperties(context.Background(), nil)
4285+
_require.NoError(err)
4286+
}

0 commit comments

Comments
 (0)