Skip to content

Commit af68cae

Browse files
siminsavani-msftsiminsavani-msft
authored
PageBlob Client: Source Content Validation (Azure#20616)
* Adding tests for source content validation in pageblob apis * Fixing tests * Updating tests for UploadPagesFromURL * Update tests for UploadPages * Recorded tests and updated tests * removing spacing issue * Updating test * fixing linting issue
1 parent 526d841 commit af68cae

File tree

2 files changed

+216
-99
lines changed

2 files changed

+216
-99
lines changed

‎sdk/storage/azblob/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "go",
44
"TagPrefix": "go/storage/azblob",
5-
"Tag": "go/storage/azblob_37ff5d95f1"
5+
"Tag": "go/storage/azblob_ff27f140f0"
66
}

‎sdk/storage/azblob/pageblob/client_test.go‎

Lines changed: 215 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -226,86 +226,180 @@ func (s *PageBlobRecordedTestsSuite) TestPutGetPages() {
226226
// }
227227
//
228228

229-
// func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithMD5() {
230-
// _require := require.New(s.T())
231-
// testName := s.T().Name()
232-
// svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
233-
// if err != nil {
234-
// _require.Fail("Unable to fetch service client because " + err.Error())
235-
// }
236-
//
237-
// containerName := testcommon.GenerateContainerName(testName)
238-
// containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
239-
// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
240-
//
241-
// contentSize := 4 * 1024 * 1024 // 4MB
242-
// r, sourceData := getRandomDataAndReader(contentSize)
243-
// md5Value := md5.Sum(sourceData)
244-
// contentMD5 := md5Value[:]
245-
// ctx := ctx // Use default Background context
246-
// srcBlob := createNewPageBlobWithSize(_require, "srcblob", containerClient, int64(contentSize))
247-
// destBlob := createNewPageBlobWithSize(_require, "dstblob", containerClient, int64(contentSize))
248-
//
249-
// // Prepare source pbClient for copy.
250-
// offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
251-
// uploadPagesOptions := pageblob.UploadPagesOptions{Offset: to.Ptr(int64(offset)), Count: to.Ptr(int64(count)),}
252-
// _, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), &uploadPagesOptions)
253-
// _require.Nil(err)
254-
// // _require.Equal(uploadSrcResp1.RawResponse.StatusCode, 201)
255-
//
256-
// // Get source pbClient URL with SAS for UploadPagesFromURL.
257-
// credential, err := getGenericCredential(nil, testcommon.TestAccountDefault)
258-
// _require.Nil(err)
259-
// srcBlobParts, _ := NewBlobURLParts(srcBlob.URL())
260-
//
261-
// srcBlobParts.SAS, err = azblob.BlobSASSignatureValues{
262-
// Protocol: SASProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
263-
// ExpiryTime: time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration
264-
// ContainerName: srcBlobParts.ContainerName,
265-
// BlobName: srcBlobParts.BlobName,
266-
// Permissions: BlobSASPermissions{Read: true}.String(),
267-
// }.Sign(credential)
268-
// if err != nil {
269-
// _require.Error(err)
270-
// }
271-
//
272-
// srcBlobURLWithSAS := srcBlobParts.URL()
273-
//
274-
// // Upload page from URL with MD5.
275-
// uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
276-
// SourceContentMD5: contentMD5,
277-
// }
278-
// pResp1, err := destBlob.UploadPagesFromURL(ctx, srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
279-
// _require.Nil(err)
280-
// // _require.Equal(pResp1.RawResponse.StatusCode, 201)
281-
// _require.NotNil(pResp1.ETag)
282-
// _require.NotNil(pResp1.LastModified)
283-
// _require.NotNil(pResp1.ContentMD5)
284-
// _require.EqualValues(pResp1.ContentMD5, contentMD5)
285-
// _require.NotNil(pResp1.RequestID)
286-
// _require.NotNil(pResp1.Version)
287-
// _require.NotNil(pResp1.Date)
288-
// _require.Equal((*pResp1.Date).IsZero(), false)
289-
// _require.Equal(*pResp1.BlobSequenceNumber, int64(0))
290-
//
291-
// // Check data integrity through downloading.
292-
// downloadResp, err := destBlob.Download(ctx, nil)
293-
// _require.Nil(err)
294-
// destData, err := io.ReadAll(downloadResp.BodyReader(&blob.RetryReaderOptions{}))
295-
// _require.Nil(err)
296-
// _require.EqualValues(destData, sourceData)
297-
//
298-
// // Upload page from URL with bad MD5
299-
// _, badMD5 := getRandomDataAndReader(16)
300-
// badContentMD5 := badMD5[:]
301-
// uploadPagesFromURLOptions = pageblob.UploadPagesFromURLOptions{
302-
// SourceContentMD5: badContentMD5,
303-
// }
304-
// _, err = destBlob.UploadPagesFromURL(ctx, srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
305-
// _require.NotNil(err)
306-
//
307-
// testcommon.ValidateBlobErrorCode(_require, err, bloberror.MD5Mismatch)
308-
// }
229+
func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithMD5() {
230+
_require := require.New(s.T())
231+
testName := s.T().Name()
232+
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
233+
_require.NoError(err)
234+
235+
containerName := testcommon.GenerateContainerName(testName)
236+
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
237+
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
238+
239+
contentSize := 4 * 1024 * 1024 // 4MB
240+
r, sourceData := testcommon.GetDataAndReader(testName, contentSize)
241+
md5Value := md5.Sum(sourceData)
242+
contentMD5 := md5Value[:]
243+
srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize))
244+
destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize))
245+
246+
// Prepare source pbClient for copy.
247+
offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
248+
_, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil)
249+
_require.Nil(err)
250+
251+
// Get source pbClient URL with SAS for UploadPagesFromURL.
252+
credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
253+
_require.Nil(err)
254+
255+
srcBlobParts, _ := blob.ParseURL(srcBlob.URL())
256+
257+
srcBlobParts.SAS, err = sas.BlobSignatureValues{
258+
Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
259+
ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
260+
ContainerName: srcBlobParts.ContainerName,
261+
BlobName: srcBlobParts.BlobName,
262+
Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(),
263+
}.SignWithSharedKey(credential)
264+
_require.Nil(err)
265+
266+
srcBlobURLWithSAS := srcBlobParts.String()
267+
268+
// Upload page from URL with MD5.
269+
uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
270+
SourceContentValidation: blob.SourceContentValidationTypeMD5(contentMD5),
271+
}
272+
pResp1, err := destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
273+
_require.Nil(err)
274+
_require.EqualValues(pResp1.ContentMD5, contentMD5)
275+
276+
// Download blob to do data integrity check.
277+
downloadResp, err := destBlob.DownloadStream(context.Background(), nil)
278+
_require.Nil(err)
279+
destData, err := io.ReadAll(downloadResp.Body)
280+
_require.Nil(err)
281+
_require.EqualValues(destData, sourceData)
282+
283+
// Upload page from URL with bad MD5
284+
_, badMD5 := testcommon.GetDataAndReader(testName+"bad-md5", contentSize)
285+
badContentMD5 := badMD5[:]
286+
uploadPagesFromURLOptions = pageblob.UploadPagesFromURLOptions{
287+
SourceContentValidation: blob.SourceContentValidationTypeMD5(badContentMD5),
288+
}
289+
_, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
290+
_require.NotNil(err)
291+
testcommon.ValidateHTTPErrorCode(_require, err, 400) // Fails with 400 (Bad Request)
292+
}
293+
294+
func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithCRC64() {
295+
_require := require.New(s.T())
296+
testName := s.T().Name()
297+
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
298+
_require.NoError(err)
299+
300+
containerName := testcommon.GenerateContainerName(testName)
301+
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
302+
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
303+
304+
contentSize := 4 * 1024 * 1024 // 4MB
305+
r, sourceData := testcommon.GetDataAndReader(testName, contentSize)
306+
crc64Value := crc64.Checksum(sourceData, shared.CRC64Table)
307+
crc := make([]byte, 8)
308+
binary.LittleEndian.PutUint64(crc, crc64Value)
309+
srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize))
310+
destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize))
311+
312+
// Prepare source pbClient for copy.
313+
offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
314+
_, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil)
315+
_require.Nil(err)
316+
317+
// Get source pbClient URL with SAS for UploadPagesFromURL.
318+
credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
319+
_require.Nil(err)
320+
321+
srcBlobParts, _ := blob.ParseURL(srcBlob.URL())
322+
323+
srcBlobParts.SAS, err = sas.BlobSignatureValues{
324+
Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
325+
ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
326+
ContainerName: srcBlobParts.ContainerName,
327+
BlobName: srcBlobParts.BlobName,
328+
Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(),
329+
}.SignWithSharedKey(credential)
330+
_require.Nil(err)
331+
332+
srcBlobURLWithSAS := srcBlobParts.String()
333+
334+
// Upload page from URL with CRC64.
335+
uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
336+
SourceContentValidation: blob.SourceContentValidationTypeCRC64(crc),
337+
}
338+
_, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
339+
_require.Nil(err)
340+
// TODO: This does not work... ContentCRC64 is not returned. Fix this later.
341+
// _require.EqualValues(pResp1.ContentCRC64, crc)
342+
343+
// Download blob to do data integrity check.
344+
downloadResp, err := destBlob.DownloadStream(context.Background(), nil)
345+
_require.Nil(err)
346+
destData, err := io.ReadAll(downloadResp.Body)
347+
_require.Nil(err)
348+
_require.EqualValues(destData, sourceData)
349+
}
350+
351+
func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithCRC64Negative() {
352+
s.T().Skip("This test is skipped because of issues in the service.")
353+
354+
_require := require.New(s.T())
355+
testName := s.T().Name()
356+
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
357+
_require.NoError(err)
358+
359+
containerName := testcommon.GenerateContainerName(testName)
360+
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
361+
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
362+
363+
contentSize := 4 * 1024 * 1024 // 4MB
364+
r, sourceData := testcommon.GetDataAndReader(testName, contentSize)
365+
crc64Value := crc64.Checksum(sourceData, shared.CRC64Table)
366+
crc := make([]byte, 8)
367+
binary.LittleEndian.PutUint64(crc, crc64Value)
368+
srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize))
369+
destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize))
370+
371+
// Prepare source pbClient for copy.
372+
offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
373+
_, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil)
374+
_require.Nil(err)
375+
376+
// Get source pbClient URL with SAS for UploadPagesFromURL.
377+
credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
378+
_require.Nil(err)
379+
380+
srcBlobParts, _ := blob.ParseURL(srcBlob.URL())
381+
382+
srcBlobParts.SAS, err = sas.BlobSignatureValues{
383+
Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
384+
ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
385+
ContainerName: srcBlobParts.ContainerName,
386+
BlobName: srcBlobParts.BlobName,
387+
Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(),
388+
}.SignWithSharedKey(credential)
389+
_require.Nil(err)
390+
391+
srcBlobURLWithSAS := srcBlobParts.String()
392+
393+
// Upload page from URL with bad CRC64
394+
badCRC64 := rand.Uint64()
395+
badcrc := make([]byte, 8)
396+
binary.LittleEndian.PutUint64(badcrc, badCRC64)
397+
uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
398+
SourceContentValidation: blob.SourceContentValidationTypeCRC64(badcrc),
399+
}
400+
_, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
401+
_require.NotNil(err) // TODO: UploadPagesFromURL should fail, but is currently not working due to service issue.
402+
}
309403

310404
func (s *PageBlobUnrecordedTestsSuite) TestClearDiffPages() {
311405
_require := require.New(s.T())
@@ -500,14 +594,50 @@ func (s *PageBlobRecordedTestsSuite) TestPageSequenceNumbers() {
500594
_require.Nil(err)
501595
}
502596

597+
func (s *PageBlobRecordedTestsSuite) TestPutPagesWithCRC64() {
598+
_require := require.New(s.T())
599+
testName := s.T().Name()
600+
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
601+
_require.NoError(err)
602+
603+
containerName := testcommon.GenerateContainerName(testName)
604+
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
605+
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
606+
607+
blobName := testcommon.GenerateBlobName(testName)
608+
pbClient := createNewPageBlob(context.Background(), _require, blobName, containerClient)
609+
610+
// put page with valid auto-generated CRC64
611+
contentSize := 1024
612+
readerToBody, body := testcommon.GetDataAndReader(testName, contentSize)
613+
offset, _, count := int64(0), int64(0)+int64(contentSize-1), int64(contentSize)
614+
crc64Value := crc64.Checksum(body, shared.CRC64Table)
615+
_ = body
616+
617+
putResp, err := pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{
618+
TransactionalValidation: blob.TransferValidationTypeCRC64(crc64Value),
619+
})
620+
_require.Nil(err)
621+
_require.NotNil(putResp.ContentCRC64)
622+
_require.EqualValues(binary.LittleEndian.Uint64(putResp.ContentCRC64), crc64Value)
623+
624+
// put page with bad CRC64
625+
readerToBody, _ = testcommon.GetDataAndReader(testName, 1024)
626+
badCRC64 := rand.Uint64()
627+
putResp, err = pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{
628+
TransactionalValidation: blob.TransferValidationTypeCRC64(badCRC64),
629+
})
630+
_require.NotNil(err)
631+
632+
// testcommon.ValidateBlobErrorCode(_require, err, bloberror.CRC64Mismatch)
633+
}
634+
503635
// nolint
504-
func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
636+
func (s *PageBlobRecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
505637
_require := require.New(s.T())
506638
testName := s.T().Name()
507639
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
508-
if err != nil {
509-
_require.Fail("Unable to fetch service client because " + err.Error())
510-
}
640+
_require.NoError(err)
511641

512642
containerName := testcommon.GenerateContainerName(testName)
513643
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
@@ -527,7 +657,6 @@ func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
527657
TransactionalValidation: blob.TransferValidationTypeComputeCRC64(),
528658
})
529659
_require.Nil(err)
530-
// _require.Equal(putResp.RawResponse.StatusCode, 201)
531660
_require.NotNil(putResp.LastModified)
532661
_require.Equal((*putResp.LastModified).IsZero(), false)
533662
_require.NotNil(putResp.ETag)
@@ -538,26 +667,14 @@ func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
538667
_require.NotNil(*putResp.Version)
539668
_require.NotNil(putResp.Date)
540669
_require.Equal((*putResp.Date).IsZero(), false)
541-
542-
// put page with bad MD5
543-
readerToBody, _ = testcommon.GetDataAndReader(testName, 1024)
544-
badCRC64 := rand.Uint64()
545-
putResp, err = pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{
546-
TransactionalValidation: blob.TransferValidationTypeCRC64(badCRC64),
547-
})
548-
_require.NotNil(err)
549-
550-
testcommon.ValidateBlobErrorCode(_require, err, bloberror.CRC64Mismatch)
551670
}
552671

553672
// nolint
554673
func (s *PageBlobRecordedTestsSuite) TestPutPagesWithMD5() {
555674
_require := require.New(s.T())
556675
testName := s.T().Name()
557676
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
558-
if err != nil {
559-
_require.Fail("Unable to fetch service client because " + err.Error())
560-
}
677+
_require.NoError(err)
561678

562679
containerName := testcommon.GenerateContainerName(testName)
563680
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)

0 commit comments

Comments
 (0)