Skip to content

Commit 2f8531f

Browse files
jalauzon-msftjalauzon-msft
authored
[Storage] Rename checkpoint constants to prepare for job plan files (Azure#38353)
1 parent 0b85496 commit 2f8531f

15 files changed

+217
-207
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ internal static async Task<BlockBlobStorageResourceOptions> GetBlockBlobResource
421421
// Get AccessTier
422422
if (!isSource)
423423
{
424-
int startIndex = DataMovementConstants.PlanFile.DstBlobBlockBlobTierIndex;
424+
int startIndex = DataMovementConstants.JobPartPlanFile.DstBlobBlockBlobTierIndex;
425425
JobPartPlanBlockBlobTier accessTier = (JobPartPlanBlockBlobTier)await checkpointer.GetByteValue(
426426
transferId,
427427
startIndex,
@@ -446,7 +446,7 @@ internal static async Task<PageBlobStorageResourceOptions> GetPageBlobResourceOp
446446
if (!isSource)
447447
{
448448
// Get AccessTier
449-
int startIndex = DataMovementConstants.PlanFile.DstBlobPageBlobTierIndex;
449+
int startIndex = DataMovementConstants.JobPartPlanFile.DstBlobPageBlobTierIndex;
450450
JobPartPlanPageBlobTier accessTier = (JobPartPlanPageBlobTier)await checkpointer.GetByteValue(
451451
transferId,
452452
startIndex,
@@ -469,24 +469,24 @@ internal static async Task<BlobStorageResourceOptions> GetBlobResourceOptionsAsy
469469
if (!isSource)
470470
{
471471
// Get Metadata
472-
int metadataIndex = DataMovementConstants.PlanFile.DstBlobMetadataLengthIndex;
473-
int metadataReadLength = DataMovementConstants.PlanFile.DstBlobTagsLengthIndex - metadataIndex;
472+
int metadataIndex = DataMovementConstants.JobPartPlanFile.DstBlobMetadataLengthIndex;
473+
int metadataReadLength = DataMovementConstants.JobPartPlanFile.DstBlobTagsLengthIndex - metadataIndex;
474474
string metadata = await checkpointer.GetHeaderUShortValue(
475475
transferId,
476476
metadataIndex,
477477
metadataReadLength,
478-
DataMovementConstants.PlanFile.MetadataStrNumBytes,
478+
DataMovementConstants.JobPartPlanFile.MetadataStrNumBytes,
479479
cancellationToken).ConfigureAwait(false);
480480
options.Metadata = metadata.ToDictionary(nameof(metadata));
481481

482482
// Get blob tags
483-
int tagsIndex = DataMovementConstants.PlanFile.DstBlobTagsLengthIndex;
484-
int tagsReadLength = DataMovementConstants.PlanFile.DstBlobIsSourceEncrypted - tagsIndex;
483+
int tagsIndex = DataMovementConstants.JobPartPlanFile.DstBlobTagsLengthIndex;
484+
int tagsReadLength = DataMovementConstants.JobPartPlanFile.DstBlobIsSourceEncrypted - tagsIndex;
485485
string tags = await checkpointer.GetHeaderLongValue(
486486
transferId,
487487
tagsIndex,
488488
tagsReadLength,
489-
DataMovementConstants.PlanFile.BlobTagsStrNumBytes,
489+
DataMovementConstants.JobPartPlanFile.BlobTagsStrNumBytes,
490490
cancellationToken).ConfigureAwait(false);
491491
options.Tags = tags.ToDictionary(nameof(tags));
492492
}

‎sdk/storage/Azure.Storage.DataMovement/src/CheckpointerExtensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ internal static async Task<bool> IsResumableAsync(
1515
{
1616
DataTransferStatus jobStatus = (DataTransferStatus)await checkpointer.GetByteValue(
1717
transferId,
18-
DataMovementConstants.PlanFile.AtomicJobStatusIndex,
18+
DataMovementConstants.JobPartPlanFile.AtomicJobStatusIndex,
1919
cancellationToken).ConfigureAwait(false);
2020

2121
// Transfers marked as fully completed are not resumable

‎sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,25 @@ internal static class Log
5353
internal const string Closing = "Closing log ";
5454
}
5555

56+
internal const int OneByte = 1;
57+
internal const int LongSizeInBytes = 8;
58+
internal const int UShortSizeInBytes = 2;
59+
5660
/// <summary>
57-
/// Constants used for plan job transfer files
61+
/// Constants used for job plan files.
5862
/// </summary>
59-
internal static class PlanFile
63+
internal static class JobPlanFile
64+
{
65+
internal const string SchemaVersion_b1 = "b1";
66+
internal const string SchemaVersion = SchemaVersion_b1;
67+
68+
internal const string FileExtension = ".ndm";
69+
}
70+
71+
/// <summary>
72+
/// Constants used for job part plan files.
73+
/// </summary>
74+
internal static class JobPartPlanFile
6075
{
6176
internal const string SchemaVersion_b1 = "b1";
6277
internal const string SchemaVersion_b2 = "b2";
@@ -68,11 +83,6 @@ internal static class PlanFile
6883
internal const int JobPartLength = 5;
6984
internal const int IdSize = 36; // Size of a guid with hyphens
7085
internal const int CustomHeaderMaxBytes = 256;
71-
internal const int Padding = 8;
72-
73-
internal const int OneByte = 1;
74-
internal const int LongSizeInBytes = 8;
75-
internal const int UShortSizeInBytes = 2;
7686

7787
// UTF-8 encoding, so 2 bytes per char
7888
internal const int VersionStrLength = 2;

‎sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementExtensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ internal static JobPartPlanHeader ToJobPartPlanHeader(this JobPartInternal jobPa
225225
string destinationPath = destinationUriBuilder.Uri.AbsoluteUri;
226226

227227
return new JobPartPlanHeader(
228-
version: DataMovementConstants.PlanFile.SchemaVersion,
228+
version: DataMovementConstants.JobPartPlanFile.SchemaVersion,
229229
startTime: DateTimeOffset.UtcNow, // TODO: update to job start time
230230
transferId: jobPart._dataTransfer.Id,
231231
partNumber: (uint)jobPart.PartNumber,

‎sdk/storage/Azure.Storage.DataMovement/src/Shared/Errors.DataMovement.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public static ArgumentException InvalidJobPartNumberFileName(string fileName)
6565
=> new ArgumentException($"Invalid Job Part Plan File: The following Job Part Plan file contains an invalid Job Part Number, could not convert to a integer: {fileName}");
6666

6767
public static ArgumentException InvalidSchemaVersionFileName(string schemaVersion)
68-
=> new ArgumentException($"Invalid Job Part Plan File: Job Part Schema version: {schemaVersion} does not match the Schema Version supported by the package: {DataMovementConstants.PlanFile.SchemaVersion}. Please consider altering the package version that supports the respective version.");
68+
=> new ArgumentException($"Invalid Job Part Plan File: Job Part Schema version: {schemaVersion} does not match the Schema Version supported by the package: {DataMovementConstants.JobPartPlanFile.SchemaVersion}. Please consider altering the package version that supports the respective version.");
6969

7070
public static ArgumentException InvalidPlanFileElement(string elementName, int expectedSize, int actualSize)
7171
=> throw new ArgumentException($"Invalid Job Part Plan File: Attempt to set element, \"{elementName}\" failed.\n Expected size: {expectedSize}\n Actual Size: {actualSize}");

‎sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanDestinationBlob.cs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ public JobPartPlanDestinationBlob(
176176
{
177177
BlobType = blobType;
178178
NoGuessMimeType = noGuessMimeType;
179-
if (contentType.Length <= DataMovementConstants.PlanFile.HeaderValueMaxLength)
179+
if (contentType.Length <= DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength)
180180
{
181181
ContentType = contentType;
182182
ContentTypeLength = (ushort) contentType.Length;
@@ -185,10 +185,10 @@ public JobPartPlanDestinationBlob(
185185
{
186186
throw Errors.InvalidPlanFileElement(
187187
elementName: nameof(ContentType),
188-
expectedSize: DataMovementConstants.PlanFile.HeaderValueMaxLength,
188+
expectedSize: DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength,
189189
actualSize: contentType.Length);
190190
}
191-
if (contentEncoding.Length <= DataMovementConstants.PlanFile.HeaderValueMaxLength)
191+
if (contentEncoding.Length <= DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength)
192192
{
193193
ContentEncoding = contentEncoding;
194194
ContentEncodingLength = (ushort) contentEncoding.Length;
@@ -197,10 +197,10 @@ public JobPartPlanDestinationBlob(
197197
{
198198
throw Errors.InvalidPlanFileElement(
199199
elementName: nameof(ContentEncoding),
200-
expectedSize: DataMovementConstants.PlanFile.HeaderValueMaxLength,
200+
expectedSize: DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength,
201201
actualSize: contentEncoding.Length);
202202
}
203-
if (contentLanguage.Length <= DataMovementConstants.PlanFile.HeaderValueMaxLength)
203+
if (contentLanguage.Length <= DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength)
204204
{
205205
ContentLanguage = contentLanguage;
206206
ContentLanguageLength = (ushort) contentLanguage.Length;
@@ -209,10 +209,10 @@ public JobPartPlanDestinationBlob(
209209
{
210210
throw Errors.InvalidPlanFileElement(
211211
elementName: nameof(ContentLanguage),
212-
expectedSize: DataMovementConstants.PlanFile.HeaderValueMaxLength,
212+
expectedSize: DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength,
213213
actualSize: contentLanguage.Length);
214214
}
215-
if (contentDisposition.Length <= DataMovementConstants.PlanFile.HeaderValueMaxLength)
215+
if (contentDisposition.Length <= DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength)
216216
{
217217
ContentDisposition = contentDisposition;
218218
ContentDispositionLength = (ushort) contentDisposition.Length;
@@ -221,10 +221,10 @@ public JobPartPlanDestinationBlob(
221221
{
222222
throw Errors.InvalidPlanFileElement(
223223
elementName: nameof(ContentDisposition),
224-
expectedSize: DataMovementConstants.PlanFile.HeaderValueMaxLength,
224+
expectedSize: DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength,
225225
actualSize: contentDisposition.Length);
226226
}
227-
if (cacheControl.Length <= DataMovementConstants.PlanFile.HeaderValueMaxLength)
227+
if (cacheControl.Length <= DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength)
228228
{
229229
CacheControl = cacheControl;
230230
CacheControlLength = (ushort) cacheControl.Length;
@@ -233,14 +233,14 @@ public JobPartPlanDestinationBlob(
233233
{
234234
throw Errors.InvalidPlanFileElement(
235235
elementName: nameof(CacheControl),
236-
expectedSize: DataMovementConstants.PlanFile.HeaderValueMaxLength,
236+
expectedSize: DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength,
237237
actualSize: cacheControl.Length);
238238
}
239239
BlockBlobTier = blockBlobTier;
240240
PageBlobTier = pageBlobTier;
241241
PutMd5 = putMd5;
242242
string metadataConvert = metadata.DictionaryToString();
243-
if (metadataConvert.Length <= DataMovementConstants.PlanFile.MetadataStrMaxLength)
243+
if (metadataConvert.Length <= DataMovementConstants.JobPartPlanFile.MetadataStrMaxLength)
244244
{
245245
Metadata = metadataConvert;
246246
MetadataLength = (ushort) metadataConvert.Length;
@@ -249,11 +249,11 @@ public JobPartPlanDestinationBlob(
249249
{
250250
throw Errors.InvalidPlanFileElement(
251251
elementName: nameof(Metadata),
252-
expectedSize: DataMovementConstants.PlanFile.MetadataStrMaxLength,
252+
expectedSize: DataMovementConstants.JobPartPlanFile.MetadataStrMaxLength,
253253
actualSize: metadataConvert.Length);
254254
}
255255
string blobTagsConvert = blobTags.DictionaryToString();
256-
if (blobTagsConvert.Length <= DataMovementConstants.PlanFile.BlobTagsStrMaxLength)
256+
if (blobTagsConvert.Length <= DataMovementConstants.JobPartPlanFile.BlobTagsStrMaxLength)
257257
{
258258
BlobTags = blobTagsConvert;
259259
BlobTagsLength = blobTagsConvert.Length;
@@ -262,11 +262,11 @@ public JobPartPlanDestinationBlob(
262262
{
263263
throw Errors.InvalidPlanFileElement(
264264
elementName: nameof(blobTags),
265-
expectedSize: DataMovementConstants.PlanFile.BlobTagsStrMaxLength,
265+
expectedSize: DataMovementConstants.JobPartPlanFile.BlobTagsStrMaxLength,
266266
actualSize: blobTagsConvert.Length);
267267
}
268268
IsSourceEncrypted = isSourceEncrypted;
269-
if (cpkScopeInfo.Length <= DataMovementConstants.PlanFile.HeaderValueMaxLength)
269+
if (cpkScopeInfo.Length <= DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength)
270270
{
271271
CpkScopeInfo = cpkScopeInfo;
272272
CpkScopeInfoLength = (ushort) cpkScopeInfo.Length;
@@ -275,7 +275,7 @@ public JobPartPlanDestinationBlob(
275275
{
276276
throw Errors.InvalidPlanFileElement(
277277
elementName: nameof(CpkScopeInfo),
278-
expectedSize: DataMovementConstants.PlanFile.HeaderValueMaxLength,
278+
expectedSize: DataMovementConstants.JobPartPlanFile.HeaderValueMaxLength,
279279
actualSize: cpkScopeInfo.Length);
280280
}
281281
BlockSize = blockSize;

‎sdk/storage/Azure.Storage.DataMovement/src/Shared/JobPlan/JobPartPlanFileName.cs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public JobPartPlanFileName(
6666
string checkpointerPath,
6767
string id,
6868
int jobPartNumber,
69-
string schemaVersion = DataMovementConstants.PlanFile.SchemaVersion)
69+
string schemaVersion = DataMovementConstants.JobPartPlanFile.SchemaVersion)
7070
{
7171
Argument.AssertNotNullOrEmpty(checkpointerPath, nameof(checkpointerPath));
7272
Argument.AssertNotNullOrEmpty(id, nameof(id));
@@ -76,7 +76,7 @@ public JobPartPlanFileName(
7676
JobPartNumber = jobPartNumber;
7777
SchemaVersion = schemaVersion;
7878

79-
string fileName = $"{Id}--{JobPartNumber.ToString("D5", NumberFormatInfo.CurrentInfo)}{DataMovementConstants.PlanFile.FileExtension}{SchemaVersion}";
79+
string fileName = $"{Id}--{JobPartNumber.ToString("D5", NumberFormatInfo.CurrentInfo)}{DataMovementConstants.JobPartPlanFile.FileExtension}{SchemaVersion}";
8080
FullPath = Path.Combine(PrefixPath, fileName);
8181
}
8282

@@ -97,23 +97,23 @@ public JobPartPlanFileName(string fullPath)
9797
// {transferid}--{jobpartNumber}.steV{schemaVersion}
9898

9999
// Check for valid Transfer Id
100-
int endTransferIdIndex = fileName.IndexOf(DataMovementConstants.PlanFile.JobPlanFileNameDelimiter, StringComparison.InvariantCultureIgnoreCase);
101-
if (endTransferIdIndex != DataMovementConstants.PlanFile.IdSize)
100+
int endTransferIdIndex = fileName.IndexOf(DataMovementConstants.JobPartPlanFile.JobPlanFileNameDelimiter, StringComparison.InvariantCultureIgnoreCase);
101+
if (endTransferIdIndex != DataMovementConstants.JobPartPlanFile.IdSize)
102102
{
103103
throw Errors.InvalidTransferIdFileName(fullPath);
104104
}
105105
Id = fileName.Substring(0, endTransferIdIndex);
106106

107107
// Check for valid transfer part number
108-
int partStartIndex = endTransferIdIndex + DataMovementConstants.PlanFile.JobPlanFileNameDelimiter.Length;
108+
int partStartIndex = endTransferIdIndex + DataMovementConstants.JobPartPlanFile.JobPlanFileNameDelimiter.Length;
109109
int endPartIndex = fileName.Length;
110110

111-
if (endPartIndex - partStartIndex != DataMovementConstants.PlanFile.JobPartLength)
111+
if (endPartIndex - partStartIndex != DataMovementConstants.JobPartPlanFile.JobPartLength)
112112
{
113113
throw Errors.InvalidJobPartFileName(fullPath);
114114
}
115115
if (!int.TryParse(
116-
fileName.Substring(partStartIndex, DataMovementConstants.PlanFile.JobPartLength),
116+
fileName.Substring(partStartIndex, DataMovementConstants.JobPartPlanFile.JobPartLength),
117117
NumberStyles.Number,
118118
CultureInfo.InvariantCulture,
119119
out int jobPartNumber))
@@ -122,12 +122,12 @@ public JobPartPlanFileName(string fullPath)
122122
}
123123
JobPartNumber = jobPartNumber;
124124

125-
string fullExtension = string.Concat(DataMovementConstants.PlanFile.FileExtension, DataMovementConstants.PlanFile.SchemaVersion);
125+
string fullExtension = string.Concat(DataMovementConstants.JobPartPlanFile.FileExtension, DataMovementConstants.JobPartPlanFile.SchemaVersion);
126126
if (!fullExtension.Equals(extension))
127127
{
128128
throw Errors.InvalidSchemaVersionFileName(extension);
129129
}
130-
SchemaVersion = DataMovementConstants.PlanFile.SchemaVersion;
130+
SchemaVersion = DataMovementConstants.JobPartPlanFile.SchemaVersion;
131131

132132
FullPath = fullPath;
133133
}

0 commit comments

Comments
 (0)