Skip to content

Commit 1a50a0d

Browse files
g2vinayg2vinay
authored
Add Storage File Share , File Datalake Perf Tests (Azure#17883)
1 parent 057f12e commit 1a50a0d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1987
-121
lines changed

‎common/perf-test-core/src/main/java/com/azure/perf/test/core/NullInputStream.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.azure.perf.test.core;
5+
6+
import java.io.IOException;
7+
import java.io.InputStream;
8+
9+
/**
10+
* The Null Input Stream class, applicable to JDK 8.
11+
*/
12+
public class NullInputStream extends InputStream {
13+
14+
@Override
15+
public int read() throws IOException {
16+
return -1;
17+
}
18+
}

‎common/perf-test-core/src/main/java/com/azure/perf/test/core/NullOutputStream.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.azure.perf.test.core;
5+
6+
import java.io.OutputStream;
7+
8+
/**
9+
* The Null Output Stream class, applicable to JDK 8.
10+
*/
11+
public class NullOutputStream extends OutputStream {
12+
@Override
13+
public void write(int b) { }
14+
15+
@Override
16+
public void write(byte[] b) { }
17+
18+
@Override
19+
public void write(byte[] b, int off, int len) { }
20+
}

‎common/perf-test-core/src/main/java/com/azure/perf/test/core/RepeatingInputStream.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.azure.perf.test.core;
5+
6+
import java.io.InputStream;
7+
import java.util.Random;
8+
9+
/**
10+
* Represents a repeating input stream with mark support enabled.
11+
*/
12+
public class RepeatingInputStream extends InputStream {
13+
private static final int RANDOM_BYTES_LENGTH = 1024 * 1024; // 1MB
14+
private static final byte[] RANDOM_BYTES;
15+
private final int size;
16+
17+
private int mark = 0;
18+
private int readLimit = Integer.MAX_VALUE;
19+
private int pos = 0;
20+
21+
static {
22+
Random random = new Random(0);
23+
RANDOM_BYTES = new byte[RANDOM_BYTES_LENGTH];
24+
random.nextBytes(RANDOM_BYTES);
25+
}
26+
27+
/**
28+
* Creates an Instance of the repeating input stream.
29+
* @param size the size of the stream.
30+
*/
31+
public RepeatingInputStream(int size) {
32+
this.size = size;
33+
}
34+
35+
@Override
36+
public synchronized int read() {
37+
return (pos < size) ? (RANDOM_BYTES[pos++ % RANDOM_BYTES_LENGTH] & 0xFF) : -1;
38+
}
39+
40+
@Override
41+
public synchronized int read(byte[] b) {
42+
return read(b, 0, b.length);
43+
}
44+
45+
@Override
46+
public synchronized int read(byte[] b, int off, int len) {
47+
if (pos >= size || pos >= readLimit) {
48+
return -1;
49+
}
50+
51+
int readCount = Math.min(len, RANDOM_BYTES_LENGTH);
52+
System.arraycopy(RANDOM_BYTES, 0, b, off, readCount);
53+
pos += readCount;
54+
55+
return readCount;
56+
}
57+
58+
@Override
59+
public synchronized void mark(int readLimit) {
60+
this.readLimit = readLimit;
61+
this.mark = this.pos;
62+
}
63+
64+
@Override
65+
public boolean markSupported() {
66+
return true;
67+
}
68+
69+
@Override
70+
public synchronized void reset() {
71+
this.pos = this.mark;
72+
}
73+
}
74+

‎common/perf-test-core/src/main/java/com/azure/perf/test/core/TestDataCreationHelper.java‎

Lines changed: 35 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import java.io.IOException;
99
import java.io.InputStream;
10+
import java.io.FileOutputStream;
1011
import java.io.OutputStream;
1112
import java.nio.ByteBuffer;
1213
import java.util.Random;
@@ -93,6 +94,28 @@ public static void writeBytesToOutputStream(OutputStream outputStream, long size
9394
outputStream.write(RANDOM_BYTES, 0, remainder);
9495
}
9596

97+
/**
98+
* Writes the data from InputStream into the OutputStream.
99+
*
100+
* @param inputStream stream to read from.
101+
* @param outputStream stream to write into.
102+
* @param bufferSize number of bytes to read in a single read.
103+
* @throws IOException If an IO error occurs.
104+
* @return the number of bytes transferred.
105+
*/
106+
public static long copyStream(InputStream inputStream, OutputStream outputStream, int bufferSize) throws IOException {
107+
long transferred = 0;
108+
byte[] buffer = new byte[bufferSize];
109+
int read;
110+
while ((read = inputStream.read(buffer, 0, bufferSize)) >= 0) {
111+
outputStream.write(buffer, 0, read);
112+
transferred += read;
113+
114+
}
115+
return transferred;
116+
}
117+
118+
96119
/**
97120
* Generate random string of given {@code targetLength length}. The string will only have lower case alphabets.
98121
*
@@ -111,52 +134,17 @@ public static String generateRandomString(int targetLength) {
111134
return generatedString;
112135
}
113136

114-
private static final class RepeatingInputStream extends InputStream {
115-
private final int size;
116-
117-
private int mark = 0;
118-
private int pos = 0;
119-
120-
private RepeatingInputStream(int size) {
121-
this.size = size;
122-
}
123-
124-
@Override
125-
public synchronized int read() {
126-
return (pos < size) ? (RANDOM_BYTES[pos++ % RANDOM_BYTES_LENGTH] & 0xFF) : -1;
127-
}
128-
129-
@Override
130-
public synchronized int read(byte[] b) {
131-
return read(b, 0, b.length);
132-
}
133-
134-
@Override
135-
public synchronized int read(byte[] b, int off, int len) {
136-
if (pos >= size) {
137-
return -1;
138-
}
139-
140-
int readCount = Math.min(len, RANDOM_BYTES_LENGTH);
141-
System.arraycopy(RANDOM_BYTES, 0, b, off, len);
142-
pos += readCount;
143-
144-
return readCount;
145-
}
146-
147-
@Override
148-
public synchronized void reset() {
149-
this.pos = this.mark;
150-
}
151-
152-
@Override
153-
public synchronized void mark(int readlimit) {
154-
this.mark = readlimit;
155-
}
156-
157-
@Override
158-
public boolean markSupported() {
159-
return true;
160-
}
137+
/**
138+
* Writes contents of the specified size to the specified file path.
139+
*
140+
* @param filePath the path of the file to write to contents to
141+
* @param size the size of the contents to write to the file.
142+
* @param bufferSize the size of the buffer to use to write to the file.
143+
* @throws IOException when an error occurs when writing to the file.
144+
*/
145+
public static void writeToFile(String filePath, long size, int bufferSize) throws IOException {
146+
InputStream inputStream = createRandomInputStream(size);
147+
OutputStream outputStream = new FileOutputStream(filePath);
148+
copyStream(inputStream, outputStream, bufferSize);
161149
}
162150
}

‎eng/versioning/version_data.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,10 @@ com.microsoft.azure:azure-keyvault-extensions;1.2.4;1.3.0-beta.1
3535
com.microsoft.azure:azure-keyvault-test;1.2.3;1.2.4
3636
com.microsoft.azure:azure-keyvault-webkey;1.2.4;1.3.0-beta.1
3737
com.microsoft.azure:azure-servicebus;3.6.3;3.7.0-beta.1
38+
com.microsoft.azure:azure-storage;8.6.5;8.6.5
3839
com.microsoft.azure:azure-storage-blob;11.0.2;11.0.2
3940
com.microsoft.azure.msi_auth_token_provider:azure-authentication-msi-token-provider;1.1.0-beta.1;1.1.0-beta.1
4041
com.microsoft.azure:azure-eventgrid;1.4.0-beta.1;1.4.0-beta.1
4142
com.microsoft.azure:azure-loganalytics;1.0.0-beta-2;1.0.0-beta.2
4243
com.microsoft.azure:azure-media;1.0.0-beta.1;1.0.0-beta.1
44+
com.microsoft.azure:microsoft-azure-storage-perf;1.0.0-beta.1;1.0.0-beta.1

‎sdk/appconfiguration/azure-data-appconfiguration/CHANGELOG.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -72,14 +72,6 @@
7272
### Breaking changes
7373
- SettingSelector takes a filter instead of taking a list of strings. Supported `SettingSelector` literal special character and wild card functions.
7474

75-
## 1.0.0-beta.7 (2019-11-26)
76-
- Added support for Azure Activity Directory authentication.
77-
- Added service API version support
78-
79-
### Breaking Changes
80-
- Removed clearReadOnly API, updated setReadOnly API to support setting and clearing read only based on the flag passed.
81-
- Removed Range class, SettingSelector no longer supports Range.
82-
8375
## 1.0.0-preview.6 (2019-10-31)
8476
- Renamed addSetting, getSetting, deleteSetting, setSetting, listSettings, listSettingRevisions to
8577
addConfigurationSetting, getConfigurationSetting, deleteConfigurationSetting, setConfigurationSetting,
@@ -157,3 +149,11 @@ and
157149
demonstrate the new API.
158150

159151
- Initial release. Please see the README and wiki for information on the new design.
152+
## 1.0.0-beta.7 (2019-11-26)
153+
- Added support for Azure Activity Directory authentication.
154+
- Added service API version support
155+
156+
### Breaking Changes
157+
- Removed clearReadOnly API, updated setReadOnly API to support setting and clearing read only based on the flag passed.
158+
- Removed Range class, SettingSelector no longer supports Range.
159+

‎sdk/storage/azure-storage-perf/pom.xml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,20 @@
2929
<artifactId>azure-storage-blob</artifactId>
3030
<version>12.12.0-beta.1</version> <!-- {x-version-update;com.azure:azure-storage-blob;current} -->
3131
</dependency>
32+
33+
<dependency>
34+
<groupId>com.azure</groupId>
35+
<artifactId>azure-storage-file-datalake</artifactId>
36+
<version>12.6.0-beta.1</version> <!-- {x-version-update;com.azure:azure-storage-file-datalake;current} -->
37+
</dependency>
38+
39+
<dependency>
40+
<groupId>com.azure</groupId>
41+
<artifactId>azure-storage-file-share</artifactId>
42+
<version>12.10.0-beta.1</version> <!-- {x-version-update;com.azure:azure-storage-file-share;current} -->
43+
</dependency>
44+
45+
3246
<dependency>
3347
<groupId>com.azure</groupId>
3448
<artifactId>perf-test-core</artifactId>
@@ -52,7 +66,7 @@
5266
<archive>
5367
<manifest>
5468
<mainClass>
55-
com.azure.storage.blob.perf.App
69+
com.azure.storage.App
5670
</mainClass>
5771
</manifest>
5872
</archive>

‎sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/App.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.azure.storage;
5+
6+
import com.azure.perf.test.core.PerfStressProgram;
7+
import com.azure.storage.blob.perf.DownloadBlobTest;
8+
import com.azure.storage.blob.perf.ListBlobsTest;
9+
import com.azure.storage.blob.perf.UploadBlobTest;
10+
import com.azure.storage.blob.perf.UploadBlockBlobTest;
11+
import com.azure.storage.blob.perf.UploadFromFileTest;
12+
import com.azure.storage.blob.perf.UploadOutputStreamTest;
13+
import com.azure.storage.file.datalake.perf.AppendFileDatalakeTest;
14+
import com.azure.storage.file.datalake.perf.ReadFileDatalakeTest;
15+
import com.azure.storage.file.datalake.perf.UploadFileDatalakeTest;
16+
import com.azure.storage.file.datalake.perf.UploadFromFileDatalakeTest;
17+
import com.azure.storage.file.share.perf.DownloadFileShareTest;
18+
import com.azure.storage.file.share.perf.DownloadToFileShareTest;
19+
import com.azure.storage.file.share.perf.UploadFileShareTest;
20+
import com.azure.storage.file.share.perf.UploadFromFileShareTest;
21+
22+
/**
23+
* Runs the Storage performance test.
24+
*
25+
* <p>To run from command line. Package the project into a jar with dependencies via mvn clean package.
26+
* Then run the program via java -jar 'compiled-jar-with-dependencies-path' </p>
27+
*
28+
* <p> To run from IDE, set all the required environment variables in IntelliJ via Run -&gt; EditConfigurations
29+
* section.
30+
* Then run the App's main method via IDE.</p>
31+
*/
32+
public class App {
33+
public static void main(String[] args) {
34+
PerfStressProgram.run(new Class<?>[]{
35+
DownloadBlobTest.class,
36+
ListBlobsTest.class,
37+
UploadBlobTest.class,
38+
UploadBlockBlobTest.class,
39+
UploadFromFileTest.class,
40+
UploadOutputStreamTest.class,
41+
DownloadFileShareTest.class,
42+
DownloadToFileShareTest.class,
43+
UploadFileShareTest.class,
44+
UploadFromFileShareTest.class,
45+
AppendFileDatalakeTest.class,
46+
ReadFileDatalakeTest.class,
47+
UploadFileDatalakeTest.class,
48+
UploadFromFileDatalakeTest.class
49+
}, args);
50+
}
51+
}

‎sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/App.java‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/DownloadBlobTest.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
package com.azure.storage.blob.perf;
55

6+
import com.azure.perf.test.core.NullOutputStream;
67
import com.azure.perf.test.core.PerfStressOptions;
78
import com.azure.storage.blob.BlobAsyncClient;
89
import com.azure.storage.blob.BlobClient;
@@ -42,20 +43,6 @@ public void run() {
4243
blobClient.download(DEV_NULL);
4344
}
4445

45-
static class NullOutputStream extends OutputStream {
46-
@Override
47-
public void write(int b) {
48-
49-
}
50-
51-
@Override
52-
public void write(byte[] b) {
53-
}
54-
55-
@Override
56-
public void write(byte[] b, int off, int len) {
57-
}
58-
}
5946

6047
@Override
6148
public Mono<Void> runAsync() {

0 commit comments

Comments
 (0)