Skip to content

Commit 5d04258

Browse files
yaricomyaricom
authored
Merge pull request #73 from comet-ml/CM-4553-api-experiment-by-name
[CM-4553]: Implement API to get experiment(-s) by name
2 parents 2680a35 + b69d494 commit 5d04258

File tree

11 files changed

+216
-17
lines changed

11 files changed

+216
-17
lines changed

‎comet-java-client/src/main/java/ml/comet/experiment/CometApi.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,29 @@ public interface CometApi extends Closeable {
4646
*/
4747
List<ExperimentMetadata> getAllExperiments(String projectId);
4848

49+
/**
50+
* Gets metadata of all experiments matching the following searching criteria.
51+
*
52+
* @param workspaceName the name of workspace where experiments defined.
53+
* @param projectName the name of project associated with experiments (optional).
54+
* @param experimentNamePattern the regex pattern for name of specific experiment (optional).
55+
* @return the list of metadata objects associated with experiments matching provided search criteria.
56+
*/
57+
List<ExperimentMetadata> getExperiments(String workspaceName, String projectName, String experimentNamePattern);
58+
59+
List<ExperimentMetadata> getExperiments(String workspaceName, String projectName);
60+
61+
List<ExperimentMetadata> getExperiments(String workspaceName);
62+
63+
/**
64+
* Allows to load metadata of the particular Comet experiment using provided {@code experimentKey}.
65+
*
66+
* @param experimentKey the ID of the Comet experiment.
67+
* @return the initialized {@code ExperimentMetadata}.
68+
* @throws ExperimentNotFoundException if Comet experiment with specified {@code experimentKey} not found.
69+
*/
70+
ExperimentMetadata getExperimentMetadata(String experimentKey) throws ExperimentNotFoundException;
71+
4972
/**
5073
* Register model defined in the specified experiment in the Comet's model registry.
5174
*

‎comet-java-client/src/main/java/ml/comet/experiment/ExperimentNotFoundException.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package ml.comet.experiment;
2+
3+
import ml.comet.experiment.exception.CometApiException;
4+
5+
/**
6+
* Exception to be thrown when Comet experiment was not found.
7+
*/
8+
public class ExperimentNotFoundException extends CometApiException {
9+
10+
/**
11+
* Constructs a new ExperimentNotFoundException exception with the specified detail message.
12+
* The cause is not initialized, and may subsequently be initialized by a
13+
* call to {@link #initCause}.
14+
*
15+
* @param message the detail message. The detail message is saved for
16+
* later retrieval by the {@link #getMessage()} method.
17+
*/
18+
public ExperimentNotFoundException(String message) {
19+
super(message);
20+
}
21+
}

‎comet-java-client/src/main/java/ml/comet/experiment/impl/BaseExperiment.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -797,7 +797,7 @@ public ExperimentMetadata getMetadata() {
797797
getLogger().debug("get metadata for experiment {}", this.experimentKey);
798798
}
799799

800-
return loadRemote(restApiClient::getMetadata, "METADATA").toExperimentMetadata();
800+
return loadRemote(restApiClient::getExperimentMetadata, "METADATA").toExperimentMetadata();
801801
}
802802

803803
@Override

‎comet-java-client/src/main/java/ml/comet/experiment/impl/CometApiImpl.java‎

Lines changed: 77 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import lombok.NonNull;
66
import lombok.SneakyThrows;
77
import ml.comet.experiment.CometApi;
8+
import ml.comet.experiment.ExperimentNotFoundException;
89
import ml.comet.experiment.builder.BaseCometBuilder;
910
import ml.comet.experiment.builder.CometApiBuilder;
1011
import ml.comet.experiment.exception.CometApiException;
@@ -54,6 +55,7 @@
5455
import java.util.Objects;
5556
import java.util.Optional;
5657
import java.util.function.Function;
58+
import java.util.regex.Pattern;
5759
import java.util.stream.Collectors;
5860
import java.util.zip.ZipInputStream;
5961

@@ -66,6 +68,7 @@
6668
import static ml.comet.experiment.impl.resources.LogMessages.DOWNLOADING_REGISTRY_MODEL_TO_DIR;
6769
import static ml.comet.experiment.impl.resources.LogMessages.DOWNLOADING_REGISTRY_MODEL_TO_FILE;
6870
import static ml.comet.experiment.impl.resources.LogMessages.EXPERIMENT_HAS_NO_MODELS;
71+
import static ml.comet.experiment.impl.resources.LogMessages.EXPERIMENT_WITH_KEY_NOT_FOUND;
6972
import static ml.comet.experiment.impl.resources.LogMessages.EXTRACTED_N_REGISTRY_MODEL_FILES;
7073
import static ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_DELETE_REGISTRY_MODEL;
7174
import static ml.comet.experiment.impl.resources.LogMessages.FAILED_TO_DELETE_REGISTRY_MODEL_VERSION;
@@ -153,6 +156,77 @@ public List<ExperimentMetadata> getAllExperiments(@NonNull String projectId) {
153156
ArrayList::addAll);
154157
}
155158

159+
@Override
160+
public List<ExperimentMetadata> getExperiments(@NonNull String workspaceName, @NonNull String projectName) {
161+
return restApiClient.getAllExperiments(projectName, workspaceName)
162+
.doOnError(ex -> this.logger.error(
163+
"Failed to read experiments found in the project {} of workspace {}",
164+
projectName, workspaceName, ex))
165+
.blockingGet()
166+
.getExperiments()
167+
.stream()
168+
.collect(ArrayList::new,
169+
(metadataList, metadataRest) -> metadataList.add(metadataRest.toExperimentMetadata()),
170+
ArrayList::addAll);
171+
}
172+
173+
@Override
174+
public List<ExperimentMetadata> getExperiments(@NonNull String workspaceName) {
175+
return this.getAllProjects(workspaceName)
176+
.stream()
177+
.map(project -> this.getAllExperiments(project.getProjectId()))
178+
.collect(ArrayList::new, ArrayList::addAll, ArrayList::addAll);
179+
}
180+
181+
@Override
182+
public List<ExperimentMetadata> getExperiments(
183+
@NonNull String workspaceName, String projectName, String experimentNamePattern) {
184+
185+
if (StringUtils.isBlank(projectName)) {
186+
// no project name provided
187+
if (!StringUtils.isBlank(experimentNamePattern)) {
188+
throw new IllegalArgumentException(
189+
"you must specify projectName when experimentNamePattern is provided");
190+
}
191+
// get experiments for all projects in the workspace
192+
return this.getExperiments(workspaceName);
193+
}
194+
195+
if (StringUtils.isBlank(experimentNamePattern)) {
196+
// no experiment name pattern provided - all experiments under project
197+
return this.getExperiments(workspaceName, projectName);
198+
}
199+
200+
// return list of experiments with names matching provided regex
201+
Pattern p = Pattern.compile(experimentNamePattern);
202+
return this.getExperiments(workspaceName, projectName)
203+
.stream()
204+
.filter(experimentMetadata -> {
205+
if (StringUtils.isEmpty(experimentMetadata.getExperimentName())) {
206+
return false;
207+
} else {
208+
return p.matcher(experimentMetadata.getExperimentName()).matches();
209+
}
210+
})
211+
.collect(Collectors.toList());
212+
}
213+
214+
@Override
215+
public ExperimentMetadata getExperimentMetadata(@NonNull final String experimentKey)
216+
throws ExperimentNotFoundException {
217+
try {
218+
return this.restApiClient
219+
.getExperimentMetadata(experimentKey)
220+
.blockingGet()
221+
.toExperimentMetadata();
222+
} catch (CometApiException ex) {
223+
if (ex.getStatusCode() == 400) {
224+
throw new ExperimentNotFoundException(getString(EXPERIMENT_WITH_KEY_NOT_FOUND, experimentKey));
225+
}
226+
throw ex;
227+
}
228+
}
229+
156230
@Override
157231
public ModelRegistryRecord registerModel(@NonNull final Model model, @NonNull final String experimentKey) {
158232
// get list of experiment models
@@ -183,7 +257,7 @@ public ModelRegistryRecord registerModel(@NonNull final Model model, @NonNull fi
183257
modelImpl.setExperimentModelId(details.get().getExperimentModelId());
184258

185259
// check if model already registered in the experiment's workspace records
186-
Boolean modelInRegistry = this.restApiClient.getMetadata(experimentKey)
260+
Boolean modelInRegistry = this.restApiClient.getExperimentMetadata(experimentKey)
187261
.concatMap(experimentMetadataRest -> {
188262
modelImpl.setWorkspace(experimentMetadataRest.getWorkspaceName());
189263
return this.restApiClient.getRegistryModelsForWorkspace(experimentMetadataRest.getWorkspaceName());
@@ -393,8 +467,8 @@ public void updateRegistryModelVersion(@NonNull String registryName, @NonNull St
393467
Optional<ModelVersionOverview> versionOverviewOptional = this.getRegistryModelVersion(
394468
registryName, workspace, version);
395469
if (!versionOverviewOptional.isPresent()) {
396-
throw new ModelVersionNotFoundException(
397-
getString(REGISTRY_MODEL_VERSION_NOT_FOUND, version, workspace, registryName));
470+
throw new ModelVersionNotFoundException(
471+
getString(REGISTRY_MODEL_VERSION_NOT_FOUND, version, workspace, registryName));
398472
}
399473

400474
// update model version details

‎comet-java-client/src/main/java/ml/comet/experiment/impl/RestApiClient.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@
115115
import static ml.comet.experiment.impl.constants.QueryParamName.MODEL_ITEM_ID;
116116
import static ml.comet.experiment.impl.constants.QueryParamName.MODEL_NAME;
117117
import static ml.comet.experiment.impl.constants.QueryParamName.PROJECT_ID;
118+
import static ml.comet.experiment.impl.constants.QueryParamName.PROJECT_NAME;
118119
import static ml.comet.experiment.impl.constants.QueryParamName.TYPE;
119120
import static ml.comet.experiment.impl.constants.QueryParamName.WORKSPACE_NAME;
120121
import static ml.comet.experiment.impl.http.ConnectionUtils.checkResponseStatus;
@@ -164,8 +165,18 @@ Single<GetExperimentsResponse> getAllExperiments(String projectId) {
164165
EXPERIMENTS, Collections.singletonMap(PROJECT_ID, projectId), GetExperimentsResponse.class);
165166
}
166167

167-
Single<ExperimentMetadataRest> getMetadata(String experimentKey) {
168-
return singleFromSyncGetWithRetries(GET_METADATA, experimentKey, ExperimentMetadataRest.class);
168+
Single<GetExperimentsResponse> getAllExperiments(String projectName, String workspaceName) {
169+
HashMap<QueryParamName, String> params = new HashMap<>();
170+
params.put(WORKSPACE_NAME, workspaceName);
171+
params.put(PROJECT_NAME, projectName);
172+
return singleFromSyncGetWithRetries(EXPERIMENTS, params, GetExperimentsResponse.class);
173+
}
174+
175+
Single<ExperimentMetadataRest> getExperimentMetadata(String experimentKey) {
176+
return singleFromSyncGetWithRetries(GET_METADATA,
177+
Collections.singletonMap(EXPERIMENT_KEY, experimentKey),
178+
true,
179+
ExperimentMetadataRest.class);
169180
}
170181

171182
Single<GitMetadataRest> getGitMetadata(String experimentKey) {
@@ -495,7 +506,7 @@ private <T> Single<T> singleFromSyncPostWithRetries(@NonNull Object payload,
495506
return this.connection.sendPostWithRetries(request, endpoint, throwOnFailure)
496507
.map(body -> Single.just(JsonUtils.fromJson(body, clazz)))
497508
.orElse(Single.error(new CometApiException(
498-
String.format("No response was returned by endpoint: %s", endpoint))));
509+
getString(NO_RESPONSE_RETURNED_BY_REMOTE_ENDPOINT, endpoint))));
499510
}
500511

501512
private Single<RestApiResponse> singleFromSyncPostWithRetriesEmptyBody(@NonNull Object payload,
@@ -541,7 +552,7 @@ private <T> Single<T> singleFromSyncGetWithRetries(@NonNull String endpoint,
541552
return this.connection.sendGetWithRetries(endpoint, queryParams, throwOnFailure)
542553
.map(body -> Single.just(JsonUtils.fromJson(body, clazz)))
543554
.orElse(Single.error(new CometApiException(
544-
String.format("No response was returned by endpoint: %s", endpoint))));
555+
getString(NO_RESPONSE_RETURNED_BY_REMOTE_ENDPOINT, endpoint))));
545556
}
546557

547558
private static RestApiResponse mapResponse(Response response) {

‎comet-java-client/src/main/java/ml/comet/experiment/impl/constants/QueryParamName.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public enum QueryParamName {
2020
OVERWRITE("overwrite"), // boolean
2121
PROJECT_ID("projectId"), // string
2222
WORKSPACE_NAME("workspaceName"), // string
23+
PROJECT_NAME("projectName"), // string
2324

2425
WORKSPACE("workspace"), // string
2526
PROJECT("project"), // string

‎comet-java-client/src/main/java/ml/comet/experiment/impl/http/Connection.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -428,9 +428,14 @@ Optional<String> executeRequestSyncWithRetries(
428428
return Optional.empty();
429429
} catch (Throwable e) {
430430
// unexpected error - throw or return
431-
this.logger.error("Failed to execute request: {}, unexpected error", request, e);
432-
if (throwOnFailure) {
433-
throw new CometApiException("failed to execute request, unexpected error", e);
431+
if (e instanceof InterruptedException) {
432+
// ignore InterruptedException as it can be thrown during shutdown - just debug it
433+
this.logger.debug("Failed to execute request: {}, unexpected error: {}", request, e);
434+
} else {
435+
this.logger.error("Failed to execute request: {}, unexpected error", request, e);
436+
if (throwOnFailure) {
437+
throw new CometApiException("failed to execute request, unexpected error", e);
438+
}
434439
}
435440
return Optional.empty();
436441
}

‎comet-java-client/src/main/java/ml/comet/experiment/impl/resources/LogMessages.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ public class LogMessages {
102102
public static final String FAILED_TO_DELETE_REGISTRY_MODEL = "FAILED_TO_DELETE_REGISTRY_MODEL";
103103
public static final String NO_RESPONSE_RETURNED_BY_REMOTE_ENDPOINT = "NO_RESPONSE_RETURNED_BY_REMOTE_ENDPOINT";
104104
public static final String FAILED_TO_DELETE_REGISTRY_MODEL_VERSION = "FAILED_TO_DELETE_REGISTRY_MODEL_VERSION";
105+
public static final String EXPERIMENT_WITH_KEY_NOT_FOUND = "EXPERIMENT_WITH_KEY_NOT_FOUND";
105106

106107

107108
/**

‎comet-java-client/src/main/resources/messages.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,4 @@ REGISTRY_MODEL_VERSION_NOT_FOUND=Version '%s' of the registry model '%s/%s' is n
8787
FAILED_TO_DELETE_REGISTRY_MODEL=Failed to delete registry model '%s/%s'.
8888
NO_RESPONSE_RETURNED_BY_REMOTE_ENDPOINT=No response was returned by endpoint '%s'
8989
FAILED_TO_DELETE_REGISTRY_MODEL_VERSION=Failed to delete registry model '%s/%s:%s'.
90+
EXPERIMENT_WITH_KEY_NOT_FOUND=Failed to get Comet experiment with key '%s'.

‎comet-java-client/src/test/java/ml/comet/experiment/impl/CometApiTest.java‎

Lines changed: 67 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import io.reactivex.rxjava3.core.Observable;
44
import ml.comet.experiment.CometApi;
5+
import ml.comet.experiment.ExperimentNotFoundException;
56
import ml.comet.experiment.OnlineExperiment;
67
import ml.comet.experiment.exception.CometApiException;
78
import ml.comet.experiment.impl.rest.RegistryModelItemOverview;
@@ -37,6 +38,7 @@
3738
import java.util.Map;
3839
import java.util.Objects;
3940
import java.util.Optional;
41+
import java.util.UUID;
4042
import java.util.concurrent.TimeUnit;
4143
import java.util.stream.Stream;
4244
import java.util.zip.ZipInputStream;
@@ -115,9 +117,63 @@ public void testGetsAllExperiments() {
115117

116118
List<ExperimentMetadata> experiments = experimentsOpt.get();
117119
assertFalse(experiments.isEmpty());
118-
boolean experimentExists = experiments.stream()
119-
.anyMatch(experiment -> SHARED_EXPERIMENT.getExperimentKey().equals(experiment.getExperimentKey()));
120-
assertTrue(experimentExists);
120+
assertExperimentInList(SHARED_EXPERIMENT.getExperimentKey(), experiments);
121+
}
122+
123+
@Test
124+
public void testGetExperiments_Workspace() {
125+
List<ExperimentMetadata> experiments = COMET_API.getExperiments(WORKSPACE_NAME);
126+
assertFalse(experiments.isEmpty());
127+
assertExperimentInList(SHARED_EXPERIMENT.getExperimentKey(), experiments);
128+
}
129+
130+
@Test
131+
public void testGetExperiments_Workspace_Project() {
132+
List<ExperimentMetadata> experiments = COMET_API.getExperiments(WORKSPACE_NAME, PROJECT_NAME);
133+
assertFalse(experiments.isEmpty());
134+
assertExperimentInList(SHARED_EXPERIMENT.getExperimentKey(), experiments);
135+
}
136+
137+
@Test
138+
public void testGetExperiments() throws Exception {
139+
String experimentName = UUID.randomUUID().toString();
140+
String experimentKey;
141+
try (OnlineExperiment experiment = createOnlineExperiment()) {
142+
experiment.setExperimentName(experimentName);
143+
experimentKey = experiment.getExperimentKey();
144+
145+
// wait for experiment name to be updated
146+
awaitForCondition(() -> experimentName.equals(experiment.getMetadata().getExperimentName()),
147+
"Experiment name update timeout");
148+
}
149+
150+
List<ExperimentMetadata> experiments = COMET_API.getExperiments(WORKSPACE_NAME, PROJECT_NAME, experimentName);
151+
assertEquals(1, experiments.size(), "one experiment expected");
152+
153+
ExperimentMetadata experimentMetadata = experiments.get(0);
154+
assertEquals(experimentKey, experimentMetadata.getExperimentKey());
155+
}
156+
157+
@Test
158+
public void testGetExperiments_emptyProject_with_experimentName() {
159+
assertThrows(IllegalArgumentException.class, () ->
160+
COMET_API.getExperiments(WORKSPACE_NAME, null, "someExperiment*."));
161+
}
162+
163+
@Test
164+
public void testGetExperimentMetadata() {
165+
ExperimentMetadata experimentMetadata = COMET_API.getExperimentMetadata(SHARED_EXPERIMENT.getExperimentKey());
166+
167+
assertEquals(SHARED_EXPERIMENT.getExperimentKey(), experimentMetadata.getExperimentKey());
168+
assertEquals(SHARED_EXPERIMENT.getExperimentName(), experimentMetadata.getExperimentName());
169+
assertEquals(SHARED_EXPERIMENT.getProjectName(), PROJECT_NAME);
170+
assertEquals(SHARED_EXPERIMENT.getWorkspaceName(), WORKSPACE_NAME);
171+
}
172+
173+
@Test
174+
public void testGetExperimentMetadata_not_found() {
175+
assertThrows(ExperimentNotFoundException.class, () ->
176+
COMET_API.getExperimentMetadata("not existing experiment key"));
121177
}
122178

123179
@Test
@@ -638,7 +694,7 @@ public void testDeleteRegistryModelVersion_model_doesnt_exists() {
638694
// try to delete version of not existing model
639695
//
640696
String modelName = "not existing model";
641-
assertThrows(CometApiException.class, ()->
697+
assertThrows(CometApiException.class, () ->
642698
COMET_API.deleteRegistryModelVersion(modelName, SHARED_EXPERIMENT.getWorkspaceName(),
643699
DEFAULT_MODEL_VERSION));
644700
}
@@ -653,7 +709,7 @@ public void testDeleteRegistryModelVersion_version_doesnt_exists() {
653709

654710
// try to delete not existing version of the model
655711
//
656-
assertThrows(ModelVersionNotFoundException.class, ()->
712+
assertThrows(ModelVersionNotFoundException.class, () ->
657713
COMET_API.deleteRegistryModelVersion(modelName, SHARED_EXPERIMENT.getWorkspaceName(),
658714
"1.0.1"));
659715
}
@@ -752,4 +808,10 @@ private static void logModelFolder(String modelName) {
752808
awaitForCondition(() -> !SHARED_EXPERIMENT.getAssetList(MODEL_ELEMENT.type()).isEmpty(),
753809
"Failed to get logged model file");
754810
}
811+
812+
private static void assertExperimentInList(String experimentKey, List<ExperimentMetadata> experiments) {
813+
boolean experimentExists = experiments.stream()
814+
.anyMatch(experiment -> experimentKey.equals(experiment.getExperimentKey()));
815+
assertTrue(experimentExists);
816+
}
755817
}

0 commit comments

Comments
 (0)