Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/continuous-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ jobs:
distribution: 'temurin'
cache: 'maven'

- name: Cache Docker images.
uses: ScribeMD/docker-cache@0.3.6
with:
key: docker-${{ runner.os }}-${{ hashFiles('**/OllamaImage.java') }}

- name: Build with Maven and deploy to Artifactory
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
Expand Down
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I'd like to speed up the IT tests as well. I'm curious as to the rationale in going from a one liner to use test containers

	@Container
	static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);

To now using two static classes and a static method. I'm not quite up on my testcontainer knowledge, but would have expected something more terse code-wise.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason is to create the new Ollama image with the orca-mini model in it. We are doing it programmatically but another approach could be to reuse an existing image from Docker Hub that has the model and the one line could remain just pointing to the new image, removing the rest of the code. However, not sure if there is any preference to use any image out there or creating their own.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a tricky situation I encountered in other projects as well. On the one hand, it'd be better to use official images. On the other hand, downloading the model every time is not much feasible, requiring some mitigation strategies with additional configuration. I looked for existing images out there, but I ended up publishing my own Ollama-based images to make sure they are always up-to-date and for having multi-architecture support: https://github.com/ThomasVitale/llm-images.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is an issue in ollama to provide specific images ollama/ollama#2161

Copy link
Member

@markpollack markpollack Jul 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That image has been closed with 'won't do' - so closing this issue. We will have to setup a separate integration test profile for ollama along with other model providers that aren't currently running in CI

Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@

package org.springframework.ai.autoconfigure.ollama;

import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.model.Image;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
Expand All @@ -32,12 +34,14 @@
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.web.client.RestClientAutoConfiguration;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import reactor.core.publisher.Flux;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand All @@ -46,18 +50,25 @@

/**
* @author Christian Tzolov
* @author Eddú Meléndez
* @since 0.8.0
*/
@Disabled("For manual smoke testing only.")
@Testcontainers
public class OllamaAutoConfigurationIT {

private static final Log logger = LogFactory.getLog(OllamaAutoConfigurationIT.class);

private static String MODEL_NAME = "mistral";

@Container
static GenericContainer<?> ollamaContainer = new GenericContainer<>("ollama/ollama:0.1.23").withExposedPorts(11434);
private static final String OLLAMA_WITH_MODEL = "%s-%s".formatted(MODEL_NAME, OllamaImage.IMAGE);

private static final OllamaContainer ollamaContainer;

static {
ollamaContainer = new OllamaContainer(OllamaDockerImageName.image());
ollamaContainer.start();
createImage(ollamaContainer, OLLAMA_WITH_MODEL);
}

static String baseUrl;

Expand Down Expand Up @@ -120,4 +131,74 @@ public void chatCompletionStreaming() {
});
}

static class OllamaContainer extends GenericContainer<OllamaContainer> {

private static final String MODEL = "orca-mini";

private final DockerImageName dockerImageName;

OllamaContainer(DockerImageName image) {
super(image);
this.dockerImageName = image;
withExposedPorts(11434);
withImagePullPolicy(dockerImageName -> !dockerImageName.getVersionPart().endsWith(MODEL));
}

@Override
protected void containerIsStarted(InspectContainerResponse containerInfo) {
if (!this.dockerImageName.getVersionPart().endsWith(MODEL)) {
try {
execInContainer("ollama", "pull", MODEL);
}
catch (IOException | InterruptedException e) {
throw new RuntimeException("Error pulling orca-mini model", e);
}
}
}

}

static void createImage(GenericContainer<?> container, String localImageName) {
DockerImageName dockerImageName = DockerImageName.parse(container.getDockerImageName());
if (!dockerImageName.equals(DockerImageName.parse(localImageName))) {
DockerClient dockerClient = DockerClientFactory.instance().client();
List<Image> images = dockerClient.listImagesCmd().withReferenceFilter(localImageName).exec();
if (images.isEmpty()) {
DockerImageName imageModel = DockerImageName.parse(localImageName);
dockerClient.commitCmd(container.getContainerId())
.withRepository(imageModel.getUnversionedPart())
.withLabels(Collections.singletonMap("org.testcontainers.sessionId", ""))
.withTag(imageModel.getVersionPart())
.exec();
}
}
}

static class OllamaDockerImageName {

private final String baseImage;

private final String localImageName;

OllamaDockerImageName(String baseImage, String localImageName) {
this.baseImage = baseImage;
this.localImageName = localImageName;
}

static DockerImageName image() {
return new OllamaDockerImageName(OllamaImage.IMAGE, OLLAMA_WITH_MODEL).resolve();
}

private DockerImageName resolve() {
var dockerImageName = DockerImageName.parse(this.baseImage);
var dockerClient = DockerClientFactory.instance().client();
var images = dockerClient.listImagesCmd().withReferenceFilter(this.localImageName).exec();
if (images.isEmpty()) {
return dockerImageName;
}
return DockerImageName.parse(this.localImageName);
}

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.springframework.ai.autoconfigure.ollama;

public class OllamaImage {

static final String IMAGE = "ollama/ollama:0.1.10";

}