From f376b1dfb6729c204508ccfeb7f88dc9230a0c2b Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 20 Oct 2021 12:42:07 +0200 Subject: [PATCH 001/443] initial commit Signed-off-by: Lehmann-Fabian --- .dockerignore | 7 + .gitignore | 77 +++++ Dockerfile | 18 + README.md | 1 + pom.xml | 141 ++++++++ src/main/java/fonda/scheduler/Main.java | 18 + .../java/fonda/scheduler/config/Beans.java | 21 ++ .../model/KubernetesClientHolder.java | 26 ++ .../fonda/scheduler/model/NodeWithAlloc.java | 66 ++++ .../scheduler/model/PodListWithIndex.java | 39 +++ .../fonda/scheduler/model/PodWithAge.java | 30 ++ .../scheduler/model/SchedulerConfig.java | 41 +++ .../fonda/scheduler/model/TaskConfig.java | 39 +++ .../rest/SchedulerRestController.java | 119 +++++++ .../scheduler/scheduler/RandomScheduler.java | 327 ++++++++++++++++++ .../fonda/scheduler/scheduler/Scheduler.java | 259 ++++++++++++++ 16 files changed, 1229 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/java/fonda/scheduler/Main.java create mode 100644 src/main/java/fonda/scheduler/config/Beans.java create mode 100644 src/main/java/fonda/scheduler/model/KubernetesClientHolder.java create mode 100644 src/main/java/fonda/scheduler/model/NodeWithAlloc.java create mode 100644 src/main/java/fonda/scheduler/model/PodListWithIndex.java create mode 100644 src/main/java/fonda/scheduler/model/PodWithAge.java create mode 100644 src/main/java/fonda/scheduler/model/SchedulerConfig.java create mode 100644 src/main/java/fonda/scheduler/model/TaskConfig.java create mode 100644 src/main/java/fonda/scheduler/rest/SchedulerRestController.java create mode 100644 src/main/java/fonda/scheduler/scheduler/RandomScheduler.java create mode 100644 src/main/java/fonda/scheduler/scheduler/Scheduler.java diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..2f030fad --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +target/ +.git/ +.idea/ +README.md +Dockerfile +.gitignore +.dockerignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2e0f9123 --- /dev/null +++ b/.gitignore @@ -0,0 +1,77 @@ +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +.idea/artifacts +.idea/compiler.xml +.idea/jarRepositories.xml +.idea/modules.xml +.idea/*.iml +.idea/modules +*.iml +*.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +.idea + +target + +pom-development.xml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b5fb1049 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM maven:3.8.3-jdk-11-slim AS build +WORKDIR /build +COPY pom.xml pom.xml +RUN mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository +COPY src/ src/ +RUN mvn package -Dmaven.repo.local=/mvn/.m2nrepo/repository + +# +# Package stage +# +FROM openjdk:11-jre-slim +WORKDIR /app +RUN groupadd -r javauser && useradd --no-log-init -r -g javauser javauser +COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar +RUN chown -R javauser:javauser /app +USER javauser +EXPOSE 8080 +ENTRYPOINT ["java","-jar","k8s-scheduler.jar"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..674f9b61 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# Kubernetes Workflow Scheduler diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..baed7ec8 --- /dev/null +++ b/pom.xml @@ -0,0 +1,141 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 2.4.2 + + + + org.example + k8s-scheduler + 1.0-SNAPSHOT + + + 11 + 11 + fonda.scheduler.Main + + + + + + io.fabric8 + kubernetes-client + 5.1.1 + + + + org.javatuples + javatuples + 1.2 + + + + org.postgresql + postgresql + 42.2.18 + + + + com.zaxxer + HikariCP + 3.4.5 + + + + org.projectlombok + lombok + 1.18.4 + provided + + + + junit + junit + 4.11 + + + + org.mockito + mockito-inline + 3.4.4 + test + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.jetbrains + annotations + 22.0.0 + compile + + + + commons-codec + commons-codec + 1.15 + + + + org.apache.commons + commons-compress + 1.20 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + META-INF/spring.handlers + + + META-INF/spring.factories + + + META-INF/spring.schemas + + + + fonda.scheduler.Main + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java new file mode 100644 index 00000000..fd72a1c6 --- /dev/null +++ b/src/main/java/fonda/scheduler/Main.java @@ -0,0 +1,18 @@ +package fonda.scheduler; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +@Slf4j +public class Main { + + public static void main(String[] args) { + if( System.getenv( "SCHEDULER_NAME" ) == null || System.getenv( "SCHEDULER_NAME" ).isEmpty() ){ + throw new IllegalArgumentException( "Please define environment variable: SCHEDULER_NAME" ); + } + SpringApplication.run(Main.class, args); + } + +} diff --git a/src/main/java/fonda/scheduler/config/Beans.java b/src/main/java/fonda/scheduler/config/Beans.java new file mode 100644 index 00000000..b4b35120 --- /dev/null +++ b/src/main/java/fonda/scheduler/config/Beans.java @@ -0,0 +1,21 @@ +package fonda.scheduler.config; + +import fonda.scheduler.model.KubernetesClientHolder; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class Beans { + + private static KubernetesClientHolder kubernetesClientHolder; + + @Bean + KubernetesClientHolder getClientHolder(){ + if(kubernetesClientHolder == null) { + kubernetesClientHolder = new KubernetesClientHolder(); + } + return kubernetesClientHolder; + } + +} diff --git a/src/main/java/fonda/scheduler/model/KubernetesClientHolder.java b/src/main/java/fonda/scheduler/model/KubernetesClientHolder.java new file mode 100644 index 00000000..47431832 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/KubernetesClientHolder.java @@ -0,0 +1,26 @@ +package fonda.scheduler.model; + +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClient; + +import java.util.HashMap; +import java.util.Map; + +public class KubernetesClientHolder { + + private final Map holder = new HashMap<>(); + + public KubernetesClient getClient(String namespace ){ + + synchronized ( holder ){ + if( !holder.containsKey( namespace.toLowerCase() ) ){ + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(); + kubernetesClient.getConfiguration().setNamespace(namespace); + holder.put( namespace.toLowerCase(), kubernetesClient ); + } + return holder.get( namespace.toLowerCase() ); + } + + } + +} diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java new file mode 100644 index 00000000..dfb0c852 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -0,0 +1,66 @@ +package fonda.scheduler.model; + +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Quantity; +import lombok.Getter; +import lombok.Setter; + +import java.math.BigDecimal; + +@Getter +@Setter +public class NodeWithAlloc implements Comparable { + + private String nodeName; + + private BigDecimal current_cpu_usage; + + private BigDecimal current_ram_usage; + + private BigDecimal free_cpu; + + private BigDecimal free_ram; + + private Node node; + + public NodeWithAlloc(Node node) { + this.node = node; + this.nodeName = node.getMetadata().getName(); + this.current_ram_usage = new BigDecimal(0); + this.current_cpu_usage = new BigDecimal(0); + this.free_ram = new BigDecimal(0); + this.free_cpu = new BigDecimal(0); + + } + + public void calculateAlloc() { + + Quantity max_cpu = this.node.getStatus().getAllocatable().get("cpu"); + Quantity max_ram = this.node.getStatus().getAllocatable().get("memory"); + + this.free_cpu = Quantity.getAmountInBytes(max_cpu).subtract(current_cpu_usage); + this.free_ram = Quantity.getAmountInBytes(max_ram).subtract(current_ram_usage); + + } + + @Override + public String toString() { + return "NodeWithAlloc{" + + "nodeName='" + nodeName + '\'' + + ", current_cpu_usage=" + current_cpu_usage + + ", current_ram_usage=" + current_ram_usage + + ", free_cpu=" + free_cpu + + ", free_ram=" + free_ram + + ", node=" + node + + '}'; + } + + @Override + public int compareTo(NodeWithAlloc o) { + if(getNode().getMetadata().getName().equals(o.getNode().getMetadata().getName())) { + return 0; + } else { + return -1; + } + } +} diff --git a/src/main/java/fonda/scheduler/model/PodListWithIndex.java b/src/main/java/fonda/scheduler/model/PodListWithIndex.java new file mode 100644 index 00000000..742789a6 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/PodListWithIndex.java @@ -0,0 +1,39 @@ +package fonda.scheduler.model; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodList; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PodListWithIndex extends PodList { + + private Map nameIndexMap = new HashMap<>(); + + public PodListWithIndex() {} + + public PodListWithIndex(List pods) { + pods.forEach( pod -> addPodToList(pod) ); + } + + /** + * Overrides existing pod if pod is already in list + * + * @param pod + */ + public void addPodToList(Pod pod) { + nameIndexMap.put(pod.getMetadata().getName(), pod); + } + + public void removePodFromList(Pod pod) { + nameIndexMap.remove(pod.getMetadata().getName()); + } + + + @Override + public List getItems() { + return new ArrayList<>(nameIndexMap.values()); + } +} diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java new file mode 100644 index 00000000..e59e236d --- /dev/null +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -0,0 +1,30 @@ +package fonda.scheduler.model; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodSpec; +import io.fabric8.kubernetes.api.model.PodStatus; +import lombok.Getter; +import lombok.Setter; + +import java.math.BigDecimal; + +public class PodWithAge extends Pod { + + @Getter + @Setter + private BigDecimal age; + + public PodWithAge(ObjectMeta metadata, PodSpec spec, PodStatus status) { + super("v1", "Pod", metadata, spec, status); + this.age = BigDecimal.ZERO; + } + public PodWithAge() { + super("v1", "Pod", null, null, null); + this.age = BigDecimal.ZERO; + } + public PodWithAge(Pod pod) { + super(pod.getApiVersion(), pod.getKind(), pod.getMetadata(), pod.getSpec(), pod.getStatus()); + this.age = BigDecimal.ZERO; + } +} diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java new file mode 100644 index 00000000..30d990db --- /dev/null +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -0,0 +1,41 @@ +package fonda.scheduler.model; + +import java.util.List; + +public class SchedulerConfig { + + final public String podName; + final public List localClaims; + final public List volumeClaims; + final public String workDir; + + private SchedulerConfig(){ + this.podName = null; + this.localClaims = null; + this.volumeClaims = null; + this.workDir = null; + } + + static public class LocalClaim { + final public String mountPath; + final public String hostPath; + + private LocalClaim(){ + this.mountPath = null; + this.hostPath = null; + } + } + + static public class VolumeClaim { + final public String mountPath; + final public String claimName; + final public String subPath; + + private VolumeClaim(){ + this.mountPath = null; + this.claimName = null; + this.subPath = null; + } + } + +} diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java new file mode 100644 index 00000000..623eb342 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -0,0 +1,39 @@ +package fonda.scheduler.model; + +import lombok.Getter; +import lombok.ToString; + +import java.util.List; +import java.util.Map; + +@Getter +@ToString +public class TaskConfig { + + private final String task; + private final String name; + private final Map< String, List> schedulerParams; + private final List inputs; + private final String hash; + + private TaskConfig() { + this.name = null; + this.schedulerParams = null; + this.inputs = null; + this.hash = null; + this.task = null; + } + + @Getter + @ToString + static public class Input { + + private final String name; + private final Object value; + + private Input() { + this.value = null; + name = null; + } + } +} diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java new file mode 100644 index 00000000..c2627e6f --- /dev/null +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -0,0 +1,119 @@ +package fonda.scheduler.rest; + +import fonda.scheduler.model.KubernetesClientHolder; +import fonda.scheduler.scheduler.RandomScheduler; +import fonda.scheduler.scheduler.Scheduler; +import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.model.TaskConfig; +import lombok.extern.slf4j.Slf4j; +import org.javatuples.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.HashMap; +import java.util.Map; + +@RestController +@Slf4j +public class SchedulerRestController { + + private final KubernetesClientHolder clientHolder; + + /** + * Holds the scheduler for one execution + * Execution: String in lowercase + * Scheduler: An instance of a scheduler with the requested type + */ + private final Map< Pair, Scheduler > schedulerHolder = new HashMap<>(); + + public SchedulerRestController( @Autowired KubernetesClientHolder clientHolder ){ + this.clientHolder = clientHolder; + } + + /** + * Register a sheduler for a workflow execution + * @param namespace namespace where the workflow runs + * @param execution unique name of the execution + * @param strategy which scheduling strategy should be used + * @param config Additional parameters for the scheduler + * @return + */ + @PostMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") + ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { + + log.trace("Register execution: " + execution + " strategy: " + strategy); + + Scheduler scheduler = null; + + if( schedulerHolder.containsKey( execution.toLowerCase() ) ) { + return new ResponseEntity( "There is already a scheduler for " + execution, HttpStatus.BAD_REQUEST ); + } + + switch ( strategy.toLowerCase() ){ + case "fifo" : + case "random" : + case "fifo-random" : scheduler = new RandomScheduler(execution, clientHolder.getClient( namespace ), config ); break; + default: return new ResponseEntity( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); + } + + final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + schedulerHolder.put( key, scheduler ); + + return new ResponseEntity( HttpStatus.OK ); + + } + + /** + * Register a task for the execution + * @param namespace namespace where the workflow runs + * @param execution unique name of the execution + * @param config The config contains the task name, input files, and optional task parameter the scheduler has to determine + * @return Parameters the scheduler suggests for the task + */ + @PostMapping("/scheduler/registerTask/{namespace}/{execution}") + ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody(required = true) TaskConfig config ) { + + log.trace( execution + " " + config.getTask() + " got: " + config ); + + final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + scheduler.addTaskConfig( config.getTask(), config ); + Map schedulerParams = scheduler.getSchedulerParams(config.getTask(), config.getName()); + + return new ResponseEntity( schedulerParams, HttpStatus.OK ); + + } + + /** + * Call this after the execution has finished + * @param namespace namespace where the workflow runs + * @param execution unique name of the execution + * @return + */ + @DeleteMapping ("/scheduler/{namespace}/{execution}") + ResponseEntity delete(@PathVariable String namespace, @PathVariable String execution) { + + log.info("Delete name: " + execution); + final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution, HttpStatus.NOT_FOUND ); + } + schedulerHolder.remove( key ); + scheduler.close(); + return new ResponseEntity( HttpStatus.OK ); + } + + @GetMapping ("/health") + ResponseEntity checkHealth() { + return new ResponseEntity( HttpStatus.OK ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java new file mode 100644 index 00000000..f0f1cc07 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -0,0 +1,327 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.model.TaskConfig; +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.dsl.ExecListener; +import io.fabric8.kubernetes.client.dsl.ExecWatch; +import io.fabric8.kubernetes.client.dsl.PodResource; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import okhttp3.Response; + +import java.io.*; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class RandomScheduler extends Scheduler { + + //For testing + private final boolean local = true; + private final String name; + private final Map daemonByNode = new HashMap<>(); + private final List localClaims; + private final Map workdirNode = new HashMap<>(); + private final SchedulerConfig schedulerConfig; + + public RandomScheduler( String name, KubernetesClient client, SchedulerConfig config ) { + super( name, client); + this.name = name; + this.localClaims = config.localClaims; + this.schedulerConfig = config; + log.info ( "Create scheduler: " + name ); + } + + public void schedule(Pod podToSchedule) { + log.info ( "===========" ); + if( podToSchedule != null ){ + List items = getNodeList().getItems(); + int trial = 0; + while( trial++ < 5 ) { + if(isClose()) return; + try { + Node node = items.get(new Random().nextInt(items.size())); + if( !daemonByNode.containsKey( node.getMetadata().getName() )) continue; + + String workingDir = getWorkingDir(podToSchedule); + workdirNode.put(workingDir, node.getMetadata().getName()); + //Copy files to this node + if( !copyConfig(workingDir, node) ){ + log.info( "No success to copy config for " + workingDir + " to Node " + node.getMetadata().getName() ); + continue; + } + + TaskConfig config = getConfigFor(podToSchedule.getMetadata().getName()); + if( !copyInputsToNode(config, node) ){ + log.info( "No success to copy input for " + workingDir + " to Node " + node.getMetadata().getName() ); + continue; + } + //Store Map: Workdir Node + assignPodToNode(podToSchedule, node); + break; + }catch (Exception e){ + log.info( "Try " + podToSchedule.getMetadata().getName() + " again on a different node..." ); + e.printStackTrace(); + } + } + if(trial > 5 ){ + log.info( "Failed to schedule " + podToSchedule.getMetadata().getName() + " " + daemonByNode ); + podToSchedule.getStatus().setPhase("Failed"); + } + } else { + log.info ( "Pod was null." ); + } + log.info ( "===========" ); + } + + void podEventReceived(Watcher.Action action, Pod pod){ + //TODO get current ExecutorPod else use local + if( ( "mount-" + name.replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ + String nodeName = pod.getSpec().getNodeName(); + if ( nodeName != null ){ + if( action == Watcher.Action.DELETED ){ + if ( pod.getMetadata().getName().equals( daemonByNode.get(nodeName) )){ + daemonByNode.remove( nodeName ); + } + } else if (pod.getStatus().getPhase().equals("Running")) { + daemonByNode.put(nodeName, pod.getMetadata().getName() ); + } else if (pod.getStatus().getPhase().equals("Failed")) { + daemonByNode.remove( nodeName ); + } else { + daemonByNode.remove( nodeName ); + } + } + } + } + + private boolean copyConfig( String workdir, Node node ){ + String destinationPod = daemonByNode.get( node.getMetadata().getName() ); + log.info ("Copy workdir: " + workdir + " to node: " + node.getMetadata().getName() + " (" + destinationPod + ") " + daemonByNode); + if( local ){ + //SchedulerConfig.LocalClaim localClaim = localClaims.stream().filter(x -> workdir.startsWith(x.mountPath)).findFirst().get(); + String[] files = {".command.run", ".command.sh"}; + for( String file : files) { + boolean success = false; + int failureCount = 0; + while( !success ) { + try{ + Path workdirPath = new File(workdir + File.separator + file).toPath(); + Boolean upload = findPodByName(destinationPod) + .file(workdir + File.separator + file) + .upload(workdirPath); + success = upload; + if(!success) log.info("No success uploading: " + file ); + }catch(Exception e){ + log.info ("Error for pod " + destinationPod + " on node: " + node.getMetadata().getName() ); + if ( ++failureCount > 10 ) { + e.printStackTrace(); + return false; + } + destinationPod = daemonByNode.get( node.getMetadata().getName() ); + log.info ("Try again for pod " + destinationPod + " on node: " + node.getMetadata().getName() ); + } + + } + } + return true; + } else { + throw new RuntimeException( "Currently this can not run in Kubernetes!" ); + } + } + + @Override + void onPodTermination( Pod pod ) { + if(isClose()) return; + //Copy results from Executor + PodResource daemon = findPodByName(daemonByNode.get(pod.getSpec().getNodeName())); + String workingDir = getWorkingDir(pod); + String[] files = { + ".command.sh", + ".command.out", + ".exitcode", + //".exitcode2", + ".command.trace", + ".command.infiles", + ".command.outfiles", + ".command.outfilesTmp" + }; + if( local ){ + log.info("Start copying all files for " + pod.getSpec().getNodeName() ); + for( String file : files) { + Path workdirPath = new File( workingDir + File.separator + file ).toPath(); + Boolean copy = daemon + .file(workingDir + File.separator + file) + .copy(workdirPath); + if(!copy){ + log.info("No success downloading: " + file ); + } + } + + //For debugging: download logs + try { + final PrintWriter printWriter = new PrintWriter(workingDir + File.separator + ".command.kubelogs"); + printWriter.print(daemon.getLog()); + printWriter.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + log.info("Copied all files for " + pod.getSpec().getNodeName() ); + try { + new File( workingDir + File.separator + ".command.done" ).createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + throw new RuntimeException( "Currently this can not run in Kubernetes!" ); + } + } + + private String extractWorkspace( String path ){ + if( path.startsWith( schedulerConfig.workDir ) ){ + return path.substring( 0, schedulerConfig.workDir.length() + 34 ); + } + return null; + } + + private boolean copyInputsToNode(TaskConfig config , Node node ){ + //TODO schedule copy + //Learn copy Speed + //Remember Filesize + //Time = Size * Speed, order desc, run in this order, x parallel + + Map< String, List< String > > filesByNode = new HashMap<>(); + + //Structure input files by node + config.getInputs().forEach( x -> { + log.info(x.toString()); + if( x.getValue() instanceof List ){ + for( Object o : (List) x.getValue()){ + if( o instanceof Map ){ + Map input = (Map) o; + if(input.containsKey( "sourceObj" )){ + String sourceObj = (String) input.get( "sourceObj" ); + String workspace = extractWorkspace(sourceObj); + + if( workspace != null ){ + + final String podName = daemonByNode.get(workdirNode.get(workspace)); + List inputs; + if( !filesByNode.containsKey( podName ) ){ + inputs = new LinkedList<>(); + filesByNode.put( podName, inputs ); + } else { + inputs = filesByNode.get( podName ); + } + + inputs.add( sourceObj ); + + } + } + } + } + } + } ); + + return filesByNode.entrySet().stream().allMatch( x -> { + return copyFile( x.getKey(), node.getMetadata().getName(), x.getValue() ); + }); + //Copy for x Nodes in parallel + + } + + private boolean copyFile(String fromPod, String toNode, List files){ + + String toPod = daemonByNode.get(toNode); + if( toPod.equals(fromPod) ){ + return true; + } + + //TODO reduce mkdir to the only necessary, remove files, if it already exists + String[] command = new String[ files.size() * 9 - 1 ]; + int i = 0; + for ( String file : files ) { + command[i++] = "mkdir"; + command[i++] = "-p"; + command[i++] = file.substring(0, file.lastIndexOf("/")) ; + command[i++] = "&&"; + command[i++] = "kubectl"; + command[i++] = "cp"; + command[i++] = fromPod + ":\"" + file + "\""; + //if( file.endsWith( "/" ) ){ + command[i++] = "\"" + file + "\""; + //} else { + // command[i++] = "\"" + file.substring( 0, file.lastIndexOf('/' ) ) + "\""; + //} + + if( i < command.length) command[i++] = "&&"; + } + //String command = file.stream() + //.map(x -> "mkdir -p \"" + x.substring(0, x.lastIndexOf("/")) + "\" && kubectl cp " + fromPod + ":\"" + x + "\" \"" + x + "\"") + //.collect(Collectors.joining(" && ")); + + log.info( toNode + ": " + Arrays.toString(command) ); + + + final PodResource pod = findPodByName(toPod); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream error = new ByteArrayOutputStream(); + CountDownLatch execLatch = new CountDownLatch(1); + + final MyPodExecListener listener = new MyPodExecListener(execLatch); + ExecWatch exec = pod + .writingOutput(out) + .writingError(error) + .usingListener(listener) + .exec("bash", "-c", String.join(" ", command )); + try { + execLatch.await(1, TimeUnit.DAYS ); + } catch (InterruptedException e) { + e.printStackTrace(); + } + exec.close(); + log.info("Exec Output: {} ", out); + if(listener.isFailure()){ + log.info ("Could not copy files from " + fromPod + " to " + toNode); + } + return !listener.isFailure(); + } + + private class MyPodExecListener implements ExecListener { + + private final CountDownLatch execLatch; + + @Getter + private boolean failure = false; + + private MyPodExecListener( final CountDownLatch execLatch ){ + this.execLatch = execLatch; + } + + @Override + public void onOpen(Response response) { + log.info("Shell was opened"); + } + + @Override + public void onFailure(Throwable throwable, Response response) { + log.info("Some error encountered"); + execLatch.countDown(); + failure = true; + } + + @Override + public void onClose(int i, String s) { + log.info("Shell Closing"); + execLatch.countDown(); + } + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java new file mode 100644 index 00000000..7a4fd62d --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -0,0 +1,259 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.model.PodListWithIndex; +import fonda.scheduler.model.PodWithAge; +import fonda.scheduler.model.TaskConfig; +import io.fabric8.kubernetes.api.model.*; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.base.OperationContext; +import io.fabric8.kubernetes.client.informers.ListerWatcher; +import io.fabric8.kubernetes.client.informers.ResourceEventHandler; +import io.fabric8.kubernetes.client.informers.SharedInformerEventListener; +import io.fabric8.kubernetes.client.informers.impl.DefaultSharedIndexInformer; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +public abstract class Scheduler { + + DefaultSharedIndexInformer defaultSharedIndexInformerNode; + final KubernetesClient client; + private OperationContext operationContext; + private ConcurrentLinkedQueue workerQueueNode; + PodListWithIndex podList = new PodListWithIndex(); + @Getter + private final String name; + + final Map> taskConfigs = new HashMap<>(); + final Map taskConfigsByHash = new HashMap<>(); + @Getter + private boolean close; + + final static private ExecutorService executorService = Executors.newFixedThreadPool(30); + + private final Watch watcher; + + Scheduler( String name, KubernetesClient client ){ + this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + name; + log.trace( "Register scheduler for " + this.name ); + + this.client = client; + + this.workerQueueNode = new ConcurrentLinkedQueue<>(); + + this.operationContext = new OperationContext(); + + + setUpIndexInformerNode(); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + watcher = client.pods().watch(new Watcher<>() { + @Override + public void eventReceived(Action action, Pod pod) { + + podEventReceived(action, pod); + + if (!Scheduler.this.name.equals(pod.getSpec().getSchedulerName())) return; + + PodWithAge pwa = new PodWithAge(pod); + if (pod.getMetadata().getLabels() != null) { + log.debug("Got pod: " + pod.getMetadata().getName() + + " app: " + pod.getMetadata().getLabels().getOrDefault("app", "-") + + " processName: " + pod.getMetadata().getLabels().getOrDefault("processName", "-") + + " runName: " + pod.getMetadata().getLabels().getOrDefault("runName", "-") + + " taskName: " + pod.getMetadata().getLabels().getOrDefault("taskName", "-") + + " scheduler: " + pwa.getSpec().getSchedulerName() + + " action: " + action + ); + } else { + log.debug("Got pod " + pod.getMetadata().getName() + " scheduler: " + pwa.getSpec().getSchedulerName()); + } + + switch (action) { + case ADDED: + addPod(pwa); + if (pwa.getSpec().getNodeName() == null) { + executorService.submit(() -> schedule(pwa)); + } + break; + case MODIFIED: + if (pod.getStatus().getContainerStatuses().size() > 0 && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { + executorService.submit(() -> onPodTermination(pwa)); + } + break; + case DELETED: + removePod(pwa); + executorService.submit(() -> schedule(null)); + } + + } + + @Override + public void onClose(WatcherException cause) { + log.info( "Watcher was closed" ); + } + + }); + + } + + void podEventReceived(Watcher.Action action, Pod pod){} + + void onPodTermination( Pod pod ){} + + /** + * + */ + private void setUpIndexInformerNode() { + ListerWatcher listerWatcher = new ListerWatcher() { + @Override + public Watch watch(ListOptions params, String namespace, OperationContext context, Watcher watcher) { + return client.nodes().watch(new Watcher<>() { + @Override + public void eventReceived(Action action, Node resource) { + + } + + @Override + public void onClose(WatcherException cause) { + + } + }); + } + + @Override + public Object list(ListOptions params, String namespace, OperationContext context) { + return getNodeList(); + } + }; + + defaultSharedIndexInformerNode = new DefaultSharedIndexInformer(Node.class, listerWatcher, 600000, operationContext, workerQueueNode); + + try { + defaultSharedIndexInformerNode.run(); + } catch (Exception e) { + e.printStackTrace(); + } + + defaultSharedIndexInformerNode.addEventHandler(new ResourceEventHandler<>() { + @Override + public void onAdd(Node obj) { + + } + + @Override + public void onUpdate(Node oldObj, Node newObj) { + + } + + @Override + public void onDelete(Node obj, boolean deletedFinalStateUnknown) { + + } + }); + } + + void assignPodToNode( Pod pod, Node node ){ + log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + node ); + + Binding b1 = new Binding(); + + ObjectMeta om = new ObjectMeta(); + om.setName(pod.getMetadata().getName()); + om.setNamespace(pod.getMetadata().getNamespace()); + b1.setMetadata(om); + + ObjectReference objectReference = new ObjectReference(); + objectReference.setApiVersion("v1"); + objectReference.setKind("Node"); + objectReference.setName(node.getMetadata().getName()); //hier ersetzen durch tatsächliche Node + + b1.setTarget(objectReference); + + client.bindings().create(b1); + + pod.getSpec().setNodeName( node.getMetadata().getName() ); // Hier stimmt evtl etwas nicht + log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); + } + + /** + * Close used resources + */ + public void close(){ + watcher.close(); + this.close = true; + } + + + public void addPod( Pod pod ) { + podList.addPodToList( pod ); + } + + public void removePod( Pod pod ) { + podList.removePodFromList( pod ); + } + + NodeList getNodeList(){ + return client.nodes().list(); + } + + public abstract void schedule( Pod podToSchedule ); + + String getWorkingDir( Pod pod ){ + return pod.getSpec().getContainers().get(0).getWorkingDir(); + } + + PodResource findPodByName(String name ){ + return client.pods().withName( name ); + } + + public void addTaskConfig( String name, TaskConfig config ) { + + Map< String, TaskConfig> conf; + synchronized (taskConfigs) { + if( taskConfigs.containsKey( name ) ){ + conf = taskConfigs.get( name ); + } else { + conf = new HashMap<>(); + taskConfigs.put( name, conf ); + } + } + synchronized ( conf ){ + conf.put( config.getName(), config ); + } + synchronized ( taskConfigsByHash ){ + taskConfigsByHash.put ( config.getHash(), config ); + } + } + + TaskConfig getConfigFor(String taskname, String name ){ + return taskConfigs.get( taskname ).get( name ); + } + + TaskConfig getConfigFor(String hash ){ + return taskConfigsByHash.get( hash ); + } + + public Map getSchedulerParams( String taskname, String name ){ + TaskConfig config = getConfigFor(taskname, name); + Map result = new HashMap(); + config.getSchedulerParams().entrySet().forEach( x -> result.put( x.getKey(), x.getValue().get(0)) ); + return result; + } + +} From 0c987a5a14afa5608c14e41e3ad38deaf4051155 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 25 Oct 2021 16:17:08 +0200 Subject: [PATCH 002/443] Refactored Scheduler Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/Main.java | 17 + .../scheduler/client/KubernetesClient.java | 193 +++++++++++ .../java/fonda/scheduler/config/Beans.java | 14 +- .../model/KubernetesClientHolder.java | 26 -- .../fonda/scheduler/model/NodeWithAlloc.java | 139 ++++++-- .../rest/SchedulerRestController.java | 12 +- .../scheduler/scheduler/RandomScheduler.java | 324 +----------------- .../fonda/scheduler/scheduler/Scheduler.java | 274 ++++++++------- .../scheduler/scheduler/SchedulerThread.java | 35 ++ 9 files changed, 517 insertions(+), 517 deletions(-) create mode 100644 src/main/java/fonda/scheduler/client/KubernetesClient.java delete mode 100644 src/main/java/fonda/scheduler/model/KubernetesClientHolder.java create mode 100644 src/main/java/fonda/scheduler/scheduler/SchedulerThread.java diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index fd72a1c6..d2611426 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -1,13 +1,20 @@ package fonda.scheduler; +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.scheduler.RandomScheduler; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import javax.annotation.PostConstruct; + @SpringBootApplication @Slf4j public class Main { + private final KubernetesClient client; + public static void main(String[] args) { if( System.getenv( "SCHEDULER_NAME" ) == null || System.getenv( "SCHEDULER_NAME" ).isEmpty() ){ throw new IllegalArgumentException( "Please define environment variable: SCHEDULER_NAME" ); @@ -15,4 +22,14 @@ public static void main(String[] args) { SpringApplication.run(Main.class, args); } + Main(@Autowired KubernetesClient client){ + this.client = client; + } + + @PostConstruct + public void afterStart(){ + log.info( "Started with namespace: {}", client.getNamespace() ); + new RandomScheduler("testscheduler", client, "default" ); + } + } diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java new file mode 100644 index 00000000..96b95157 --- /dev/null +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -0,0 +1,193 @@ +package fonda.scheduler.client; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.scheduler.Scheduler; +import io.fabric8.kubernetes.api.model.ListOptions; +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.NodeList; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.dsl.base.OperationContext; +import io.fabric8.kubernetes.client.informers.ListerWatcher; +import io.fabric8.kubernetes.client.informers.SharedInformerEventListener; +import io.fabric8.kubernetes.client.informers.impl.DefaultSharedIndexInformer; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +@Slf4j +public class KubernetesClient extends DefaultKubernetesClient { + + private final Map nodeHolder= new HashMap<>(); + private final List schedulerList = new LinkedList<>(); + private OperationContext operationContext; + private DefaultSharedIndexInformer defaultSharedIndexInformerNode; + private ConcurrentLinkedQueue workerQueueNode; + + + public KubernetesClient(){ + this.operationContext = new OperationContext(); + this.workerQueueNode = new ConcurrentLinkedQueue<>(); + setUpIndexInformerNode(); + for( Node node : this.nodes().list().getItems() ){ + nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node) ); + } + this.pods().inAnyNamespace().watch( new PodWatcher( this ) ); + } + + public void addScheduler( Scheduler scheduler ){ + synchronized ( schedulerList ){ + schedulerList.add( scheduler ); + } + } + + public void removeScheduler( Scheduler scheduler ){ + synchronized ( schedulerList ){ + schedulerList.remove( scheduler ); + } + } + + private void informAllScheduler(){ + synchronized ( schedulerList ){ + for (Scheduler scheduler : schedulerList) { + scheduler.informResourceChange(); + } + } + } + + private void informAllSchedulersNewNode( NodeWithAlloc node ){ + synchronized ( schedulerList ){ + for (Scheduler scheduler : schedulerList) { + scheduler.newNode( node ); + } + } + } + + private void informAllSchedulersRemovedNode( NodeWithAlloc node ){ + synchronized ( schedulerList ){ + for (Scheduler scheduler : schedulerList) { + scheduler.removedNode( node ); + } + } + } + + public List getAllNodes(){ + return new ArrayList<>(this.nodeHolder.values()); + } + + /** + * + */ + private void setUpIndexInformerNode() { + ListerWatcher listerWatcher = new ListerWatcher() { + @Override + public Watch watch(ListOptions params, String namespace, OperationContext context, Watcher watcher) { + return KubernetesClient.this.nodes().watch(new NodeWatcher( KubernetesClient.this )); + } + + @Override + public Object list(ListOptions params, String namespace, OperationContext context) { + return KubernetesClient.this.nodes().list(); + } + }; + + defaultSharedIndexInformerNode = new DefaultSharedIndexInformer(Node.class, listerWatcher, 6000, this.operationContext, this.workerQueueNode); + + try { + defaultSharedIndexInformerNode.run(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + class NodeWatcher implements Watcher{ + + private final KubernetesClient kubernetesClient; + + public NodeWatcher(KubernetesClient kubernetesClient) { + this.kubernetesClient = kubernetesClient; + } + + @Override + public void eventReceived(Action action, Node node) { + boolean change = false; + NodeWithAlloc processedNode = null; + switch (action) { + case ADDED: + log.info("New Node {} was added", node.getMetadata().getName()); + synchronized ( kubernetesClient.nodeHolder ){ + if ( ! kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ + processedNode = new NodeWithAlloc(node); + kubernetesClient.nodeHolder.put( node.getMetadata().getName(), processedNode ); + change = true; + } + } + if ( change ) kubernetesClient.informAllSchedulersNewNode( processedNode ); + break; + case DELETED: + log.info("Node {} was deleted", node.getMetadata().getName()); + synchronized ( kubernetesClient.nodeHolder ){ + if ( kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ + processedNode = kubernetesClient.nodeHolder.remove( node.getMetadata().getName() ); + change = true; + } + } + if ( change ) kubernetesClient.informAllSchedulersRemovedNode( processedNode ); + break; + case ERROR: + log.info("Node {} has an error", node.getMetadata().getName()); + //todo deal with error + break; + case MODIFIED: + log.info("Node {} was an modified", node.getMetadata().getName()); + //todo deal with changed stae + break; + } + } + + @Override + public void onClose(WatcherException cause) { + log.info( "Watcher was closed" ); + } + } + + class PodWatcher implements Watcher { + + private final KubernetesClient kubernetesClient; + + public PodWatcher(KubernetesClient kubernetesClient) { + this.kubernetesClient = kubernetesClient; + } + + @Override + public void eventReceived(Action action, Pod pod) { + String nodeName = pod.getSpec().getNodeName(); + if( nodeName != null ){ + NodeWithAlloc node = kubernetesClient.nodeHolder.get( pod.getSpec().getNodeName() ); + switch ( action ){ + case ADDED: + node.addPod( pod ); break; + case DELETED: + case ERROR: + node.removePod( pod ); + kubernetesClient.informAllScheduler(); + break; + case MODIFIED: break; + } + } + } + + + @Override + public void onClose(WatcherException cause) { + log.info( "Watcher was closed" ); + } + + } + +} diff --git a/src/main/java/fonda/scheduler/config/Beans.java b/src/main/java/fonda/scheduler/config/Beans.java index b4b35120..1eca2cda 100644 --- a/src/main/java/fonda/scheduler/config/Beans.java +++ b/src/main/java/fonda/scheduler/config/Beans.java @@ -1,21 +1,21 @@ package fonda.scheduler.config; -import fonda.scheduler.model.KubernetesClientHolder; -import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import fonda.scheduler.client.KubernetesClient; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class Beans { - private static KubernetesClientHolder kubernetesClientHolder; + private static KubernetesClient kubernetesClient; @Bean - KubernetesClientHolder getClientHolder(){ - if(kubernetesClientHolder == null) { - kubernetesClientHolder = new KubernetesClientHolder(); + KubernetesClient getClient(){ + if(kubernetesClient == null) { + kubernetesClient = new KubernetesClient(); + kubernetesClient.getConfiguration().setNamespace(null); } - return kubernetesClientHolder; + return kubernetesClient; } } diff --git a/src/main/java/fonda/scheduler/model/KubernetesClientHolder.java b/src/main/java/fonda/scheduler/model/KubernetesClientHolder.java deleted file mode 100644 index 47431832..00000000 --- a/src/main/java/fonda/scheduler/model/KubernetesClientHolder.java +++ /dev/null @@ -1,26 +0,0 @@ -package fonda.scheduler.model; - -import io.fabric8.kubernetes.client.DefaultKubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClient; - -import java.util.HashMap; -import java.util.Map; - -public class KubernetesClientHolder { - - private final Map holder = new HashMap<>(); - - public KubernetesClient getClient(String namespace ){ - - synchronized ( holder ){ - if( !holder.containsKey( namespace.toLowerCase() ) ){ - DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(); - kubernetesClient.getConfiguration().setNamespace(namespace); - holder.put( namespace.toLowerCase(), kubernetesClient ); - } - return holder.get( namespace.toLowerCase() ); - } - - } - -} diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index dfb0c852..64bd3243 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -1,66 +1,141 @@ package fonda.scheduler.model; import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Quantity; import lombok.Getter; -import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; @Getter -@Setter -public class NodeWithAlloc implements Comparable { +@Slf4j +public class NodeWithAlloc extends Node implements Comparable { - private String nodeName; + private final PodRequirements max_resources; - private BigDecimal current_cpu_usage; + Map assignedPods; - private BigDecimal current_ram_usage; + public NodeWithAlloc(Node node) { - private BigDecimal free_cpu; + this.setApiVersion( node.getApiVersion() ); + this.setKind( node.getKind() ); + this.setMetadata( node.getMetadata() ); + this.setSpec( node.getSpec() ); + this.setStatus( node.getStatus() ); + for (Map.Entry e : node.getAdditionalProperties().entrySet()) { + this.setAdditionalProperty( e.getKey(), e.getValue() ); + } - private BigDecimal free_ram; + BigDecimal max_cpu = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("cpu")); + BigDecimal max_ram = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("memory")); - private Node node; + max_resources = new PodRequirements( max_cpu, max_ram); - public NodeWithAlloc(Node node) { - this.node = node; - this.nodeName = node.getMetadata().getName(); - this.current_ram_usage = new BigDecimal(0); - this.current_cpu_usage = new BigDecimal(0); - this.free_ram = new BigDecimal(0); - this.free_cpu = new BigDecimal(0); + assignedPods = new HashMap<>(); + log.info("Node {} has RAM: {} and CPU: {}", node.getMetadata().getName(), max_ram, max_cpu); } - public void calculateAlloc() { + public void addPod( Pod pod ){ + PodRequirements request = getRequest(pod); + synchronized (assignedPods) { + assignedPods.put( pod.getMetadata().getUid(), request );; + } + } - Quantity max_cpu = this.node.getStatus().getAllocatable().get("cpu"); - Quantity max_ram = this.node.getStatus().getAllocatable().get("memory"); + public void removePod( Pod pod ){ + synchronized (assignedPods) { + assignedPods.remove( pod.getMetadata().getUid() ); + } + } - this.free_cpu = Quantity.getAmountInBytes(max_cpu).subtract(current_cpu_usage); - this.free_ram = Quantity.getAmountInBytes(max_ram).subtract(current_ram_usage); + public PodRequirements getRequestedResources(){ + synchronized (assignedPods) { + return assignedPods.values().stream() + .reduce( new PodRequirements(), PodRequirements::addToThis ); + } + } + public PodRequirements getAvailableResources(){ + return max_resources.sub(getRequestedResources()); } - @Override - public String toString() { - return "NodeWithAlloc{" + - "nodeName='" + nodeName + '\'' + - ", current_cpu_usage=" + current_cpu_usage + - ", current_ram_usage=" + current_ram_usage + - ", free_cpu=" + free_cpu + - ", free_ram=" + free_ram + - ", node=" + node + - '}'; + private PodRequirements getRequest(final Pod pod){ + return pod + .getSpec().getContainers().stream() + .filter( x -> x.getResources() != null + && x.getResources().getRequests() != null ) + .map( x -> + new PodRequirements( + x.getResources().getRequests().get("cpu") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("cpu")), + x.getResources().getRequests().get("memory") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("memory")) + ) + ).reduce( new PodRequirements(), PodRequirements::addToThis ); + } + + public boolean canSchedule( Pod pod ){ + final PodRequirements request = getRequest(pod); + PodRequirements availableResources = getAvailableResources(); + return request.getRequested_cpu().compareTo(availableResources.getRequested_cpu()) <= 0 + && request.getRequested_ram().compareTo(availableResources.getRequested_ram()) <= 0; } @Override public int compareTo(NodeWithAlloc o) { - if(getNode().getMetadata().getName().equals(o.getNode().getMetadata().getName())) { + if(getMetadata().getName().equals(o.getMetadata().getName())) { return 0; } else { return -1; } } + + private static class PodRequirements { + + @Getter + private BigDecimal requested_cpu; + @Getter + private BigDecimal requested_ram; + + public PodRequirements(BigDecimal requested_cpu, BigDecimal requested_ram) { + this.requested_cpu = requested_cpu == null ? BigDecimal.ZERO : requested_cpu; + this.requested_ram = requested_ram == null ? BigDecimal.ZERO : requested_ram; + } + + private PodRequirements(){ + this( BigDecimal.ZERO, BigDecimal.ZERO ); + } + + public PodRequirements addToThis( PodRequirements podRequirements ){ + this.requested_cpu = this.requested_cpu.add(podRequirements.requested_cpu); + this.requested_ram = this.requested_ram.add(podRequirements.requested_ram); + return this; + } + + public PodRequirements addRAMtoThis( BigDecimal requested_ram ){ + this.requested_ram = this.requested_ram.add( requested_ram ); + return this; + } + + public PodRequirements addCPUtoThis( BigDecimal requested_cpu ){ + this.requested_cpu = this.requested_cpu.add( requested_cpu ); + return this; + } + + public PodRequirements subFromThis( PodRequirements podRequirements ){ + this.requested_cpu = this.requested_cpu.subtract(podRequirements.requested_cpu); + this.requested_ram = this.requested_ram.subtract(podRequirements.requested_ram); + return this; + } + + public PodRequirements sub( PodRequirements podRequirements ){ + return new PodRequirements( + this.requested_cpu.subtract(podRequirements.requested_cpu), + this.requested_ram.subtract(podRequirements.requested_ram) + ); + } + + } } diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index c2627e6f..b6a63b50 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -1,6 +1,6 @@ package fonda.scheduler.rest; -import fonda.scheduler.model.KubernetesClientHolder; +import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.scheduler.RandomScheduler; import fonda.scheduler.scheduler.Scheduler; import fonda.scheduler.model.SchedulerConfig; @@ -19,17 +19,17 @@ @Slf4j public class SchedulerRestController { - private final KubernetesClientHolder clientHolder; + private final KubernetesClient client; /** * Holds the scheduler for one execution * Execution: String in lowercase * Scheduler: An instance of a scheduler with the requested type */ - private final Map< Pair, Scheduler > schedulerHolder = new HashMap<>(); + private static final Map< Pair, Scheduler > schedulerHolder = new HashMap<>(); - public SchedulerRestController( @Autowired KubernetesClientHolder clientHolder ){ - this.clientHolder = clientHolder; + public SchedulerRestController( @Autowired KubernetesClient client ){ + this.client = client; } /** @@ -54,7 +54,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable S switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : - case "fifo-random" : scheduler = new RandomScheduler(execution, clientHolder.getClient( namespace ), config ); break; + case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace ); break; default: return new ResponseEntity( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index f0f1cc07..51721e2c 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -1,327 +1,39 @@ package fonda.scheduler.scheduler; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.model.TaskConfig; -import io.fabric8.kubernetes.api.model.Node; +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.NodeWithAlloc; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.dsl.ExecListener; -import io.fabric8.kubernetes.client.dsl.ExecWatch; -import io.fabric8.kubernetes.client.dsl.PodResource; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import okhttp3.Response; -import java.io.*; -import java.nio.file.Path; import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; @Slf4j -public class RandomScheduler extends Scheduler { +public class RandomScheduler extends Scheduler { + - //For testing - private final boolean local = true; - private final String name; private final Map daemonByNode = new HashMap<>(); - private final List localClaims; private final Map workdirNode = new HashMap<>(); - private final SchedulerConfig schedulerConfig; - - public RandomScheduler( String name, KubernetesClient client, SchedulerConfig config ) { - super( name, client); - this.name = name; - this.localClaims = config.localClaims; - this.schedulerConfig = config; - log.info ( "Create scheduler: " + name ); - } - - public void schedule(Pod podToSchedule) { - log.info ( "===========" ); - if( podToSchedule != null ){ - List items = getNodeList().getItems(); - int trial = 0; - while( trial++ < 5 ) { - if(isClose()) return; - try { - Node node = items.get(new Random().nextInt(items.size())); - if( !daemonByNode.containsKey( node.getMetadata().getName() )) continue; - - String workingDir = getWorkingDir(podToSchedule); - workdirNode.put(workingDir, node.getMetadata().getName()); - //Copy files to this node - if( !copyConfig(workingDir, node) ){ - log.info( "No success to copy config for " + workingDir + " to Node " + node.getMetadata().getName() ); - continue; - } - - TaskConfig config = getConfigFor(podToSchedule.getMetadata().getName()); - if( !copyInputsToNode(config, node) ){ - log.info( "No success to copy input for " + workingDir + " to Node " + node.getMetadata().getName() ); - continue; - } - //Store Map: Workdir Node - assignPodToNode(podToSchedule, node); - break; - }catch (Exception e){ - log.info( "Try " + podToSchedule.getMetadata().getName() + " again on a different node..." ); - e.printStackTrace(); - } - } - if(trial > 5 ){ - log.info( "Failed to schedule " + podToSchedule.getMetadata().getName() + " " + daemonByNode ); - podToSchedule.getStatus().setPhase("Failed"); - } - } else { - log.info ( "Pod was null." ); - } - log.info ( "===========" ); - } - void podEventReceived(Watcher.Action action, Pod pod){ - //TODO get current ExecutorPod else use local - if( ( "mount-" + name.replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ - String nodeName = pod.getSpec().getNodeName(); - if ( nodeName != null ){ - if( action == Watcher.Action.DELETED ){ - if ( pod.getMetadata().getName().equals( daemonByNode.get(nodeName) )){ - daemonByNode.remove( nodeName ); - } - } else if (pod.getStatus().getPhase().equals("Running")) { - daemonByNode.put(nodeName, pod.getMetadata().getName() ); - } else if (pod.getStatus().getPhase().equals("Failed")) { - daemonByNode.remove( nodeName ); - } else { - daemonByNode.remove( nodeName ); - } - } - } - } - - private boolean copyConfig( String workdir, Node node ){ - String destinationPod = daemonByNode.get( node.getMetadata().getName() ); - log.info ("Copy workdir: " + workdir + " to node: " + node.getMetadata().getName() + " (" + destinationPod + ") " + daemonByNode); - if( local ){ - //SchedulerConfig.LocalClaim localClaim = localClaims.stream().filter(x -> workdir.startsWith(x.mountPath)).findFirst().get(); - String[] files = {".command.run", ".command.sh"}; - for( String file : files) { - boolean success = false; - int failureCount = 0; - while( !success ) { - try{ - Path workdirPath = new File(workdir + File.separator + file).toPath(); - Boolean upload = findPodByName(destinationPod) - .file(workdir + File.separator + file) - .upload(workdirPath); - success = upload; - if(!success) log.info("No success uploading: " + file ); - }catch(Exception e){ - log.info ("Error for pod " + destinationPod + " on node: " + node.getMetadata().getName() ); - if ( ++failureCount > 10 ) { - e.printStackTrace(); - return false; - } - destinationPod = daemonByNode.get( node.getMetadata().getName() ); - log.info ("Try again for pod " + destinationPod + " on node: " + node.getMetadata().getName() ); - } - - } - } - return true; - } else { - throw new RuntimeException( "Currently this can not run in Kubernetes!" ); - } + public RandomScheduler(String name, KubernetesClient client, String namespace) { + super(name, client, namespace); } @Override - void onPodTermination( Pod pod ) { - if(isClose()) return; - //Copy results from Executor - PodResource daemon = findPodByName(daemonByNode.get(pod.getSpec().getNodeName())); - String workingDir = getWorkingDir(pod); - String[] files = { - ".command.sh", - ".command.out", - ".exitcode", - //".exitcode2", - ".command.trace", - ".command.infiles", - ".command.outfiles", - ".command.outfilesTmp" - }; - if( local ){ - log.info("Start copying all files for " + pod.getSpec().getNodeName() ); - for( String file : files) { - Path workdirPath = new File( workingDir + File.separator + file ).toPath(); - Boolean copy = daemon - .file(workingDir + File.separator + file) - .copy(workdirPath); - if(!copy){ - log.info("No success downloading: " + file ); - } - } - - //For debugging: download logs - try { - final PrintWriter printWriter = new PrintWriter(workingDir + File.separator + ".command.kubelogs"); - printWriter.print(daemon.getLog()); - printWriter.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - - log.info("Copied all files for " + pod.getSpec().getNodeName() ); - try { - new File( workingDir + File.separator + ".command.done" ).createNewFile(); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - throw new RuntimeException( "Currently this can not run in Kubernetes!" ); - } - } - - private String extractWorkspace( String path ){ - if( path.startsWith( schedulerConfig.workDir ) ){ - return path.substring( 0, schedulerConfig.workDir.length() + 34 ); - } - return null; - } - - private boolean copyInputsToNode(TaskConfig config , Node node ){ - //TODO schedule copy - //Learn copy Speed - //Remember Filesize - //Time = Size * Speed, order desc, run in this order, x parallel - - Map< String, List< String > > filesByNode = new HashMap<>(); - - //Structure input files by node - config.getInputs().forEach( x -> { - log.info(x.toString()); - if( x.getValue() instanceof List ){ - for( Object o : (List) x.getValue()){ - if( o instanceof Map ){ - Map input = (Map) o; - if(input.containsKey( "sourceObj" )){ - String sourceObj = (String) input.get( "sourceObj" ); - String workspace = extractWorkspace(sourceObj); - - if( workspace != null ){ - - final String podName = daemonByNode.get(workdirNode.get(workspace)); - List inputs; - if( !filesByNode.containsKey( podName ) ){ - inputs = new LinkedList<>(); - filesByNode.put( podName, inputs ); - } else { - inputs = filesByNode.get( podName ); - } - - inputs.add( sourceObj ); - - } - } - } - } + public void schedule( final List unscheduledPods ) { + List items = getNodeList(); + for ( final Pod pod : unscheduledPods) { + if(isClose()) return; + Optional node = items.stream().filter(x -> x.canSchedule(pod)).findFirst(); + if( node.isPresent() ){ + assignPodToNode( pod, node.get() ); + super.scheduledPod( pod ); + } else { + log.info( "No node with enough resources for {}", pod.getMetadata().getName() ); } - } ); - - return filesByNode.entrySet().stream().allMatch( x -> { - return copyFile( x.getKey(), node.getMetadata().getName(), x.getValue() ); - }); - //Copy for x Nodes in parallel - - } - - private boolean copyFile(String fromPod, String toNode, List files){ - - String toPod = daemonByNode.get(toNode); - if( toPod.equals(fromPod) ){ - return true; - } - - //TODO reduce mkdir to the only necessary, remove files, if it already exists - String[] command = new String[ files.size() * 9 - 1 ]; - int i = 0; - for ( String file : files ) { - command[i++] = "mkdir"; - command[i++] = "-p"; - command[i++] = file.substring(0, file.lastIndexOf("/")) ; - command[i++] = "&&"; - command[i++] = "kubectl"; - command[i++] = "cp"; - command[i++] = fromPod + ":\"" + file + "\""; - //if( file.endsWith( "/" ) ){ - command[i++] = "\"" + file + "\""; - //} else { - // command[i++] = "\"" + file.substring( 0, file.lastIndexOf('/' ) ) + "\""; - //} - - if( i < command.length) command[i++] = "&&"; - } - //String command = file.stream() - //.map(x -> "mkdir -p \"" + x.substring(0, x.lastIndexOf("/")) + "\" && kubectl cp " + fromPod + ":\"" + x + "\" \"" + x + "\"") - //.collect(Collectors.joining(" && ")); - - log.info( toNode + ": " + Arrays.toString(command) ); - - - final PodResource pod = findPodByName(toPod); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ByteArrayOutputStream error = new ByteArrayOutputStream(); - CountDownLatch execLatch = new CountDownLatch(1); - - final MyPodExecListener listener = new MyPodExecListener(execLatch); - ExecWatch exec = pod - .writingOutput(out) - .writingError(error) - .usingListener(listener) - .exec("bash", "-c", String.join(" ", command )); - try { - execLatch.await(1, TimeUnit.DAYS ); - } catch (InterruptedException e) { - e.printStackTrace(); - } - exec.close(); - log.info("Exec Output: {} ", out); - if(listener.isFailure()){ - log.info ("Could not copy files from " + fromPod + " to " + toNode); - } - return !listener.isFailure(); - } - - private class MyPodExecListener implements ExecListener { - - private final CountDownLatch execLatch; - - @Getter - private boolean failure = false; - - private MyPodExecListener( final CountDownLatch execLatch ){ - this.execLatch = execLatch; - } - - @Override - public void onOpen(Response response) { - log.info("Shell was opened"); - } - - @Override - public void onFailure(Throwable throwable, Response response) { - log.info("Some error encountered"); - execLatch.countDown(); - failure = true; - } - - @Override - public void onClose(int i, String s) { - log.info("Shell Closing"); - execLatch.countDown(); } } + @Override + void podEventReceived(Watcher.Action action, Pod pod){} } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 7a4fd62d..7735c87f 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -1,175 +1,84 @@ package fonda.scheduler.scheduler; -import fonda.scheduler.model.PodListWithIndex; +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; import fonda.scheduler.model.TaskConfig; -import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.api.model.Binding; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectReference; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; import io.fabric8.kubernetes.client.dsl.PodResource; -import io.fabric8.kubernetes.client.dsl.base.OperationContext; -import io.fabric8.kubernetes.client.informers.ListerWatcher; -import io.fabric8.kubernetes.client.informers.ResourceEventHandler; -import io.fabric8.kubernetes.client.informers.SharedInformerEventListener; -import io.fabric8.kubernetes.client.informers.impl.DefaultSharedIndexInformer; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; @Slf4j public abstract class Scheduler { - DefaultSharedIndexInformer defaultSharedIndexInformerNode; - final KubernetesClient client; - private OperationContext operationContext; - private ConcurrentLinkedQueue workerQueueNode; - PodListWithIndex podList = new PodListWithIndex(); + //Visible variables @Getter private final String name; + @Getter + private boolean close; + + final KubernetesClient client; + private final Map podsByUID = new HashMap<>(); + private final String namespace; + final private List unscheduledPods = new ArrayList<>(100); + final Map> taskConfigs = new HashMap<>(); final Map taskConfigsByHash = new HashMap<>(); - @Getter - private boolean close; - final static private ExecutorService executorService = Executors.newFixedThreadPool(30); + + private final SchedulerThread schedulerThread; private final Watch watcher; - Scheduler( String name, KubernetesClient client ){ + Scheduler(String name, KubernetesClient client, String namespace){ this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + name; + this.namespace = namespace; log.trace( "Register scheduler for " + this.name ); - this.client = client; - this.workerQueueNode = new ConcurrentLinkedQueue<>(); - - this.operationContext = new OperationContext(); - - - setUpIndexInformerNode(); - - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - watcher = client.pods().watch(new Watcher<>() { - @Override - public void eventReceived(Action action, Pod pod) { - - podEventReceived(action, pod); - - if (!Scheduler.this.name.equals(pod.getSpec().getSchedulerName())) return; - - PodWithAge pwa = new PodWithAge(pod); - if (pod.getMetadata().getLabels() != null) { - log.debug("Got pod: " + pod.getMetadata().getName() + - " app: " + pod.getMetadata().getLabels().getOrDefault("app", "-") + - " processName: " + pod.getMetadata().getLabels().getOrDefault("processName", "-") + - " runName: " + pod.getMetadata().getLabels().getOrDefault("runName", "-") + - " taskName: " + pod.getMetadata().getLabels().getOrDefault("taskName", "-") + - " scheduler: " + pwa.getSpec().getSchedulerName() + - " action: " + action - ); - } else { - log.debug("Got pod " + pod.getMetadata().getName() + " scheduler: " + pwa.getSpec().getSchedulerName()); - } - - switch (action) { - case ADDED: - addPod(pwa); - if (pwa.getSpec().getNodeName() == null) { - executorService.submit(() -> schedule(pwa)); - } - break; - case MODIFIED: - if (pod.getStatus().getContainerStatuses().size() > 0 && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { - executorService.submit(() -> onPodTermination(pwa)); - } - break; - case DELETED: - removePod(pwa); - executorService.submit(() -> schedule(null)); - } - - } - - @Override - public void onClose(WatcherException cause) { - log.info( "Watcher was closed" ); - } - - }); + PodWatcher podWatcher = new PodWatcher(this); + log.info("Start watching"); + watcher = client.pods().inNamespace( this.namespace ).watch(podWatcher); + log.info("Watching"); + schedulerThread = new SchedulerThread( unscheduledPods,this ); + schedulerThread.start(); } + public abstract void schedule( final List unscheduledPods ); void podEventReceived(Watcher.Action action, Pod pod){} - - void onPodTermination( Pod pod ){} + void onPodTermination( Pod pod ){ + //todo run in thread + } /** - * + * You may extend this method + * @param pod + * @param node + * @return */ - private void setUpIndexInformerNode() { - ListerWatcher listerWatcher = new ListerWatcher() { - @Override - public Watch watch(ListOptions params, String namespace, OperationContext context, Watcher watcher) { - return client.nodes().watch(new Watcher<>() { - @Override - public void eventReceived(Action action, Node resource) { - - } - - @Override - public void onClose(WatcherException cause) { - - } - }); - } - - @Override - public Object list(ListOptions params, String namespace, OperationContext context) { - return getNodeList(); - } - }; - - defaultSharedIndexInformerNode = new DefaultSharedIndexInformer(Node.class, listerWatcher, 600000, operationContext, workerQueueNode); - - try { - defaultSharedIndexInformerNode.run(); - } catch (Exception e) { - e.printStackTrace(); - } - - defaultSharedIndexInformerNode.addEventHandler(new ResourceEventHandler<>() { - @Override - public void onAdd(Node obj) { - - } + boolean canPodBeScheduled( Pod pod, NodeWithAlloc node ){ + return node.canSchedule( pod ); + } - @Override - public void onUpdate(Node oldObj, Node newObj) { + void assignPodToNode( Pod pod, NodeWithAlloc node ){ - } + node.addPod( pod ); - @Override - public void onDelete(Node obj, boolean deletedFinalStateUnknown) { - - } - }); - } - - void assignPodToNode( Pod pod, Node node ){ - log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + node ); + log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + node.getMetadata().getName() ); Binding b1 = new Binding(); @@ -181,13 +90,13 @@ void assignPodToNode( Pod pod, Node node ){ ObjectReference objectReference = new ObjectReference(); objectReference.setApiVersion("v1"); objectReference.setKind("Node"); - objectReference.setName(node.getMetadata().getName()); //hier ersetzen durch tatsächliche Node + objectReference.setName(node.getMetadata().getName()); b1.setTarget(objectReference); client.bindings().create(b1); - pod.getSpec().setNodeName( node.getMetadata().getName() ); // Hier stimmt evtl etwas nicht + pod.getSpec().setNodeName( node.getMetadata().getName() ); log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); } @@ -196,24 +105,46 @@ void assignPodToNode( Pod pod, Node node ){ */ public void close(){ watcher.close(); + schedulerThread.interrupt(); this.close = true; } + public void addUnscheduledPod( Pod pod ) { + synchronized ( unscheduledPods ){ + unscheduledPods.add( pod ); + unscheduledPods.notifyAll(); + } + } + + public void scheduledPod( Pod pod ) { + synchronized ( unscheduledPods ){ + unscheduledPods.remove( pod ); + } + } + + public void informResourceChange() { + synchronized ( unscheduledPods ){ + unscheduledPods.notifyAll(); + } + } + public void addPod( Pod pod ) { - podList.addPodToList( pod ); + synchronized (podsByUID){ + podsByUID.put( pod.getMetadata().getUid(), pod ); + } } public void removePod( Pod pod ) { - podList.removePodFromList( pod ); + synchronized (podsByUID){ + podsByUID.remove( pod.getMetadata().getUid() ); + } } - NodeList getNodeList(){ - return client.nodes().list(); + List getNodeList(){ + return client.getAllNodes(); } - public abstract void schedule( Pod podToSchedule ); - String getWorkingDir( Pod pod ){ return pod.getSpec().getContainers().get(0).getWorkingDir(); } @@ -256,4 +187,67 @@ public Map getSchedulerParams( String taskname, String name ){ return result; } + public void newNode(NodeWithAlloc node) { + informResourceChange(); + } + + public void removedNode(NodeWithAlloc node) {} + + + static class PodWatcher implements Watcher { + + private final Scheduler scheduler; + + public PodWatcher(Scheduler scheduler) { + this.scheduler = scheduler; + } + + @Override + public void eventReceived(Action action, Pod pod) { + + scheduler.podEventReceived(action, pod); + + if (!scheduler.name.equals(pod.getSpec().getSchedulerName())) return; + + PodWithAge pwa = new PodWithAge(pod); + if (pod.getMetadata().getLabels() != null) { + log.debug("Got pod: " + pod.getMetadata().getName() + + " app: " + pod.getMetadata().getLabels().getOrDefault("app", "-") + + " processName: " + pod.getMetadata().getLabels().getOrDefault("processName", "-") + + " runName: " + pod.getMetadata().getLabels().getOrDefault("runName", "-") + + " taskName: " + pod.getMetadata().getLabels().getOrDefault("taskName", "-") + + " scheduler: " + pwa.getSpec().getSchedulerName() + + " action: " + action + ); + } else { + log.debug("Got pod " + pod.getMetadata().getName() + " scheduler: " + pwa.getSpec().getSchedulerName()); + } + + switch (action) { + case ADDED: + scheduler.addPod(pwa); + if (pwa.getSpec().getNodeName() == null && pwa.getSpec().getSchedulerName().equalsIgnoreCase( scheduler.name )) { + scheduler.addUnscheduledPod ( pwa ); + } + break; + case MODIFIED: + if (pod.getStatus().getContainerStatuses().size() > 0 && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { + scheduler.onPodTermination(pwa); + } + break; + case DELETED: + scheduler.removePod(pwa); + } + + } + + + @Override + public void onClose(WatcherException cause) { + log.info( "Watcher was closed" ); + } + + } + + } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerThread.java b/src/main/java/fonda/scheduler/scheduler/SchedulerThread.java new file mode 100644 index 00000000..76a2597b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerThread.java @@ -0,0 +1,35 @@ +package fonda.scheduler.scheduler; + +import io.fabric8.kubernetes.api.model.Pod; +import lombok.extern.slf4j.Slf4j; +import java.util.*; + +@Slf4j +public class SchedulerThread extends Thread { + + private final List unscheduledPods; + private final Scheduler scheduler; + + public SchedulerThread(List unscheduledPods, Scheduler scheduler) { + this.unscheduledPods = unscheduledPods; + this.scheduler = scheduler; + } + + @Override + public void run() { + while(!Thread.interrupted()){ + try{ + synchronized (unscheduledPods) { + if (unscheduledPods.isEmpty()) { + unscheduledPods.wait( 10000 ); + } + } + scheduler.schedule( new LinkedList<>(unscheduledPods) ); + } catch (InterruptedException e){ + Thread.currentThread().interrupt(); + } catch (Exception e){ + log.info("Error while scheduling",e); + } + } + } +} From d5f1bf2166dcd2308523b8a6d8638572693f5587 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 27 Oct 2021 16:20:06 +0200 Subject: [PATCH 003/443] restructured Scheduler.java & wrap pods into Tasks Signed-off-by: Lehmann-Fabian --- .../java/fonda/scheduler/model/State.java | 7 + src/main/java/fonda/scheduler/model/Task.java | 25 ++ .../java/fonda/scheduler/model/TaskState.java | 24 ++ .../rest/SchedulerRestController.java | 22 +- .../scheduler/scheduler/RandomScheduler.java | 29 ++- .../fonda/scheduler/scheduler/Scheduler.java | 244 +++++++++++------- .../scheduler/scheduler/SchedulerThread.java | 35 --- .../scheduler/TaskprocessingThread.java | 42 +++ 8 files changed, 292 insertions(+), 136 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/State.java create mode 100644 src/main/java/fonda/scheduler/model/Task.java create mode 100644 src/main/java/fonda/scheduler/model/TaskState.java delete mode 100644 src/main/java/fonda/scheduler/scheduler/SchedulerThread.java create mode 100644 src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java diff --git a/src/main/java/fonda/scheduler/model/State.java b/src/main/java/fonda/scheduler/model/State.java new file mode 100644 index 00000000..48b12b50 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/State.java @@ -0,0 +1,7 @@ +package fonda.scheduler.model; + +public enum State { + + RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, ERROR, PROCESSING_FINISHED, FINISHED, DELETED; + +} diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java new file mode 100644 index 00000000..a75cc7f0 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -0,0 +1,25 @@ +package fonda.scheduler.model; + +import io.fabric8.kubernetes.api.model.Pod; +import lombok.Getter; +import lombok.Setter; + +public class Task { + + @Getter + private final TaskConfig config; + @Getter + private final TaskState state = new TaskState(); + + @Getter + @Setter + private Pod pod = null; + + @Getter + @Setter + private String nodeName = null; + + public Task(TaskConfig config) { + this.config = config; + } +} diff --git a/src/main/java/fonda/scheduler/model/TaskState.java b/src/main/java/fonda/scheduler/model/TaskState.java new file mode 100644 index 00000000..99335ce4 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/TaskState.java @@ -0,0 +1,24 @@ +package fonda.scheduler.model; + +import lombok.Getter; +import lombok.Setter; + +public class TaskState { + + @Getter + @Setter + private State state; + + @Getter + private String error; + + public TaskState() { + this.state = State.RECEIVED_CONFIG; + } + + public void error (String error){ + this.state = State.ERROR; + this.error = error; + } +} + diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index b6a63b50..639b9e62 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -83,13 +83,33 @@ ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } - scheduler.addTaskConfig( config.getTask(), config ); + scheduler.addTask( config ); Map schedulerParams = scheduler.getSchedulerParams(config.getTask(), config.getName()); return new ResponseEntity( schedulerParams, HttpStatus.OK ); } + /** + * Check Task state + * @param namespace namespace where the workflow runs + * @param execution unique name of the execution + * @param taskid unique name of task (nf-XYZ...) + * @return boolean + */ + @GetMapping("/scheduler/taskstate/{namespace}/{execution}/{taskid}") + ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { + + final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + return new ResponseEntity( scheduler.getTaskState( taskid ), HttpStatus.OK ); + + } + /** * Call this after the execution has finished * @param namespace namespace where the workflow runs diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 51721e2c..9312a56e 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -2,6 +2,8 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.State; +import fonda.scheduler.model.Task; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; import lombok.extern.slf4j.Slf4j; @@ -20,18 +22,31 @@ public RandomScheduler(String name, KubernetesClient client, String namespace) { } @Override - public void schedule( final List unscheduledPods ) { + public int schedule( final List unscheduledTasks ) { + log.info("Schedule " + this.getName()); List items = getNodeList(); - for ( final Pod pod : unscheduledPods) { - if(isClose()) return; - Optional node = items.stream().filter(x -> x.canSchedule(pod)).findFirst(); + int unscheduled = 0; + for ( final Task task : unscheduledTasks) { + if(isClose()) return -1; + Optional node = items.stream().filter(x -> x.canSchedule(task.getPod())).findFirst(); if( node.isPresent() ){ - assignPodToNode( pod, node.get() ); - super.scheduledPod( pod ); + log.info("Task needs: " + task.getConfig().getInputs().toString()); + assignPodToNode( task.getPod(), node.get() ); + super.taskWasScheduled( task ); } else { - log.info( "No node with enough resources for {}", pod.getMetadata().getName() ); + log.info( "No node with enough resources for {}", task.getPod().getMetadata().getName() ); + unscheduled++; } } + return unscheduled; + } + + @Override + int terminateTasks(List finishedTasks) { + for (Task finishedTask : finishedTasks) { + super.taskWasFinished( finishedTask ); + } + return 0; } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 7735c87f..070d195b 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -1,9 +1,7 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.model.TaskConfig; +import fonda.scheduler.model.*; import io.fabric8.kubernetes.api.model.Binding; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectReference; @@ -30,18 +28,13 @@ public abstract class Scheduler { private boolean close; final KubernetesClient client; - private final Map podsByUID = new HashMap<>(); private final String namespace; - final private List unscheduledPods = new ArrayList<>(100); - - - final Map> taskConfigs = new HashMap<>(); - final Map taskConfigsByHash = new HashMap<>(); - - - private final SchedulerThread schedulerThread; - + final private List unscheduledTasks = new ArrayList<>(100); + final private List unfinishedTasks = new ArrayList<>(100); + final Map tasksByHash = new HashMap<>(); private final Watch watcher; + private final TaskprocessingThread schedulingThread; + private final TaskprocessingThread finishThread; Scheduler(String name, KubernetesClient client, String namespace){ this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + name; @@ -54,14 +47,112 @@ public abstract class Scheduler { watcher = client.pods().inNamespace( this.namespace ).watch(podWatcher); log.info("Watching"); - schedulerThread = new SchedulerThread( unscheduledPods,this ); - schedulerThread.start(); + schedulingThread = new TaskprocessingThread( unscheduledTasks, this::schedule ); + schedulingThread.start(); + + finishThread = new TaskprocessingThread(unfinishedTasks, this::terminateTasks ); + finishThread.start(); } - public abstract void schedule( final List unscheduledPods ); + /* Abstract methods */ + + /** + * + * @param unscheduledTasks + * @return the number of unscheduled Tasks + */ + abstract int schedule( final List unscheduledTasks ); + + abstract int terminateTasks( final List finishedTasks ); + + /* Pod Event */ + void podEventReceived(Watcher.Action action, Pod pod){} + void onPodTermination( Pod pod ){ - //todo run in thread + Task t = changeStateOfTask( pod, State.PROCESSING_FINISHED ); + + //If null, task was already changed + if( t == null ) return; + + synchronized (unfinishedTasks){ + unfinishedTasks.add( t ); + unfinishedTasks.notifyAll(); + } + } + + void taskWasFinished( Task task ){ + synchronized (unfinishedTasks){ + unfinishedTasks.remove( task ); + } + task.getState().setState(State.FINISHED); + } + + public void schedulePod(Pod pod ) { + Task t = changeStateOfTask( pod, State.UNSCHEDULED ); + //If null, task was already unscheduled + if ( t == null ) return; + t.setPod( pod ); + synchronized (unscheduledTasks){ + unscheduledTasks.add( t ); + unscheduledTasks.notifyAll(); + } + } + + public void taskWasScheduled(Task task ) { + synchronized (unscheduledTasks){ + unscheduledTasks.remove( task ); + } + task.getState().setState( State.SCHEDULED ); + } + + public void markPodAsDeleted( Pod pod ) { + changeStateOfTask( pod, State.DELETED ); + } + + /* External access to Tasks */ + + public void addTask( TaskConfig conf ) { + synchronized (tasksByHash) { + if( ! tasksByHash.containsKey( conf.getHash() ) ){ + tasksByHash.put( conf.getHash(), new Task(conf) ); + } + } + } + + TaskConfig getConfigFor( String hash ){ + synchronized (tasksByHash) { + if( tasksByHash.containsKey( hash ) ){ + return tasksByHash.get( hash ).getConfig(); + } + } + return null; + } + + public Map getSchedulerParams( String taskname, String name ){ + Map result = new HashMap(); + return result; + } + + public TaskState getTaskState(String taskid) { + synchronized (tasksByHash) { + if( tasksByHash.containsKey( taskid ) ){ + return tasksByHash.get( taskid ).getState(); + } + } + return null; + } + + /* Nodes */ + + public void newNode(NodeWithAlloc node) { + informResourceChange(); + } + + public void removedNode(NodeWithAlloc node) {} + + List getNodeList(){ + return client.getAllNodes(); } /** @@ -100,100 +191,68 @@ void assignPodToNode( Pod pod, NodeWithAlloc node ){ log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); } + /* Helper */ + /** - * Close used resources + * + * @param pod + * @param state + * @return returns the task, if the state was changed */ - public void close(){ - watcher.close(); - schedulerThread.interrupt(); - this.close = true; - } - - public void addUnscheduledPod( Pod pod ) { - synchronized ( unscheduledPods ){ - unscheduledPods.add( pod ); - unscheduledPods.notifyAll(); - } - } - - public void scheduledPod( Pod pod ) { - synchronized ( unscheduledPods ){ - unscheduledPods.remove( pod ); - } - } - - public void informResourceChange() { - synchronized ( unscheduledPods ){ - unscheduledPods.notifyAll(); - } - } - - - public void addPod( Pod pod ) { - synchronized (podsByUID){ - podsByUID.put( pod.getMetadata().getUid(), pod ); - } - } - - public void removePod( Pod pod ) { - synchronized (podsByUID){ - podsByUID.remove( pod.getMetadata().getUid() ); + private Task changeStateOfTask( Pod pod, State state ){ + Task t = getTaskByPod( pod ); + if( t != null ){ + synchronized ( t.getState() ){ + if( t.getState().getState() != state ){ + t.getState().setState( state ); + return t; + } else { + return null; + } + } } - } - - List getNodeList(){ - return client.getAllNodes(); + return null; } String getWorkingDir( Pod pod ){ return pod.getSpec().getContainers().get(0).getWorkingDir(); } + @Deprecated PodResource findPodByName(String name ){ return client.pods().withName( name ); } + + public void informResourceChange() { + synchronized (unscheduledTasks){ + unscheduledTasks.notifyAll(); + } + } - public void addTaskConfig( String name, TaskConfig config ) { - - Map< String, TaskConfig> conf; - synchronized (taskConfigs) { - if( taskConfigs.containsKey( name ) ){ - conf = taskConfigs.get( name ); - } else { - conf = new HashMap<>(); - taskConfigs.put( name, conf ); + private Task getTaskByPod( Pod pod ) { + Task t = null; + synchronized (tasksByHash) { + if( tasksByHash.containsKey( pod.getMetadata().getName() ) ){ + t = tasksByHash.get( pod.getMetadata().getName() ); } } - synchronized ( conf ){ - conf.put( config.getName(), config ); - } - synchronized ( taskConfigsByHash ){ - taskConfigsByHash.put ( config.getHash(), config ); - } - } - TaskConfig getConfigFor(String taskname, String name ){ - return taskConfigs.get( taskname ).get( name ); - } + if ( t == null ){ + throw new IllegalStateException( "No task with config found for: " + pod.getMetadata().getName() ); + } - TaskConfig getConfigFor(String hash ){ - return taskConfigsByHash.get( hash ); + return t; } - public Map getSchedulerParams( String taskname, String name ){ - TaskConfig config = getConfigFor(taskname, name); - Map result = new HashMap(); - config.getSchedulerParams().entrySet().forEach( x -> result.put( x.getKey(), x.getValue().get(0)) ); - return result; - } - - public void newNode(NodeWithAlloc node) { - informResourceChange(); + /** + * Close used resources + */ + public void close(){ + watcher.close(); + schedulingThread.interrupt(); + this.close = true; } - public void removedNode(NodeWithAlloc node) {} - - static class PodWatcher implements Watcher { private final Scheduler scheduler; @@ -225,9 +284,8 @@ public void eventReceived(Action action, Pod pod) { switch (action) { case ADDED: - scheduler.addPod(pwa); if (pwa.getSpec().getNodeName() == null && pwa.getSpec().getSchedulerName().equalsIgnoreCase( scheduler.name )) { - scheduler.addUnscheduledPod ( pwa ); + scheduler.schedulePod( pwa ); } break; case MODIFIED: @@ -236,7 +294,7 @@ public void eventReceived(Action action, Pod pod) { } break; case DELETED: - scheduler.removePod(pwa); + scheduler.markPodAsDeleted(pwa); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerThread.java b/src/main/java/fonda/scheduler/scheduler/SchedulerThread.java deleted file mode 100644 index 76a2597b..00000000 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerThread.java +++ /dev/null @@ -1,35 +0,0 @@ -package fonda.scheduler.scheduler; - -import io.fabric8.kubernetes.api.model.Pod; -import lombok.extern.slf4j.Slf4j; -import java.util.*; - -@Slf4j -public class SchedulerThread extends Thread { - - private final List unscheduledPods; - private final Scheduler scheduler; - - public SchedulerThread(List unscheduledPods, Scheduler scheduler) { - this.unscheduledPods = unscheduledPods; - this.scheduler = scheduler; - } - - @Override - public void run() { - while(!Thread.interrupted()){ - try{ - synchronized (unscheduledPods) { - if (unscheduledPods.isEmpty()) { - unscheduledPods.wait( 10000 ); - } - } - scheduler.schedule( new LinkedList<>(unscheduledPods) ); - } catch (InterruptedException e){ - Thread.currentThread().interrupt(); - } catch (Exception e){ - log.info("Error while scheduling",e); - } - } - } -} diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java new file mode 100644 index 00000000..5a92ec5c --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java @@ -0,0 +1,42 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.model.Task; +import lombok.extern.slf4j.Slf4j; +import java.util.*; +import java.util.function.Function; + +@Slf4j +public class TaskprocessingThread extends Thread { + + private final List unprocessedTasks; + private final Function, Integer> function; + + public TaskprocessingThread(List unprocessedTasks, Function, Integer> function ) { + this.unprocessedTasks = unprocessedTasks; + this.function = function; + } + + @Override + public void run() { + int unscheduled = 0; + while(!Thread.interrupted()){ + try{ + LinkedList tasks; + synchronized (unprocessedTasks) { + do { + if (unscheduled == unprocessedTasks.size()) { + unprocessedTasks.wait( 10000 ); + } + if( Thread.interrupted() ) return; + } while ( unprocessedTasks.isEmpty() ); + tasks = new LinkedList<>(unprocessedTasks); + } + unscheduled = function.apply( tasks ); + } catch (InterruptedException e){ + Thread.currentThread().interrupt(); + } catch (Exception e){ + log.info("Error while processing",e); + } + } + } +} From 2a06e896f98d30edfe1c77b8e031fc560bbbe551 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 4 Nov 2021 17:13:23 +0100 Subject: [PATCH 004/443] Added HierarchyHolder Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/File.java | 15 ++ .../model/location/hierachy/Folder.java | 67 +++++++ .../location/hierachy/HierarchyWrapper.java | 93 ++++++++++ .../hierachy/HierarchyWrapperTest.java | 173 ++++++++++++++++++ 4 files changed, 348 insertions(+) create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/File.java create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/Folder.java create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java create mode 100644 src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/File.java b/src/main/java/fonda/scheduler/model/location/hierachy/File.java new file mode 100644 index 00000000..8acd76e4 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/File.java @@ -0,0 +1,15 @@ +package fonda.scheduler.model.location.hierachy; + +public class File { + + private final static File fileInstance = new File(); + + public static File get(){ + return fileInstance; + } + + public boolean isDirectory(){ + return false; + } + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java new file mode 100644 index 00000000..7a3ec979 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -0,0 +1,67 @@ +package fonda.scheduler.model.location.hierachy; + +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Slf4j +public class Folder extends File { + + private final ConcurrentMap children = new ConcurrentHashMap<>(); + + @Override + public boolean isDirectory() { + return true; + } + + public File get( String name ){ + return children.get( name ); + } + + /** + * Creates folder if not existing + * @param name name of the folder to create + * @return the file with the name, or the new created folder + */ + public File getFileOrCreateFolder( String name ){ + final File file = children.get(name); + if( file == null ){ + children.putIfAbsent( name, new Folder() ); + return children.get( name ); + } + return file; + } + + public List getAllChildren(Path currentPath ){ + List result = new LinkedList<>(); + getAllChildren( result, currentPath ); + return result; + } + + private void getAllChildren(final List result, Path currentPath){ + for (Map.Entry entry : children.entrySet()) { + Path resolve = currentPath.resolve(entry.getKey()); + if ( entry.getValue().isDirectory() ){ + final Folder value = (Folder) entry.getValue(); + value.getAllChildren( result, resolve ); + } else { + result.add( resolve ); + } + } + } + + public boolean addFile( String p ) { + File file = children.get(p); + if( file == null ){ + children.putIfAbsent( p, File.get() ); + file = children.get( p ); + } + return !file.isDirectory(); + } + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java new file mode 100644 index 00000000..ecaec71e --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -0,0 +1,93 @@ +package fonda.scheduler.model.location.hierachy; + +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Slf4j +public class HierarchyWrapper { + + private final Path workdir; + + private final ConcurrentMap workDirs = new ConcurrentHashMap<>(2); + + public HierarchyWrapper( String workdir ) { + this.workdir = Paths.get( workdir ).normalize(); + } + + private Path relativize( String path ){ + return workdir.relativize(Paths.get( path )).normalize(); + } + + private Folder getWorkdir( Iterator iterator, boolean create ){ + if(!iterator.hasNext()) return null; + final String hash1 = iterator.next().toString(); + if(!iterator.hasNext()) return null; + final String hash2 = iterator.next().toString(); + final String key = hash1 + hash2; + final Folder folder = workDirs.get( key ); + if( create && folder == null ){ + workDirs.putIfAbsent( key, new Folder()); + return workDirs.get( key ); + } + return folder; + } + + /** + * + * @param path get all files recursively in this folder (absolute path) + * @return Null if folder is empty, or not found + */ + public List getAllFilesInDir( String path ){ + final Path relativePath = relativize( path ); + Iterator iterator = relativePath.iterator(); + File current = getWorkdir( iterator, false ); + if( current == null ) return null; + while(iterator.hasNext()){ + Path p = iterator.next(); + if ( current != null && current.isDirectory() ){ + current = ((Folder) current).get( p.toString() ); + } else { + return null; + } + } + if( current.isDirectory() ) + return ((Folder) current).getAllChildren( Paths.get(path).normalize() ); + else + return null; + } + + /** + * + * @param path file to add (absolute path) + * @return false if file can not be created + */ + public boolean addFile( String path ){ + final Path relativePath = relativize( path ); + if (relativePath.startsWith("..")){ + return false; + } + Iterator iterator = relativePath.iterator(); + Folder current = getWorkdir( iterator, true ); + if( current == null ) return false; + while(iterator.hasNext()) { + Path p = iterator.next(); + if( iterator.hasNext() ){ + //folder + final File file = current.getFileOrCreateFolder( p.toString() ); + if ( !file.isDirectory() ) return false; + current = (Folder) file; + } else { + //file + return current.addFile( p.toString() ); + } + } + //This would add a file in working hierarchy + return false; + } +} diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java new file mode 100644 index 00000000..f0c8c6af --- /dev/null +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -0,0 +1,173 @@ +package fonda.scheduler.model.location.hierachy; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +@Slf4j +public class HierarchyWrapperTest { + + final String workdir = "/folder/localworkdir/"; + HierarchyWrapper hw; + List files; + List result; + String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; + + @Before + public void init() { + hw = new HierarchyWrapper(workdir); + files = new LinkedList(); + + files.add( temporaryDir + "test" ); + files.add( temporaryDir + "file.abc" ); + files.add( temporaryDir + "a/test.abc" ); + files.add( temporaryDir + "a/file.abc" ); + files.add( temporaryDir + "d/test" ); + files.add( temporaryDir + "d/e/file.txt" ); + files.add( temporaryDir + "b/c/test.abc" ); + files.add( temporaryDir + "bc/file.abc" ); + + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x))); + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + } + + private void compare( List a, List b){ + assertEquals( + new HashSet<>(a.stream().map(x -> Paths.get(x).normalize()).collect(Collectors.toList())), + new HashSet<>(b) + ); + assertEquals(a.size(),b.size()); + } + + + @Test + public void getAllFilesInDir() { + log.info("{}", this.result); + List result; + + result = hw.getAllFilesInDir( temporaryDir + "b/" ); + compare(Arrays.asList(temporaryDir + "b/c/test.abc"), result); + + log.info("{}", result); + + result = hw.getAllFilesInDir( temporaryDir + "b" ); + compare(Arrays.asList(temporaryDir + "b/c/test.abc"), result); + + log.info("{}", result); + } + + @Test + public void getAllFilesInFile() { + assertNull(hw.getAllFilesInDir( temporaryDir + "test/" )); + assertNull(hw.getAllFilesInDir( temporaryDir + "d/test/" )); + assertNull(hw.getAllFilesInDir( temporaryDir + "d/test/c/" )); + } + + @Test + public void getAllFilesOutOfScrope() { + assertNull(hw.getAllFilesInDir( "/somewhere/" )); + assertNull(hw.getAllFilesInDir( "/somewhere/on/the/machine/very/deep/hierarchy/" )); + } + + @Test + public void getAllFilesinAllWorkdirs() { + assertNull(hw.getAllFilesInDir( workdir )); + assertNull(hw.getAllFilesInDir( workdir + "/ab/" )); + } + + @Test + public void createFileinFile() { + assertFalse(hw.addFile(temporaryDir + "test/b.txt")); + + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + } + + @Test + public void createFileinWorkdir() { + assertFalse(hw.addFile(workdir + "ab/b.txt")); + + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + } + + @Test + public void createFileOutOfScope() { + assertFalse(hw.addFile("/somewhere/test.txt")); + assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt")); + + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + } + + @Test + public void createFileTwice() { + assertTrue(hw.addFile(temporaryDir + "bc/file.abc")); + + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + } + + @Test + public void createFileButWasFolder() { + assertFalse(hw.addFile(temporaryDir + "bc")); + + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + + } + + @Test + public void testParallelAdd() { + System.gc(); + long intialMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + + List files = new LinkedList(); + String wd = workdir + "ab/abcdefg/"; + Map> m = new HashMap(); + + int iters = 1_000_000; + + for (int i = 0; i < iters; i++) { + String p = wd + (i % 3) + "/" + (i % 4); + String file = p + "/" + "file-" + i; + files.add( file ); + final List currentData = m.getOrDefault(p, new LinkedList()); + currentData.add(file); + m.put(p, currentData); + } + + System.gc(); + + long finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + + log.info( "Memory input: {}mb", (finalMem - intialMem) / 1024 / 1024 ); + + intialMem = finalMem; + + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x))); + + finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); + System.gc(); + finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + log.info( "Memory hierachy after gc: {}mb", (finalMem - intialMem) / 1024 / 1024 ); + log.info( "Memory per entry: {}b", (finalMem - intialMem) / iters ); + + result = hw.getAllFilesInDir( wd ); + compare(files, result); + + for (Map.Entry> entry : m.entrySet()) { + result = hw.getAllFilesInDir( entry.getKey() ); + compare( entry.getValue(), result); + } + } +} \ No newline at end of file From 4230c394d5ea4582a662bfca696ef233d24cf5d3 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:04:20 +0100 Subject: [PATCH 005/443] created LocationWrapper Signed-off-by: Lehmann-Fabian --- .../scheduler/model/location/Location.java | 25 +++++++++++ .../model/location/LocationType.java | 7 +++ .../model/location/NodeLocation.java | 44 +++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 src/main/java/fonda/scheduler/model/location/Location.java create mode 100644 src/main/java/fonda/scheduler/model/location/LocationType.java create mode 100644 src/main/java/fonda/scheduler/model/location/NodeLocation.java diff --git a/src/main/java/fonda/scheduler/model/location/Location.java b/src/main/java/fonda/scheduler/model/location/Location.java new file mode 100644 index 00000000..a0feabce --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/Location.java @@ -0,0 +1,25 @@ +package fonda.scheduler.model.location; + +import java.util.Objects; + +public abstract class Location { + + abstract String getIdentifier(); + + abstract LocationType getType(); + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || !(o instanceof Location) ) return false; + Location that = (Location) o; + return ( getType() == that.getType() ) && ( getIdentifier().equals( that.getIdentifier() )); + } + + @Override + public int hashCode() { + return Objects.hash(getIdentifier(),getType()); + } + + +} diff --git a/src/main/java/fonda/scheduler/model/location/LocationType.java b/src/main/java/fonda/scheduler/model/location/LocationType.java new file mode 100644 index 00000000..ff58776b --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/LocationType.java @@ -0,0 +1,7 @@ +package fonda.scheduler.model.location; + +public enum LocationType { + + NODE + +} diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java new file mode 100644 index 00000000..62b8de37 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -0,0 +1,44 @@ +package fonda.scheduler.model.location; + +import io.fabric8.kubernetes.api.model.Node; +import lombok.Getter; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class NodeLocation extends Location { + + static final private ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap(); + + @Getter + private final String identifier; + + private NodeLocation(String identifier) { + this.identifier = identifier; + } + + public static NodeLocation getLocation( Node node ){ + return getLocation( node.getMetadata().getName() ); + } + + public static NodeLocation getLocation( String node ){ + final NodeLocation nodeLocation = locationHolder.get(node); + if ( nodeLocation == null ){ + locationHolder.putIfAbsent( node, new NodeLocation( node ) ); + return locationHolder.get( node ); + } + return nodeLocation; + } + + @Override + public LocationType getType() { + return LocationType.NODE; + } + + @Override + public String toString() { + return "Node(" + identifier + ")"; + } + +} From f00c05aef0533d29883aad659813ae9b2dc7aa78 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:05:17 +0100 Subject: [PATCH 006/443] store location in hierarchy, new class RealFile Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/File.java | 12 +-- .../model/location/hierachy/Folder.java | 8 +- .../location/hierachy/HierarchyWrapper.java | 34 ++++++- .../model/location/hierachy/RealFile.java | 66 +++++++++++++ .../hierachy/HierarchyWrapperTest.java | 69 ++++++++++++-- .../model/location/hierachy/RealFileTest.java | 93 +++++++++++++++++++ 6 files changed, 260 insertions(+), 22 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java create mode 100644 src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/File.java b/src/main/java/fonda/scheduler/model/location/hierachy/File.java index 8acd76e4..d090d141 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/File.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/File.java @@ -1,15 +1,7 @@ package fonda.scheduler.model.location.hierachy; -public class File { +public abstract class File { - private final static File fileInstance = new File(); - - public static File get(){ - return fileInstance; - } - - public boolean isDirectory(){ - return false; - } + public abstract boolean isDirectory(); } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 7a3ec979..5ad04c49 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -1,5 +1,6 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.location.Location; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; @@ -14,6 +15,8 @@ public class Folder extends File { private final ConcurrentMap children = new ConcurrentHashMap<>(); + Folder(){} + @Override public boolean isDirectory() { return true; @@ -55,12 +58,13 @@ private void getAllChildren(final List result, Path currentPath){ } } - public boolean addFile( String p ) { + public boolean addFile( String p, Location... locations ) { File file = children.get(p); if( file == null ){ - children.putIfAbsent( p, File.get() ); + children.putIfAbsent( p, new RealFile() ); file = children.get( p ); } + if ( !file.isDirectory() ) ((RealFile) file).addLocation( locations ); return !file.isDirectory(); } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index ecaec71e..65aeee52 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -1,5 +1,6 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.location.Location; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; @@ -65,9 +66,10 @@ public List getAllFilesInDir( String path ){ /** * * @param path file to add (absolute path) + * @param locations locations where the file is located * @return false if file can not be created */ - public boolean addFile( String path ){ + public boolean addFile( String path, Location... locations ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return false; @@ -84,10 +86,38 @@ public boolean addFile( String path ){ current = (Folder) file; } else { //file - return current.addFile( p.toString() ); + return current.addFile( p.toString(), locations ); } } //This would add a file in working hierarchy return false; } + + /** + * + * @param path file to get (absolute path) + * @return File or null if file does not exist + */ + public RealFile getFile( String path ){ + final Path relativePath = relativize( path ); + if (relativePath.startsWith("..")){ + return null; + } + Iterator iterator = relativePath.iterator(); + Folder current = getWorkdir( iterator, true ); + if( current == null ) return null; + while(iterator.hasNext()) { + Path p = iterator.next(); + final File file = current.get( p.toString() ); + if( iterator.hasNext() && file.isDirectory() ){ + //folder + current = (Folder) file; + } else if ( !iterator.hasNext() && !file.isDirectory() ) { + //file + return (RealFile) file; + } else + break; + } + return null; + } } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java new file mode 100644 index 00000000..98bd72db --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -0,0 +1,66 @@ +package fonda.scheduler.model.location.hierachy; + +import fonda.scheduler.model.location.Location; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class RealFile extends File { + + @Getter + private Location[] locations = null; + + RealFile(){} + + @Override + public boolean isDirectory(){ + return false; + } + + private void checkIfValidInput( Location[] location ){ + if ( location == null || location.length == 0 ) + throw new IllegalArgumentException( "location was null or empty" ); + for (Location loc : location) { + if ( loc == null ) throw new IllegalArgumentException( "location contains null value" ); + } + } + + public void addLocation( Location... location ){ + checkIfValidInput( location ); + synchronized ( this ){ + if ( locations == null ){ + locations = location; + return; + } + List newLocationList = new ArrayList<>( location.length ); + for (Location newLoc : location) { + boolean foundEqual = false; + for (Location l : locations) { + if ( newLoc.equals(l) ) { + foundEqual = true; + break; + } + } + if ( !foundEqual ){ + newLocationList.add( newLoc ); + } + } + final Location[] newLocation = Arrays.copyOf(locations, locations.length + newLocationList.size() ); + int index = locations.length; + for (Location l : newLocationList) { + newLocation[index ++ ] = l; + } + locations = newLocation; + } + } + + public void changeFile( Location... location ){ + checkIfValidInput( location ); + synchronized ( this ) { + locations = location; + } + } + +} diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index f0c8c6af..ab26a355 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -1,5 +1,7 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; @@ -19,6 +21,7 @@ public class HierarchyWrapperTest { List files; List result; String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; + Location node1 = NodeLocation.getLocation("Node1"); @Before public void init() { @@ -34,7 +37,7 @@ public void init() { files.add( temporaryDir + "b/c/test.abc" ); files.add( temporaryDir + "bc/file.abc" ); - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x))); + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x,node1))); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); } @@ -85,7 +88,7 @@ public void getAllFilesinAllWorkdirs() { @Test public void createFileinFile() { - assertFalse(hw.addFile(temporaryDir + "test/b.txt")); + assertFalse(hw.addFile(temporaryDir + "test/b.txt", node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -93,7 +96,7 @@ public void createFileinFile() { @Test public void createFileinWorkdir() { - assertFalse(hw.addFile(workdir + "ab/b.txt")); + assertFalse(hw.addFile(workdir + "ab/b.txt", node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -101,8 +104,8 @@ public void createFileinWorkdir() { @Test public void createFileOutOfScope() { - assertFalse(hw.addFile("/somewhere/test.txt")); - assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt")); + assertFalse(hw.addFile("/somewhere/test.txt", node1)); + assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt", node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -110,7 +113,7 @@ public void createFileOutOfScope() { @Test public void createFileTwice() { - assertTrue(hw.addFile(temporaryDir + "bc/file.abc")); + assertTrue(hw.addFile(temporaryDir + "bc/file.abc", node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -118,7 +121,7 @@ public void createFileTwice() { @Test public void createFileButWasFolder() { - assertFalse(hw.addFile(temporaryDir + "bc")); + assertFalse(hw.addFile(temporaryDir + "bc", node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -153,7 +156,7 @@ public void testParallelAdd() { intialMem = finalMem; - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x))); + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, node1))); finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); @@ -170,4 +173,54 @@ public void testParallelAdd() { compare( entry.getValue(), result); } } + + @Test + public void testGetFile() { + + hw = new HierarchyWrapper(workdir); + files = new LinkedList(); + + files.add( temporaryDir + "test" ); + files.add( temporaryDir + "file.abc" ); + files.add( temporaryDir + "a/test.abc" ); + files.add( temporaryDir + "a/file.abc" ); + files.add( temporaryDir + "d/test" ); + files.add( temporaryDir + "d/e/file.txt" ); + files.add( temporaryDir + "b/c/test.abc" ); + + int index = 0; + for (String file : files) { + assertTrue(hw.addFile(file,NodeLocation.getLocation( "Node" + index ))); + } + + result = hw.getAllFilesInDir(temporaryDir); + compare( files, result); + + index = 0; + for (String file : files) { + RealFile realFile = hw.getFile(file); + + final Location[] locations = realFile.getLocations(); + assertEquals(NodeLocation.getLocation( "Node" + index ), locations[0]); + assertEquals(1,locations.length); + } + } + + @Test + public void testGetFileOutOfScope() { + assertNull(hw.getFile( "/file.txt" )); + assertNull(hw.getFile( "/somewhere/on/the/machine/very/deep/hierarchy/file.txt" )); + } + + @Test + public void testGetFileWorkdir() { + assertNull(hw.getFile( workdir + "ab/test.txt" )); + } + + @Test + public void testGetFileButIsDir() { + assertNull(hw.getFile( temporaryDir + "d" )); + } + + } \ No newline at end of file diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java new file mode 100644 index 00000000..461ea2ae --- /dev/null +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -0,0 +1,93 @@ +package fonda.scheduler.model.location.hierachy; + +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class RealFileTest { + + + @Test + public void addLocation() { + + final RealFile realFile = new RealFile(); + List locations = new LinkedList<>(); + + locations.add( NodeLocation.getLocation("Node1") ); + realFile.addLocation(NodeLocation.getLocation("Node1")); + assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + locations.add( NodeLocation.getLocation("Node2") ); + realFile.addLocation(NodeLocation.getLocation("Node2")); + assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + locations.add( NodeLocation.getLocation("Node3") ); + realFile.addLocation(NodeLocation.getLocation("Node3")); + assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + realFile.addLocation(NodeLocation.getLocation("Node1")); + assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + } + + @Test + public void addEmptyLocation() { + final RealFile realFile = new RealFile(); + assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( null )); + assertThrows(IllegalArgumentException.class, () -> realFile.addLocation()); + assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( new Location[0] )); + assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( new Location[1] )); + } + + @Test + public void addInParallel() { + final RealFile realFile = new RealFile(); + + List locations = new LinkedList<>(); + for (int i = 0; i < 10_000; i++) { + locations.add( NodeLocation.getLocation( "Node" + i ) ); + } + + Collections.shuffle(locations); + + locations.parallelStream().forEach( realFile::addLocation ); + + assertEquals( + new HashSet<>(locations), + new HashSet<>(Arrays.asList(realFile.getLocations())) + ); + + } + + @Test + public void changeFile() { + + final RealFile realFile = new RealFile(); + realFile.addLocation( NodeLocation.getLocation("Node1") ); + realFile.addLocation( NodeLocation.getLocation("Node2") ); + realFile.addLocation( NodeLocation.getLocation("Node3") ); + + realFile.changeFile( NodeLocation.getLocation("NodeNew") ); + Location[] expected = { NodeLocation.getLocation("NodeNew") }; + assertArrayEquals( expected, realFile.getLocations() ); + + } + + @Test + public void changeFileEmpty() { + + final RealFile realFile = new RealFile(); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile(null )); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile() ); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( new Location[0] )); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( new Location[1] )); + + } + +} \ No newline at end of file From 37396cc57aeb5414af6f2f25b3a95bcb3a73728c Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:40:47 +0100 Subject: [PATCH 007/443] check if workdir is set Signed-off-by: Lehmann-Fabian --- .../scheduler/model/location/hierachy/HierarchyWrapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 65aeee52..d9c21951 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -18,6 +18,7 @@ public class HierarchyWrapper { private final ConcurrentMap workDirs = new ConcurrentHashMap<>(2); public HierarchyWrapper( String workdir ) { + if ( workdir == null ) throw new IllegalArgumentException( "Workdir is not defined" ); this.workdir = Paths.get( workdir ).normalize(); } From a45b7cf5d39279191622276f0704ec4a919ee545 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:41:54 +0100 Subject: [PATCH 008/443] retry scheduling if exception was raised Signed-off-by: Lehmann-Fabian --- .../java/fonda/scheduler/scheduler/TaskprocessingThread.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java index 5a92ec5c..bf9b4981 100644 --- a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java @@ -35,6 +35,7 @@ public void run() { } catch (InterruptedException e){ Thread.currentThread().interrupt(); } catch (Exception e){ + unscheduled = -1; log.info("Error while processing",e); } } From de1ec80e11e1970ca78db0ed75750edb9b7da64c Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:44:47 +0100 Subject: [PATCH 009/443] getWorkdir method to task Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/Task.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index a75cc7f0..1672cd13 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -22,4 +22,8 @@ public class Task { public Task(TaskConfig config) { this.config = config; } + + public String getWorkingDir(){ + return pod.getSpec().getContainers().get(0).getWorkingDir(); + } } From 34ecfbee3b02a72791d971db7a6054c0c1ceb9c4 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:45:30 +0100 Subject: [PATCH 010/443] toString in SchedulerConfig Signed-off-by: Lehmann-Fabian --- .../scheduler/model/SchedulerConfig.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 30d990db..a753c4d7 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -4,13 +4,11 @@ public class SchedulerConfig { - final public String podName; final public List localClaims; final public List volumeClaims; final public String workDir; private SchedulerConfig(){ - this.podName = null; this.localClaims = null; this.volumeClaims = null; this.workDir = null; @@ -24,6 +22,14 @@ private LocalClaim(){ this.mountPath = null; this.hostPath = null; } + + @Override + public String toString() { + return "LocalClaim{" + + "mountPath='" + mountPath + '\'' + + ", hostPath='" + hostPath + '\'' + + '}'; + } } static public class VolumeClaim { @@ -36,6 +42,23 @@ private VolumeClaim(){ this.claimName = null; this.subPath = null; } + + @Override + public String toString() { + return "VolumeClaim{" + + "mountPath='" + mountPath + '\'' + + ", claimName='" + claimName + '\'' + + ", subPath='" + subPath + '\'' + + '}'; + } } + @Override + public String toString() { + return "SchedulerConfig{" + + "localClaims=" + localClaims + + ", volumeClaims=" + volumeClaims + + ", workDir='" + workDir + '\'' + + '}'; + } } From 36954ee7cb5ac60d7bc16b38f16c7cf5d358fef8 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:49:14 +0100 Subject: [PATCH 011/443] added Dockerfile for development Signed-off-by: Lehmann-Fabian --- Dockerfile | 2 +- Dockerfile-development | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Dockerfile-development diff --git a/Dockerfile b/Dockerfile index b5fb1049..dd3f75e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN mvn package -Dmaven.repo.local=/mvn/.m2nrepo/repository # FROM openjdk:11-jre-slim WORKDIR /app -RUN groupadd -r javauser && useradd --no-log-init -r -g javauser javauser +RUN groupadd -r javauser && useradd --no-log-init -r -g javauser javauser && mkdir data COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar RUN chown -R javauser:javauser /app USER javauser diff --git a/Dockerfile-development b/Dockerfile-development new file mode 100644 index 00000000..08cf9ce8 --- /dev/null +++ b/Dockerfile-development @@ -0,0 +1,6 @@ +FROM maven:3.8.3-jdk-11-slim AS build +WORKDIR /build +COPY pom.xml pom.xml +RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository +COPY src/ src/ +RUN mvn package -Dmaven.repo.local=/mvn/.m2nrepo/repository From a03222d6be450a1dcbb6fa37929f4f6e4c2fe395 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 15:50:17 +0100 Subject: [PATCH 012/443] Dockerignore data folder Signed-off-by: Lehmann-Fabian --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 2f030fad..151bcdaf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,4 @@ README.md Dockerfile .gitignore .dockerignore +data/ From 7d4664bc00a6dc1b6b9bcf331c042fea122d52ec Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 16:03:33 +0100 Subject: [PATCH 013/443] Get Schedulerconfig Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/Main.java | 3 +- .../rest/SchedulerRestController.java | 4 +- .../scheduler/scheduler/RandomScheduler.java | 26 +++-- .../fonda/scheduler/scheduler/Scheduler.java | 24 +++-- .../scheduler/SchedulerWithDaemonSet.java | 98 +++++++++++++++++++ 5 files changed, 132 insertions(+), 23 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index d2611426..c003b1e6 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -1,6 +1,7 @@ package fonda.scheduler; import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.scheduler.RandomScheduler; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -29,7 +30,7 @@ public static void main(String[] args) { @PostConstruct public void afterStart(){ log.info( "Started with namespace: {}", client.getNamespace() ); - new RandomScheduler("testscheduler", client, "default" ); + new RandomScheduler("testscheduler", client, "default", null); } } diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 639b9e62..b76345fb 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -43,7 +43,7 @@ public SchedulerRestController( @Autowired KubernetesClient client ){ @PostMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { - log.trace("Register execution: " + execution + " strategy: " + strategy); + log.trace("Register execution: {} strategy: {} config: {}", execution, strategy, config); Scheduler scheduler = null; @@ -54,7 +54,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable S switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : - case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace ); break; + case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace, config ); break; default: return new ResponseEntity( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 9312a56e..17cf3874 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -1,24 +1,29 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.State; +import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.Task; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.Watcher; import lombok.extern.slf4j.Slf4j; -import java.util.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; @Slf4j -public class RandomScheduler extends Scheduler { +public class RandomScheduler extends SchedulerWithDaemonSet { - - private final Map daemonByNode = new HashMap<>(); private final Map workdirNode = new HashMap<>(); - public RandomScheduler(String name, KubernetesClient client, String namespace) { - super(name, client, namespace); + public RandomScheduler(String name, KubernetesClient client, String namespace, SchedulerConfig config) { + super(name, client, namespace, config); } @Override @@ -28,13 +33,14 @@ public int schedule( final List unscheduledTasks ) { int unscheduled = 0; for ( final Task task : unscheduledTasks) { if(isClose()) return -1; - Optional node = items.stream().filter(x -> x.canSchedule(task.getPod())).findFirst(); + final Pod pod = task.getPod(); + Optional node = items.stream().filter(x -> x.canSchedule(pod) && this.getDaemonOnNode(x) != null).findFirst(); if( node.isPresent() ){ log.info("Task needs: " + task.getConfig().getInputs().toString()); assignPodToNode( task.getPod(), node.get() ); super.taskWasScheduled( task ); } else { - log.info( "No node with enough resources for {}", task.getPod().getMetadata().getName() ); + log.info( "No node with enough resources for {}", pod.getMetadata().getName() ); unscheduled++; } } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 070d195b..f8dec54c 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -2,6 +2,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.hierachy.HierarchyWrapper; import io.fabric8.kubernetes.api.model.Binding; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectReference; @@ -25,33 +26,40 @@ public abstract class Scheduler { @Getter private final String name; @Getter + private final String execution; + @Getter + private final String namespace; + @Getter private boolean close; final KubernetesClient client; - private final String namespace; final private List unscheduledTasks = new ArrayList<>(100); final private List unfinishedTasks = new ArrayList<>(100); final Map tasksByHash = new HashMap<>(); private final Watch watcher; private final TaskprocessingThread schedulingThread; private final TaskprocessingThread finishThread; + final HierarchyWrapper hierarchyWrapper; - Scheduler(String name, KubernetesClient client, String namespace){ - this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + name; + Scheduler(String execution, KubernetesClient client, String namespace, SchedulerConfig config){ + this.execution = execution; + this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + execution; this.namespace = namespace; log.trace( "Register scheduler for " + this.name ); this.client = client; + this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); PodWatcher podWatcher = new PodWatcher(this); - log.info("Start watching"); - watcher = client.pods().inNamespace( this.namespace ).watch(podWatcher); - log.info("Watching"); schedulingThread = new TaskprocessingThread( unscheduledTasks, this::schedule ); schedulingThread.start(); finishThread = new TaskprocessingThread(unfinishedTasks, this::terminateTasks ); finishThread.start(); + + log.info("Start watching"); + watcher = client.pods().inNamespace( this.namespace ).watch(podWatcher); + log.info("Watching"); } /* Abstract methods */ @@ -214,10 +222,6 @@ private Task changeStateOfTask( Pod pod, State state ){ return null; } - String getWorkingDir( Pod pod ){ - return pod.getSpec().getContainers().get(0).getWorkingDir(); - } - @Deprecated PodResource findPodByName(String name ){ return client.pods().withName( name ); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java new file mode 100644 index 00000000..8af711b1 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -0,0 +1,98 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.SchedulerConfig; +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.Watcher; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public abstract class SchedulerWithDaemonSet extends Scheduler { + + private final Map daemonByNode = new HashMap<>(); + + SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { + super(execution, client, namespace, config); + } + + String getDaemonOnNode( String node ){ + synchronized ( daemonByNode ) { + return daemonByNode.get( node ); + } + } + + String getDaemonOnNode( Node node ){ + return getDaemonOnNode( node.getMetadata().getName() ); + } + + String getOneDaemon() { + synchronized ( daemonByNode ) { + final Collection values = daemonByNode.values(); + return values.stream().skip(values.size()).findFirst().get(); + } + } + + void uploadDataToNode ( Node node, File file, Path dest ) { + int i = 0; + do { + log.info( "Upload {} to {} ({})", file, dest, node.getMetadata().getName() ); + try { + final boolean result = client.pods() + .inNamespace(getNamespace()) + .withName(getDaemonOnNode(node)) + .file(dest.toString()) + .upload(file.toPath()); + if (result) return; + } catch (Exception e){ + e.printStackTrace(); + } + try { + Thread.sleep((long) (Math.pow(2,i) * 100) ); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } while ( i++ < 10 ); + throw new RuntimeException( "Can not upload file: " + file + " to node: " + node.getMetadata().getName() ); + } + + @Override + void podEventReceived(Watcher.Action action, Pod pod){ + while ( daemonByNode == null ){ + //The Watcher can be started before the class is initialized + try { + Thread.sleep(20); + } catch (InterruptedException e) {} + } + if( ( "mount-" + this.getExecution().replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ + final String nodeName = pod.getSpec().getNodeName(); + if ( nodeName != null ){ + synchronized ( daemonByNode ) { + final String podName = pod.getMetadata().getName(); + final boolean podIsCurrentDaemon = podName.equals(daemonByNode.get(nodeName)); + if ( action == Watcher.Action.DELETED ) { + if (podIsCurrentDaemon) { + daemonByNode.remove(nodeName); + } + } else if ( pod.getStatus().getPhase().equals("Running") ) { + daemonByNode.put( nodeName, podName); + informResourceChange(); + } else if ( podIsCurrentDaemon ) { + daemonByNode.remove(nodeName); + if( !pod.getStatus().getPhase().equals("Failed") ){ + log.info( "Unexpected phase {} for daemon: {}", pod.getStatus().getPhase(), podName ); + } + } + } + } + } + } + + +} From 65277cc0b11a5ea3136810806e58201acdc32f33 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 16:55:19 +0100 Subject: [PATCH 014/443] store filesize Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/Folder.java | 4 ++-- .../location/hierachy/HierarchyWrapper.java | 4 ++-- .../model/location/hierachy/RealFile.java | 9 ++++++-- .../hierachy/HierarchyWrapperTest.java | 18 ++++++++-------- .../model/location/hierachy/RealFileTest.java | 21 ++++++++++--------- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 5ad04c49..86d64827 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -58,10 +58,10 @@ private void getAllChildren(final List result, Path currentPath){ } } - public boolean addFile( String p, Location... locations ) { + public boolean addFile( String p, long sizeInBytes, Location... locations ) { File file = children.get(p); if( file == null ){ - children.putIfAbsent( p, new RealFile() ); + children.putIfAbsent( p, new RealFile(sizeInBytes) ); file = children.get( p ); } if ( !file.isDirectory() ) ((RealFile) file).addLocation( locations ); diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index d9c21951..e551f803 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -70,7 +70,7 @@ public List getAllFilesInDir( String path ){ * @param locations locations where the file is located * @return false if file can not be created */ - public boolean addFile( String path, Location... locations ){ + public boolean addFile( String path, long sizeInBytes, Location... locations ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return false; @@ -87,7 +87,7 @@ public boolean addFile( String path, Location... locations ){ current = (Folder) file; } else { //file - return current.addFile( p.toString(), locations ); + return current.addFile( p.toString(), sizeInBytes, locations ); } } //This would add a file in working hierarchy diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 98bd72db..51a3941f 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -11,8 +11,12 @@ public class RealFile extends File { @Getter private Location[] locations = null; + @Getter + private long sizeInBytes; - RealFile(){} + RealFile(long sizeInBytes) { + this.sizeInBytes = sizeInBytes; + } @Override public boolean isDirectory(){ @@ -56,9 +60,10 @@ public void addLocation( Location... location ){ } } - public void changeFile( Location... location ){ + public void changeFile( long sizeInBytes, Location... location ){ checkIfValidInput( location ); synchronized ( this ) { + this.sizeInBytes = sizeInBytes; locations = location; } } diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index ab26a355..ae7e594b 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -37,7 +37,7 @@ public void init() { files.add( temporaryDir + "b/c/test.abc" ); files.add( temporaryDir + "bc/file.abc" ); - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x,node1))); + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x,0, node1))); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); } @@ -88,7 +88,7 @@ public void getAllFilesinAllWorkdirs() { @Test public void createFileinFile() { - assertFalse(hw.addFile(temporaryDir + "test/b.txt", node1)); + assertFalse(hw.addFile(temporaryDir + "test/b.txt", 10, node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -96,7 +96,7 @@ public void createFileinFile() { @Test public void createFileinWorkdir() { - assertFalse(hw.addFile(workdir + "ab/b.txt", node1)); + assertFalse(hw.addFile(workdir + "ab/b.txt", 10, node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -104,8 +104,8 @@ public void createFileinWorkdir() { @Test public void createFileOutOfScope() { - assertFalse(hw.addFile("/somewhere/test.txt", node1)); - assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt", node1)); + assertFalse(hw.addFile("/somewhere/test.txt", 10, node1)); + assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt", 10, node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -113,7 +113,7 @@ public void createFileOutOfScope() { @Test public void createFileTwice() { - assertTrue(hw.addFile(temporaryDir + "bc/file.abc", node1)); + assertTrue(hw.addFile(temporaryDir + "bc/file.abc", 10, node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -121,7 +121,7 @@ public void createFileTwice() { @Test public void createFileButWasFolder() { - assertFalse(hw.addFile(temporaryDir + "bc", node1)); + assertFalse(hw.addFile(temporaryDir + "bc", 10, node1)); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -156,7 +156,7 @@ public void testParallelAdd() { intialMem = finalMem; - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, node1))); + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, 10, node1))); finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); @@ -190,7 +190,7 @@ public void testGetFile() { int index = 0; for (String file : files) { - assertTrue(hw.addFile(file,NodeLocation.getLocation( "Node" + index ))); + assertTrue(hw.addFile(file,10, NodeLocation.getLocation( "Node" + index ))); } result = hw.getAllFilesInDir(temporaryDir); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 461ea2ae..38d77b37 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -16,7 +16,7 @@ public class RealFileTest { @Test public void addLocation() { - final RealFile realFile = new RealFile(); + final RealFile realFile = new RealFile( 10 ); List locations = new LinkedList<>(); locations.add( NodeLocation.getLocation("Node1") ); @@ -38,7 +38,7 @@ public void addLocation() { @Test public void addEmptyLocation() { - final RealFile realFile = new RealFile(); + final RealFile realFile = new RealFile( 10 ); assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( null )); assertThrows(IllegalArgumentException.class, () -> realFile.addLocation()); assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( new Location[0] )); @@ -47,7 +47,7 @@ public void addEmptyLocation() { @Test public void addInParallel() { - final RealFile realFile = new RealFile(); + final RealFile realFile = new RealFile( 10 ); List locations = new LinkedList<>(); for (int i = 0; i < 10_000; i++) { @@ -68,13 +68,14 @@ public void addInParallel() { @Test public void changeFile() { - final RealFile realFile = new RealFile(); + final RealFile realFile = new RealFile( 4 ); realFile.addLocation( NodeLocation.getLocation("Node1") ); realFile.addLocation( NodeLocation.getLocation("Node2") ); realFile.addLocation( NodeLocation.getLocation("Node3") ); - realFile.changeFile( NodeLocation.getLocation("NodeNew") ); + realFile.changeFile( 5, NodeLocation.getLocation("NodeNew") ); Location[] expected = { NodeLocation.getLocation("NodeNew") }; + assertEquals( 5, realFile.getSizeInBytes() ); assertArrayEquals( expected, realFile.getLocations() ); } @@ -82,11 +83,11 @@ public void changeFile() { @Test public void changeFileEmpty() { - final RealFile realFile = new RealFile(); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile(null )); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile() ); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( new Location[0] )); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( new Location[1] )); + final RealFile realFile = new RealFile( 10 ); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile(1,null )); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile(1) ); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( 1,new Location[0] )); + assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( 1,new Location[1] )); } From c986c1678c9a07ebf07eebe30f7f409775c66643 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 5 Nov 2021 16:58:34 +0100 Subject: [PATCH 015/443] import Watcher Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/scheduler/RandomScheduler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 17cf3874..5dd18ab3 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -6,6 +6,7 @@ import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.Task; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.Watcher; import lombok.extern.slf4j.Slf4j; import java.io.File; From 01e87099c2c2952d8e0981eed92f3e10c6f3b376 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 9 Nov 2021 14:50:29 +0100 Subject: [PATCH 016/443] fixed Nullpointer after startup Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/Main.java | 2 +- src/main/java/fonda/scheduler/model/SchedulerConfig.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index c003b1e6..46234ad0 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -30,7 +30,7 @@ public static void main(String[] args) { @PostConstruct public void afterStart(){ log.info( "Started with namespace: {}", client.getNamespace() ); - new RandomScheduler("testscheduler", client, "default", null); + new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null,null,"/localwork/")); } } diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index a753c4d7..f8752d89 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -8,6 +8,12 @@ public class SchedulerConfig { final public List volumeClaims; final public String workDir; + public SchedulerConfig(List localClaims, List volumeClaims, String workDir) { + this.localClaims = localClaims; + this.volumeClaims = volumeClaims; + this.workDir = workDir; + } + private SchedulerConfig(){ this.localClaims = null; this.volumeClaims = null; From 9ad14bd34566910fcce1c8b23150a11f5bcee973 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 9 Nov 2021 14:52:14 +0100 Subject: [PATCH 017/443] Create initfile Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/Task.java | 10 ++++++ .../scheduler/scheduler/RandomScheduler.java | 31 +++++++++++++++---- .../fonda/scheduler/scheduler/Scheduler.java | 4 ++- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 1672cd13..b36b4666 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -26,4 +26,14 @@ public Task(TaskConfig config) { public String getWorkingDir(){ return pod.getSpec().getContainers().get(0).getWorkingDir(); } + + @Override + public String toString() { + return "Task{" + + "state=" + state + + ", pod=" + pod.getMetadata().getName() + + ", nodeName='" + nodeName + '\'' + + ", workDir='" + getWorkingDir() + '\'' + + '}'; + } } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 5dd18ab3..93cd8222 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -1,18 +1,15 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.Task; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.Watcher; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.FileNotFoundException; import java.io.PrintWriter; -import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,7 +35,7 @@ public int schedule( final List unscheduledTasks ) { Optional node = items.stream().filter(x -> x.canSchedule(pod) && this.getDaemonOnNode(x) != null).findFirst(); if( node.isPresent() ){ log.info("Task needs: " + task.getConfig().getInputs().toString()); - assignPodToNode( task.getPod(), node.get() ); + assignTaskToNode( task, node.get() ); super.taskWasScheduled( task ); } else { log.info( "No node with enough resources for {}", pod.getMetadata().getName() ); @@ -48,14 +45,36 @@ public int schedule( final List unscheduledTasks ) { return unscheduled; } + @Override + void assignTaskToNode( Task task, NodeWithAlloc node ) { + + //Create initData + log.info( "Task: {}", task ); + + File file = new File(task.getWorkingDir() + '/' + ".command.init"); + + PrintWriter pw = null; + try { + pw = new PrintWriter( file ); + pw.println( "echo \"Task init successful\"" ); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + if ( pw != null ){ + pw.close(); + } + } + + super.assignTaskToNode(task, node); + } + @Override int terminateTasks(List finishedTasks) { for (Task finishedTask : finishedTasks) { + //TODO store new Data super.taskWasFinished( finishedTask ); } return 0; } - @Override - void podEventReceived(Watcher.Action action, Pod pod){} } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index f8dec54c..84f727f8 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -173,7 +173,9 @@ boolean canPodBeScheduled( Pod pod, NodeWithAlloc node ){ return node.canSchedule( pod ); } - void assignPodToNode( Pod pod, NodeWithAlloc node ){ + void assignTaskToNode( Task task, NodeWithAlloc node ){ + + final Pod pod = task.getPod(); node.addPod( pod ); From 2341124736a8898d778280d4b387d07953e88a08 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 9 Nov 2021 16:00:34 +0100 Subject: [PATCH 018/443] Changed logic for changed files Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/Folder.java | 20 ++++++++----------- .../location/hierachy/HierarchyWrapper.java | 4 +--- .../hierachy/HierarchyWrapperTest.java | 20 ++++++++++++++++--- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 86d64827..eef973e5 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -31,13 +31,12 @@ public File get( String name ){ * @param name name of the folder to create * @return the file with the name, or the new created folder */ - public File getFileOrCreateFolder( String name ){ + public Folder getOrCreateFolder(String name ){ final File file = children.get(name); - if( file == null ){ - children.putIfAbsent( name, new Folder() ); - return children.get( name ); + if( file == null || !file.isDirectory() ){ + return (Folder) children.compute( name, (key,value) -> (value == null || !value.isDirectory()) ? new Folder() : value ); } - return file; + return (Folder) file; } public List getAllChildren(Path currentPath ){ @@ -59,13 +58,10 @@ private void getAllChildren(final List result, Path currentPath){ } public boolean addFile( String p, long sizeInBytes, Location... locations ) { - File file = children.get(p); - if( file == null ){ - children.putIfAbsent( p, new RealFile(sizeInBytes) ); - file = children.get( p ); - } - if ( !file.isDirectory() ) ((RealFile) file).addLocation( locations ); - return !file.isDirectory(); + final RealFile realFile = new RealFile(sizeInBytes); + realFile.addLocation( locations ); + children.put ( p, realFile ); + return true; } } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index e551f803..263aeda7 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -82,9 +82,7 @@ public boolean addFile( String path, long sizeInBytes, Location... locations ){ Path p = iterator.next(); if( iterator.hasNext() ){ //folder - final File file = current.getFileOrCreateFolder( p.toString() ); - if ( !file.isDirectory() ) return false; - current = (Folder) file; + current = current.getOrCreateFolder( p.toString() ); } else { //file return current.addFile( p.toString(), sizeInBytes, locations ); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index ae7e594b..e372e729 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -9,6 +9,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -88,7 +90,10 @@ public void getAllFilesinAllWorkdirs() { @Test public void createFileinFile() { - assertFalse(hw.addFile(temporaryDir + "test/b.txt", 10, node1)); + assertTrue(hw.addFile(temporaryDir + "test/b.txt", 10, node1)); + + files.remove( temporaryDir + "test" ); + files.add( temporaryDir + "test/b.txt" ); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -121,7 +126,10 @@ public void createFileTwice() { @Test public void createFileButWasFolder() { - assertFalse(hw.addFile(temporaryDir + "bc", 10, node1)); + assertTrue(hw.addFile(temporaryDir + "bc", 10, node1)); + + files.remove( temporaryDir + "bc/file.abc" ); + files.add( temporaryDir + "bc" ); result = hw.getAllFilesInDir(temporaryDir); compare( files, result); @@ -222,5 +230,11 @@ public void testGetFileButIsDir() { assertNull(hw.getFile( temporaryDir + "d" )); } - + @Test + public void testFileIsNowDir(){ + assertTrue(hw.addFile( temporaryDir + "d", 120, NodeLocation.getLocation("nodeXY") )); + result = hw.getAllFilesInDir( temporaryDir ); + assertNotNull( hw.getFile( temporaryDir + "d" ) ); + assertNull( hw.getFile( temporaryDir + "d/e/file.txt" ) ); + } } \ No newline at end of file From 320c9d1711bfc75841741b9b991e16c57b03ebd2 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 11 Nov 2021 15:28:28 +0100 Subject: [PATCH 019/443] store size and process for each node/file Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/Folder.java | 22 +++-- .../location/hierachy/HierarchyWrapper.java | 7 +- .../model/location/hierachy/RealFile.java | 47 ++++------ .../hierachy/HierarchyWrapperTest.java | 69 +++++++------- .../model/location/hierachy/RealFileTest.java | 90 +++++++++++-------- 5 files changed, 125 insertions(+), 110 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index eef973e5..c5990434 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -1,12 +1,12 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.location.Location; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -39,28 +39,32 @@ public Folder getOrCreateFolder(String name ){ return (Folder) file; } - public List getAllChildren(Path currentPath ){ - List result = new LinkedList<>(); + public Map getAllChildren( Path currentPath ){ + Map result = new TreeMap<>(); getAllChildren( result, currentPath ); return result; } - private void getAllChildren(final List result, Path currentPath){ + private void getAllChildren( final Map result, Path currentPath ){ for (Map.Entry entry : children.entrySet()) { Path resolve = currentPath.resolve(entry.getKey()); if ( entry.getValue().isDirectory() ){ final Folder value = (Folder) entry.getValue(); value.getAllChildren( result, resolve ); } else { - result.add( resolve ); + result.put( resolve, (RealFile) entry.getValue()); } } } - public boolean addFile( String p, long sizeInBytes, Location... locations ) { - final RealFile realFile = new RealFile(sizeInBytes); - realFile.addLocation( locations ); - children.put ( p, realFile ); + public boolean addOrUpdateFile( final String name, final LocationWrapper... locations ) { + children.compute( name, (k,v) -> { + if (v == null || v.isDirectory()) + return new RealFile( locations ); + final RealFile file = (RealFile) v; + file.addOrUpdateLocation( locations ); + return v; + } ); return true; } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 263aeda7..480783a8 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -7,6 +7,7 @@ import java.nio.file.Paths; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -45,7 +46,7 @@ private Folder getWorkdir( Iterator iterator, boolean create ){ * @param path get all files recursively in this folder (absolute path) * @return Null if folder is empty, or not found */ - public List getAllFilesInDir( String path ){ + public Map getAllFilesInDir(String path ){ final Path relativePath = relativize( path ); Iterator iterator = relativePath.iterator(); File current = getWorkdir( iterator, false ); @@ -70,7 +71,7 @@ public List getAllFilesInDir( String path ){ * @param locations locations where the file is located * @return false if file can not be created */ - public boolean addFile( String path, long sizeInBytes, Location... locations ){ + public boolean addFile( final String path, final LocationWrapper... locations ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return false; @@ -85,7 +86,7 @@ public boolean addFile( String path, long sizeInBytes, Location... locations ){ current = current.getOrCreateFolder( p.toString() ); } else { //file - return current.addFile( p.toString(), sizeInBytes, locations ); + return current.addOrUpdateFile( p.toString(), locations ); } } //This would add a file in working hierarchy diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 51a3941f..b8c4b7d6 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -10,12 +10,11 @@ public class RealFile extends File { @Getter - private Location[] locations = null; - @Getter - private long sizeInBytes; + private LocationWrapper[] locations; - RealFile(long sizeInBytes) { - this.sizeInBytes = sizeInBytes; + public RealFile( LocationWrapper... locations ) { + checkIfValidInput( locations ); + this.locations = locations; } @Override @@ -23,49 +22,37 @@ public boolean isDirectory(){ return false; } - private void checkIfValidInput( Location[] location ){ + private void checkIfValidInput( LocationWrapper[] location ){ if ( location == null || location.length == 0 ) throw new IllegalArgumentException( "location was null or empty" ); - for (Location loc : location) { + for (LocationWrapper loc : location) { if ( loc == null ) throw new IllegalArgumentException( "location contains null value" ); } } - public void addLocation( Location... location ){ + public void addOrUpdateLocation( LocationWrapper... location ){ checkIfValidInput( location ); synchronized ( this ){ - if ( locations == null ){ - locations = location; - return; - } - List newLocationList = new ArrayList<>( location.length ); - for (Location newLoc : location) { + int index = 0; + LocationWrapper[] newLocationsTmp = new LocationWrapper[location.length]; + for (LocationWrapper newLoc : location) { boolean foundEqual = false; - for (Location l : locations) { - if ( newLoc.equals(l) ) { + for (int i = 0; i < locations.length; i++) { + if ( newLoc.getLocation().equals( locations[i].getLocation() ) ) { foundEqual = true; + if ( newLoc.getTimestamp() > locations[i].getTimestamp() ) + locations[i] = newLoc; break; } } if ( !foundEqual ){ - newLocationList.add( newLoc ); + newLocationsTmp[index++] = newLoc; } } - final Location[] newLocation = Arrays.copyOf(locations, locations.length + newLocationList.size() ); - int index = locations.length; - for (Location l : newLocationList) { - newLocation[index ++ ] = l; - } + final LocationWrapper[] newLocation = Arrays.copyOf(locations, locations.length + index ); + System.arraycopy( newLocationsTmp, 0, newLocation, locations.length, index ); locations = newLocation; } } - public void changeFile( long sizeInBytes, Location... location ){ - checkIfValidInput( location ); - synchronized ( this ) { - this.sizeInBytes = sizeInBytes; - locations = location; - } - } - } diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index e372e729..15ef4393 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -1,6 +1,5 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; import org.junit.Before; @@ -9,8 +8,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -21,9 +18,13 @@ public class HierarchyWrapperTest { final String workdir = "/folder/localworkdir/"; HierarchyWrapper hw; List files; - List result; + Collection result; String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; - Location node1 = NodeLocation.getLocation("Node1"); + LocationWrapper node1 = getLocationWrapper("Node1"); + + private LocationWrapper getLocationWrapper( String location ){ + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, "processA" ); + } @Before public void init() { @@ -39,12 +40,12 @@ public void init() { files.add( temporaryDir + "b/c/test.abc" ); files.add( temporaryDir + "bc/file.abc" ); - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x,0, node1))); - result = hw.getAllFilesInDir(temporaryDir); + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, node1))); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); } - private void compare( List a, List b){ + private void compare( List a, Collection b){ assertEquals( new HashSet<>(a.stream().map(x -> Paths.get(x).normalize()).collect(Collectors.toList())), new HashSet<>(b) @@ -56,15 +57,15 @@ private void compare( List a, List b){ @Test public void getAllFilesInDir() { log.info("{}", this.result); - List result; + Collection result; - result = hw.getAllFilesInDir( temporaryDir + "b/" ); - compare(Arrays.asList(temporaryDir + "b/c/test.abc"), result); + result = hw.getAllFilesInDir( temporaryDir + "b/" ).keySet(); + compare(List.of(temporaryDir + "b/c/test.abc"), result); log.info("{}", result); - result = hw.getAllFilesInDir( temporaryDir + "b" ); - compare(Arrays.asList(temporaryDir + "b/c/test.abc"), result); + result = hw.getAllFilesInDir( temporaryDir + "b" ).keySet(); + compare(List.of(temporaryDir + "b/c/test.abc"), result); log.info("{}", result); } @@ -90,48 +91,48 @@ public void getAllFilesinAllWorkdirs() { @Test public void createFileinFile() { - assertTrue(hw.addFile(temporaryDir + "test/b.txt", 10, node1)); + assertTrue(hw.addFile(temporaryDir + "test/b.txt", node1)); files.remove( temporaryDir + "test" ); files.add( temporaryDir + "test/b.txt" ); - result = hw.getAllFilesInDir(temporaryDir); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); } @Test public void createFileinWorkdir() { - assertFalse(hw.addFile(workdir + "ab/b.txt", 10, node1)); + assertFalse(hw.addFile(workdir + "ab/b.txt", node1)); - result = hw.getAllFilesInDir(temporaryDir); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); } @Test public void createFileOutOfScope() { - assertFalse(hw.addFile("/somewhere/test.txt", 10, node1)); - assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt", 10, node1)); + assertFalse(hw.addFile("/somewhere/test.txt", node1)); + assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt", node1)); - result = hw.getAllFilesInDir(temporaryDir); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); } @Test public void createFileTwice() { - assertTrue(hw.addFile(temporaryDir + "bc/file.abc", 10, node1)); + assertTrue(hw.addFile(temporaryDir + "bc/file.abc", node1)); - result = hw.getAllFilesInDir(temporaryDir); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); } @Test public void createFileButWasFolder() { - assertTrue(hw.addFile(temporaryDir + "bc", 10, node1)); + assertTrue(hw.addFile(temporaryDir + "bc",node1)); files.remove( temporaryDir + "bc/file.abc" ); files.add( temporaryDir + "bc" ); - result = hw.getAllFilesInDir(temporaryDir); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); } @@ -164,7 +165,7 @@ public void testParallelAdd() { intialMem = finalMem; - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, 10, node1))); + files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, getLocationWrapper("Node1")))); finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); @@ -173,11 +174,11 @@ public void testParallelAdd() { log.info( "Memory hierachy after gc: {}mb", (finalMem - intialMem) / 1024 / 1024 ); log.info( "Memory per entry: {}b", (finalMem - intialMem) / iters ); - result = hw.getAllFilesInDir( wd ); + result = hw.getAllFilesInDir( wd ).keySet(); compare(files, result); for (Map.Entry> entry : m.entrySet()) { - result = hw.getAllFilesInDir( entry.getKey() ); + result = hw.getAllFilesInDir( entry.getKey() ).keySet(); compare( entry.getValue(), result); } } @@ -197,19 +198,21 @@ public void testGetFile() { files.add( temporaryDir + "b/c/test.abc" ); int index = 0; + LocationWrapper[] lw = new LocationWrapper[ files.size() ]; for (String file : files) { - assertTrue(hw.addFile(file,10, NodeLocation.getLocation( "Node" + index ))); + lw[index] = getLocationWrapper( "Node" + index ); + assertTrue(hw.addFile(file,lw[index++])); } - result = hw.getAllFilesInDir(temporaryDir); + result = hw.getAllFilesInDir(temporaryDir).keySet(); compare( files, result); index = 0; for (String file : files) { RealFile realFile = hw.getFile(file); - final Location[] locations = realFile.getLocations(); - assertEquals(NodeLocation.getLocation( "Node" + index ), locations[0]); + final LocationWrapper[] locations = realFile.getLocations(); + assertEquals(lw[index++], locations[0]); assertEquals(1,locations.length); } } @@ -232,8 +235,8 @@ public void testGetFileButIsDir() { @Test public void testFileIsNowDir(){ - assertTrue(hw.addFile( temporaryDir + "d", 120, NodeLocation.getLocation("nodeXY") )); - result = hw.getAllFilesInDir( temporaryDir ); + assertTrue(hw.addFile( temporaryDir + "d", getLocationWrapper("nodeXY") )); + result = hw.getAllFilesInDir( temporaryDir ).keySet(); assertNotNull( hw.getFile( temporaryDir + "d" ) ); assertNull( hw.getFile( temporaryDir + "d/e/file.txt" ) ); } diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 38d77b37..edf30277 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -1,6 +1,5 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; import org.junit.Test; @@ -12,52 +11,63 @@ public class RealFileTest { + private LocationWrapper getLocationWrapper( String location ){ + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, "processA" ); + } + @Test public void addLocation() { - final RealFile realFile = new RealFile( 10 ); - List locations = new LinkedList<>(); + List locations = new LinkedList<>(); - locations.add( NodeLocation.getLocation("Node1") ); - realFile.addLocation(NodeLocation.getLocation("Node1")); - assertArrayEquals( locations.toArray(), realFile.getLocations() ); - locations.add( NodeLocation.getLocation("Node2") ); - realFile.addLocation(NodeLocation.getLocation("Node2")); + final LocationWrapper node1 = getLocationWrapper("Node1"); + locations.add(node1); + final RealFile realFile = new RealFile(node1); assertArrayEquals( locations.toArray(), realFile.getLocations() ); - locations.add( NodeLocation.getLocation("Node3") ); - realFile.addLocation(NodeLocation.getLocation("Node3")); + final LocationWrapper node2 = getLocationWrapper("Node2"); + locations.add(node2); + realFile.addOrUpdateLocation(node2); assertArrayEquals( locations.toArray(), realFile.getLocations() ); - realFile.addLocation(NodeLocation.getLocation("Node1")); + final LocationWrapper node3 = getLocationWrapper("Node3"); + locations.add(node3); + realFile.addOrUpdateLocation(node3); assertArrayEquals( locations.toArray(), realFile.getLocations() ); + final LocationWrapper node1New = getLocationWrapper("Node1"); + realFile.addOrUpdateLocation(node1New); + assertArrayEquals( locations.toArray(), realFile.getLocations()); + } @Test public void addEmptyLocation() { - final RealFile realFile = new RealFile( 10 ); - assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( null )); - assertThrows(IllegalArgumentException.class, () -> realFile.addLocation()); - assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( new Location[0] )); - assertThrows(IllegalArgumentException.class, () -> realFile.addLocation( new Location[1] )); + final RealFile realFile = new RealFile( getLocationWrapper("node1") ); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( null )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation()); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( new LocationWrapper[0] )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( new LocationWrapper[1] )); } @Test public void addInParallel() { - final RealFile realFile = new RealFile( 10 ); + final LocationWrapper node0 = getLocationWrapper("Node0"); + final RealFile realFile = new RealFile(node0); + + List locations = new LinkedList<>(); - List locations = new LinkedList<>(); - for (int i = 0; i < 10_000; i++) { - locations.add( NodeLocation.getLocation( "Node" + i ) ); + for (int i = 1; i < 10_000; i++) { + locations.add( getLocationWrapper( "Node" + i ) ); } Collections.shuffle(locations); - locations.parallelStream().forEach( realFile::addLocation ); + locations.parallelStream().forEach( realFile::addOrUpdateLocation ); + locations.add( node0 ); assertEquals( new HashSet<>(locations), new HashSet<>(Arrays.asList(realFile.getLocations())) @@ -68,26 +78,36 @@ public void addInParallel() { @Test public void changeFile() { - final RealFile realFile = new RealFile( 4 ); - realFile.addLocation( NodeLocation.getLocation("Node1") ); - realFile.addLocation( NodeLocation.getLocation("Node2") ); - realFile.addLocation( NodeLocation.getLocation("Node3") ); - - realFile.changeFile( 5, NodeLocation.getLocation("NodeNew") ); - Location[] expected = { NodeLocation.getLocation("NodeNew") }; - assertEquals( 5, realFile.getSizeInBytes() ); + final LocationWrapper node0 = getLocationWrapper("Node0"); + final RealFile realFile = new RealFile(node0); + final LocationWrapper node1 = getLocationWrapper("Node1"); + realFile.addOrUpdateLocation(node1); + final LocationWrapper node2 = getLocationWrapper("Node2"); + realFile.addOrUpdateLocation(node2); + final LocationWrapper node3 = getLocationWrapper("Node3"); + realFile.addOrUpdateLocation(node3); + + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, "processB"); + realFile.addOrUpdateLocation( nodeNew ); + LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); } @Test - public void changeFileEmpty() { + public void changeFileOnExistingLocation() { + + final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); + + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, "processA"); + realFile.addOrUpdateLocation( nodeNew ); + LocationWrapper[] expected = { nodeNew }; + assertArrayEquals( expected, realFile.getLocations() ); - final RealFile realFile = new RealFile( 10 ); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile(1,null )); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile(1) ); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( 1,new Location[0] )); - assertThrows(IllegalArgumentException.class, () -> realFile.changeFile( 1,new Location[1] )); + final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, "processB"); + realFile.addOrUpdateLocation( nodeNew2 ); + LocationWrapper[] expected2 = { nodeNew2 }; + assertArrayEquals( expected2, realFile.getLocations() ); } From b44c2f872ad66e75dcb59d913241f7ab81143732 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 11 Nov 2021 15:54:07 +0100 Subject: [PATCH 020/443] added LocationWrapperClass Signed-off-by: Lehmann-Fabian --- .../location/hierachy/LocationWrapper.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java new file mode 100644 index 00000000..fd97b768 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -0,0 +1,30 @@ +package fonda.scheduler.model.location.hierachy; + +import fonda.scheduler.model.location.Location; +import lombok.Getter; + +@Getter +public class LocationWrapper { + + private final Location location; + private final long timestamp; + private final long sizeInBytes; + private final String process; + + public LocationWrapper(Location location, long timestamp, long sizeInBytes, String process) { + this.location = location; + this.timestamp = timestamp; + this.sizeInBytes = sizeInBytes; + this.process = process; + } + + @Override + public String toString() { + return "LocationWrapper{" + + "location=" + location + + ", timestamp=" + timestamp + + ", sizeInBytes=" + sizeInBytes + + ", process='" + process + '\'' + + '}'; + } +} From 0091aab9d3fe6166731726aff2cf03bb36eb1674 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 11 Nov 2021 15:54:29 +0100 Subject: [PATCH 021/443] address files with pathes Signed-off-by: Lehmann-Fabian --- .../location/hierachy/HierarchyWrapper.java | 12 +- .../hierachy/HierarchyWrapperTest.java | 124 +++++++++--------- 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 480783a8..efd08680 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -23,8 +23,8 @@ public HierarchyWrapper( String workdir ) { this.workdir = Paths.get( workdir ).normalize(); } - private Path relativize( String path ){ - return workdir.relativize(Paths.get( path )).normalize(); + private Path relativize( Path path ){ + return workdir.relativize( path ).normalize(); } private Folder getWorkdir( Iterator iterator, boolean create ){ @@ -46,7 +46,7 @@ private Folder getWorkdir( Iterator iterator, boolean create ){ * @param path get all files recursively in this folder (absolute path) * @return Null if folder is empty, or not found */ - public Map getAllFilesInDir(String path ){ + public Map getAllFilesInDir( final Path path ){ final Path relativePath = relativize( path ); Iterator iterator = relativePath.iterator(); File current = getWorkdir( iterator, false ); @@ -60,7 +60,7 @@ public Map getAllFilesInDir(String path ){ } } if( current.isDirectory() ) - return ((Folder) current).getAllChildren( Paths.get(path).normalize() ); + return ((Folder) current).getAllChildren( path.normalize() ); else return null; } @@ -71,7 +71,7 @@ public Map getAllFilesInDir(String path ){ * @param locations locations where the file is located * @return false if file can not be created */ - public boolean addFile( final String path, final LocationWrapper... locations ){ + public boolean addFile( final Path path, final LocationWrapper... locations ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return false; @@ -98,7 +98,7 @@ public boolean addFile( final String path, final LocationWrapper... locations ){ * @param path file to get (absolute path) * @return File or null if file does not exist */ - public RealFile getFile( String path ){ + public RealFile getFile( Path path ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return null; diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index 15ef4393..96137182 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -17,7 +17,7 @@ public class HierarchyWrapperTest { final String workdir = "/folder/localworkdir/"; HierarchyWrapper hw; - List files; + List files; Collection result; String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; LocationWrapper node1 = getLocationWrapper("Node1"); @@ -31,23 +31,23 @@ public void init() { hw = new HierarchyWrapper(workdir); files = new LinkedList(); - files.add( temporaryDir + "test" ); - files.add( temporaryDir + "file.abc" ); - files.add( temporaryDir + "a/test.abc" ); - files.add( temporaryDir + "a/file.abc" ); - files.add( temporaryDir + "d/test" ); - files.add( temporaryDir + "d/e/file.txt" ); - files.add( temporaryDir + "b/c/test.abc" ); - files.add( temporaryDir + "bc/file.abc" ); + files.add( Paths.get(temporaryDir + "test" )); + files.add( Paths.get(temporaryDir + "file.abc" )); + files.add( Paths.get(temporaryDir + "a/test.abc" )); + files.add( Paths.get(temporaryDir + "a/file.abc" )); + files.add( Paths.get(temporaryDir + "d/test" )); + files.add( Paths.get(temporaryDir + "d/e/file.txt" )); + files.add( Paths.get(temporaryDir + "b/c/test.abc" )); + files.add( Paths.get(temporaryDir + "bc/file.abc" )); files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, node1))); - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } - private void compare( List a, Collection b){ + private void compare( List a, Collection b){ assertEquals( - new HashSet<>(a.stream().map(x -> Paths.get(x).normalize()).collect(Collectors.toList())), + new HashSet<>(a.stream().map(x -> x.normalize()).collect(Collectors.toList())), new HashSet<>(b) ); assertEquals(a.size(),b.size()); @@ -59,80 +59,80 @@ public void getAllFilesInDir() { log.info("{}", this.result); Collection result; - result = hw.getAllFilesInDir( temporaryDir + "b/" ).keySet(); - compare(List.of(temporaryDir + "b/c/test.abc"), result); + result = hw.getAllFilesInDir( Paths.get(temporaryDir + "b/" )).keySet(); + compare(List.of(Paths.get(temporaryDir + "b/c/test.abc")), result); log.info("{}", result); - result = hw.getAllFilesInDir( temporaryDir + "b" ).keySet(); - compare(List.of(temporaryDir + "b/c/test.abc"), result); + result = hw.getAllFilesInDir(Paths.get( temporaryDir + "b" )).keySet(); + compare(List.of(Paths.get(temporaryDir + "b/c/test.abc")), result); log.info("{}", result); } @Test public void getAllFilesInFile() { - assertNull(hw.getAllFilesInDir( temporaryDir + "test/" )); - assertNull(hw.getAllFilesInDir( temporaryDir + "d/test/" )); - assertNull(hw.getAllFilesInDir( temporaryDir + "d/test/c/" )); + assertNull(hw.getAllFilesInDir( Paths.get(temporaryDir + "test/" ) )); + assertNull(hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/" ) )); + assertNull(hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/c/" ) )); } @Test public void getAllFilesOutOfScrope() { - assertNull(hw.getAllFilesInDir( "/somewhere/" )); - assertNull(hw.getAllFilesInDir( "/somewhere/on/the/machine/very/deep/hierarchy/" )); + assertNull(hw.getAllFilesInDir( Paths.get("/somewhere/" ) )); + assertNull(hw.getAllFilesInDir( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/" ) )); } @Test public void getAllFilesinAllWorkdirs() { - assertNull(hw.getAllFilesInDir( workdir )); - assertNull(hw.getAllFilesInDir( workdir + "/ab/" )); + assertNull(hw.getAllFilesInDir( Paths.get(workdir ) )); + assertNull(hw.getAllFilesInDir( Paths.get(workdir + "/ab/" ) )); } @Test public void createFileinFile() { - assertTrue(hw.addFile(temporaryDir + "test/b.txt", node1)); + assertTrue(hw.addFile( Paths.get(temporaryDir + "test/b.txt"), node1)); - files.remove( temporaryDir + "test" ); - files.add( temporaryDir + "test/b.txt" ); + files.remove( Paths.get(temporaryDir + "test" )); + files.add( Paths.get(temporaryDir + "test/b.txt") ); - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir( Paths.get(temporaryDir) ).keySet(); compare( files, result); } @Test public void createFileinWorkdir() { - assertFalse(hw.addFile(workdir + "ab/b.txt", node1)); + assertFalse(hw.addFile(Paths.get(workdir + "ab/b.txt"), node1)); - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @Test public void createFileOutOfScope() { - assertFalse(hw.addFile("/somewhere/test.txt", node1)); - assertFalse(hw.addFile("/somewhere/on/the/machine/very/deep/hierarchy/test.txt", node1)); + assertFalse(hw.addFile(Paths.get("/somewhere/test.txt"), node1)); + assertFalse(hw.addFile(Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/test.txt"), node1)); - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @Test public void createFileTwice() { - assertTrue(hw.addFile(temporaryDir + "bc/file.abc", node1)); + assertTrue(hw.addFile(Paths.get(temporaryDir + "bc/file.abc"), node1)); - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @Test public void createFileButWasFolder() { - assertTrue(hw.addFile(temporaryDir + "bc",node1)); + assertTrue(hw.addFile(Paths.get(temporaryDir + "bc"),node1)); - files.remove( temporaryDir + "bc/file.abc" ); - files.add( temporaryDir + "bc" ); + files.remove( Paths.get(temporaryDir + "bc/file.abc") ); + files.add( Paths.get(temporaryDir + "bc") ); - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @@ -142,15 +142,15 @@ public void testParallelAdd() { System.gc(); long intialMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - List files = new LinkedList(); + List files = new LinkedList(); String wd = workdir + "ab/abcdefg/"; - Map> m = new HashMap(); + Map> m = new HashMap(); int iters = 1_000_000; for (int i = 0; i < iters; i++) { String p = wd + (i % 3) + "/" + (i % 4); - String file = p + "/" + "file-" + i; + Path file = Paths.get(p + "/" + "file-" + i); files.add( file ); final List currentData = m.getOrDefault(p, new LinkedList()); currentData.add(file); @@ -174,11 +174,11 @@ public void testParallelAdd() { log.info( "Memory hierachy after gc: {}mb", (finalMem - intialMem) / 1024 / 1024 ); log.info( "Memory per entry: {}b", (finalMem - intialMem) / iters ); - result = hw.getAllFilesInDir( wd ).keySet(); + result = hw.getAllFilesInDir( Paths.get( wd ) ).keySet(); compare(files, result); - for (Map.Entry> entry : m.entrySet()) { - result = hw.getAllFilesInDir( entry.getKey() ).keySet(); + for (Map.Entry> entry : m.entrySet()) { + result = hw.getAllFilesInDir( Paths.get( entry.getKey() )).keySet(); compare( entry.getValue(), result); } } @@ -189,26 +189,26 @@ public void testGetFile() { hw = new HierarchyWrapper(workdir); files = new LinkedList(); - files.add( temporaryDir + "test" ); - files.add( temporaryDir + "file.abc" ); - files.add( temporaryDir + "a/test.abc" ); - files.add( temporaryDir + "a/file.abc" ); - files.add( temporaryDir + "d/test" ); - files.add( temporaryDir + "d/e/file.txt" ); - files.add( temporaryDir + "b/c/test.abc" ); + files.add( Paths.get( temporaryDir + "test" ) ); + files.add( Paths.get( temporaryDir + "file.abc" ) ); + files.add( Paths.get( temporaryDir + "a/test.abc" ) ); + files.add( Paths.get( temporaryDir + "a/file.abc" ) ); + files.add( Paths.get( temporaryDir + "d/test" ) ); + files.add( Paths.get( temporaryDir + "d/e/file.txt" ) ); + files.add( Paths.get( temporaryDir + "b/c/test.abc" ) ); int index = 0; LocationWrapper[] lw = new LocationWrapper[ files.size() ]; - for (String file : files) { + for ( Path file : files) { lw[index] = getLocationWrapper( "Node" + index ); assertTrue(hw.addFile(file,lw[index++])); } - result = hw.getAllFilesInDir(temporaryDir).keySet(); + result = hw.getAllFilesInDir( Paths.get(temporaryDir) ).keySet(); compare( files, result); index = 0; - for (String file : files) { + for ( Path file : files) { RealFile realFile = hw.getFile(file); final LocationWrapper[] locations = realFile.getLocations(); @@ -219,25 +219,25 @@ public void testGetFile() { @Test public void testGetFileOutOfScope() { - assertNull(hw.getFile( "/file.txt" )); - assertNull(hw.getFile( "/somewhere/on/the/machine/very/deep/hierarchy/file.txt" )); + assertNull(hw.getFile( Paths.get("/file.txt" ) )); + assertNull(hw.getFile( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/file.txt" ) )); } @Test public void testGetFileWorkdir() { - assertNull(hw.getFile( workdir + "ab/test.txt" )); + assertNull(hw.getFile( Paths.get(workdir + "ab/test.txt" ))); } @Test public void testGetFileButIsDir() { - assertNull(hw.getFile( temporaryDir + "d" )); + assertNull(hw.getFile( Paths.get(temporaryDir + "d" ))); } @Test public void testFileIsNowDir(){ - assertTrue(hw.addFile( temporaryDir + "d", getLocationWrapper("nodeXY") )); - result = hw.getAllFilesInDir( temporaryDir ).keySet(); - assertNotNull( hw.getFile( temporaryDir + "d" ) ); - assertNull( hw.getFile( temporaryDir + "d/e/file.txt" ) ); + assertTrue(hw.addFile( Paths.get(temporaryDir + "d"), getLocationWrapper("nodeXY") )); + result = hw.getAllFilesInDir( Paths.get(temporaryDir ) ).keySet(); + assertNotNull( hw.getFile( Paths.get(temporaryDir + "d" )) ); + assertNull( hw.getFile( Paths.get(temporaryDir + "d/e/file.txt" )) ); } } \ No newline at end of file From ebb59880e5b46e2400293b06256685ed2b3515ce Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 11 Nov 2021 16:22:50 +0100 Subject: [PATCH 022/443] store process as Singleton Signed-off-by: Lehmann-Fabian --- .../java/fonda/scheduler/model/Process.java | 28 +++++++++++++++++++ .../location/hierachy/HierarchyWrapper.java | 2 -- .../location/hierachy/LocationWrapper.java | 5 ++-- .../hierachy/HierarchyWrapperTest.java | 3 +- .../model/location/hierachy/RealFileTest.java | 9 +++--- 5 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/Process.java diff --git a/src/main/java/fonda/scheduler/model/Process.java b/src/main/java/fonda/scheduler/model/Process.java new file mode 100644 index 00000000..61689308 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/Process.java @@ -0,0 +1,28 @@ +package fonda.scheduler.model; + +import lombok.Getter; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class Process { + + static final private ConcurrentMap< String, Process > processHolder = new ConcurrentHashMap(); + + @Getter + private final String name; + + private Process(String name) { + this.name = name; + } + + public static Process getProcess ( final String process ){ + final Process processFound = processHolder.get( process ); + if ( processFound == null ){ + processHolder.putIfAbsent( process, new Process( process ) ); + return processHolder.get( process ); + } + return processFound; + } + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index efd08680..80f698cc 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -1,12 +1,10 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.location.Location; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index fd97b768..598cc9b2 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -1,5 +1,6 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.Process; import fonda.scheduler.model.location.Location; import lombok.Getter; @@ -9,9 +10,9 @@ public class LocationWrapper { private final Location location; private final long timestamp; private final long sizeInBytes; - private final String process; + private final Process process; - public LocationWrapper(Location location, long timestamp, long sizeInBytes, String process) { + public LocationWrapper(Location location, long timestamp, long sizeInBytes, Process process) { this.location = location; this.timestamp = timestamp; this.sizeInBytes = sizeInBytes; diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index 96137182..b618f7c7 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -1,5 +1,6 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.Process; import fonda.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; import org.junit.Before; @@ -23,7 +24,7 @@ public class HierarchyWrapperTest { LocationWrapper node1 = getLocationWrapper("Node1"); private LocationWrapper getLocationWrapper( String location ){ - return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, "processA" ); + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, Process.getProcess("processA") ); } @Before diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index edf30277..eac081dc 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -1,5 +1,6 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.Process; import fonda.scheduler.model.location.NodeLocation; import org.junit.Test; @@ -12,7 +13,7 @@ public class RealFileTest { private LocationWrapper getLocationWrapper( String location ){ - return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, "processA" ); + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, Process.getProcess("processA") ); } @@ -87,7 +88,7 @@ public void changeFile() { final LocationWrapper node3 = getLocationWrapper("Node3"); realFile.addOrUpdateLocation(node3); - final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, "processB"); + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, Process.getProcess("processB") ); realFile.addOrUpdateLocation( nodeNew ); LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); @@ -99,12 +100,12 @@ public void changeFileOnExistingLocation() { final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); - final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, "processA"); + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, Process.getProcess("processA") ); realFile.addOrUpdateLocation( nodeNew ); LocationWrapper[] expected = { nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); - final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, "processB"); + final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, Process.getProcess("processB") ); realFile.addOrUpdateLocation( nodeNew2 ); LocationWrapper[] expected2 = { nodeNew2 }; assertArrayEquals( expected2, realFile.getLocations() ); From 1b73ee8a0c76ec08b772f5355d1060dc5c521a91 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 12 Nov 2021 16:44:00 +0100 Subject: [PATCH 023/443] Parse outputfile Signed-off-by: Lehmann-Fabian --- .../model/PathLocationWrapperPair.java | 40 +++++ .../java/fonda/scheduler/model/Process.java | 13 ++ .../scheduler/model/TaskResultParser.java | 102 ++++++++++++ .../location/hierachy/LocationWrapper.java | 18 ++ .../scheduler/model/TaskResultParserTest.java | 155 ++++++++++++++++++ 5 files changed, 328 insertions(+) create mode 100644 src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java create mode 100644 src/main/java/fonda/scheduler/model/TaskResultParser.java create mode 100644 src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java diff --git a/src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java b/src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java new file mode 100644 index 00000000..b01eff31 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java @@ -0,0 +1,40 @@ +package fonda.scheduler.model; + +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import lombok.Getter; + +import java.nio.file.Path; +import java.util.Objects; + +@Getter +public class PathLocationWrapperPair { + + private final Path path; + private final LocationWrapper locationWrapper; + + public PathLocationWrapperPair(Path path, LocationWrapper locationWrapper) { + this.path = path; + this.locationWrapper = locationWrapper; + } + + @Override + public String toString() { + return "PathLocationWrapperPair{" + + "path=" + path + + ", locationWrapper=" + locationWrapper + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof PathLocationWrapperPair)) return false; + PathLocationWrapperPair that = (PathLocationWrapperPair) o; + return getPath().equals(that.getPath()) && getLocationWrapper().equals(that.getLocationWrapper()); + } + + @Override + public int hashCode() { + return Objects.hash(getPath(), getLocationWrapper()); + } +} diff --git a/src/main/java/fonda/scheduler/model/Process.java b/src/main/java/fonda/scheduler/model/Process.java index 61689308..b87ee692 100644 --- a/src/main/java/fonda/scheduler/model/Process.java +++ b/src/main/java/fonda/scheduler/model/Process.java @@ -2,6 +2,7 @@ import lombok.Getter; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -25,4 +26,16 @@ public static Process getProcess ( final String process ){ return processFound; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Process)) return false; + Process process = (Process) o; + return getName().equals(process.getName()); + } + + @Override + public int hashCode() { + return Objects.hash(getName()); + } } diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java new file mode 100644 index 00000000..04ace5ca --- /dev/null +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -0,0 +1,102 @@ +package fonda.scheduler.model; + +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; + +@Slf4j +public class TaskResultParser { + + private String getRootDir( File file ){ + try { + Scanner sc = new Scanner( file ); + if( sc.hasNext() ) return sc.next().split(";")[0]; + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + return null; + } + + /** + * + * @param workdir + * @return A list of all new or updated files + */ + public Set getNewAndUpdatedFiles(final Path workdir, Location location, Process process ){ + + final Path infile = workdir.resolve(".command.infiles"); + final Path outfile = workdir.resolve(".command.outfiles"); + + String taskRootDir = getRootDir( infile.toFile() ); + String outputRootDir = getRootDir( outfile.toFile() ); + + final Map inputdata = new HashMap<>(); + + Set newOrUpdated = new HashSet<>(); + + try { + Files.lines( infile ).skip( 1 ) + .forEach( line -> { + String[] data = line.split(";"); + String path = data[1].equals("") ? data[0].substring( taskRootDir.length() + 1 ) : data[1]; + String modificationDate = data[6]; + inputdata.put( path , modificationDate ); + }); + + log.info( "{}", inputdata ); + + Files.lines( outfile ).skip( 1 ) + .forEach( line -> { + String[] data = line.split(";"); + boolean realFile = data[1].equals(""); + String path = realFile ? data[0] : data[1]; + String modificationDate = data[6]; + if ( "directory".equals(data[3]) ) return; + String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; + if ( !inputdata.containsKey(lockupPath) + || + !modificationDate.equals( inputdata.get( lockupPath ) )) + { + final LocationWrapper locationWrapper = new LocationWrapper( + location, + fileTimeFromString(modificationDate), + Long.parseLong(data[2]), + process + ); + newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); + } + }); + + } catch (IOException e) { + e.printStackTrace(); + } + return newOrUpdated; + + } + + private long fileTimeFromString(String date) { + if( date == null || date == "-" ) { + return -1; + } + String[] parts = date.split(" "); + parts[1] = parts[1].substring(0, 12); + String shortenedDate = String.join(" ", parts); + try { + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); + } catch (ParseException e) { + e.printStackTrace(); + } + return -1; + } + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 598cc9b2..9183a496 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -4,6 +4,8 @@ import fonda.scheduler.model.location.Location; import lombok.Getter; +import java.util.Objects; + @Getter public class LocationWrapper { @@ -28,4 +30,20 @@ public String toString() { ", process='" + process + '\'' + '}'; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof LocationWrapper)) return false; + LocationWrapper that = (LocationWrapper) o; + return getTimestamp() == that.getTimestamp() + && getSizeInBytes() == that.getSizeInBytes() + && Objects.equals(getLocation(), that.getLocation()) + && Objects.equals(getProcess(), that.getProcess()); + } + + @Override + public int hashCode() { + return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getProcess()); + } } diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java new file mode 100644 index 00000000..73b0d760 --- /dev/null +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -0,0 +1,155 @@ +package fonda.scheduler.model; + +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import static org.junit.Assert.*; + +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; + +@Slf4j +public class TaskResultParserTest { + + private final String NL = "\n"; + + private Path storeData( String[] inputdata, String[] outputdata ){ + Path tmpDir = null; + + try { + + tmpDir = Files.createTempDirectory( "results" ); + log.info("Path: {}", tmpDir ); + PrintWriter pw = new PrintWriter( tmpDir.resolve(".command.infiles").toString() ); + for (String s : inputdata) { + pw.println( s ); + } + pw.close(); + + pw = new PrintWriter( tmpDir.resolve(".command.outfiles").toString() ); + for (String s : outputdata) { + pw.println( s ); + } + pw.close(); + } catch ( Exception e ){} + return tmpDir; + } + + + @Test + public void test1(){ + + String infiles[] = { + "/tmp/nxf.3iuGDWr6Id;;4096;directory;-;2021-11-10 12:58:11.210414589 +0000;2021-11-10 12:58:11.222414603 +0000", + "/tmp/nxf.3iuGDWr6Id/file.txt;/pvcdata/testfile.txt;6;regular file;-;2021-11-10 12:58:07.485035700 +0000;2021-11-10 12:58:07.219065500 +0000", + "/tmp/nxf.3iuGDWr6Id/.command.err;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000", + "/tmp/nxf.3iuGDWr6Id/.command.out;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000" + }; + + String[] outfiles = { + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650;;4096;directory;-;2021-11-10 12:58:11.278414667 +0000;2021-11-10 12:58:11.278414667 +0000", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000" + }; + + final Path path = storeData(infiles, outfiles); + + final TaskResultParser taskResultParser = new TaskResultParser(); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), Process.getProcess("P1")); + + log.info("{}", newAndUpdatedFiles); + + final PathLocationWrapperPair[] expected = { + new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230l, 13, Process.getProcess("P1"))) + }; + + assertArrayEquals( expected, newAndUpdatedFiles.toArray() ); + + } + + @Test + public void test2(){ + + String infiles[] = { + "/tmp/nxf.IANFIlM3Kv;;4096;directory;-;2021-11-12 12:42:42.155614026 +0000;2021-11-12 12:42:42.171614019 +0000", + "/tmp/nxf.IANFIlM3Kv/file.txt;/pvcdata/testfile.txt;0;regular empty file;-;2021-11-12 12:42:29.000000000 +0000;2021-11-12 12:42:29.000000000 +0000", + "/tmp/nxf.IANFIlM3Kv/.command.err;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000", + "/tmp/nxf.IANFIlM3Kv/.command.out;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000" + }; + + String[] outfiles = { + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa;;4096;directory;-;2021-11-12 12:42:42.239613991 +0000;2021-11-12 12:42:42.243613989 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t;;4096;directory;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt;;0;regular empty file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000" + }; + + final Path path = storeData(infiles, outfiles); + + final TaskResultParser taskResultParser = new TaskResultParser(); + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Process p1 = Process.getProcess("P1"); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1); + + log.info("{}", newAndUpdatedFiles); + + final PathLocationWrapperPair[] expected = { + new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223l, 0, p1)), + new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)), + new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)), + new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) + + }; + + assertArrayEquals( expected, newAndUpdatedFiles.toArray() ); + + } + + @Test + public void test3(){ + + String infiles[] = { + "/tmp/nxf.9J6Y5mcXRD;;4096;directory;-;2021-11-12 14:42:20.941053482 +0000;2021-11-12 14:42:20.937053480 +0000", + "/tmp/nxf.9J6Y5mcXRD/t;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/b.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/c.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/a.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/.command.err;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000", + "/tmp/nxf.9J6Y5mcXRD/.command.out;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000" + }; + + String[] outfiles = { + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6;;4096;directory;-;2021-11-12 14:42:21.005053535 +0000;2021-11-12 14:42:21.009053539 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;4;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:20.993053525 +0000" + }; + + final Path path = storeData(infiles, outfiles); + + final TaskResultParser taskResultParser = new TaskResultParser(); + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Process p1 = Process.getProcess("P1"); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1); + + log.info("{}", newAndUpdatedFiles); + + final PathLocationWrapperPair[] expected = { + new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper(node1, 1636728140993l, 4, p1)) + + }; + + assertArrayEquals( expected, newAndUpdatedFiles.toArray() ); + + } + +} \ No newline at end of file From 19312fe02ca28bded9ec9d83dd082385e06aa633 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 12 Nov 2021 16:44:47 +0100 Subject: [PATCH 024/443] NodeWithAlloc stores its Location Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 64bd3243..90ac1320 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -1,5 +1,6 @@ package fonda.scheduler.model; +import fonda.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Quantity; @@ -18,6 +19,9 @@ public class NodeWithAlloc extends Node implements Comparable { Map assignedPods; + @Getter + private final NodeLocation nodeLocation; + public NodeWithAlloc(Node node) { this.setApiVersion( node.getApiVersion() ); @@ -36,6 +40,8 @@ public NodeWithAlloc(Node node) { assignedPods = new HashMap<>(); + this.nodeLocation = NodeLocation.getLocation( node ); + log.info("Node {} has RAM: {} and CPU: {}", node.getMetadata().getName(), max_ram, max_cpu); } From a1866d88459a91bff1f4382cc796d7c07c7400a0 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 12 Nov 2021 16:45:30 +0100 Subject: [PATCH 025/443] Task stores the Process Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/Task.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index b36b4666..e748df52 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -1,8 +1,10 @@ package fonda.scheduler.model; +import fonda.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Pod; import lombok.Getter; import lombok.Setter; +import org.apache.tomcat.jni.Proc; public class Task { @@ -11,16 +13,20 @@ public class Task { @Getter private final TaskState state = new TaskState(); + @Getter + private final Process process; + @Getter @Setter private Pod pod = null; @Getter @Setter - private String nodeName = null; + private NodeLocation node = null; public Task(TaskConfig config) { this.config = config; + this.process = Process.getProcess( config.getTask() ); } public String getWorkingDir(){ @@ -32,7 +38,7 @@ public String toString() { return "Task{" + "state=" + state + ", pod=" + pod.getMetadata().getName() + - ", nodeName='" + nodeName + '\'' + + ", node='" + (node != null ? node.getIdentifier() : "--") + '\'' + ", workDir='" + getWorkingDir() + '\'' + '}'; } From c1e0cbf00575813975e20d716276993911ed5ebe Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Fri, 12 Nov 2021 16:51:54 +0100 Subject: [PATCH 026/443] read output file and add to hierarchy holder Signed-off-by: Lehmann-Fabian --- .../scheduler/scheduler/RandomScheduler.java | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 93cd8222..2ff0ebac 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -1,25 +1,19 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.model.Task; +import fonda.scheduler.model.*; import io.fabric8.kubernetes.api.model.Pod; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.FileNotFoundException; import java.io.PrintWriter; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.nio.file.Paths; +import java.util.*; @Slf4j public class RandomScheduler extends SchedulerWithDaemonSet { - private final Map workdirNode = new HashMap<>(); - public RandomScheduler(String name, KubernetesClient client, String namespace, SchedulerConfig config) { super(name, client, namespace, config); } @@ -49,31 +43,34 @@ public int schedule( final List unscheduledTasks ) { void assignTaskToNode( Task task, NodeWithAlloc node ) { //Create initData - log.info( "Task: {}", task ); File file = new File(task.getWorkingDir() + '/' + ".command.init"); - PrintWriter pw = null; - try { - pw = new PrintWriter( file ); - pw.println( "echo \"Task init successful\"" ); + try (PrintWriter pw = new PrintWriter(file)) { + pw.println("echo \"Task init successful\""); } catch (FileNotFoundException e) { e.printStackTrace(); - } finally { - if ( pw != null ){ - pw.close(); - } } + task.setNode( node.getNodeLocation() ); + super.assignTaskToNode(task, node); } @Override int terminateTasks(List finishedTasks) { - for (Task finishedTask : finishedTasks) { - //TODO store new Data + final TaskResultParser taskResultParser = new TaskResultParser(); + finishedTasks.parallelStream().forEach( finishedTask -> { + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + Paths.get(finishedTask.getWorkingDir()), + finishedTask.getNode(), + finishedTask.getProcess() + ); + for (PathLocationWrapperPair newAndUpdatedFile : newAndUpdatedFiles) { + hierarchyWrapper.addFile( newAndUpdatedFile.getPath(), newAndUpdatedFile.getLocationWrapper() ); + } super.taskWasFinished( finishedTask ); - } + }); return 0; } From a13fdcb139a772743bc5ef9d447733ed7318033c Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 15:08:34 +0100 Subject: [PATCH 027/443] close input- and outputfiles file Signed-off-by: Lehmann-Fabian --- .../java/fonda/scheduler/model/TaskResultParser.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 04ace5ca..d2143c0a 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -18,11 +18,17 @@ public class TaskResultParser { private String getRootDir( File file ){ + Scanner sc = null; try { - Scanner sc = new Scanner( file ); + sc = new Scanner( file ); if( sc.hasNext() ) return sc.next().split(";")[0]; } catch (FileNotFoundException e) { e.printStackTrace(); + } finally { + if ( sc != null ) + try{ + sc.close(); + } catch ( Exception e ){} } return null; } @@ -30,6 +36,8 @@ private String getRootDir( File file ){ /** * * @param workdir + * @param location + * @param process * @return A list of all new or updated files */ public Set getNewAndUpdatedFiles(final Path workdir, Location location, Process process ){ @@ -85,7 +93,7 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo } private long fileTimeFromString(String date) { - if( date == null || date == "-" ) { + if( date == null || date.equals("-")) { return -1; } String[] parts = date.split(" "); From 6691982c639a9b8cac11e6ae3aed4a72ecfdb857 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 15:09:31 +0100 Subject: [PATCH 028/443] store podIP instead of podname in daemonByNode map Signed-off-by: Lehmann-Fabian --- .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 8af711b1..7c261bca 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -22,14 +22,14 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { super(execution, client, namespace, config); } - String getDaemonOnNode( String node ){ + public String getDaemonOnNode( String node ){ synchronized ( daemonByNode ) { return daemonByNode.get( node ); } } String getDaemonOnNode( Node node ){ - return getDaemonOnNode( node.getMetadata().getName() ); + return getDaemonOnNode( node.getMetadata().getName() ); } String getOneDaemon() { @@ -81,7 +81,7 @@ void podEventReceived(Watcher.Action action, Pod pod){ daemonByNode.remove(nodeName); } } else if ( pod.getStatus().getPhase().equals("Running") ) { - daemonByNode.put( nodeName, podName); + daemonByNode.put( nodeName, pod.getStatus().getPodIP()); informResourceChange(); } else if ( podIsCurrentDaemon ) { daemonByNode.remove(nodeName); @@ -94,5 +94,4 @@ void podEventReceived(Watcher.Action action, Pod pod){ } } - } From a235b27cdcc510ab74a9315702845d49c2944c88 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 15:10:29 +0100 Subject: [PATCH 029/443] added ftpDaemon Signed-off-by: Lehmann-Fabian --- daemons/ftp/Dockerfile | 10 +++ daemons/ftp/vsftpd.conf | 162 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 daemons/ftp/Dockerfile create mode 100644 daemons/ftp/vsftpd.conf diff --git a/daemons/ftp/Dockerfile b/daemons/ftp/Dockerfile new file mode 100644 index 00000000..c2db711a --- /dev/null +++ b/daemons/ftp/Dockerfile @@ -0,0 +1,10 @@ +FROM python:slim +RUN apt update && apt install -y \ + vsftpd \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /var/run/vsftpd/empty + +COPY vsftpd.conf /etc/vsftpd.conf + +ENTRYPOINT ["sh","-c","/usr/sbin/vsftpd /etc/vsftpd.conf"] \ No newline at end of file diff --git a/daemons/ftp/vsftpd.conf b/daemons/ftp/vsftpd.conf new file mode 100644 index 00000000..9e659d66 --- /dev/null +++ b/daemons/ftp/vsftpd.conf @@ -0,0 +1,162 @@ +# Example config file /etc/vsftpd.conf +# +# The default compiled in settings are fairly paranoid. This sample file +# loosens things up a bit, to make the ftp daemon more usable. +# Please see vsftpd.conf.5 for all compiled in defaults. +# +# READ THIS: This example file is NOT an exhaustive list of vsftpd options. +# Please read the vsftpd.conf.5 manual page to get a full idea of vsftpd's +# capabilities. +# +# +# Run standalone? vsftpd can run either from an inetd or as a standalone +# daemon started from an initscript. +listen=YES +# +# This directive enables listening on IPv6 sockets. By default, listening +# on the IPv6 "any" address (::) will accept connections from both IPv6 +# and IPv4 clients. It is not necessary to listen on *both* IPv4 and IPv6 +# sockets. If you want that (perhaps because you want to listen on specific +# addresses) then you must run two copies of vsftpd with two configuration +# files. +listen_ipv6=NO +# +# Allow anonymous FTP? (Disabled by default). +anonymous_enable=YES +# +# Uncomment this to allow local users to log in. +local_enable=NO +# +# Uncomment this to enable any form of FTP write command. +write_enable=NO +# +# Default umask for local users is 077. You may wish to change this to 022, +# if your users expect that (022 is used by most other ftpd's) +#local_umask=022 +# +# Uncomment this to allow the anonymous FTP user to upload files. This only +# has an effect if the above global write enable is activated. Also, you will +# obviously need to create a directory writable by the FTP user. +anon_upload_enable=NO +# +# Uncomment this if you want the anonymous FTP user to be able to create +# new directories. +anon_mkdir_write_enable=NO +# +# Activate directory messages - messages given to remote users when they +# go into a certain directory. +dirmessage_enable=YES +# +# If enabled, vsftpd will display directory listings with the time +# in your local time zone. The default is to display GMT. The +# times returned by the MDTM FTP command are also affected by this +# option. +use_localtime=YES +# +# Activate logging of uploads/downloads. +xferlog_enable=YES +# +# Make sure PORT transfer connections originate from port 20 (ftp-data). +connect_from_port_20=NO +# +# If you want, you can arrange for uploaded anonymous files to be owned by +# a different user. Note! Using "root" for uploaded files is not +# recommended! +#chown_uploads=YES +#chown_username=whoever +# +# You may override where the log file goes if you like. The default is shown +# below. +#xferlog_file=/var/log/vsftpd.log +# +# If you want, you can have your log file in standard ftpd xferlog format. +# Note that the default log file location is /var/log/xferlog in this case. +#xferlog_std_format=YES +# +# You may change the default value for timing out an idle session. +#idle_session_timeout=600 +# +# You may change the default value for timing out a data connection. +#data_connection_timeout=120 +# +# It is recommended that you define on your system a unique user which the +# ftp server can use as a totally isolated and unprivileged user. +#nopriv_user=ftpsecure +# +# Enable this and the server will recognise asynchronous ABOR requests. Not +# recommended for security (the code is non-trivial). Not enabling it, +# however, may confuse older FTP clients. +#async_abor_enable=YES +# +# By default the server will pretend to allow ASCII mode but in fact ignore +# the request. Turn on the below options to have the server actually do ASCII +# mangling on files when in ASCII mode. +# Beware that on some FTP servers, ASCII support allows a denial of service +# attack (DoS) via the command "SIZE /big/file" in ASCII mode. vsftpd +# predicted this attack and has always been safe, reporting the size of the +# raw file. +# ASCII mangling is a horrible feature of the protocol. +#ascii_upload_enable=YES +#ascii_download_enable=YES +# +# You may fully customise the login banner string: +#ftpd_banner=Welcome to blah FTP service. +# +# You may specify a file of disallowed anonymous e-mail addresses. Apparently +# useful for combatting certain DoS attacks. +#deny_email_enable=YES +# (default follows) +#banned_email_file=/etc/vsftpd.banned_emails +# +# You may restrict local users to their home directories. See the FAQ for +# the possible risks in this before using chroot_local_user or +# chroot_list_enable below. +#chroot_local_user=YES +# +# You may specify an explicit list of local users to chroot() to their home +# directory. If chroot_local_user is YES, then this list becomes a list of +# users to NOT chroot(). +# (Warning! chroot'ing can be very dangerous. If using chroot, make sure that +# the user does not have write access to the top level directory within the +# chroot) +#chroot_local_user=YES +#chroot_list_enable=YES +# (default follows) +#chroot_list_file=/etc/vsftpd.chroot_list +# +# You may activate the "-R" option to the builtin ls. This is disabled by +# default to avoid remote users being able to cause excessive I/O on large +# sites. However, some broken FTP clients such as "ncftp" and "mirror" assume +# the presence of the "-R" option, so there is a strong case for enabling it. +#ls_recurse_enable=YES +# +# Customization +# +# Some of vsftpd's settings don't fit the filesystem layout by +# default. +# +# This option should be the name of a directory which is empty. Also, the +# directory should not be writable by the ftp user. This directory is used +# as a secure chroot() jail at times vsftpd does not require filesystem +# access. +secure_chroot_dir=/var/run/vsftpd/empty +# +# This string is the name of the PAM service vsftpd will use. +pam_service_name=vsftpd +# +# This option specifies the location of the RSA certificate to use for SSL +# encrypted connections. +rsa_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +rsa_private_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +ssl_enable=NO + +anonymous_enable=yes +anon_root=/ + +pasv_enable=Yes +pasv_max_port=10090 +pasv_min_port=10090 + +# +# Uncomment this to indicate that vsftpd use a utf8 filesystem. +#utf8_filesystem=YES \ No newline at end of file From b7efefc90bb3d3f0ee49b182d3a40c5323342aab Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 15:11:45 +0100 Subject: [PATCH 030/443] save scheduler dns Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/SchedulerConfig.java | 5 ++++- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index f8752d89..0f1912ba 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -7,17 +7,20 @@ public class SchedulerConfig { final public List localClaims; final public List volumeClaims; final public String workDir; + final public String dns; - public SchedulerConfig(List localClaims, List volumeClaims, String workDir) { + public SchedulerConfig(List localClaims, List volumeClaims, String workDir, String dns) { this.localClaims = localClaims; this.volumeClaims = volumeClaims; this.workDir = workDir; + this.dns = dns; } private SchedulerConfig(){ this.localClaims = null; this.volumeClaims = null; this.workDir = null; + this.dns = null; } static public class LocalClaim { diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 84f727f8..3516a40b 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -30,6 +30,8 @@ public abstract class Scheduler { @Getter private final String namespace; @Getter + private final String dns; + @Getter private boolean close; final KubernetesClient client; @@ -48,6 +50,7 @@ public abstract class Scheduler { log.trace( "Register scheduler for " + this.name ); this.client = client; this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); + this.dns = config.dns; PodWatcher podWatcher = new PodWatcher(this); From 31ade01dc2ced54c600edbdc6d0d1a636e114041 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 15:15:48 +0100 Subject: [PATCH 031/443] rest api to get IP of daemon on node Signed-off-by: Lehmann-Fabian --- .../rest/SchedulerRestController.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index b76345fb..3bebcfe1 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -5,6 +5,7 @@ import fonda.scheduler.scheduler.Scheduler; import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.TaskConfig; +import fonda.scheduler.scheduler.SchedulerWithDaemonSet; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; @@ -131,6 +132,27 @@ ResponseEntity delete(@PathVariable String namespace, @PathVariable String exec return new ResponseEntity( HttpStatus.OK ); } + @GetMapping("/daemon/{namespace}/{execution}/{node}") + ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable String execution, @PathVariable String node ) { + + log.info( "Got request: {}{}{}", namespace, execution, node ); + + final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + String daemon = ((SchedulerWithDaemonSet) scheduler).getDaemonOnNode( node ); + + if ( daemon == null ){ + return new ResponseEntity( "No daemon for node found: " + node , HttpStatus.NOT_FOUND ); + } + + return new ResponseEntity( daemon, HttpStatus.OK ); + + } + @GetMapping ("/health") ResponseEntity checkHealth() { return new ResponseEntity( HttpStatus.OK ); From a031e787fa0e2b9deaaa4f5dfbce19f8901e20cc Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 15:18:39 +0100 Subject: [PATCH 032/443] added dns to schedulerconfig constructor call in Main Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 46234ad0..f39d6b3e 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -30,7 +30,7 @@ public static void main(String[] args) { @PostConstruct public void afterStart(){ log.info( "Started with namespace: {}", client.getNamespace() ); - new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null,null,"/localwork/")); + final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null, null, "/localwork/", null)); } } From 9a764d47785349dd3c7ad2a4c1958fb543ee0c67 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 16:15:15 +0100 Subject: [PATCH 033/443] copyStrategy configurable Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/Main.java | 9 +++++++-- src/main/java/fonda/scheduler/model/SchedulerConfig.java | 9 ++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index f39d6b3e..b84f4676 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -29,8 +29,13 @@ public static void main(String[] args) { @PostConstruct public void afterStart(){ - log.info( "Started with namespace: {}", client.getNamespace() ); - final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null, null, "/localwork/", null)); + try{ + log.info( "Started with namespace: {}", client.getNamespace() ); + final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null, null, "/localwork/", null, "ftp")); + } catch (Exception e){ + e.printStackTrace(); + System.exit(1); + } } } diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 0f1912ba..8890c0b8 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -8,12 +8,18 @@ public class SchedulerConfig { final public List volumeClaims; final public String workDir; final public String dns; + final public String copyStrategy; - public SchedulerConfig(List localClaims, List volumeClaims, String workDir, String dns) { + public SchedulerConfig(List localClaims, + List volumeClaims, + String workDir, + String dns, + String copyStrategy) { this.localClaims = localClaims; this.volumeClaims = volumeClaims; this.workDir = workDir; this.dns = dns; + this.copyStrategy = copyStrategy; } private SchedulerConfig(){ @@ -21,6 +27,7 @@ private SchedulerConfig(){ this.volumeClaims = null; this.workDir = null; this.dns = null; + this.copyStrategy = null; } static public class LocalClaim { From 65d163397a52d0f8cbe2ce114fd4a756c8431f5d Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 16:16:43 +0100 Subject: [PATCH 034/443] run testscheduler Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/Main.java | 4 ++++ .../java/fonda/scheduler/rest/SchedulerRestController.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index b84f4676..77802858 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -2,8 +2,10 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.rest.SchedulerRestController; import fonda.scheduler.scheduler.RandomScheduler; import lombok.extern.slf4j.Slf4j; +import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -32,6 +34,8 @@ public void afterStart(){ try{ log.info( "Started with namespace: {}", client.getNamespace() ); final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null, null, "/localwork/", null, "ftp")); + final Pair key = new Pair( "default", "test-run" ); + SchedulerRestController.addScheduler( key, randomScheduler ); } catch (Exception e){ e.printStackTrace(); System.exit(1); diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 3bebcfe1..20284908 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -33,6 +33,10 @@ public SchedulerRestController( @Autowired KubernetesClient client ){ this.client = client; } + public static void addScheduler(Pair key, Scheduler scheduler ){ + schedulerHolder.put( key, scheduler ); + } + /** * Register a sheduler for a workflow execution * @param namespace namespace where the workflow runs From fd32bfd6fd59f48a268c70b506e35c2a0b291cdb Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 16:20:25 +0100 Subject: [PATCH 035/443] added copyStrategies Signed-off-by: Lehmann-Fabian --- .../scheduler/copystrategy/CopyStrategy.java | 40 +++++++ .../scheduler/copystrategy/FTPstrategy.java | 10 ++ src/main/resources/copystrategies/ftp.py | 110 ++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java create mode 100644 src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java create mode 100644 src/main/resources/copystrategies/ftp.py diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java new file mode 100644 index 00000000..7a5ec7dd --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -0,0 +1,40 @@ +package fonda.scheduler.scheduler.copystrategy; + +import fonda.scheduler.model.Task; + +import java.io.*; +import java.nio.charset.StandardCharsets; + +public abstract class CopyStrategy { + + public void generateCopyScript( Task task ){ + + File file = new File(task.getWorkingDir() + '/' + ".command.init.run"); + + try (PrintWriter pw = new PrintWriter(file)) { + + ClassLoader classLoader = getClass().getClassLoader(); + + try (InputStream inputStream = classLoader.getResourceAsStream( getResource() ); + InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(streamReader)) { + + String line; + while ((line = reader.readLine()) != null) { + pw.println(line); + } + + } catch (IOException e) { + e.printStackTrace(); + } + + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + } + + abstract String getResource(); + +} diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java new file mode 100644 index 00000000..c7cf8263 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java @@ -0,0 +1,10 @@ +package fonda.scheduler.scheduler.copystrategy; + +public class FTPstrategy extends CopyStrategy { + + @Override + String getResource() { + return "copystrategies/ftp.py"; + } + +} \ No newline at end of file diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py new file mode 100644 index 00000000..c7a9e03c --- /dev/null +++ b/src/main/resources/copystrategies/ftp.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +import ftplib +import time +import os +import json +import urllib.request +import sys + +def getIP( node ): + ip = urllib.request.urlopen(dns + node).read() + return str(ip.decode("utf-8") ) + + +def download( node, files ): + + ftp = None + size = len(files) + + connectionProblem = 0 + + while len(files) > 0: + + if connectionProblem > 1: + ftp = None + if connectionProblem == 8: + exit(8) + print( "Connection refused! Try again..." ) + time.sleep(2**connectionProblem) + + + if ftp is None: + try: + connectionProblem += 1 + ip = getIP( node ) + print( "Try to connect to", ip ) + ftp = ftplib.FTP( ip ) + ftp.login("ftp", "pythonclient") + ftp.set_pasv(True) + ftp.encoding='utf-8' + print( "Connection established" ) + connectionProblem = 0 + except ConnectionRefusedError as err: + continue + except BaseException as err: + print(f"Unexpected {err=}, {type(err)=}") + continue + + + filename = files[0] + index = size - len(files) + 1 + print("Download", "[" + str( index ).rjust( len( str( size ) ) ) + "/" + str( size ) + "]" , filename) + + try: + if os.path.exists( filename ): + os.remove(filename) + ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) + files.pop(0) + except ftplib.error_perm as err: + if( str(err) == "550 Failed to open file." ): + print( "File not found:", node + filename ) + if exitIfFileWasNotFound: + exit(550) + except FileNotFoundError as err: + print( "File not found:", node + filename ) + if exitIfFileWasNotFound: + exit(404) + except EOFError as err: + print( "It seems the connection was lost! Try again..." ) + ftp = None + continue + except BaseException as err: + print(f"Unexpected {err=}, {type(err)=}") + ftp = None + continue + + files.pop(0) + + if ftp is not None: + try: + ftp.quit() + ftp.close() + except: + print(f"Unexpected {err=}, {type(err)=}") + + + +print("Start to setup the environment") + +exitIfFileWasNotFound = False + +if ( len(sys.argv) < 2 ): + print( "Configfile not specified" ) + exit(101) + +configFilePath = sys.argv[1] + +if not os.path.isfile( configFilePath ): + print ("Config file not found:", configFilePath ) + exit(102) + +configFile = open(configFile,) +config = json.load( configFile ) +print( config ) +dns = config["dns"] + +data = config[ "data" ] + +for d in data: + files = d["files"] + download( d["node"], files ) \ No newline at end of file From 0a093eabd3d442723feeb584e2d24744171f401d Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Tue, 16 Nov 2021 16:24:06 +0100 Subject: [PATCH 036/443] moved assignTaskToNode and terminateTask into SchedulerWithDaemonSet.java Signed-off-by: Lehmann-Fabian --- .../scheduler/scheduler/RandomScheduler.java | 39 --------------- .../scheduler/SchedulerWithDaemonSet.java | 48 +++++++++++++++++-- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 2ff0ebac..aaa9c29d 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -5,10 +5,6 @@ import io.fabric8.kubernetes.api.model.Pod; import lombok.extern.slf4j.Slf4j; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.PrintWriter; -import java.nio.file.Paths; import java.util.*; @Slf4j @@ -39,39 +35,4 @@ public int schedule( final List unscheduledTasks ) { return unscheduled; } - @Override - void assignTaskToNode( Task task, NodeWithAlloc node ) { - - //Create initData - - File file = new File(task.getWorkingDir() + '/' + ".command.init"); - - try (PrintWriter pw = new PrintWriter(file)) { - pw.println("echo \"Task init successful\""); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - - task.setNode( node.getNodeLocation() ); - - super.assignTaskToNode(task, node); - } - - @Override - int terminateTasks(List finishedTasks) { - final TaskResultParser taskResultParser = new TaskResultParser(); - finishedTasks.parallelStream().forEach( finishedTask -> { - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( - Paths.get(finishedTask.getWorkingDir()), - finishedTask.getNode(), - finishedTask.getProcess() - ); - for (PathLocationWrapperPair newAndUpdatedFile : newAndUpdatedFiles) { - hierarchyWrapper.addFile( newAndUpdatedFile.getPath(), newAndUpdatedFile.getLocationWrapper() ); - } - super.taskWasFinished( finishedTask ); - }); - return 0; - } - } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 7c261bca..78e0dbd7 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -1,25 +1,38 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.model.*; +import fonda.scheduler.scheduler.copystrategy.CopyStrategy; +import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.nio.file.Path; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.nio.file.Paths; +import java.util.*; @Slf4j public abstract class SchedulerWithDaemonSet extends Scheduler { private final Map daemonByNode = new HashMap<>(); + @Getter + private final CopyStrategy copyStrategy; SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); + if ( config.copyStrategy == null ) throw new IllegalArgumentException( "Copy strategy is null" ); + switch ( config.copyStrategy ){ + case "ftp": + case "copy": + copyStrategy = new FTPstrategy(); + break; + default: + throw new IllegalArgumentException( "Copy strategy is unknown " + config.copyStrategy ); + } } public String getDaemonOnNode( String node ){ @@ -62,6 +75,33 @@ void uploadDataToNode ( Node node, File file, Path dest ) { throw new RuntimeException( "Can not upload file: " + file + " to node: " + node.getMetadata().getName() ); } + @Override + void assignTaskToNode(Task task, NodeWithAlloc node ) { + + getCopyStrategy().generateCopyScript( task ); + + task.setNode( node.getNodeLocation() ); + + super.assignTaskToNode(task, node); + } + + @Override + int terminateTasks(List finishedTasks) { + final TaskResultParser taskResultParser = new TaskResultParser(); + finishedTasks.parallelStream().forEach( finishedTask -> { + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + Paths.get(finishedTask.getWorkingDir()), + finishedTask.getNode(), + finishedTask.getProcess() + ); + for (PathLocationWrapperPair newAndUpdatedFile : newAndUpdatedFiles) { + hierarchyWrapper.addFile( newAndUpdatedFile.getPath(), newAndUpdatedFile.getLocationWrapper() ); + } + super.taskWasFinished( finishedTask ); + }); + return 0; + } + @Override void podEventReceived(Watcher.Action action, Pod pod){ while ( daemonByNode == null ){ From b3494ca79c27d6ad1e6963ad9ed54a3c5584851e Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 22 Nov 2021 10:59:28 +0100 Subject: [PATCH 037/443] fixed permission problems Signed-off-by: Lehmann-Fabian --- .../scheduler/scheduler/copystrategy/CopyStrategy.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java index 7a5ec7dd..42d63780 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -4,6 +4,10 @@ import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; public abstract class CopyStrategy { @@ -24,6 +28,8 @@ public void generateCopyScript( Task task ){ pw.println(line); } + Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); + Files.setPosixFilePermissions( file.toPath(), executable ); } catch (IOException e) { e.printStackTrace(); } From 953b0f3d7ffae97e0cb8c876e453074702ac6f49 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 22 Nov 2021 17:29:23 +0100 Subject: [PATCH 038/443] assume fixed configfile name Signed-off-by: Lehmann-Fabian --- src/main/resources/copystrategies/ftp.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index c7a9e03c..ed81d3c0 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -88,17 +88,13 @@ def download( node, files ): exitIfFileWasNotFound = False -if ( len(sys.argv) < 2 ): - print( "Configfile not specified" ) - exit(101) - -configFilePath = sys.argv[1] +configFilePath = ".command.inputs.json" if not os.path.isfile( configFilePath ): print ("Config file not found:", configFilePath ) exit(102) -configFile = open(configFile,) +configFile = open(configFilePath) config = json.load( configFile ) print( config ) dns = config["dns"] From de76d18c27c66884199f7be4ffe913a5e8a062b4 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 22 Nov 2021 17:30:29 +0100 Subject: [PATCH 039/443] hierarchyWrapper getFile not RealFile Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/HierarchyWrapper.java | 11 +++++++---- .../model/location/hierachy/HierarchyWrapperTest.java | 6 ++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 80f698cc..c1ed64ff 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -96,7 +96,7 @@ public boolean addFile( final Path path, final LocationWrapper... locations ){ * @param path file to get (absolute path) * @return File or null if file does not exist */ - public RealFile getFile( Path path ){ + public File getFile( Path path ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return null; @@ -110,12 +110,15 @@ public RealFile getFile( Path path ){ if( iterator.hasNext() && file.isDirectory() ){ //folder current = (Folder) file; - } else if ( !iterator.hasNext() && !file.isDirectory() ) { - //file - return (RealFile) file; + } else if ( !iterator.hasNext() ) { + return file; } else break; } return null; } + + public boolean isInScope( Path path ){ + return path.startsWith( workdir ); + } } diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index b618f7c7..d1591c2d 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -210,7 +210,7 @@ public void testGetFile() { index = 0; for ( Path file : files) { - RealFile realFile = hw.getFile(file); + RealFile realFile = (RealFile) hw.getFile(file); final LocationWrapper[] locations = realFile.getLocations(); assertEquals(lw[index++], locations[0]); @@ -231,7 +231,9 @@ public void testGetFileWorkdir() { @Test public void testGetFileButIsDir() { - assertNull(hw.getFile( Paths.get(temporaryDir + "d" ))); + final File file = hw.getFile(Paths.get(temporaryDir + "d")); + assertNotNull( file ); + assertTrue( file.isDirectory() ); } @Test From a1162750ee961bca28f7c6dd03b3fdb7dce70314 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 22 Nov 2021 17:32:05 +0100 Subject: [PATCH 040/443] move HierarchyWrapper into SchedulerWithDaemonSet Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 2 -- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 3516a40b..1efff231 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -41,7 +41,6 @@ public abstract class Scheduler { private final Watch watcher; private final TaskprocessingThread schedulingThread; private final TaskprocessingThread finishThread; - final HierarchyWrapper hierarchyWrapper; Scheduler(String execution, KubernetesClient client, String namespace, SchedulerConfig config){ this.execution = execution; @@ -49,7 +48,6 @@ public abstract class Scheduler { this.namespace = namespace; log.trace( "Register scheduler for " + this.name ); this.client = client; - this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); this.dns = config.dns; PodWatcher podWatcher = new PodWatcher(this); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 78e0dbd7..35265cb2 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -2,6 +2,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.hierachy.HierarchyWrapper; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import io.fabric8.kubernetes.api.model.Node; @@ -21,9 +22,11 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final Map daemonByNode = new HashMap<>(); @Getter private final CopyStrategy copyStrategy; + final HierarchyWrapper hierarchyWrapper; SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); + this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); if ( config.copyStrategy == null ) throw new IllegalArgumentException( "Copy strategy is null" ); switch ( config.copyStrategy ){ case "ftp": From e825ebf4310c99577255bfa173af7f93351b3b40 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 22 Nov 2021 17:32:54 +0100 Subject: [PATCH 041/443] select file locations for task randomly Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/RealFile.java | 9 ++ .../scheduler/scheduler/RandomScheduler.java | 29 ++++++- .../scheduler/SchedulerWithDaemonSet.java | 84 +++++++++++++++++++ 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index b8c4b7d6..5951c01d 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -1,14 +1,19 @@ package fonda.scheduler.model.location.hierachy; +import fonda.scheduler.model.Process; import fonda.scheduler.model.location.Location; import lombok.Getter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class RealFile extends File { + /** + * This field contains the newest LocationWrapper of one file for each node. + */ @Getter private LocationWrapper[] locations; @@ -55,4 +60,8 @@ public void addOrUpdateLocation( LocationWrapper... location ){ } } + public List getFilesForProcess( Process process ){ + return Arrays.asList( locations ); + } + } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index aaa9c29d..dc85e3b5 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -1,11 +1,17 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.*; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; import io.fabric8.kubernetes.api.model.Pod; import lombok.extern.slf4j.Slf4j; +import java.nio.file.Path; import java.util.*; +import java.util.stream.Collectors; @Slf4j public class RandomScheduler extends SchedulerWithDaemonSet { @@ -22,9 +28,13 @@ public int schedule( final List unscheduledTasks ) { for ( final Task task : unscheduledTasks) { if(isClose()) return -1; final Pod pod = task.getPod(); - Optional node = items.stream().filter(x -> x.canSchedule(pod) && this.getDaemonOnNode(x) != null).findFirst(); + final List matchingNodes = items.stream().filter(x -> x.canSchedule(pod) && this.getDaemonOnNode(x) != null).collect(Collectors.toList()); + Optional node = matchingNodes.isEmpty() + ? Optional.empty() + : Optional.of(matchingNodes.get(new Random().nextInt(matchingNodes.size()))); if( node.isPresent() ){ log.info("Task needs: " + task.getConfig().getInputs().toString()); + if ( !getInputsFromNodes( task, node.get() ) ) continue; assignTaskToNode( task, node.get() ); super.taskWasScheduled( task ); } else { @@ -35,4 +45,19 @@ public int schedule( final List unscheduledTasks ) { return unscheduled; } + @Override + Map> scheduleFiles(Task task, Map> inputsOfTask, NodeWithAlloc node) { + final HashMap> map = new HashMap<>(); + for (Map.Entry> entry : inputsOfTask.entrySet()) { + final LocationWrapper locationWrapper = entry.getValue().get(new Random().nextInt(entry.getValue().size())); + final String nodeIdentifier = ((NodeLocation) locationWrapper.getLocation()).getIdentifier(); + if ( !map.containsKey( nodeIdentifier )){ + map.put( nodeIdentifier, new LinkedList<>() ); + } + final List pathsOfNode = map.get( nodeIdentifier ); + pathsOfNode.add( entry.getKey().toString() ); + } + return map; + } + } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 35265cb2..8d12e774 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -1,10 +1,16 @@ package fonda.scheduler.scheduler; +import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.hierachy.Folder; import fonda.scheduler.model.location.hierachy.HierarchyWrapper; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.RealFile; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; +import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; +import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; @@ -12,9 +18,12 @@ import lombok.extern.slf4j.Slf4j; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j public abstract class SchedulerWithDaemonSet extends Scheduler { @@ -105,6 +114,81 @@ int terminateTasks(List finishedTasks) { return 0; } + void generateConfigFile( File config, Map> filesByNode ) throws IOException { + final Inputs inputs = new Inputs(this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/"); + for (Map.Entry> entry : filesByNode.entrySet()) { + inputs.data.add( new InputEntry( getDaemonOnNode( entry.getKey() ), entry.getKey(), entry.getValue() ) ); + } + new ObjectMapper().writeValue( config, inputs ); + } + + private Stream>> streamFile( + final fonda.scheduler.model.location.hierachy.File file, + final Task task, + final Path sourcePath ) + { + if( file.isDirectory() ){ + return ((Folder) file).getAllChildren( sourcePath ) + .entrySet() + .stream() + .map( y -> new AbstractMap.SimpleImmutableEntry<>( + y.getKey(), + ((RealFile) y.getValue()).getFilesForProcess( task.getProcess() ) + ) + ); + } + return Stream.of( new AbstractMap.SimpleImmutableEntry<>( + sourcePath, + ((RealFile) file).getFilesForProcess( task.getProcess() ) + ) + ); + } + + private Map> getInputsOfTask(Task task ){ + return task.getConfig() + .getInputs() + .parallelStream() + .flatMap(x -> { + if ( !(x.getValue() instanceof List) ) return Stream.empty(); + for (Object o : (List) x.getValue()) { + if (o instanceof Map) { + Map input = (Map) o; + if (input.containsKey("sourceObj")) { + String sourceObj = (String) input.get("sourceObj"); + Path sourcePath = Paths.get(sourceObj); + log.info("Src: {}", sourceObj); + if (this.hierarchyWrapper.isInScope(sourcePath)) { + return streamFile( hierarchyWrapper.getFile(sourcePath), task, sourcePath ); + } + } + } + } + return Stream.empty(); + }) + .collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue )); + } + + boolean getInputsFromNodes( Task task, NodeWithAlloc node ) { + + Map> inputsOfTask = getInputsOfTask(task); + Map< String, List > inputsByNode = scheduleFiles( task, inputsOfTask, node ); + + final File config = new File(task.getWorkingDir() + '/' + ".command.inputs.json"); + try { + generateConfigFile( config, inputsByNode ); + return true; + } catch (IOException e) { + e.printStackTrace(); + } + return false; + } + + abstract Map> scheduleFiles( + Task task, + Map> inputsOfTask, + NodeWithAlloc node + ); + @Override void podEventReceived(Watcher.Action action, Pod pod){ while ( daemonByNode == null ){ From d2adbcba6a938a4f2f723c18c82d51ea6e03475b Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 22 Nov 2021 17:33:16 +0100 Subject: [PATCH 042/443] select file locations for task randomly data structure Signed-off-by: Lehmann-Fabian --- .../schedulingstrategy/InputEntry.java | 17 +++++++++++++++++ .../scheduler/schedulingstrategy/Inputs.java | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java create mode 100644 src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java new file mode 100644 index 00000000..9205c196 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java @@ -0,0 +1,17 @@ +package fonda.scheduler.scheduler.schedulingstrategy; + +import java.util.LinkedList; +import java.util.List; + +public class InputEntry { + + public final String currentIP; + public final String node; + public final List files; + + public InputEntry( String currentIP, String node, List files ) { + this.currentIP = currentIP; + this.node = node; + this.files = files; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java new file mode 100644 index 00000000..bbbb6b5b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -0,0 +1,16 @@ +package fonda.scheduler.scheduler.schedulingstrategy; + +import java.util.LinkedList; +import java.util.List; + +public class Inputs { + + public final String dns; + public final List data; + + public Inputs( String dns ) { + this.dns = dns; + this.data = new LinkedList<>(); + } + +} From ba669647de6ecf5b0c00853b25543d9b93ba22c0 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 24 Nov 2021 11:37:43 +0100 Subject: [PATCH 043/443] fix problem files are not closed Signed-off-by: Lehmann-Fabian --- .../scheduler/model/TaskResultParser.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index d2143c0a..511540c4 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -13,22 +13,16 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; +import java.util.stream.Stream; @Slf4j public class TaskResultParser { private String getRootDir( File file ){ - Scanner sc = null; - try { - sc = new Scanner( file ); + try ( Scanner sc = new Scanner( file ) ) { if( sc.hasNext() ) return sc.next().split(";")[0]; } catch (FileNotFoundException e) { e.printStackTrace(); - } finally { - if ( sc != null ) - try{ - sc.close(); - } catch ( Exception e ){} } return null; } @@ -52,8 +46,12 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo Set newOrUpdated = new HashSet<>(); - try { - Files.lines( infile ).skip( 1 ) + try ( + Stream in = Files.lines(infile); + Stream out = Files.lines(outfile) + ) { + + in.skip( 1 ) .forEach( line -> { String[] data = line.split(";"); String path = data[1].equals("") ? data[0].substring( taskRootDir.length() + 1 ) : data[1]; @@ -63,7 +61,7 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo log.info( "{}", inputdata ); - Files.lines( outfile ).skip( 1 ) + out.skip( 1 ) .forEach( line -> { String[] data = line.split(";"); boolean realFile = data[1].equals(""); From 067ae4ed900f497835dc7bda3fb8f4ebdf8e27f3 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 24 Nov 2021 11:49:52 +0100 Subject: [PATCH 044/443] fix: Not all files were copied, create all parent dirs Signed-off-by: Lehmann-Fabian --- src/main/resources/copystrategies/ftp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index ed81d3c0..4d8d23ba 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -5,6 +5,7 @@ import json import urllib.request import sys +from pathlib import Path def getIP( node ): ip = urllib.request.urlopen(dns + node).read() @@ -53,8 +54,8 @@ def download( node, files ): try: if os.path.exists( filename ): os.remove(filename) + Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) - files.pop(0) except ftplib.error_perm as err: if( str(err) == "550 Failed to open file." ): print( "File not found:", node + filename ) From afc5f4cf2df066b616c2b8d872b4f58813312a4c Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 24 Nov 2021 11:50:35 +0100 Subject: [PATCH 045/443] Refactor Scheduling logic Signed-off-by: Lehmann-Fabian --- .../scheduler/scheduler/RandomScheduler.java | 33 +++--- .../fonda/scheduler/scheduler/Scheduler.java | 35 ++++-- .../scheduler/SchedulerWithDaemonSet.java | 109 ++++++------------ .../scheduler/util/NodeTaskAlignment.java | 15 +++ .../util/NodeTaskFilesAlignment.java | 19 +++ .../util/PathFileLocationTriple.java | 20 ++++ .../java/fonda/scheduler/util/FilePath.java | 14 +++ 7 files changed, 148 insertions(+), 97 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java create mode 100644 src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java create mode 100644 src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java create mode 100644 src/main/java/fonda/scheduler/util/FilePath.java diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index dc85e3b5..79aac856 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -6,10 +6,13 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.scheduler.util.PathFileLocationTriple; +import fonda.scheduler.util.FilePath; import io.fabric8.kubernetes.api.model.Pod; import lombok.extern.slf4j.Slf4j; -import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -21,12 +24,10 @@ public RandomScheduler(String name, KubernetesClient client, String namespace, S } @Override - public int schedule( final List unscheduledTasks ) { - log.info("Schedule " + this.getName()); + public List getTaskNodeAlignment( final List unscheduledTasks ){ List items = getNodeList(); - int unscheduled = 0; + List alignment = new LinkedList<>(); for ( final Task task : unscheduledTasks) { - if(isClose()) return -1; final Pod pod = task.getPod(); final List matchingNodes = items.stream().filter(x -> x.canSchedule(pod) && this.getDaemonOnNode(x) != null).collect(Collectors.toList()); Optional node = matchingNodes.isEmpty() @@ -34,28 +35,26 @@ public int schedule( final List unscheduledTasks ) { : Optional.of(matchingNodes.get(new Random().nextInt(matchingNodes.size()))); if( node.isPresent() ){ log.info("Task needs: " + task.getConfig().getInputs().toString()); - if ( !getInputsFromNodes( task, node.get() ) ) continue; - assignTaskToNode( task, node.get() ); - super.taskWasScheduled( task ); + final List inputsOfTask = getInputsOfTask(task); + final Map> stringListMap = scheduleFiles(task, inputsOfTask, node.get()); + alignment.add( new NodeTaskFilesAlignment(node.get(),task,stringListMap)); } else { log.info( "No node with enough resources for {}", pod.getMetadata().getName() ); - unscheduled++; } } - return unscheduled; + return alignment; } - @Override - Map> scheduleFiles(Task task, Map> inputsOfTask, NodeWithAlloc node) { - final HashMap> map = new HashMap<>(); - for (Map.Entry> entry : inputsOfTask.entrySet()) { - final LocationWrapper locationWrapper = entry.getValue().get(new Random().nextInt(entry.getValue().size())); + Map> scheduleFiles(Task task, List inputsOfTask, NodeWithAlloc node) { + final HashMap> map = new HashMap<>(); + for ( PathFileLocationTriple entry : inputsOfTask ) { + final LocationWrapper locationWrapper = entry.locations.get(new Random().nextInt(entry.locations.size())); final String nodeIdentifier = ((NodeLocation) locationWrapper.getLocation()).getIdentifier(); if ( !map.containsKey( nodeIdentifier )){ map.put( nodeIdentifier, new LinkedList<>() ); } - final List pathsOfNode = map.get( nodeIdentifier ); - pathsOfNode.add( entry.getKey().toString() ); + final List pathsOfNode = map.get( nodeIdentifier ); + pathsOfNode.add( new FilePath( entry.path.toString(), entry.file ) ); } return map; } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 1efff231..4694c14d 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -2,7 +2,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.hierachy.HierarchyWrapper; +import fonda.scheduler.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Binding; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectReference; @@ -70,7 +70,22 @@ public abstract class Scheduler { * @param unscheduledTasks * @return the number of unscheduled Tasks */ - abstract int schedule( final List unscheduledTasks ); + public int schedule( final List unscheduledTasks ) { + final List taskNodeAlignment = getTaskNodeAlignment(unscheduledTasks); + //check if still possible... + boolean possible = true; + if (!possible) return taskNodeAlignment.size(); + + for (NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment) { + if (isClose()) return -1; + assignTaskToNode( nodeTaskAlignment ); + taskWasScheduled(nodeTaskAlignment.task); + + } + return unscheduledTasks.size() - taskNodeAlignment.size(); + } + + abstract List getTaskNodeAlignment( final List unscheduledTasks ); abstract int terminateTasks( final List finishedTasks ); @@ -108,7 +123,7 @@ public void schedulePod(Pod pod ) { } } - public void taskWasScheduled(Task task ) { + void taskWasScheduled(Task task ) { synchronized (unscheduledTasks){ unscheduledTasks.remove( task ); } @@ -174,13 +189,15 @@ boolean canPodBeScheduled( Pod pod, NodeWithAlloc node ){ return node.canSchedule( pod ); } - void assignTaskToNode( Task task, NodeWithAlloc node ){ + void assignTaskToNode( NodeTaskAlignment alignment ){ + + alignment.task.setNode( alignment.node.getNodeLocation() ); - final Pod pod = task.getPod(); + final Pod pod = alignment.task.getPod(); - node.addPod( pod ); + alignment.node.addPod( pod ); - log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + node.getMetadata().getName() ); + log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + alignment.node.getMetadata().getName() ); Binding b1 = new Binding(); @@ -192,13 +209,13 @@ void assignTaskToNode( Task task, NodeWithAlloc node ){ ObjectReference objectReference = new ObjectReference(); objectReference.setApiVersion("v1"); objectReference.setKind("Node"); - objectReference.setName(node.getMetadata().getName()); + objectReference.setName(alignment.node.getMetadata().getName()); b1.setTarget(objectReference); client.bindings().create(b1); - pod.getSpec().setNodeName( node.getMetadata().getName() ); + pod.getSpec().setNodeName( alignment.node.getMetadata().getName() ); log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 8d12e774..d445b63c 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -5,12 +5,15 @@ import fonda.scheduler.model.*; import fonda.scheduler.model.location.hierachy.Folder; import fonda.scheduler.model.location.hierachy.HierarchyWrapper; -import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealFile; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; +import fonda.scheduler.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.scheduler.util.PathFileLocationTriple; +import fonda.scheduler.util.FilePath; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; @@ -57,44 +60,11 @@ String getDaemonOnNode( Node node ){ return getDaemonOnNode( node.getMetadata().getName() ); } - String getOneDaemon() { - synchronized ( daemonByNode ) { - final Collection values = daemonByNode.values(); - return values.stream().skip(values.size()).findFirst().get(); - } - } - - void uploadDataToNode ( Node node, File file, Path dest ) { - int i = 0; - do { - log.info( "Upload {} to {} ({})", file, dest, node.getMetadata().getName() ); - try { - final boolean result = client.pods() - .inNamespace(getNamespace()) - .withName(getDaemonOnNode(node)) - .file(dest.toString()) - .upload(file.toPath()); - if (result) return; - } catch (Exception e){ - e.printStackTrace(); - } - try { - Thread.sleep((long) (Math.pow(2,i) * 100) ); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } while ( i++ < 10 ); - throw new RuntimeException( "Can not upload file: " + file + " to node: " + node.getMetadata().getName() ); - } - @Override - void assignTaskToNode(Task task, NodeWithAlloc node ) { - - getCopyStrategy().generateCopyScript( task ); - - task.setNode( node.getNodeLocation() ); - - super.assignTaskToNode(task, node); + void assignTaskToNode( NodeTaskAlignment alignment ) { + writeInitConfig( (NodeTaskFilesAlignment) alignment ); + getCopyStrategy().generateCopyScript( alignment.task ); + super.assignTaskToNode( alignment ); } @Override @@ -114,15 +84,30 @@ int terminateTasks(List finishedTasks) { return 0; } - void generateConfigFile( File config, Map> filesByNode ) throws IOException { - final Inputs inputs = new Inputs(this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/"); - for (Map.Entry> entry : filesByNode.entrySet()) { - inputs.data.add( new InputEntry( getDaemonOnNode( entry.getKey() ), entry.getKey(), entry.getValue() ) ); + boolean writeInitConfig( NodeTaskFilesAlignment alignment ) { + + final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); + try { + final Inputs inputs = new Inputs( + this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/" + ); + for (Map.Entry> entry : alignment.nodeFileAlignment.entrySet()) { + if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) continue; + final List collect = entry .getValue() + .stream() + .map(x -> x.path) + .collect(Collectors.toList()); + inputs.data.add( new InputEntry( getDaemonOnNode( entry.getKey() ), entry.getKey(), collect ) ); + } + new ObjectMapper().writeValue( config, inputs ); + return true; + } catch (IOException e) { + e.printStackTrace(); } - new ObjectMapper().writeValue( config, inputs ); + return false; } - private Stream>> streamFile( + private Stream streamFile( final fonda.scheduler.model.location.hierachy.File file, final Task task, final Path sourcePath ) @@ -131,20 +116,23 @@ private Stream>> st return ((Folder) file).getAllChildren( sourcePath ) .entrySet() .stream() - .map( y -> new AbstractMap.SimpleImmutableEntry<>( + .map( y -> new PathFileLocationTriple( y.getKey(), - ((RealFile) y.getValue()).getFilesForProcess( task.getProcess() ) + y.getValue(), + y.getValue().getFilesForProcess( task.getProcess() ) ) ); } - return Stream.of( new AbstractMap.SimpleImmutableEntry<>( + final RealFile realFile = (RealFile) file; + return Stream.of( new PathFileLocationTriple( sourcePath, - ((RealFile) file).getFilesForProcess( task.getProcess() ) + realFile, + realFile.getFilesForProcess( task.getProcess() ) ) ); } - private Map> getInputsOfTask(Task task ){ + List getInputsOfTask( Task task ){ return task.getConfig() .getInputs() .parallelStream() @@ -165,30 +153,9 @@ private Map> getInputsOfTask(Task task ){ } return Stream.empty(); }) - .collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue )); - } - - boolean getInputsFromNodes( Task task, NodeWithAlloc node ) { - - Map> inputsOfTask = getInputsOfTask(task); - Map< String, List > inputsByNode = scheduleFiles( task, inputsOfTask, node ); - - final File config = new File(task.getWorkingDir() + '/' + ".command.inputs.json"); - try { - generateConfigFile( config, inputsByNode ); - return true; - } catch (IOException e) { - e.printStackTrace(); - } - return false; + .collect(Collectors.toList()); } - abstract Map> scheduleFiles( - Task task, - Map> inputsOfTask, - NodeWithAlloc node - ); - @Override void podEventReceived(Watcher.Action action, Pod pod){ while ( daemonByNode == null ){ diff --git a/src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java b/src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java new file mode 100644 index 00000000..62a9f0fa --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java @@ -0,0 +1,15 @@ +package fonda.scheduler.scheduler.util; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; + +public class NodeTaskAlignment { + + public final NodeWithAlloc node; + public final Task task; + + public NodeTaskAlignment( NodeWithAlloc node, Task task ) { + this.node = node; + this.task = task; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java new file mode 100644 index 00000000..a5507a0d --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java @@ -0,0 +1,19 @@ +package fonda.scheduler.scheduler.util; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.util.FilePath; + +import java.util.List; +import java.util.Map; + +public class NodeTaskFilesAlignment extends NodeTaskAlignment { + + + public final Map> nodeFileAlignment; + + public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, Map> nodeFileAlignment ) { + super(node, task); + this.nodeFileAlignment = nodeFileAlignment; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java new file mode 100644 index 00000000..105d9ef9 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java @@ -0,0 +1,20 @@ +package fonda.scheduler.scheduler.util; + +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.RealFile; + +import java.nio.file.Path; +import java.util.List; + +public class PathFileLocationTriple { + + public final Path path; + public final RealFile file; + public final List locations; + + public PathFileLocationTriple(Path path, RealFile file, List locations) { + this.path = path; + this.file = file; + this.locations = locations; + } +} diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/fonda/scheduler/util/FilePath.java new file mode 100644 index 00000000..f4e3e65d --- /dev/null +++ b/src/main/java/fonda/scheduler/util/FilePath.java @@ -0,0 +1,14 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.location.hierachy.RealFile; + +public class FilePath { + + public final String path; + public final RealFile file; + + public FilePath(String path, RealFile file) { + this.path = path; + this.file = file; + } +} From fce1f83155796cf137e2701d3cc93a54373cc89a Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 24 Nov 2021 15:46:41 +0100 Subject: [PATCH 046/443] new Pair into method Signed-off-by: Lehmann-Fabian --- .../scheduler/rest/SchedulerRestController.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 20284908..afe7fdca 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -63,7 +63,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable S default: return new ResponseEntity( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } - final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Pair key = getKey( namespace, execution ); schedulerHolder.put( key, scheduler ); return new ResponseEntity( HttpStatus.OK ); @@ -82,7 +82,7 @@ ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String log.trace( execution + " " + config.getTask() + " got: " + config ); - final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); @@ -105,7 +105,7 @@ ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String @GetMapping("/scheduler/taskstate/{namespace}/{execution}/{taskid}") ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { - final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); @@ -125,7 +125,7 @@ ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String ResponseEntity delete(@PathVariable String namespace, @PathVariable String execution) { log.info("Delete name: " + execution); - final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ @@ -141,7 +141,7 @@ ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable Strin log.info( "Got request: {}{}{}", namespace, execution, node ); - final Pair key = new Pair(namespace.toLowerCase(), execution.toLowerCase()); + final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); @@ -162,4 +162,8 @@ ResponseEntity checkHealth() { return new ResponseEntity( HttpStatus.OK ); } + private Pair getKey(String namespace, String execution ){ + return new Pair(namespace.toLowerCase(), execution.toLowerCase()); + } + } From 0d47fdb0721023f46299cdf6431200a4120c1ac6 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 24 Nov 2021 16:32:29 +0100 Subject: [PATCH 047/443] submit tasks as batch Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/Task.java | 5 ++ .../rest/SchedulerRestController.java | 26 +++++++ .../fonda/scheduler/scheduler/Scheduler.java | 73 ++++++++++++++++--- .../fonda/scheduler/scheduler/util/Batch.java | 59 +++++++++++++++ 4 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/util/Batch.java diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index e748df52..2526ed47 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -1,6 +1,7 @@ package fonda.scheduler.model; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.scheduler.util.Batch; import io.fabric8.kubernetes.api.model.Pod; import lombok.Getter; import lombok.Setter; @@ -24,6 +25,10 @@ public class Task { @Setter private NodeLocation node = null; + @Getter + @Setter + private Batch batch; + public Task(TaskConfig config) { this.config = config; this.process = Process.getProcess( config.getTask() ); diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index afe7fdca..7777fe5f 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -95,6 +95,32 @@ ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String } + @PostMapping("/scheduler/startBatch/{namespace}/{execution}") + ResponseEntity startBatch(@PathVariable String namespace, @PathVariable String execution ) { + + final Pair key = getKey( namespace, execution ); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + scheduler.startBatch(); + return new ResponseEntity( HttpStatus.OK ); + + } + + @PostMapping("/scheduler/endBatch/{namespace}/{execution}") + ResponseEntity endBatch(@PathVariable String namespace, @PathVariable String execution, @RequestBody(required = true) int tasksInBatch ) { + + final Pair key = getKey( namespace, execution ); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + scheduler.endBatch( tasksInBatch ); + return new ResponseEntity( HttpStatus.OK ); + + } + /** * Check Task state * @param namespace namespace where the workflow runs diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 4694c14d..2d6fea6a 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -2,6 +2,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.scheduler.util.Batch; import fonda.scheduler.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Binding; import io.fabric8.kubernetes.api.model.ObjectMeta; @@ -14,10 +15,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Slf4j public abstract class Scheduler { @@ -34,7 +32,12 @@ public abstract class Scheduler { @Getter private boolean close; + private final Object batchHelper = new Object(); + private int currentBatch = 0; + private Batch currentBatchInstance = null; + final KubernetesClient client; + final private Set upcomingTasks = new HashSet<>(); final private List unscheduledTasks = new ArrayList<>(100); final private List unfinishedTasks = new ArrayList<>(100); final Map tasksByHash = new HashMap<>(); @@ -113,13 +116,35 @@ void taskWasFinished( Task task ){ } public void schedulePod(Pod pod ) { - Task t = changeStateOfTask( pod, State.UNSCHEDULED ); + Task task = changeStateOfTask( pod, State.UNSCHEDULED ); //If null, task was already unscheduled - if ( t == null ) return; - t.setPod( pod ); - synchronized (unscheduledTasks){ - unscheduledTasks.add( t ); - unscheduledTasks.notifyAll(); + if ( task == null ) return; + task.setPod( pod ); + if ( task.getBatch() == null ){ + synchronized (unscheduledTasks){ + unscheduledTasks.add( task ); + unscheduledTasks.notifyAll(); + synchronized ( upcomingTasks ){ + upcomingTasks.remove( task ); + } + } + } else { + Batch batch = task.getBatch(); + batch.informScheduable( task ); + tryToScheduleBatch( batch ); + } + } + + private void tryToScheduleBatch( Batch batch ){ + if ( batch.canSchedule() ){ + synchronized (unscheduledTasks){ + final List tasksToScheduleAndDestroy = batch.getTasksToScheduleAndDestroy(); + unscheduledTasks.addAll(tasksToScheduleAndDestroy); + unscheduledTasks.notifyAll(); + synchronized ( upcomingTasks ){ + upcomingTasks.removeAll( tasksToScheduleAndDestroy ); + } + } } } @@ -137,11 +162,18 @@ public void markPodAsDeleted( Pod pod ) { /* External access to Tasks */ public void addTask( TaskConfig conf ) { + final Task task = new Task(conf); synchronized (tasksByHash) { if( ! tasksByHash.containsKey( conf.getHash() ) ){ - tasksByHash.put( conf.getHash(), new Task(conf) ); + tasksByHash.put( conf.getHash(), task ); } } + synchronized ( upcomingTasks ){ + upcomingTasks.add( task ); + } + if( currentBatchInstance != null ){ + currentBatchInstance.registerTask( task ); + } } TaskConfig getConfigFor( String hash ){ @@ -221,6 +253,21 @@ void assignTaskToNode( NodeTaskAlignment alignment ){ /* Helper */ + public void startBatch(){ + synchronized (batchHelper){ + if ( currentBatchInstance == null || currentBatchInstance.isClosed() ){ + currentBatchInstance = new Batch( currentBatch++ ); + } + } + } + + public void endBatch( int tasksInBatch ){ + synchronized (batchHelper){ + currentBatchInstance.close( tasksInBatch ); + tryToScheduleBatch( currentBatchInstance ); + } + } + /** * * @param pod @@ -268,6 +315,10 @@ private Task getTaskByPod( Pod pod ) { return t; } + LinkedList getUpcomingTasksCopy() { + return new LinkedList<>( upcomingTasks ); + } + /** * Close used resources */ diff --git a/src/main/java/fonda/scheduler/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/scheduler/util/Batch.java new file mode 100644 index 00000000..68c4494b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/util/Batch.java @@ -0,0 +1,59 @@ +package fonda.scheduler.scheduler.util; + +import fonda.scheduler.model.Task; +import lombok.Getter; + +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +public class Batch { + + @Getter + boolean closed = false; + + public final int id; + private List ready = new LinkedList<>(); + private Set unready = new HashSet<>(); + private int tasksInBatch = -1; + + public Batch(int id) { + this.id = id; + } + + public void close( int tasksInBatch ){ + this.closed = true; + this.tasksInBatch = tasksInBatch; + } + + public void registerTask( Task task ){ + synchronized ( unready ){ + if ( closed && ready.size() + unready.size() >= tasksInBatch ) { + throw new IllegalStateException("Batch was closed!"); + } + unready.add( task ); + } + task.setBatch( this ); + } + + public void informScheduable( Task task ){ + synchronized ( unready ){ + final boolean remove = unready.remove(task); + if ( remove ) ready.add( task ); + } + } + + public boolean canSchedule(){ + return closed && unready.isEmpty(); + } + + public List getTasksToScheduleAndDestroy(){ + if ( !closed ) throw new IllegalStateException("Batch was not yet closed!"); + final List ready = this.ready; + this.ready = null; + this.unready = null; + return ready; + } + +} From 50ce91b89c488510f93eeea9804353eb1cea9f2c Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 24 Nov 2021 16:32:44 +0100 Subject: [PATCH 048/443] submit cpus and memory in task config Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/TaskConfig.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 623eb342..50a5f1ba 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -15,6 +15,8 @@ public class TaskConfig { private final Map< String, List> schedulerParams; private final List inputs; private final String hash; + private final int cpus; + private final long memoryInBytes; private TaskConfig() { this.name = null; @@ -22,6 +24,8 @@ private TaskConfig() { this.inputs = null; this.hash = null; this.task = null; + this.cpus = 0; + this.memoryInBytes = 0; } @Getter From 7d93153a596c3edf93f6c92511f061bde543e068 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 25 Nov 2021 14:24:04 +0100 Subject: [PATCH 049/443] only process successful outputs of tasks Signed-off-by: Lehmann-Fabian --- src/main/java/fonda/scheduler/model/State.java | 2 +- src/main/java/fonda/scheduler/model/Task.java | 4 ++++ .../fonda/scheduler/scheduler/Scheduler.java | 12 +++++++++--- .../scheduler/SchedulerWithDaemonSet.java | 16 +++++++++------- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/State.java b/src/main/java/fonda/scheduler/model/State.java index 48b12b50..cbafc138 100644 --- a/src/main/java/fonda/scheduler/model/State.java +++ b/src/main/java/fonda/scheduler/model/State.java @@ -2,6 +2,6 @@ public enum State { - RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, ERROR, PROCESSING_FINISHED, FINISHED, DELETED; + RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, DELETED; } diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 2526ed47..904e32ec 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -38,6 +38,10 @@ public String getWorkingDir(){ return pod.getSpec().getContainers().get(0).getWorkingDir(); } + public boolean wasSuccessfullyExecuted(){ + return pod.getStatus().getContainerStatuses().get( 0 ).getState().getTerminated().getExitCode() == 0; + } + @Override public String toString() { return "Task{" + diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 2d6fea6a..0c818103 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -101,6 +101,7 @@ void onPodTermination( Pod pod ){ //If null, task was already changed if( t == null ) return; + t.setPod( pod ); synchronized (unfinishedTasks){ unfinishedTasks.add( t ); @@ -112,7 +113,7 @@ void taskWasFinished( Task task ){ synchronized (unfinishedTasks){ unfinishedTasks.remove( task ); } - task.getState().setState(State.FINISHED); + task.getState().setState(task.wasSuccessfullyExecuted() ? State.FINISHED : State.FINISHED_WITH_ERROR); } public void schedulePod(Pod pod ) { @@ -156,7 +157,8 @@ void taskWasScheduled(Task task ) { } public void markPodAsDeleted( Pod pod ) { - changeStateOfTask( pod, State.DELETED ); + final Task task = changeStateOfTask(pod, State.DELETED); + task.setPod( pod ); } /* External access to Tasks */ @@ -357,15 +359,19 @@ public void eventReceived(Action action, Pod pod) { log.debug("Got pod " + pod.getMetadata().getName() + " scheduler: " + pwa.getSpec().getSchedulerName()); } + switch (action) { case ADDED: - if (pwa.getSpec().getNodeName() == null && pwa.getSpec().getSchedulerName().equalsIgnoreCase( scheduler.name )) { + if ( pwa.getSpec().getNodeName() == null ) { scheduler.schedulePod( pwa ); } break; case MODIFIED: if (pod.getStatus().getContainerStatuses().size() > 0 && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { scheduler.onPodTermination(pwa); + } else { + final Task task = scheduler.getTaskByPod(pwa); + task.setPod( pwa ); } break; case DELETED: diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index d445b63c..25559dfc 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -71,13 +71,15 @@ void assignTaskToNode( NodeTaskAlignment alignment ) { int terminateTasks(List finishedTasks) { final TaskResultParser taskResultParser = new TaskResultParser(); finishedTasks.parallelStream().forEach( finishedTask -> { - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( - Paths.get(finishedTask.getWorkingDir()), - finishedTask.getNode(), - finishedTask.getProcess() - ); - for (PathLocationWrapperPair newAndUpdatedFile : newAndUpdatedFiles) { - hierarchyWrapper.addFile( newAndUpdatedFile.getPath(), newAndUpdatedFile.getLocationWrapper() ); + if( finishedTask.wasSuccessfullyExecuted() ) { + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + Paths.get(finishedTask.getWorkingDir()), + finishedTask.getNode(), + finishedTask.getProcess() + ); + for (PathLocationWrapperPair newAndUpdatedFile : newAndUpdatedFiles) { + hierarchyWrapper.addFile(newAndUpdatedFile.getPath(), newAndUpdatedFile.getLocationWrapper()); + } } super.taskWasFinished( finishedTask ); }); From 6fe5d7c4105381c1d0324f5d0a47a1e7b09deb1a Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 25 Nov 2021 14:24:34 +0100 Subject: [PATCH 050/443] fix exception handling problem Signed-off-by: Lehmann-Fabian --- src/main/resources/copystrategies/ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 4d8d23ba..2b1d7363 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -80,7 +80,7 @@ def download( node, files ): try: ftp.quit() ftp.close() - except: + except BaseException as err: print(f"Unexpected {err=}, {type(err)=}") From 7c03e365cccde3eb2e21e4be95c0f95a1592c18b Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Thu, 25 Nov 2021 14:25:03 +0100 Subject: [PATCH 051/443] task inputs in a better parsable format Signed-off-by: Lehmann-Fabian --- .../fonda/scheduler/model/FileHolder.java | 22 +++++++++++++++ .../fonda/scheduler/model/InputParam.java | 20 +++++++++++++ .../fonda/scheduler/model/TaskConfig.java | 2 +- .../java/fonda/scheduler/model/TaskInput.java | 28 +++++++++++++++++++ .../scheduler/SchedulerWithDaemonSet.java | 21 +++----------- 5 files changed, 75 insertions(+), 18 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/FileHolder.java create mode 100644 src/main/java/fonda/scheduler/model/InputParam.java create mode 100644 src/main/java/fonda/scheduler/model/TaskInput.java diff --git a/src/main/java/fonda/scheduler/model/FileHolder.java b/src/main/java/fonda/scheduler/model/FileHolder.java new file mode 100644 index 00000000..66ef7089 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/FileHolder.java @@ -0,0 +1,22 @@ +package fonda.scheduler.model; + +public class FileHolder { + + public final String storePath; + public final String sourceObj; + public final String stageName; + + private FileHolder() { + this.storePath = null; + this.sourceObj = null; + this.stageName = null; + } + + @Override + public String toString() { + return "FileHolder{" + + "sourceObj='" + sourceObj + '\'' + + '}'; + } + +} diff --git a/src/main/java/fonda/scheduler/model/InputParam.java b/src/main/java/fonda/scheduler/model/InputParam.java new file mode 100644 index 00000000..b3bef8a9 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/InputParam.java @@ -0,0 +1,20 @@ +package fonda.scheduler.model; + +public class InputParam { + + public final String name; + public final T value; + + private InputParam() { + this.name = null; + this.value = null; + } + + @Override + public String toString() { + return "InputParam{" + + "name='" + name + '\'' + + ", value=" + value + + '}'; + } +} diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 50a5f1ba..4e715043 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -13,7 +13,7 @@ public class TaskConfig { private final String task; private final String name; private final Map< String, List> schedulerParams; - private final List inputs; + private final TaskInput inputs; private final String hash; private final int cpus; private final long memoryInBytes; diff --git a/src/main/java/fonda/scheduler/model/TaskInput.java b/src/main/java/fonda/scheduler/model/TaskInput.java new file mode 100644 index 00000000..79558a9b --- /dev/null +++ b/src/main/java/fonda/scheduler/model/TaskInput.java @@ -0,0 +1,28 @@ +package fonda.scheduler.model; + +import java.util.List; + +public class TaskInput { + + public final List> booleanInputs; + public final List> numberInputs; + public final List> stringInputs; + public final List> fileInputs; + + private TaskInput() { + this.booleanInputs = null; + this.numberInputs = null; + this.stringInputs = null; + this.fileInputs = null; + } + + @Override + public String toString() { + return "TaskInput{" + + "booleanInputs=" + booleanInputs + + ", numberInputs=" + numberInputs + + ", stringInputs=" + stringInputs + + ", fileInputs=" + fileInputs + + '}'; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 25559dfc..e29c6d91 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -137,24 +137,11 @@ private Stream streamFile( List getInputsOfTask( Task task ){ return task.getConfig() .getInputs() + .fileInputs .parallelStream() - .flatMap(x -> { - if ( !(x.getValue() instanceof List) ) return Stream.empty(); - for (Object o : (List) x.getValue()) { - if (o instanceof Map) { - Map input = (Map) o; - if (input.containsKey("sourceObj")) { - String sourceObj = (String) input.get("sourceObj"); - Path sourcePath = Paths.get(sourceObj); - log.info("Src: {}", sourceObj); - if (this.hierarchyWrapper.isInScope(sourcePath)) { - return streamFile( hierarchyWrapper.getFile(sourcePath), task, sourcePath ); - } - } - } - } - return Stream.empty(); - }) + .map( x -> Path.of(x.value.storePath) ) + .filter( x -> this.hierarchyWrapper.isInScope(x) ) + .flatMap( sourcePath -> streamFile( hierarchyWrapper.getFile(sourcePath), task, sourcePath )) .collect(Collectors.toList()); } From 0acb0d59e56decd67db563c19d508497e22bb3b2 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 09:31:18 +0100 Subject: [PATCH 052/443] Allow 1000 parallel connections Signed-off-by: Lehmann-Fabian --- daemons/ftp/vsftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemons/ftp/vsftpd.conf b/daemons/ftp/vsftpd.conf index 9e659d66..8f851a17 100644 --- a/daemons/ftp/vsftpd.conf +++ b/daemons/ftp/vsftpd.conf @@ -155,7 +155,7 @@ anon_root=/ pasv_enable=Yes pasv_max_port=10090 -pasv_min_port=10090 +pasv_min_port=11090 # # Uncomment this to indicate that vsftpd use a utf8 filesystem. From 88876444588f149a939bc23a01ddcc6e23ba20d3 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 09:36:13 +0100 Subject: [PATCH 053/443] Allow to overwrite a file's locations Signed-off-by: Lehmann-Fabian --- .../model/location/hierachy/Folder.java | 4 +-- .../location/hierachy/HierarchyWrapper.java | 6 +++- .../model/location/hierachy/RealFile.java | 6 +++- .../model/location/hierachy/RealFileTest.java | 28 +++++++++---------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index c5990434..2fe95de5 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -57,12 +57,12 @@ private void getAllChildren( final Map result, Path currentPath ) } } - public boolean addOrUpdateFile( final String name, final LocationWrapper... locations ) { + public boolean addOrUpdateFile( final String name, boolean overwrite, final LocationWrapper... locations ) { children.compute( name, (k,v) -> { if (v == null || v.isDirectory()) return new RealFile( locations ); final RealFile file = (RealFile) v; - file.addOrUpdateLocation( locations ); + file.addOrUpdateLocation( overwrite, locations ); return v; } ); return true; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index c1ed64ff..f5fafc16 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -70,6 +70,10 @@ public Map getAllFilesInDir( final Path path ){ * @return false if file can not be created */ public boolean addFile( final Path path, final LocationWrapper... locations ){ + return addFile( path, false, locations ); + } + + public boolean addFile( final Path path, boolean overwrite, final LocationWrapper... locations ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return false; @@ -84,7 +88,7 @@ public boolean addFile( final Path path, final LocationWrapper... locations ){ current = current.getOrCreateFolder( p.toString() ); } else { //file - return current.addOrUpdateFile( p.toString(), locations ); + return current.addOrUpdateFile( p.toString(), overwrite, locations ); } } //This would add a file in working hierarchy diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 5951c01d..13af2c55 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -35,8 +35,12 @@ private void checkIfValidInput( LocationWrapper[] location ){ } } - public void addOrUpdateLocation( LocationWrapper... location ){ + public void addOrUpdateLocation( boolean overwrite, LocationWrapper... location ){ checkIfValidInput( location ); + if ( overwrite ){ + this.locations = location; + return; + } synchronized ( this ){ int index = 0; LocationWrapper[] newLocationsTmp = new LocationWrapper[location.length]; diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index eac081dc..c09e0af5 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -30,16 +30,16 @@ public void addLocation() { final LocationWrapper node2 = getLocationWrapper("Node2"); locations.add(node2); - realFile.addOrUpdateLocation(node2); + realFile.addOrUpdateLocation( false, node2); assertArrayEquals( locations.toArray(), realFile.getLocations() ); final LocationWrapper node3 = getLocationWrapper("Node3"); locations.add(node3); - realFile.addOrUpdateLocation(node3); + realFile.addOrUpdateLocation( false, node3); assertArrayEquals( locations.toArray(), realFile.getLocations() ); final LocationWrapper node1New = getLocationWrapper("Node1"); - realFile.addOrUpdateLocation(node1New); + realFile.addOrUpdateLocation( false, node1New); assertArrayEquals( locations.toArray(), realFile.getLocations()); } @@ -47,10 +47,10 @@ public void addLocation() { @Test public void addEmptyLocation() { final RealFile realFile = new RealFile( getLocationWrapper("node1") ); - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( null )); - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation()); - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( new LocationWrapper[0] )); - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( new LocationWrapper[1] )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, new LocationWrapper[0] )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, new LocationWrapper[1] )); } @Test @@ -66,7 +66,7 @@ public void addInParallel() { Collections.shuffle(locations); - locations.parallelStream().forEach( realFile::addOrUpdateLocation ); + locations.parallelStream().forEach( r -> realFile.addOrUpdateLocation( false, r ) ); locations.add( node0 ); assertEquals( @@ -82,14 +82,14 @@ public void changeFile() { final LocationWrapper node0 = getLocationWrapper("Node0"); final RealFile realFile = new RealFile(node0); final LocationWrapper node1 = getLocationWrapper("Node1"); - realFile.addOrUpdateLocation(node1); + realFile.addOrUpdateLocation( false, node1); final LocationWrapper node2 = getLocationWrapper("Node2"); - realFile.addOrUpdateLocation(node2); + realFile.addOrUpdateLocation( false, node2); final LocationWrapper node3 = getLocationWrapper("Node3"); - realFile.addOrUpdateLocation(node3); + realFile.addOrUpdateLocation( false, node3); final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, Process.getProcess("processB") ); - realFile.addOrUpdateLocation( nodeNew ); + realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); @@ -101,12 +101,12 @@ public void changeFileOnExistingLocation() { final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, Process.getProcess("processA") ); - realFile.addOrUpdateLocation( nodeNew ); + realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, Process.getProcess("processB") ); - realFile.addOrUpdateLocation( nodeNew2 ); + realFile.addOrUpdateLocation( false, nodeNew2 ); LocationWrapper[] expected2 = { nodeNew2 }; assertArrayEquals( expected2, realFile.getLocations() ); From 7a980df98d885fa87e243d842e4bbc79406c77a1 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 09:44:49 +0100 Subject: [PATCH 054/443] Use PodWithAge everywhere Signed-off-by: Lehmann-Fabian --- .../scheduler/client/KubernetesClient.java | 3 +- .../fonda/scheduler/model/NodeWithAlloc.java | 73 +++---------------- .../scheduler/model/PodRequirements.java | 64 ++++++++++++++++ .../fonda/scheduler/model/PodWithAge.java | 24 +++++- src/main/java/fonda/scheduler/model/Task.java | 2 +- .../scheduler/scheduler/RandomScheduler.java | 41 ++++++++--- .../fonda/scheduler/scheduler/Scheduler.java | 10 +-- 7 files changed, 132 insertions(+), 85 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/PodRequirements.java diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 96b95157..947e57d1 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -1,6 +1,7 @@ package fonda.scheduler.client; import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.PodWithAge; import fonda.scheduler.scheduler.Scheduler; import io.fabric8.kubernetes.api.model.ListOptions; import io.fabric8.kubernetes.api.model.Node; @@ -171,7 +172,7 @@ public void eventReceived(Action action, Pod pod) { NodeWithAlloc node = kubernetesClient.nodeHolder.get( pod.getSpec().getNodeName() ); switch ( action ){ case ADDED: - node.addPod( pod ); break; + node.addPod( new PodWithAge(pod) ); break; case DELETED: case ERROR: node.removePod( pod ); diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 90ac1320..7a470d99 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -45,8 +45,8 @@ public NodeWithAlloc(Node node) { log.info("Node {} has RAM: {} and CPU: {}", node.getMetadata().getName(), max_ram, max_cpu); } - public void addPod( Pod pod ){ - PodRequirements request = getRequest(pod); + public void addPod( PodWithAge pod ){ + PodRequirements request = pod.getRequest(); synchronized (assignedPods) { assignedPods.put( pod.getMetadata().getUid(), request );; } @@ -69,24 +69,15 @@ public PodRequirements getAvailableResources(){ return max_resources.sub(getRequestedResources()); } - private PodRequirements getRequest(final Pod pod){ - return pod - .getSpec().getContainers().stream() - .filter( x -> x.getResources() != null - && x.getResources().getRequests() != null ) - .map( x -> - new PodRequirements( - x.getResources().getRequests().get("cpu") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("cpu")), - x.getResources().getRequests().get("memory") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("memory")) - ) - ).reduce( new PodRequirements(), PodRequirements::addToThis ); + public boolean canSchedule( PodWithAge pod ){ + final PodRequirements request = pod.getRequest(); + PodRequirements availableResources = getAvailableResources(); + return request.getCpu().compareTo(availableResources.getCpu()) <= 0 + && request.getRam().compareTo(availableResources.getRam()) <= 0; } - public boolean canSchedule( Pod pod ){ - final PodRequirements request = getRequest(pod); - PodRequirements availableResources = getAvailableResources(); - return request.getRequested_cpu().compareTo(availableResources.getRequested_cpu()) <= 0 - && request.getRequested_ram().compareTo(availableResources.getRequested_ram()) <= 0; + public String getName(){ + return this.getMetadata().getName(); } @Override @@ -98,50 +89,4 @@ public int compareTo(NodeWithAlloc o) { } } - private static class PodRequirements { - - @Getter - private BigDecimal requested_cpu; - @Getter - private BigDecimal requested_ram; - - public PodRequirements(BigDecimal requested_cpu, BigDecimal requested_ram) { - this.requested_cpu = requested_cpu == null ? BigDecimal.ZERO : requested_cpu; - this.requested_ram = requested_ram == null ? BigDecimal.ZERO : requested_ram; - } - - private PodRequirements(){ - this( BigDecimal.ZERO, BigDecimal.ZERO ); - } - - public PodRequirements addToThis( PodRequirements podRequirements ){ - this.requested_cpu = this.requested_cpu.add(podRequirements.requested_cpu); - this.requested_ram = this.requested_ram.add(podRequirements.requested_ram); - return this; - } - - public PodRequirements addRAMtoThis( BigDecimal requested_ram ){ - this.requested_ram = this.requested_ram.add( requested_ram ); - return this; - } - - public PodRequirements addCPUtoThis( BigDecimal requested_cpu ){ - this.requested_cpu = this.requested_cpu.add( requested_cpu ); - return this; - } - - public PodRequirements subFromThis( PodRequirements podRequirements ){ - this.requested_cpu = this.requested_cpu.subtract(podRequirements.requested_cpu); - this.requested_ram = this.requested_ram.subtract(podRequirements.requested_ram); - return this; - } - - public PodRequirements sub( PodRequirements podRequirements ){ - return new PodRequirements( - this.requested_cpu.subtract(podRequirements.requested_cpu), - this.requested_ram.subtract(podRequirements.requested_ram) - ); - } - - } } diff --git a/src/main/java/fonda/scheduler/model/PodRequirements.java b/src/main/java/fonda/scheduler/model/PodRequirements.java new file mode 100644 index 00000000..9ea2ae60 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/PodRequirements.java @@ -0,0 +1,64 @@ +package fonda.scheduler.model; + +import lombok.Getter; + +import java.math.BigDecimal; + +public class PodRequirements { + + @Getter + private BigDecimal cpu; + @Getter + private BigDecimal ram; + + public PodRequirements( BigDecimal cpu, BigDecimal ram ) { + this.cpu = cpu == null ? BigDecimal.ZERO : cpu; + this.ram = ram == null ? BigDecimal.ZERO : ram; + } + + public PodRequirements(){ + this( BigDecimal.ZERO, BigDecimal.ZERO ); + } + + public PodRequirements addToThis( PodRequirements podRequirements ){ + this.cpu = this.cpu.add(podRequirements.cpu); + this.ram = this.ram.add(podRequirements.ram); + return this; + } + + public PodRequirements addRAMtoThis( BigDecimal ram ){ + this.ram = this.ram.add( ram ); + return this; + } + + public PodRequirements addCPUtoThis( BigDecimal cpu ){ + this.cpu = this.cpu.add( cpu ); + return this; + } + + public PodRequirements subFromThis( PodRequirements podRequirements ){ + this.cpu = this.cpu.subtract(podRequirements.getCpu()); + this.ram = this.ram.subtract(podRequirements.getRam()); + return this; + } + + public PodRequirements sub( PodRequirements podRequirements ){ + return new PodRequirements( + this.cpu.subtract(podRequirements.getCpu()), + this.ram.subtract(podRequirements.getRam()) + ); + } + + public boolean higherOrEquals( PodRequirements podRequirements ){ + return this.cpu.compareTo( podRequirements.cpu ) >= 0 + && this.ram.compareTo( podRequirements.ram ) >= 0; + } + + @Override + public String toString() { + return "PodRequirements{" + + "cpu=" + cpu + + ", ram=" + ram + + '}'; + } +} diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index e59e236d..04b9b75d 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -1,9 +1,6 @@ package fonda.scheduler.model; -import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodSpec; -import io.fabric8.kubernetes.api.model.PodStatus; +import io.fabric8.kubernetes.api.model.*; import lombok.Getter; import lombok.Setter; @@ -27,4 +24,23 @@ public PodWithAge(Pod pod) { super(pod.getApiVersion(), pod.getKind(), pod.getMetadata(), pod.getSpec(), pod.getStatus()); this.age = BigDecimal.ZERO; } + + public PodRequirements getRequest(){ + return this + .getSpec().getContainers().stream() + .filter( x -> x.getResources() != null + && x.getResources().getRequests() != null ) + .map( x -> + new PodRequirements( + x.getResources().getRequests().get("cpu") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("cpu")), + x.getResources().getRequests().get("memory") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("memory")) + ) + ).reduce( new PodRequirements(), PodRequirements::addToThis ); + } + + public String getName(){ + return this.getMetadata().getName(); + } + + } diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 904e32ec..c5f4526f 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -19,7 +19,7 @@ public class Task { @Getter @Setter - private Pod pod = null; + private PodWithAge pod = null; @Getter @Setter diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 79aac856..db142060 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -1,16 +1,13 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.model.Task; +import fonda.scheduler.model.*; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.scheduler.util.NodeTaskAlignment; import fonda.scheduler.scheduler.util.NodeTaskFilesAlignment; import fonda.scheduler.scheduler.util.PathFileLocationTriple; import fonda.scheduler.util.FilePath; -import io.fabric8.kubernetes.api.model.Pod; import lombok.extern.slf4j.Slf4j; import java.util.*; @@ -27,21 +24,45 @@ public RandomScheduler(String name, KubernetesClient client, String namespace, S public List getTaskNodeAlignment( final List unscheduledTasks ){ List items = getNodeList(); List alignment = new LinkedList<>(); + + Map availableByNode = new HashMap<>(); + + List logInfo = new LinkedList(); + logInfo.add("------------------------------------"); + for (NodeWithAlloc item : items) { + final PodRequirements availableResources = item.getAvailableResources(); + availableByNode.put(item.getName(), availableResources); + logInfo.add("Node: " + item.getName() + " " + availableResources); + } + logInfo.add("------------------------------------"); + System.out.println(String.join("\n", logInfo)); + + for ( final Task task : unscheduledTasks) { - final Pod pod = task.getPod(); - final List matchingNodes = items.stream().filter(x -> x.canSchedule(pod) && this.getDaemonOnNode(x) != null).collect(Collectors.toList()); + final PodWithAge pod = task.getPod(); + final List matchingNodes = items + .stream() + .filter( + x -> availableByNode.get(x.getName()).higherOrEquals(pod.getRequest()) + && this.getDaemonOnNode(x) != null + && !x.getName().equals(this.getWorkflowEngineNode())) + .collect(Collectors.toList()); + System.out.println("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); Optional node = matchingNodes.isEmpty() ? Optional.empty() : Optional.of(matchingNodes.get(new Random().nextInt(matchingNodes.size()))); if( node.isPresent() ){ - log.info("Task needs: " + task.getConfig().getInputs().toString()); + //log.info("Task needs: " + task.getConfig().getInputs().toString()); final List inputsOfTask = getInputsOfTask(task); final Map> stringListMap = scheduleFiles(task, inputsOfTask, node.get()); - alignment.add( new NodeTaskFilesAlignment(node.get(),task,stringListMap)); + alignment.add( new NodeTaskFilesAlignment(node.get(),task,stringListMap) ); + availableByNode.get(node.get().getName()).subFromThis(pod.getRequest()); + System.out.println("--> " + node.get().getName()); } else { - log.info( "No node with enough resources for {}", pod.getMetadata().getName() ); + log.info( "No node with enough resources for {}", pod.getName() ); } } + System.out.flush(); return alignment; } @@ -49,7 +70,7 @@ Map> scheduleFiles(Task task, List> map = new HashMap<>(); for ( PathFileLocationTriple entry : inputsOfTask ) { final LocationWrapper locationWrapper = entry.locations.get(new Random().nextInt(entry.locations.size())); - final String nodeIdentifier = ((NodeLocation) locationWrapper.getLocation()).getIdentifier(); + final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); if ( !map.containsKey( nodeIdentifier )){ map.put( nodeIdentifier, new LinkedList<>() ); } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 0c818103..1eaabc40 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -96,7 +96,7 @@ public int schedule( final List unscheduledTasks ) { void podEventReceived(Watcher.Action action, Pod pod){} - void onPodTermination( Pod pod ){ + void onPodTermination( PodWithAge pod ){ Task t = changeStateOfTask( pod, State.PROCESSING_FINISHED ); //If null, task was already changed @@ -116,7 +116,7 @@ void taskWasFinished( Task task ){ task.getState().setState(task.wasSuccessfullyExecuted() ? State.FINISHED : State.FINISHED_WITH_ERROR); } - public void schedulePod(Pod pod ) { + public void schedulePod(PodWithAge pod ) { Task task = changeStateOfTask( pod, State.UNSCHEDULED ); //If null, task was already unscheduled if ( task == null ) return; @@ -156,7 +156,7 @@ void taskWasScheduled(Task task ) { task.getState().setState( State.SCHEDULED ); } - public void markPodAsDeleted( Pod pod ) { + public void markPodAsDeleted( PodWithAge pod ) { final Task task = changeStateOfTask(pod, State.DELETED); task.setPod( pod ); } @@ -219,7 +219,7 @@ List getNodeList(){ * @param node * @return */ - boolean canPodBeScheduled( Pod pod, NodeWithAlloc node ){ + boolean canPodBeScheduled( PodWithAge pod, NodeWithAlloc node ){ return node.canSchedule( pod ); } @@ -227,7 +227,7 @@ void assignTaskToNode( NodeTaskAlignment alignment ){ alignment.task.setNode( alignment.node.getNodeLocation() ); - final Pod pod = alignment.task.getPod(); + final PodWithAge pod = alignment.task.getPod(); alignment.node.addPod( pod ); From bac7819cb8c88e71096d238bd9f5b701e9fe88f4 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 09:48:41 +0100 Subject: [PATCH 055/443] Get last file version Signed-off-by: Lehmann-Fabian --- .../scheduler/model/location/Location.java | 6 +++-- .../model/location/NodeDaemonPair.java | 14 ++++++++++++ .../location/hierachy/LocationWrapper.java | 1 + .../model/location/hierachy/RealFile.java | 11 ++++++++++ .../rest/SchedulerRestController.java | 22 ++++++++++++++++++- .../scheduler/SchedulerWithDaemonSet.java | 11 ++++++++++ 6 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java diff --git a/src/main/java/fonda/scheduler/model/location/Location.java b/src/main/java/fonda/scheduler/model/location/Location.java index a0feabce..e671f84d 100644 --- a/src/main/java/fonda/scheduler/model/location/Location.java +++ b/src/main/java/fonda/scheduler/model/location/Location.java @@ -1,12 +1,14 @@ package fonda.scheduler.model.location; +import lombok.Getter; + import java.util.Objects; public abstract class Location { - abstract String getIdentifier(); + public abstract String getIdentifier(); - abstract LocationType getType(); + public abstract LocationType getType(); @Override public boolean equals(Object o) { diff --git a/src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java b/src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java new file mode 100644 index 00000000..c1ae8f1f --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java @@ -0,0 +1,14 @@ +package fonda.scheduler.model.location; + +public class NodeDaemonPair { + + public final boolean sameAsEngine; + public final String node; + public final String daemon; + + public NodeDaemonPair( String node, String daemon, boolean sameAsEngine ) { + this.sameAsEngine = sameAsEngine; + this.node = node; + this.daemon = daemon; + } +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 9183a496..89b49e03 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -13,6 +13,7 @@ public class LocationWrapper { private final long timestamp; private final long sizeInBytes; private final Process process; + private final long createTime = System.currentTimeMillis(); public LocationWrapper(Location location, long timestamp, long sizeInBytes, Process process) { this.location = location; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 13af2c55..e35bd24b 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -2,6 +2,7 @@ import fonda.scheduler.model.Process; import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.LocationType; import lombok.Getter; import java.util.ArrayList; @@ -68,4 +69,14 @@ public List getFilesForProcess( Process process ){ return Arrays.asList( locations ); } + public LocationWrapper getLastUpdate( LocationType type ){ + LocationWrapper lastLocation = null; + for (LocationWrapper location : locations) { + if( location.getLocation().getType() == type && (lastLocation == null || lastLocation.getCreateTime() < location.getCreateTime() )){ + lastLocation = location; + } + } + return lastLocation; + } + } diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 7777fe5f..2562f0e3 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -1,6 +1,7 @@ package fonda.scheduler.rest; import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.location.NodeDaemonPair; import fonda.scheduler.scheduler.RandomScheduler; import fonda.scheduler.scheduler.Scheduler; import fonda.scheduler.model.SchedulerConfig; @@ -183,7 +184,26 @@ ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable Strin } - @GetMapping ("/health") + @PostMapping("/file/{namespace}/{execution}") + ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestBody String path ) { + + log.info( "Get file location request: {} {} {}", namespace, execution, path ); + + final Pair key = getKey( namespace, execution ); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + NodeDaemonPair nodeDaemonPair = ((SchedulerWithDaemonSet) scheduler).nodeOfLastFileVersion( path ); + + if ( nodeDaemonPair == null ){ + return new ResponseEntity( "No node for file found: " + path , HttpStatus.NOT_FOUND ); + } + + return new ResponseEntity( nodeDaemonPair, HttpStatus.OK ); + + } @GetMapping ("/health") ResponseEntity checkHealth() { return new ResponseEntity( HttpStatus.OK ); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index e29c6d91..6c91a3cf 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -3,8 +3,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.LocationType; +import fonda.scheduler.model.location.NodeDaemonPair; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.Folder; import fonda.scheduler.model.location.hierachy.HierarchyWrapper; +import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealFile; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; @@ -145,6 +150,12 @@ List getInputsOfTask( Task task ){ .collect(Collectors.toList()); } + public NodeDaemonPair nodeOfLastFileVersion(String path ){ + final RealFile file = (RealFile) hierarchyWrapper.getFile(Paths.get(path)); + String node = file.getLastUpdate( LocationType.NODE ).getLocation().getIdentifier(); + return new NodeDaemonPair( node, getDaemonOnNode( node ), node.equals( workflowEngineNode ) ); + } + @Override void podEventReceived(Watcher.Action action, Pod pod){ while ( daemonByNode == null ){ From aa5598ecaa55da531aacd38f72c794a906558f77 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 09:49:09 +0100 Subject: [PATCH 056/443] Update file location Signed-off-by: Lehmann-Fabian --- .../fonda/scheduler/rest/PathAttributes.java | 17 ++++++++++ .../rest/SchedulerRestController.java | 34 ++++++++++++++++++- .../scheduler/SchedulerWithDaemonSet.java | 12 +++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 src/main/java/fonda/scheduler/rest/PathAttributes.java diff --git a/src/main/java/fonda/scheduler/rest/PathAttributes.java b/src/main/java/fonda/scheduler/rest/PathAttributes.java new file mode 100644 index 00000000..e79f69d8 --- /dev/null +++ b/src/main/java/fonda/scheduler/rest/PathAttributes.java @@ -0,0 +1,17 @@ +package fonda.scheduler.rest; + +import lombok.Getter; + +@Getter +public class PathAttributes { + + private final String path; + private final long size; + private final long timestamp; + + private PathAttributes() { + this.path = null; + this.size = -1; + this.timestamp = -1; + } +} diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 2562f0e3..1054ee95 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -203,7 +203,39 @@ ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable Stri return new ResponseEntity( nodeDaemonPair, HttpStatus.OK ); - } @GetMapping ("/health") + } + + @PostMapping("/file/location/{method}/{namespace}/{execution}") + ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @RequestBody PathAttributes pa ) { + return changeLocationForFile(method,namespace,execution,null,pa); + } + + @PostMapping("/file/location/{method}/{namespace}/{execution}/{node}") + ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @PathVariable String node, @RequestBody PathAttributes pa ) { + + log.info( "Change file location request: {} {} {} {}", method, namespace, execution, pa ); + + final Pair key = getKey( namespace, execution ); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ + log.info("No scheduler for: " + execution); + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + if ( !method.equals("add") && !method.equals("overwrite") ) { + log.info("Method not found: " + method); + return new ResponseEntity( "Method not found: " + method , HttpStatus.NOT_FOUND ); + } + + boolean overwrite = method.equals("overwrite"); + + ((SchedulerWithDaemonSet) scheduler).addFile( pa.getPath(), pa.getSize(), pa.getTimestamp(), overwrite, node ); + + return new ResponseEntity( HttpStatus.OK ); + + } + + @GetMapping ("/health") ResponseEntity checkHealth() { return new ResponseEntity( HttpStatus.OK ); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 6c91a3cf..e8a93e1a 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -38,6 +38,8 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final Map daemonByNode = new HashMap<>(); @Getter + private String workflowEngineNode = null; + @Getter private final CopyStrategy copyStrategy; final HierarchyWrapper hierarchyWrapper; @@ -156,8 +158,18 @@ public NodeDaemonPair nodeOfLastFileVersion(String path ){ return new NodeDaemonPair( node, getDaemonOnNode( node ), node.equals( workflowEngineNode ) ); } + public void addFile( String path, long size, long timestamp, boolean overwrite, String node ){ + final NodeLocation location = NodeLocation.getLocation( node == null ? workflowEngineNode : node ); + final LocationWrapper locationWrapper = new LocationWrapper( location, timestamp, size, null); + hierarchyWrapper.addFile( Paths.get( path ), overwrite, locationWrapper ); + } + @Override void podEventReceived(Watcher.Action action, Pod pod){ + if ( pod.getMetadata().getName().equals( this.getExecution().replace('_', '-') ) ){ + this.workflowEngineNode = pod.getSpec().getNodeName(); + log.info( "WorkflowEngineNode was set to {}", workflowEngineNode ); + } while ( daemonByNode == null ){ //The Watcher can be started before the class is initialized try { From 153883b5f1a17618940ab73f1ab8b54eca343339 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 15:38:29 +0100 Subject: [PATCH 057/443] Adjusted TaskResultParser Signed-off-by: Lehmann-Fabian --- .../scheduler/model/TaskResultParser.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 511540c4..8d1073cf 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -18,6 +18,15 @@ @Slf4j public class TaskResultParser { + static final int VIRTUAL_PATH = 0; + static final int FILE_EXISTS = 1; + static final int REAL_PATH = 2; + static final int SIZE = 3; + static final int FILE_TYPE = 4; + static final int CREATION_DATE = 5; + static final int ACCESS_DATE = 6; + static final int MODIFICATION_DATE = 7; + private String getRootDir( File file ){ try ( Scanner sc = new Scanner( file ) ) { if( sc.hasNext() ) return sc.next().split(";")[0]; @@ -54,8 +63,9 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo in.skip( 1 ) .forEach( line -> { String[] data = line.split(";"); - String path = data[1].equals("") ? data[0].substring( taskRootDir.length() + 1 ) : data[1]; - String modificationDate = data[6]; + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; + String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ].substring( taskRootDir.length() + 1 ) : data[ REAL_PATH ]; + String modificationDate = data[ MODIFICATION_DATE ]; inputdata.put( path , modificationDate ); }); @@ -64,10 +74,11 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo out.skip( 1 ) .forEach( line -> { String[] data = line.split(";"); - boolean realFile = data[1].equals(""); - String path = realFile ? data[0] : data[1]; - String modificationDate = data[6]; - if ( "directory".equals(data[3]) ) return; + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; + boolean realFile = data[ REAL_PATH ].equals(""); + String path = realFile ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; + String modificationDate = data[ MODIFICATION_DATE ]; + if ( "directory".equals(data[ FILE_TYPE ]) ) return; String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; if ( !inputdata.containsKey(lockupPath) || @@ -76,7 +87,7 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo final LocationWrapper locationWrapper = new LocationWrapper( location, fileTimeFromString(modificationDate), - Long.parseLong(data[2]), + Long.parseLong(data[ SIZE ]), process ); newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); From d04b3336a8cc8a01e0c5fd4b5397e48a34c7fe98 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Wed, 1 Dec 2021 15:38:56 +0100 Subject: [PATCH 058/443] Deal with files, that were not found Signed-off-by: Lehmann-Fabian --- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index e8a93e1a..b315e44d 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -121,6 +121,10 @@ private Stream streamFile( final Task task, final Path sourcePath ) { + if( file == null ){ + log.info( "File to stream was null: {}", sourcePath ); + return Stream.empty(); + } if( file.isDirectory() ){ return ((Folder) file).getAllChildren( sourcePath ) .entrySet() @@ -154,7 +158,10 @@ List getInputsOfTask( Task task ){ public NodeDaemonPair nodeOfLastFileVersion(String path ){ final RealFile file = (RealFile) hierarchyWrapper.getFile(Paths.get(path)); - String node = file.getLastUpdate( LocationType.NODE ).getLocation().getIdentifier(); + if( file == null ) return null; + final LocationWrapper lastUpdate = file.getLastUpdate(LocationType.NODE); + if( lastUpdate == null ) return null; + String node = lastUpdate.getLocation().getIdentifier(); return new NodeDaemonPair( node, getDaemonOnNode( node ), node.equals( workflowEngineNode ) ); } From 10b50106413bdd5bbfb5fb051e17232d412a4b01 Mon Sep 17 00:00:00 2001 From: Lehmann-Fabian Date: Mon, 6 Dec 2021 15:19:16 +0100 Subject: [PATCH 059/443] Release resources of failed pod Signed-off-by: Lehmann-Fabian --- .../fonda/scheduler/client/KubernetesClient.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 947e57d1..fbcc98c2 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -3,10 +3,7 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; import fonda.scheduler.scheduler.Scheduler; -import io.fabric8.kubernetes.api.model.ListOptions; -import io.fabric8.kubernetes.api.model.Node; -import io.fabric8.kubernetes.api.model.NodeList; -import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; @@ -173,13 +170,22 @@ public void eventReceived(Action action, Pod pod) { switch ( action ){ case ADDED: node.addPod( new PodWithAge(pod) ); break; + case MODIFIED: + final List containerStatuses = pod.getStatus().getContainerStatuses(); + if ( containerStatuses.size() == 0 + || + containerStatuses.get(0).getState().getTerminated() == null ) { + break; + } case DELETED: case ERROR: + log.info("Pod has released its resources: {}", pod.getMetadata().getName()); + //Delete Pod in any case node.removePod( pod ); kubernetesClient.informAllScheduler(); break; - case MODIFIED: break; } + } } From a66505eef6b9e20140eecb9b8972fef45025086c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Dec 2021 10:42:42 +0100 Subject: [PATCH 060/443] Moved files Signed-off-by: Lehmann_Fabian --- .../model/location/NodeDaemonPair.java | 14 ------------- .../PathLocationWrapperPair.java | 2 +- .../scheduler/{scheduler => }/util/Batch.java | 2 +- .../util/NodeTaskAlignment.java | 2 +- .../util/NodeTaskFilesAlignment.java | 18 +++++++++++++++++ .../util/inputs/PathFileLocationTriple.java | 20 +++++++++++++++++++ 6 files changed, 41 insertions(+), 17 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java rename src/main/java/fonda/scheduler/model/{ => outfiles}/PathLocationWrapperPair.java (96%) rename src/main/java/fonda/scheduler/{scheduler => }/util/Batch.java (97%) rename src/main/java/fonda/scheduler/{scheduler => }/util/NodeTaskAlignment.java (88%) create mode 100644 src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java create mode 100644 src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java diff --git a/src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java b/src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java deleted file mode 100644 index c1ae8f1f..00000000 --- a/src/main/java/fonda/scheduler/model/location/NodeDaemonPair.java +++ /dev/null @@ -1,14 +0,0 @@ -package fonda.scheduler.model.location; - -public class NodeDaemonPair { - - public final boolean sameAsEngine; - public final String node; - public final String daemon; - - public NodeDaemonPair( String node, String daemon, boolean sameAsEngine ) { - this.sameAsEngine = sameAsEngine; - this.node = node; - this.daemon = daemon; - } -} diff --git a/src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java similarity index 96% rename from src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java rename to src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java index b01eff31..ba43df26 100644 --- a/src/main/java/fonda/scheduler/model/PathLocationWrapperPair.java +++ b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package fonda.scheduler.model.outfiles; import fonda.scheduler.model.location.hierachy.LocationWrapper; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/util/Batch.java similarity index 97% rename from src/main/java/fonda/scheduler/scheduler/util/Batch.java rename to src/main/java/fonda/scheduler/util/Batch.java index 68c4494b..50136dbd 100644 --- a/src/main/java/fonda/scheduler/scheduler/util/Batch.java +++ b/src/main/java/fonda/scheduler/util/Batch.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.util; +package fonda.scheduler.util; import fonda.scheduler.model.Task; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java similarity index 88% rename from src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java rename to src/main/java/fonda/scheduler/util/NodeTaskAlignment.java index 62a9f0fa..94867c5f 100644 --- a/src/main/java/fonda/scheduler/scheduler/util/NodeTaskAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.util; +package fonda.scheduler.util; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java new file mode 100644 index 00000000..ab1f208e --- /dev/null +++ b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java @@ -0,0 +1,18 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; + +import java.util.List; +import java.util.Map; + +public class NodeTaskFilesAlignment extends NodeTaskAlignment { + + + public final Map> nodeFileAlignment; + + public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, Map> nodeFileAlignment ) { + super(node, task); + this.nodeFileAlignment = nodeFileAlignment; + } +} diff --git a/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java new file mode 100644 index 00000000..f6c27df5 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java @@ -0,0 +1,20 @@ +package fonda.scheduler.util.inputs; + +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.RealFile; + +import java.nio.file.Path; +import java.util.List; + +public class PathFileLocationTriple { + + public final Path path; + public final RealFile file; + public final List locations; + + public PathFileLocationTriple(Path path, RealFile file, List locations) { + this.path = path; + this.file = file; + this.locations = locations; + } +} From 04000bb0c32d67f07511aec4e06c5f97cc4b8fd3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Dec 2021 10:43:32 +0100 Subject: [PATCH 061/443] Moved files Signed-off-by: Lehmann_Fabian --- .../util/NodeTaskFilesAlignment.java | 19 ------------------ .../util/PathFileLocationTriple.java | 20 ------------------- 2 files changed, 39 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java delete mode 100644 src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java diff --git a/src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java deleted file mode 100644 index a5507a0d..00000000 --- a/src/main/java/fonda/scheduler/scheduler/util/NodeTaskFilesAlignment.java +++ /dev/null @@ -1,19 +0,0 @@ -package fonda.scheduler.scheduler.util; - -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.util.FilePath; - -import java.util.List; -import java.util.Map; - -public class NodeTaskFilesAlignment extends NodeTaskAlignment { - - - public final Map> nodeFileAlignment; - - public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, Map> nodeFileAlignment ) { - super(node, task); - this.nodeFileAlignment = nodeFileAlignment; - } -} diff --git a/src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java deleted file mode 100644 index 105d9ef9..00000000 --- a/src/main/java/fonda/scheduler/scheduler/util/PathFileLocationTriple.java +++ /dev/null @@ -1,20 +0,0 @@ -package fonda.scheduler.scheduler.util; - -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealFile; - -import java.nio.file.Path; -import java.util.List; - -public class PathFileLocationTriple { - - public final Path path; - public final RealFile file; - public final List locations; - - public PathFileLocationTriple(Path path, RealFile file, List locations) { - this.path = path; - this.file = file; - this.locations = locations; - } -} From 598aa2ea7745ab339b73e3f2b15e9825732d30fe Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Dec 2021 10:46:05 +0100 Subject: [PATCH 062/443] Work with and keep symlink structure Signed-off-by: Lehmann_Fabian --- .../scheduler/model/TaskResultParser.java | 19 +++- .../model/location/hierachy/AbstractFile.java | 5 + .../model/location/hierachy/File.java | 2 + .../model/location/hierachy/Folder.java | 26 +++-- .../location/hierachy/HierarchyWrapper.java | 28 ++++- .../model/location/hierachy/LinkFile.java | 25 +++++ .../model/location/hierachy/RealFile.java | 18 +++- .../scheduler/model/outfiles/OutputFile.java | 29 +++++ .../outfiles/PathLocationWrapperPair.java | 14 +-- .../model/outfiles/SymlinkOutput.java | 32 ++++++ .../rest/SchedulerRestController.java | 15 ++- .../exceptions/NotARealFileException.java | 8 ++ .../rest/response/getfile/FileResponse.java | 45 ++++++++ .../scheduler/scheduler/RandomScheduler.java | 42 +++++--- .../scheduler/SchedulerWithDaemonSet.java | 102 +++++++++++++----- .../scheduler/schedulingstrategy/Inputs.java | 4 + .../fonda/scheduler/util/FileAlignment.java | 17 +++ .../util/NodeTaskFilesAlignment.java | 6 +- .../fonda/scheduler/util/inputs/Input.java | 4 + .../util/inputs/PathFileLocationTriple.java | 2 +- .../scheduler/util/inputs/SymlinkInput.java | 17 +++ src/main/resources/copystrategies/ftp.py | 23 +++- .../scheduler/model/TaskResultParserTest.java | 9 +- 23 files changed, 408 insertions(+), 84 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java create mode 100644 src/main/java/fonda/scheduler/model/outfiles/OutputFile.java create mode 100644 src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java create mode 100644 src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java create mode 100644 src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java create mode 100644 src/main/java/fonda/scheduler/util/FileAlignment.java create mode 100644 src/main/java/fonda/scheduler/util/inputs/Input.java create mode 100644 src/main/java/fonda/scheduler/util/inputs/SymlinkInput.java diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 8d1073cf..9d351f56 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -2,6 +2,9 @@ import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.outfiles.OutputFile; +import fonda.scheduler.model.outfiles.PathLocationWrapperPair; +import fonda.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; import java.io.File; @@ -43,7 +46,12 @@ private String getRootDir( File file ){ * @param process * @return A list of all new or updated files */ - public Set getNewAndUpdatedFiles(final Path workdir, Location location, Process process ){ + public Set getNewAndUpdatedFiles( + final Path workdir, + final Location location, + final Process process, + final boolean onlyUpdated + ){ final Path infile = workdir.resolve(".command.infiles"); final Path outfile = workdir.resolve(".command.outfiles"); @@ -53,7 +61,7 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo final Map inputdata = new HashMap<>(); - Set newOrUpdated = new HashSet<>(); + Set newOrUpdated = new HashSet<>(); try ( Stream in = Files.lines(infile); @@ -69,7 +77,7 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo inputdata.put( path , modificationDate ); }); - log.info( "{}", inputdata ); + log.trace( "{}", inputdata ); out.skip( 1 ) .forEach( line -> { @@ -80,7 +88,7 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo String modificationDate = data[ MODIFICATION_DATE ]; if ( "directory".equals(data[ FILE_TYPE ]) ) return; String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; - if ( !inputdata.containsKey(lockupPath) + if ( ( !inputdata.containsKey(lockupPath) && !onlyUpdated ) || !modificationDate.equals( inputdata.get( lockupPath ) )) { @@ -92,6 +100,9 @@ public Set getNewAndUpdatedFiles(final Path workdir, Lo ); newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); } + if( !realFile ){ + newOrUpdated.add( new SymlinkOutput( data[ VIRTUAL_PATH ], data[ REAL_PATH ])); + } }); } catch (IOException e) { diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java new file mode 100644 index 00000000..212c4f63 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java @@ -0,0 +1,5 @@ +package fonda.scheduler.model.location.hierachy; + +public abstract class AbstractFile extends File { + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/File.java b/src/main/java/fonda/scheduler/model/location/hierachy/File.java index d090d141..79de9243 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/File.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/File.java @@ -4,4 +4,6 @@ public abstract class File { public abstract boolean isDirectory(); + public abstract boolean isSymlink(); + } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 2fe95de5..f6e87794 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -22,6 +22,11 @@ public boolean isDirectory() { return true; } + @Override + public boolean isSymlink() { + return false; + } + public File get( String name ){ return children.get( name ); } @@ -39,27 +44,27 @@ public Folder getOrCreateFolder(String name ){ return (Folder) file; } - public Map getAllChildren( Path currentPath ){ - Map result = new TreeMap<>(); + public Map getAllChildren( Path currentPath ){ + Map result = new TreeMap<>(); getAllChildren( result, currentPath ); return result; } - private void getAllChildren( final Map result, Path currentPath ){ + private void getAllChildren( final Map result, Path currentPath ){ for (Map.Entry entry : children.entrySet()) { Path resolve = currentPath.resolve(entry.getKey()); - if ( entry.getValue().isDirectory() ){ + if ( !entry.getValue().isSymlink() && entry.getValue().isDirectory() ){ final Folder value = (Folder) entry.getValue(); value.getAllChildren( result, resolve ); } else { - result.put( resolve, (RealFile) entry.getValue()); + result.put( resolve, (AbstractFile) entry.getValue()); } } } public boolean addOrUpdateFile( final String name, boolean overwrite, final LocationWrapper... locations ) { children.compute( name, (k,v) -> { - if (v == null || v.isDirectory()) + if (v == null || v.isDirectory() || v.isSymlink() ) return new RealFile( locations ); final RealFile file = (RealFile) v; file.addOrUpdateLocation( overwrite, locations ); @@ -68,4 +73,13 @@ public boolean addOrUpdateFile( final String name, boolean overwrite, final Loca return true; } + public boolean addSymlink( final String name, final Path dst ){ + children.compute( name, (k,v) -> { + if ( v == null || !v.isSymlink() || !((LinkFile) v).getDst().equals(dst) ) + return new LinkFile( dst ); + return v; + } ); + return true; + } + } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index f5fafc16..cf305ae3 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -44,7 +44,7 @@ private Folder getWorkdir( Iterator iterator, boolean create ){ * @param path get all files recursively in this folder (absolute path) * @return Null if folder is empty, or not found */ - public Map getAllFilesInDir( final Path path ){ + public Map getAllFilesInDir( final Path path ){ final Path relativePath = relativize( path ); Iterator iterator = relativePath.iterator(); File current = getWorkdir( iterator, false ); @@ -74,13 +74,31 @@ public boolean addFile( final Path path, final LocationWrapper... locations ){ } public boolean addFile( final Path path, boolean overwrite, final LocationWrapper... locations ){ + + final Folder folderToInsert = findFolderToInsert(path); + + if( folderToInsert == null ) return false; + else return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, locations ); + + } + + public boolean addSymlink( final Path src, final Path dst ){ + + final Folder folderToInsert = findFolderToInsert( src ); + + if( folderToInsert == null ) return false; + else return folderToInsert.addSymlink( src.getFileName().toString(), dst ); + + } + + private Folder findFolderToInsert( final Path path ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ - return false; + return null; } Iterator iterator = relativePath.iterator(); Folder current = getWorkdir( iterator, true ); - if( current == null ) return false; + if( current == null ) return null; while(iterator.hasNext()) { Path p = iterator.next(); if( iterator.hasNext() ){ @@ -88,11 +106,11 @@ public boolean addFile( final Path path, boolean overwrite, final LocationWrappe current = current.getOrCreateFolder( p.toString() ); } else { //file - return current.addOrUpdateFile( p.toString(), overwrite, locations ); + return current; } } //This would add a file in working hierarchy - return false; + return null; } /** diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java new file mode 100644 index 00000000..db5e04f4 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java @@ -0,0 +1,25 @@ +package fonda.scheduler.model.location.hierachy; + +import lombok.Getter; + +import java.nio.file.Path; + +public class LinkFile extends AbstractFile { + + @Getter + private final Path dst; + + public LinkFile( Path dst ) { + this.dst = dst; + } + + @Override + public boolean isDirectory() { + throw new IllegalStateException("Call on link"); + } + + @Override + public boolean isSymlink() { + return true; + } +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index e35bd24b..e54cafb8 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -1,16 +1,13 @@ package fonda.scheduler.model.location.hierachy; import fonda.scheduler.model.Process; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import lombok.Getter; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; -public class RealFile extends File { +public class RealFile extends AbstractFile { /** * This field contains the newest LocationWrapper of one file for each node. @@ -28,6 +25,11 @@ public boolean isDirectory(){ return false; } + @Override + public boolean isSymlink() { + return false; + } + private void checkIfValidInput( LocationWrapper[] location ){ if ( location == null || location.length == 0 ) throw new IllegalArgumentException( "location was null or empty" ); @@ -66,7 +68,13 @@ public void addOrUpdateLocation( boolean overwrite, LocationWrapper... location } public List getFilesForProcess( Process process ){ - return Arrays.asList( locations ); + LocationWrapper lastUpdated = null; + for (LocationWrapper location : locations) { + if ( lastUpdated == null || lastUpdated.getCreateTime() < location.getCreateTime() ) { + lastUpdated = location; + } + } + return List.of( lastUpdated ); } public LocationWrapper getLastUpdate( LocationType type ){ diff --git a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java b/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java new file mode 100644 index 00000000..22e5ea1a --- /dev/null +++ b/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java @@ -0,0 +1,29 @@ +package fonda.scheduler.model.outfiles; + +import lombok.Getter; + +import java.nio.file.Path; +import java.util.Objects; + +@Getter +public class OutputFile { + + private final Path path; + + public OutputFile(Path path) { + this.path = path; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof OutputFile)) return false; + OutputFile that = (OutputFile) o; + return Objects.equals(getPath(), that.getPath()); + } + + @Override + public int hashCode() { + return Objects.hash(getPath()); + } +} diff --git a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java index ba43df26..42e2c1e8 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java +++ b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java @@ -7,20 +7,20 @@ import java.util.Objects; @Getter -public class PathLocationWrapperPair { +public class PathLocationWrapperPair extends OutputFile { + - private final Path path; private final LocationWrapper locationWrapper; public PathLocationWrapperPair(Path path, LocationWrapper locationWrapper) { - this.path = path; + super( path ); this.locationWrapper = locationWrapper; } @Override public String toString() { return "PathLocationWrapperPair{" + - "path=" + path + + "path=" + getPath() + ", locationWrapper=" + locationWrapper + '}'; } @@ -29,12 +29,14 @@ public String toString() { public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof PathLocationWrapperPair)) return false; + if (!super.equals(o)) return false; PathLocationWrapperPair that = (PathLocationWrapperPair) o; - return getPath().equals(that.getPath()) && getLocationWrapper().equals(that.getLocationWrapper()); + return Objects.equals(getLocationWrapper(), that.getLocationWrapper()); } @Override public int hashCode() { - return Objects.hash(getPath(), getLocationWrapper()); + return Objects.hash(super.hashCode(), getLocationWrapper()); } + } diff --git a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java b/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java new file mode 100644 index 00000000..0aef4c5a --- /dev/null +++ b/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java @@ -0,0 +1,32 @@ +package fonda.scheduler.model.outfiles; + +import lombok.Getter; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +@Getter +public class SymlinkOutput extends OutputFile { + + private final Path dst; + + public SymlinkOutput( String path, String dst ) { + super( Paths.get(path) ); + this.dst = Paths.get(dst); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SymlinkOutput)) return false; + if (!super.equals(o)) return false; + SymlinkOutput that = (SymlinkOutput) o; + return Objects.equals(dst, that.dst); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dst); + } +} diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 1054ee95..d04fafa9 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -1,7 +1,8 @@ package fonda.scheduler.rest; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.location.NodeDaemonPair; +import fonda.scheduler.rest.exceptions.NotARealFileException; +import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.RandomScheduler; import fonda.scheduler.scheduler.Scheduler; import fonda.scheduler.model.SchedulerConfig; @@ -195,13 +196,19 @@ ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable Stri return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } - NodeDaemonPair nodeDaemonPair = ((SchedulerWithDaemonSet) scheduler).nodeOfLastFileVersion( path ); + FileResponse fileResponse = null; + try { + fileResponse = ((SchedulerWithDaemonSet) scheduler).nodeOfLastFileVersion( path ); + log.info(fileResponse.toString()); + } catch (NotARealFileException e) { + return new ResponseEntity( "Requested path is not a real file: " + path , HttpStatus.BAD_REQUEST ); + } - if ( nodeDaemonPair == null ){ + if ( fileResponse == null ){ return new ResponseEntity( "No node for file found: " + path , HttpStatus.NOT_FOUND ); } - return new ResponseEntity( nodeDaemonPair, HttpStatus.OK ); + return new ResponseEntity( fileResponse, HttpStatus.OK ); } diff --git a/src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java b/src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java new file mode 100644 index 00000000..5b552ac2 --- /dev/null +++ b/src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java @@ -0,0 +1,8 @@ +package fonda.scheduler.rest.exceptions; + +public class NotARealFileException extends Exception { + + public NotARealFileException() { + super( "Not a real file" ); + } +} diff --git a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java new file mode 100644 index 00000000..a83f1dba --- /dev/null +++ b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java @@ -0,0 +1,45 @@ +package fonda.scheduler.rest.response.getfile; + +import fonda.scheduler.util.inputs.SymlinkInput; + +import java.util.List; + +public class FileResponse { + + public final String path; + public final boolean sameAsEngine; + public final String node; + public final String daemon; + public final List symlinks; + public final boolean notInContext; + + public FileResponse( String path, String node, String daemon, boolean sameAsEngine, List symlinks) { + this.path = path; + this.sameAsEngine = sameAsEngine; + this.node = node; + this.daemon = daemon; + this.symlinks = symlinks; + notInContext = false; + } + + public FileResponse( String path, List symlinks) { + this.path = path; + this.sameAsEngine = true; + this.node = null; + this.daemon = null; + this.symlinks = symlinks; + notInContext = true; + } + + @Override + public String toString() { + return "FileResponse{" + + "path='" + path + '\'' + + ", sameAsEngine=" + sameAsEngine + + ", node='" + node + '\'' + + ", daemon='" + daemon + '\'' + + ", symlinks=" + symlinks + + ", notInContext=" + notInContext + + '}'; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index db142060..1d81d993 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -2,12 +2,14 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.scheduler.util.NodeTaskAlignment; -import fonda.scheduler.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.scheduler.util.PathFileLocationTriple; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.inputs.Input; import fonda.scheduler.util.FilePath; +import fonda.scheduler.util.inputs.PathFileLocationTriple; +import fonda.scheduler.util.inputs.SymlinkInput; import lombok.extern.slf4j.Slf4j; import java.util.*; @@ -53,9 +55,9 @@ public List getTaskNodeAlignment( final List unschedule : Optional.of(matchingNodes.get(new Random().nextInt(matchingNodes.size()))); if( node.isPresent() ){ //log.info("Task needs: " + task.getConfig().getInputs().toString()); - final List inputsOfTask = getInputsOfTask(task); - final Map> stringListMap = scheduleFiles(task, inputsOfTask, node.get()); - alignment.add( new NodeTaskFilesAlignment(node.get(),task,stringListMap) ); + final List inputsOfTask = getInputsOfTask(task); + final FileAlignment fileAlignment = scheduleFiles(task, inputsOfTask, node.get()); + alignment.add( new NodeTaskFilesAlignment(node.get(),task, fileAlignment ) ); availableByNode.get(node.get().getName()).subFromThis(pod.getRequest()); System.out.println("--> " + node.get().getName()); } else { @@ -66,18 +68,26 @@ public List getTaskNodeAlignment( final List unschedule return alignment; } - Map> scheduleFiles(Task task, List inputsOfTask, NodeWithAlloc node) { + FileAlignment scheduleFiles(Task task, List inputsOfTask, NodeWithAlloc node) { final HashMap> map = new HashMap<>(); - for ( PathFileLocationTriple entry : inputsOfTask ) { - final LocationWrapper locationWrapper = entry.locations.get(new Random().nextInt(entry.locations.size())); - final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - if ( !map.containsKey( nodeIdentifier )){ - map.put( nodeIdentifier, new LinkedList<>() ); + final List symlinkInputs = new LinkedList<>(); + for ( Input entry : inputsOfTask ) { + if( entry instanceof PathFileLocationTriple ){ + final PathFileLocationTriple pathFileLocationTriple = (PathFileLocationTriple) entry; + final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( + new Random().nextInt( pathFileLocationTriple.locations.size() ) + ); + final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); + if ( !map.containsKey( nodeIdentifier )){ + map.put( nodeIdentifier, new LinkedList<>() ); + } + final List pathsOfNode = map.get( nodeIdentifier ); + pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file ) ); + } else if ( entry instanceof SymlinkInput ){ + symlinkInputs.add( (SymlinkInput) entry ); } - final List pathsOfNode = map.get( nodeIdentifier ); - pathsOfNode.add( new FilePath( entry.path.toString(), entry.file ) ); } - return map; + return new FileAlignment( map, symlinkInputs ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index b315e44d..95c3a081 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -3,21 +3,23 @@ import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; -import fonda.scheduler.model.location.NodeDaemonPair; +import fonda.scheduler.rest.exceptions.NotARealFileException; +import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.Folder; -import fonda.scheduler.model.location.hierachy.HierarchyWrapper; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealFile; +import fonda.scheduler.model.location.hierachy.*; +import fonda.scheduler.model.outfiles.OutputFile; +import fonda.scheduler.model.outfiles.PathLocationWrapperPair; +import fonda.scheduler.model.outfiles.SymlinkOutput; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; -import fonda.scheduler.scheduler.util.NodeTaskAlignment; -import fonda.scheduler.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.scheduler.util.PathFileLocationTriple; +import fonda.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.inputs.Input; +import fonda.scheduler.util.inputs.PathFileLocationTriple; +import fonda.scheduler.util.inputs.SymlinkInput; import fonda.scheduler.util.FilePath; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; @@ -78,15 +80,28 @@ void assignTaskToNode( NodeTaskAlignment alignment ) { int terminateTasks(List finishedTasks) { final TaskResultParser taskResultParser = new TaskResultParser(); finishedTasks.parallelStream().forEach( finishedTask -> { - if( finishedTask.wasSuccessfullyExecuted() ) { - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + try{ + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( Paths.get(finishedTask.getWorkingDir()), finishedTask.getNode(), - finishedTask.getProcess() + finishedTask.getProcess(), + !finishedTask.wasSuccessfullyExecuted() ); - for (PathLocationWrapperPair newAndUpdatedFile : newAndUpdatedFiles) { - hierarchyWrapper.addFile(newAndUpdatedFile.getPath(), newAndUpdatedFile.getLocationWrapper()); + for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { + if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { + hierarchyWrapper.addFile( + newAndUpdatedFile.getPath(), + ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() + ); + } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ + hierarchyWrapper.addSymlink( + newAndUpdatedFile.getPath(), + ((SymlinkOutput) newAndUpdatedFile).getDst() + ); + } } + } catch ( Exception e ){ + log.info( "Problem while finishing task: " + finishedTask.getConfig().getName(), e ); } super.taskWasFinished( finishedTask ); }); @@ -100,7 +115,7 @@ boolean writeInitConfig( NodeTaskFilesAlignment alignment ) { final Inputs inputs = new Inputs( this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/" ); - for (Map.Entry> entry : alignment.nodeFileAlignment.entrySet()) { + for (Map.Entry> entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) continue; final List collect = entry .getValue() .stream() @@ -108,6 +123,10 @@ boolean writeInitConfig( NodeTaskFilesAlignment alignment ) { .collect(Collectors.toList()); inputs.data.add( new InputEntry( getDaemonOnNode( entry.getKey() ), entry.getKey(), collect ) ); } + for (SymlinkInput symlink : alignment.fileAlignment.symlinks) { + inputs.symlinks.add( symlink ); + } + new ObjectMapper().writeValue( config, inputs ); return true; } catch (IOException e) { @@ -116,7 +135,7 @@ boolean writeInitConfig( NodeTaskFilesAlignment alignment ) { return false; } - private Stream streamFile( + private Stream streamFile( final fonda.scheduler.model.location.hierachy.File file, final Task task, final Path sourcePath ) @@ -125,15 +144,27 @@ private Stream streamFile( log.info( "File to stream was null: {}", sourcePath ); return Stream.empty(); } + if ( file.isSymlink() ){ + final Path linkTo = ((LinkFile) file).getDst(); + final fonda.scheduler.model.location.hierachy.File destFile = hierarchyWrapper.getFile(linkTo); + final Stream inputStream = streamFile(destFile, task, linkTo); + return Stream.concat(Stream.of( new SymlinkInput( sourcePath, linkTo ) ), inputStream); + } if( file.isDirectory() ){ return ((Folder) file).getAllChildren( sourcePath ) .entrySet() .stream() - .map( y -> new PathFileLocationTriple( - y.getKey(), - y.getValue(), - y.getValue().getFilesForProcess( task.getProcess() ) - ) + .flatMap( y -> { + if ( !y.getValue().isSymlink() ) + return Stream.of(new PathFileLocationTriple( + y.getKey(), + ((RealFile) y.getValue()), + ((RealFile) y.getValue()).getFilesForProcess(task.getProcess()) + )); + else { + return streamFile( y.getValue(), task, y.getKey() ); + } + } ); } final RealFile realFile = (RealFile) file; @@ -145,24 +176,41 @@ private Stream streamFile( ); } - List getInputsOfTask( Task task ){ + List getInputsOfTask( Task task ){ return task.getConfig() .getInputs() .fileInputs .parallelStream() - .map( x -> Path.of(x.value.storePath) ) + .map( x -> Path.of(x.value.sourceObj) ) .filter( x -> this.hierarchyWrapper.isInScope(x) ) .flatMap( sourcePath -> streamFile( hierarchyWrapper.getFile(sourcePath), task, sourcePath )) .collect(Collectors.toList()); } - public NodeDaemonPair nodeOfLastFileVersion(String path ){ - final RealFile file = (RealFile) hierarchyWrapper.getFile(Paths.get(path)); - if( file == null ) return null; + public FileResponse nodeOfLastFileVersion(String path ) throws NotARealFileException { + LinkedList symlinks = new LinkedList<>(); + Path currentPath = Paths.get(path); + fonda.scheduler.model.location.hierachy.File currentFile = hierarchyWrapper.getFile( currentPath ); + while ( currentFile instanceof LinkFile ){ + final LinkFile linkFile = (LinkFile) currentFile; + symlinks.add( new SymlinkInput( currentPath, linkFile.getDst() ) ); + currentPath = linkFile.getDst(); + currentFile = hierarchyWrapper.getFile( currentPath ); + } + Collections.reverse( symlinks ); + //File is maybe out of scope + if ( currentFile == null ) { + return new FileResponse( currentPath.toString(), symlinks ); + } + if ( ! (currentFile instanceof RealFile) ){ + log.info( "File was: {}", currentFile ); + throw new NotARealFileException(); + } + final RealFile file = (RealFile) currentFile; final LocationWrapper lastUpdate = file.getLastUpdate(LocationType.NODE); if( lastUpdate == null ) return null; String node = lastUpdate.getLocation().getIdentifier(); - return new NodeDaemonPair( node, getDaemonOnNode( node ), node.equals( workflowEngineNode ) ); + return new FileResponse( currentPath.toString(), node, getDaemonOnNode(node), node.equals(workflowEngineNode), symlinks ); } public void addFile( String path, long size, long timestamp, boolean overwrite, String node ){ diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index bbbb6b5b..f4fe317e 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -1,5 +1,7 @@ package fonda.scheduler.scheduler.schedulingstrategy; +import fonda.scheduler.util.inputs.SymlinkInput; + import java.util.LinkedList; import java.util.List; @@ -7,10 +9,12 @@ public class Inputs { public final String dns; public final List data; + public final List symlinks; public Inputs( String dns ) { this.dns = dns; this.data = new LinkedList<>(); + this.symlinks = new LinkedList<>(); } } diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java new file mode 100644 index 00000000..91016747 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -0,0 +1,17 @@ +package fonda.scheduler.util; + +import fonda.scheduler.util.inputs.SymlinkInput; + +import java.util.List; +import java.util.Map; + +public class FileAlignment { + + public final Map> nodeFileAlignment; + public final List symlinks; + + public FileAlignment(Map> nodeFileAlignment, List symlinks) { + this.nodeFileAlignment = nodeFileAlignment; + this.symlinks = symlinks; + } +} diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java index ab1f208e..e8e8a201 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java @@ -9,10 +9,10 @@ public class NodeTaskFilesAlignment extends NodeTaskAlignment { - public final Map> nodeFileAlignment; + public final FileAlignment fileAlignment; - public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, Map> nodeFileAlignment ) { + public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, FileAlignment fileAlignment ) { super(node, task); - this.nodeFileAlignment = nodeFileAlignment; + this.fileAlignment = fileAlignment; } } diff --git a/src/main/java/fonda/scheduler/util/inputs/Input.java b/src/main/java/fonda/scheduler/util/inputs/Input.java new file mode 100644 index 00000000..35c3e48c --- /dev/null +++ b/src/main/java/fonda/scheduler/util/inputs/Input.java @@ -0,0 +1,4 @@ +package fonda.scheduler.util.inputs; + +public interface Input { +} diff --git a/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java index f6c27df5..a96354f2 100644 --- a/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java @@ -6,7 +6,7 @@ import java.nio.file.Path; import java.util.List; -public class PathFileLocationTriple { +public class PathFileLocationTriple implements Input { public final Path path; public final RealFile file; diff --git a/src/main/java/fonda/scheduler/util/inputs/SymlinkInput.java b/src/main/java/fonda/scheduler/util/inputs/SymlinkInput.java new file mode 100644 index 00000000..4a6f9bcf --- /dev/null +++ b/src/main/java/fonda/scheduler/util/inputs/SymlinkInput.java @@ -0,0 +1,17 @@ +package fonda.scheduler.util.inputs; + +import lombok.Getter; + +import java.nio.file.Path; + +@Getter +public class SymlinkInput implements Input { + + private final String src; + private final String dst; + + public SymlinkInput(Path src, Path dst) { + this.src = src.toAbsolutePath().toString(); + this.dst = dst.toAbsolutePath().toString(); + } +} diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 2b1d7363..f5d05956 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -6,11 +6,21 @@ import urllib.request import sys from pathlib import Path +import shutil def getIP( node ): ip = urllib.request.urlopen(dns + node).read() return str(ip.decode("utf-8") ) +def clearLocatation( path ): + if os.path.exists( path ): + if os.path.islink( path ): + os.unlink( path ) + elif os.path.isdir( path ): + shutil.rmtree( path ) + else: + os.remove( path ) + def download( node, files ): @@ -52,8 +62,7 @@ def download( node, files ): print("Download", "[" + str( index ).rjust( len( str( size ) ) ) + "/" + str( size ) + "]" , filename) try: - if os.path.exists( filename ): - os.remove(filename) + clearLocatation( filename ) Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) except ftplib.error_perm as err: @@ -101,7 +110,15 @@ def download( node, files ): dns = config["dns"] data = config[ "data" ] +symlinks = config[ "symlinks" ] for d in data: files = d["files"] - download( d["node"], files ) \ No newline at end of file + download( d["node"], files ) + +for s in symlinks: + src = s["src"] + dst = s["dst"] + clearLocatation( src ) + Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) + os.symlink( dst, src ) \ No newline at end of file diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java index 73b0d760..7f303c96 100644 --- a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -2,6 +2,8 @@ import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.outfiles.OutputFile; +import fonda.scheduler.model.outfiles.PathLocationWrapperPair; import lombok.extern.slf4j.Slf4j; import org.junit.Test; import static org.junit.Assert.*; @@ -9,7 +11,6 @@ import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Set; @Slf4j @@ -59,7 +60,7 @@ public void test1(){ final Path path = storeData(infiles, outfiles); final TaskResultParser taskResultParser = new TaskResultParser(); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), Process.getProcess("P1")); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), Process.getProcess("P1"), false); log.info("{}", newAndUpdatedFiles); @@ -95,7 +96,7 @@ public void test2(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); final Process p1 = Process.getProcess("P1"); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); log.info("{}", newAndUpdatedFiles); @@ -139,7 +140,7 @@ public void test3(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); final Process p1 = Process.getProcess("P1"); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); log.info("{}", newAndUpdatedFiles); From c5fba2b459a1ad7b567ac638c80d7a95fbd99d0f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Dec 2021 10:46:49 +0100 Subject: [PATCH 063/443] Save node location Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/Scheduler.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 1eaabc40..56f1d1c7 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -2,8 +2,8 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; -import fonda.scheduler.scheduler.util.Batch; -import fonda.scheduler.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.util.Batch; +import fonda.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Binding; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectReference; @@ -15,6 +15,9 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; import java.util.*; @Slf4j @@ -225,6 +228,14 @@ boolean canPodBeScheduled( PodWithAge pod, NodeWithAlloc node ){ void assignTaskToNode( NodeTaskAlignment alignment ){ + final File nodeFile = new File(alignment.task.getWorkingDir() + '/' + ".command.node"); + + try(PrintWriter printWriter = new PrintWriter(nodeFile)){ + printWriter.println( alignment.node.getName() ); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + alignment.task.setNode( alignment.node.getNodeLocation() ); final PodWithAge pod = alignment.task.getPod(); From 857f9269cd722de4a8f211a81ebd7a99094d2e2b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Dec 2021 10:47:03 +0100 Subject: [PATCH 064/443] cpus as float Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/TaskConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 4e715043..77319c67 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -15,7 +15,7 @@ public class TaskConfig { private final Map< String, List> schedulerParams; private final TaskInput inputs; private final String hash; - private final int cpus; + private final float cpus; private final long memoryInBytes; private TaskConfig() { From 05818ccf07bdabd6a99a289f69a342ec40336453 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Dec 2021 10:47:21 +0100 Subject: [PATCH 065/443] Removed unused dependencies Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index c5f4526f..2a0a8737 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -1,11 +1,9 @@ package fonda.scheduler.model; import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.scheduler.util.Batch; -import io.fabric8.kubernetes.api.model.Pod; +import fonda.scheduler.util.Batch; import lombok.Getter; import lombok.Setter; -import org.apache.tomcat.jni.Proc; public class Task { From 5f80cc821b195a7a787deece64266add53b49117 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Jan 2022 11:56:23 +0100 Subject: [PATCH 066/443] getTaskNodeAlignment returns a ScheduleObject Signed-off-by: Lehmann_Fabian --- .../scheduler/model/PodRequirements.java | 2 ++ .../fonda/scheduler/model/ScheduleObject.java | 20 +++++++++++++ .../scheduler/scheduler/RandomScheduler.java | 9 ++++-- .../fonda/scheduler/scheduler/Scheduler.java | 29 +++++++++++++++---- 4 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/ScheduleObject.java diff --git a/src/main/java/fonda/scheduler/model/PodRequirements.java b/src/main/java/fonda/scheduler/model/PodRequirements.java index 9ea2ae60..b6b56fe2 100644 --- a/src/main/java/fonda/scheduler/model/PodRequirements.java +++ b/src/main/java/fonda/scheduler/model/PodRequirements.java @@ -6,6 +6,8 @@ public class PodRequirements { + public static final PodRequirements ZERO = new PodRequirements(); + @Getter private BigDecimal cpu; @Getter diff --git a/src/main/java/fonda/scheduler/model/ScheduleObject.java b/src/main/java/fonda/scheduler/model/ScheduleObject.java new file mode 100644 index 00000000..602c6db3 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/ScheduleObject.java @@ -0,0 +1,20 @@ +package fonda.scheduler.model; + +import fonda.scheduler.util.NodeTaskAlignment; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Setter +@Getter +public class ScheduleObject { + + private final List taskAlignments; + private boolean checkStillPossible = false; + + public ScheduleObject( List taskAlignments ) { + this.taskAlignments = taskAlignments; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 1d81d993..a15c77c8 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -23,7 +23,7 @@ public RandomScheduler(String name, KubernetesClient client, String namespace, S } @Override - public List getTaskNodeAlignment( final List unscheduledTasks ){ + public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ List items = getNodeList(); List alignment = new LinkedList<>(); @@ -47,7 +47,8 @@ public List getTaskNodeAlignment( final List unschedule .filter( x -> availableByNode.get(x.getName()).higherOrEquals(pod.getRequest()) && this.getDaemonOnNode(x) != null - && !x.getName().equals(this.getWorkflowEngineNode())) + //&& !x.getName().equals(this.getWorkflowEngineNode()) + ) .collect(Collectors.toList()); System.out.println("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); Optional node = matchingNodes.isEmpty() @@ -65,7 +66,9 @@ public List getTaskNodeAlignment( final List unschedule } } System.out.flush(); - return alignment; + final ScheduleObject scheduleObject = new ScheduleObject(alignment); + scheduleObject.setCheckStillPossible( true ); + return scheduleObject; } FileAlignment scheduleFiles(Task task, List inputsOfTask, NodeWithAlloc node) { diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 56f1d1c7..bf426a20 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -77,21 +77,40 @@ public abstract class Scheduler { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { - final List taskNodeAlignment = getTaskNodeAlignment(unscheduledTasks); + final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks); + final List taskNodeAlignment = scheduleObject.getTaskAlignments(); + //check if still possible... - boolean possible = true; - if (!possible) return taskNodeAlignment.size(); + if ( scheduleObject.isCheckStillPossible() ) { + boolean possible = validSchedulePlan ( taskNodeAlignment ); + if (!possible) return taskNodeAlignment.size(); + } for (NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment) { if (isClose()) return -1; assignTaskToNode( nodeTaskAlignment ); taskWasScheduled(nodeTaskAlignment.task); - } return unscheduledTasks.size() - taskNodeAlignment.size(); } - abstract List getTaskNodeAlignment( final List unscheduledTasks ); + public boolean validSchedulePlan( List taskNodeAlignment ){ + List items = getNodeList(); + Map< NodeWithAlloc,PodRequirements> availableByNode = new HashMap<>(); + for ( NodeWithAlloc item : items ) { + final PodRequirements availableResources = item.getAvailableResources(); + availableByNode.put(item, availableResources); + } + for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { + availableByNode.get(nodeTaskAlignment.node).subFromThis(nodeTaskAlignment.task.getPod().getRequest()); + } + for ( Map.Entry e : availableByNode.entrySet() ) { + if ( ! e.getValue().higherOrEquals( PodRequirements.ZERO ) ) return false; + } + return true; + } + + abstract ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ); abstract int terminateTasks( final List finishedTasks ); From c14e4f395780459031cd0314cd9db89fb02d5ec8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 6 Jan 2022 15:41:18 +0100 Subject: [PATCH 067/443] Add new location after task was initialized Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/State.java | 2 +- src/main/java/fonda/scheduler/model/Task.java | 8 +++ .../model/TaskInputFileLocationWrapper.java | 22 ++++++++ .../model/location/hierachy/RealFile.java | 8 +++ .../fonda/scheduler/scheduler/Scheduler.java | 10 ++-- .../scheduler/SchedulerWithDaemonSet.java | 54 +++++++++++++++++-- .../fonda/scheduler/util/FileAlignment.java | 4 ++ 7 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java diff --git a/src/main/java/fonda/scheduler/model/State.java b/src/main/java/fonda/scheduler/model/State.java index cbafc138..c1068e73 100644 --- a/src/main/java/fonda/scheduler/model/State.java +++ b/src/main/java/fonda/scheduler/model/State.java @@ -2,6 +2,6 @@ public enum State { - RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, DELETED; + RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, PREPARED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, DELETED; } diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 2a0a8737..b470ad04 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -1,10 +1,14 @@ package fonda.scheduler.model; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.RealFile; import fonda.scheduler.util.Batch; import lombok.Getter; import lombok.Setter; +import java.util.List; + public class Task { @Getter @@ -15,6 +19,10 @@ public class Task { @Getter private final Process process; + @Getter + @Setter + private List< TaskInputFileLocationWrapper > inputFiles; + @Getter @Setter private PodWithAge pod = null; diff --git a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java new file mode 100644 index 00000000..fd745e71 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java @@ -0,0 +1,22 @@ +package fonda.scheduler.model; + +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.RealFile; +import lombok.Getter; + +@Getter +public class TaskInputFileLocationWrapper { + + final private RealFile file; + final private LocationWrapper wrapper; + + public TaskInputFileLocationWrapper(RealFile file, LocationWrapper wrapper) { + this.file = file; + this.wrapper = wrapper; + } + + public void apply(){ + file.addOrUpdateLocation( false, wrapper ); + } + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index e54cafb8..2b16abd1 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -1,6 +1,7 @@ package fonda.scheduler.model.location.hierachy; import fonda.scheduler.model.Process; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import lombok.Getter; @@ -87,4 +88,11 @@ public LocationWrapper getLastUpdate( LocationType type ){ return lastLocation; } + public LocationWrapper getLocationWrapper( Location location ){ + for (LocationWrapper locationWrapper : locations) { + if ( locationWrapper.getLocation() == location ) return locationWrapper; + } + throw new RuntimeException( "Not found: " + location.getIdentifier() ); + } + } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index bf426a20..3000b3a6 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -175,7 +175,11 @@ void taskWasScheduled(Task task ) { synchronized (unscheduledTasks){ unscheduledTasks.remove( task ); } - task.getState().setState( State.SCHEDULED ); + taskWasScheduledSetState( task ); + } + + void taskWasScheduledSetState( Task task ){ + task.getState().setState( State.PREPARED ); } public void markPodAsDeleted( PodWithAge pod ) { @@ -306,7 +310,7 @@ public void endBatch( int tasksInBatch ){ * @param state * @return returns the task, if the state was changed */ - private Task changeStateOfTask( Pod pod, State state ){ + Task changeStateOfTask(Pod pod, State state){ Task t = getTaskByPod( pod ); if( t != null ){ synchronized ( t.getState() ){ @@ -332,7 +336,7 @@ public void informResourceChange() { } } - private Task getTaskByPod( Pod pod ) { + Task getTaskByPod( Pod pod ) { Task t = null; synchronized (tasksByHash) { if( tasksByHash.containsKey( pod.getMetadata().getName() ) ){ diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 95c3a081..9e81805a 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; @@ -21,6 +22,7 @@ import fonda.scheduler.util.inputs.PathFileLocationTriple; import fonda.scheduler.util.inputs.SymlinkInput; import fonda.scheduler.util.FilePath; +import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; @@ -71,7 +73,8 @@ String getDaemonOnNode( Node node ){ @Override void assignTaskToNode( NodeTaskAlignment alignment ) { - writeInitConfig( (NodeTaskFilesAlignment) alignment ); + final List< TaskInputFileLocationWrapper > locationWrappers = writeInitConfig((NodeTaskFilesAlignment) alignment); + alignment.task.setInputFiles( locationWrappers ); getCopyStrategy().generateCopyScript( alignment.task ); super.assignTaskToNode( alignment ); } @@ -108,9 +111,10 @@ int terminateTasks(List finishedTasks) { return 0; } - boolean writeInitConfig( NodeTaskFilesAlignment alignment ) { + List writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); + LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList(); try { final Inputs inputs = new Inputs( this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/" @@ -122,17 +126,34 @@ boolean writeInitConfig( NodeTaskFilesAlignment alignment ) { .map(x -> x.path) .collect(Collectors.toList()); inputs.data.add( new InputEntry( getDaemonOnNode( entry.getKey() ), entry.getKey(), collect ) ); + + final NodeLocation location = NodeLocation.getLocation( entry.getKey() ); + for (FilePath filePath : entry.getValue()) { + final LocationWrapper locationWrapper = filePath.file.getLocationWrapper(location); + inputFiles.add( + new TaskInputFileLocationWrapper( + filePath.file, + new LocationWrapper( + location, + locationWrapper.getTimestamp(), + locationWrapper.getSizeInBytes(), + locationWrapper.getProcess() + ) + ) + ); + } + } for (SymlinkInput symlink : alignment.fileAlignment.symlinks) { inputs.symlinks.add( symlink ); } new ObjectMapper().writeValue( config, inputs ); - return true; + return inputFiles; } catch (IOException e) { e.printStackTrace(); } - return false; + return null; } private Stream streamFile( @@ -219,6 +240,19 @@ public void addFile( String path, long size, long timestamp, boolean overwrite, hierarchyWrapper.addFile( Paths.get( path ), overwrite, locationWrapper ); } + private void podWasInitialized( Pod pod ){ + final Task task = changeStateOfTask(pod, State.PREPARED); + task.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); + } + + /** + * Since task was not yet initialized: set scheduled + * @param task + */ + void taskWasScheduledSetState( Task task ){ + task.getState().setState( State.SCHEDULED ); + } + @Override void podEventReceived(Watcher.Action action, Pod pod){ if ( pod.getMetadata().getName().equals( this.getExecution().replace('_', '-') ) ){ @@ -242,7 +276,7 @@ void podEventReceived(Watcher.Action action, Pod pod){ daemonByNode.remove(nodeName); } } else if ( pod.getStatus().getPhase().equals("Running") ) { - daemonByNode.put( nodeName, pod.getStatus().getPodIP()); + daemonByNode.put( nodeName, pod.getStatus().getPodIP() ); informResourceChange(); } else if ( podIsCurrentDaemon ) { daemonByNode.remove(nodeName); @@ -252,6 +286,16 @@ void podEventReceived(Watcher.Action action, Pod pod){ } } } + } else if ( this.getName().equals(pod.getSpec().getSchedulerName())) { + if ( action == Watcher.Action.MODIFIED ){ + Task t = getTaskByPod( pod ); + if ( t.getState().getState() == State.SCHEDULED ) { + final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); + if ( ! initContainerStatuses.isEmpty() && initContainerStatuses.get(0).getState().getTerminated() != null ) { + podWasInitialized( pod ); + } + } + } } } diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 91016747..71a57c98 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -7,6 +7,10 @@ public class FileAlignment { + /* + Key: node + Value: Files from the node + */ public final Map> nodeFileAlignment; public final List symlinks; From 462a8d5955ddac6e9eb4e8c9174c2a765e22d014 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 4 Feb 2022 12:39:42 +0100 Subject: [PATCH 068/443] Changed request methods Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/rest/SchedulerRestController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index d04fafa9..622737dd 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -47,7 +47,7 @@ public static void addScheduler(Pair key, Scheduler scheduler ){ * @param config Additional parameters for the scheduler * @return */ - @PostMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") + @PutMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { log.trace("Register execution: {} strategy: {} config: {}", execution, strategy, config); @@ -79,7 +79,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable S * @param config The config contains the task name, input files, and optional task parameter the scheduler has to determine * @return Parameters the scheduler suggests for the task */ - @PostMapping("/scheduler/registerTask/{namespace}/{execution}") + @PutMapping("/scheduler/registerTask/{namespace}/{execution}") ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody(required = true) TaskConfig config ) { log.trace( execution + " " + config.getTask() + " got: " + config ); @@ -185,8 +185,8 @@ ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable Strin } - @PostMapping("/file/{namespace}/{execution}") - ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestBody String path ) { + @GetMapping("/file/{namespace}/{execution}") + ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestParam String path ) { log.info( "Get file location request: {} {} {}", namespace, execution, path ); From 537df0aafbc64eb46e3eb7ecf9bc1832c6edcf9a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 4 Feb 2022 16:15:56 +0100 Subject: [PATCH 069/443] Receive DAG Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/DAG.java | 33 +++++++++++++++ src/main/java/fonda/scheduler/dag/Edge.java | 19 +++++++++ .../java/fonda/scheduler/dag/InputEdge.java | 20 ++++++++++ src/main/java/fonda/scheduler/dag/Type.java | 10 +++++ src/main/java/fonda/scheduler/dag/Vertex.java | 35 ++++++++++++++++ .../rest/SchedulerRestController.java | 40 +++++++++++++++++++ .../fonda/scheduler/scheduler/Scheduler.java | 4 ++ 7 files changed, 161 insertions(+) create mode 100644 src/main/java/fonda/scheduler/dag/DAG.java create mode 100644 src/main/java/fonda/scheduler/dag/Edge.java create mode 100644 src/main/java/fonda/scheduler/dag/InputEdge.java create mode 100644 src/main/java/fonda/scheduler/dag/Type.java create mode 100644 src/main/java/fonda/scheduler/dag/Vertex.java diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/fonda/scheduler/dag/DAG.java new file mode 100644 index 00000000..3eddcccb --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/DAG.java @@ -0,0 +1,33 @@ +package fonda.scheduler.dag; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DAG { + + private Map vertices = new HashMap<>(); + + private Vertex getByUid( int uid ){ + final Vertex vertex = vertices.get(uid); + if ( vertex == null ) throw new IllegalStateException( "Cannot find vertex with id " + uid ); + return vertex; + } + + public void registerVertices( List vertices ){ + for (Vertex vertex : vertices) { + synchronized ( this.vertices ) { + this.vertices.put( vertex.getUid(), vertex ); + } + } + } + + public void registerEdges( List edges ){ + for (InputEdge edge : edges) { + final Edge edgeNew = new Edge(edge.getLabel(), getByUid(edge.getFrom()), getByUid(edge.getTo())); + edgeNew.getFrom().addOutbound( edgeNew ); + edgeNew.getTo().addInbound( edgeNew ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/dag/Edge.java b/src/main/java/fonda/scheduler/dag/Edge.java new file mode 100644 index 00000000..3dcb92a5 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Edge.java @@ -0,0 +1,19 @@ +package fonda.scheduler.dag; + +import lombok.Getter; +import lombok.ToString; + +@Getter +@ToString +public class Edge { + + private final String label; + private final Vertex from; + private final Vertex to; + + public Edge(String label, Vertex from, Vertex to) { + this.label = label; + this.from = from; + this.to = to; + } +} diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/fonda/scheduler/dag/InputEdge.java new file mode 100644 index 00000000..0cf2cf6e --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/InputEdge.java @@ -0,0 +1,20 @@ +package fonda.scheduler.dag; + +import lombok.Getter; +import lombok.ToString; + +@Getter +@ToString +public class InputEdge { + + private final String label; + private final int from; + private final int to; + + private InputEdge() { + this.label = null; + this.from = -1; + this.to = -1; + } + +} diff --git a/src/main/java/fonda/scheduler/dag/Type.java b/src/main/java/fonda/scheduler/dag/Type.java new file mode 100644 index 00000000..edf4d6dc --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Type.java @@ -0,0 +1,10 @@ +package fonda.scheduler.dag; + +public enum Type { + + PROCESS, + OPERATOR, + ORIGIN, + NODE + +} diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java new file mode 100644 index 00000000..876a2f85 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -0,0 +1,35 @@ +package fonda.scheduler.dag; + +import lombok.Getter; +import lombok.ToString; + +import java.util.LinkedList; +import java.util.List; + +@Getter +@ToString +public class Vertex { + + private final String label; + private final Type type; + private final int uid; + private List in = new LinkedList<>(); + private List out = new LinkedList<>(); + + private Vertex() { + label = null; + type = null; + uid = -1; + } + + public void addInbound( Edge e ) { + synchronized ( in) { + in.add(e); + } + } + + public void addOutbound( Edge e ) { + out.add( e ); + } + +} diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 622737dd..6ad1977e 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -1,6 +1,10 @@ package fonda.scheduler.rest; import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.Edge; +import fonda.scheduler.dag.InputEdge; +import fonda.scheduler.dag.Vertex; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.RandomScheduler; @@ -16,6 +20,7 @@ import org.springframework.web.bind.annotation.*; import java.util.HashMap; +import java.util.List; import java.util.Map; @RestController @@ -251,4 +256,39 @@ private Pair getKey(String namespace, String execution ){ return new Pair(namespace.toLowerCase(), execution.toLowerCase()); } + @PutMapping("/scheduler/DAG/addVertices/{namespace}/{execution}") + ResponseEntity addVertices(@PathVariable String namespace, @PathVariable String execution, @RequestBody List vertices ) { + + log.trace( "submit vertices: {}", vertices ); + + final Pair key = getKey( namespace, execution ); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + scheduler.getDag().registerVertices( vertices ); + + return new ResponseEntity( HttpStatus.OK ); + + } + + @PutMapping("/scheduler/DAG/addEdges/{namespace}/{execution}") + ResponseEntity addEdges(@PathVariable String namespace, @PathVariable String execution, @RequestBody List edges ) { + + log.trace( "submit edges: {}", edges ); + + final Pair key = getKey( namespace, execution ); + final Scheduler scheduler = schedulerHolder.get( key ); + if( scheduler == null ){ + return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + } + + final DAG dag = scheduler.getDag(); + dag.registerEdges( edges ); + + return new ResponseEntity( HttpStatus.OK ); + + } + } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 3000b3a6..4944a0d3 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -1,6 +1,7 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.dag.DAG; import fonda.scheduler.model.*; import fonda.scheduler.util.Batch; import fonda.scheduler.util.NodeTaskAlignment; @@ -34,6 +35,8 @@ public abstract class Scheduler { private final String dns; @Getter private boolean close; + @Getter + private DAG dag; private final Object batchHelper = new Object(); private int currentBatch = 0; @@ -55,6 +58,7 @@ public abstract class Scheduler { log.trace( "Register scheduler for " + this.name ); this.client = client; this.dns = config.dns; + this.dag = new DAG(); PodWatcher podWatcher = new PodWatcher(this); From a2e4d40db11773a6d4635ab502662dc70fcbceef Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 4 Feb 2022 16:16:55 +0100 Subject: [PATCH 070/443] ToString Method for PathAttributes Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/rest/PathAttributes.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/fonda/scheduler/rest/PathAttributes.java b/src/main/java/fonda/scheduler/rest/PathAttributes.java index e79f69d8..17398f09 100644 --- a/src/main/java/fonda/scheduler/rest/PathAttributes.java +++ b/src/main/java/fonda/scheduler/rest/PathAttributes.java @@ -1,8 +1,10 @@ package fonda.scheduler.rest; import lombok.Getter; +import lombok.ToString; @Getter +@ToString public class PathAttributes { private final String path; From 398fbca82b69f44efaee47f9e573fae82b1321d9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Feb 2022 14:43:33 +0100 Subject: [PATCH 071/443] Resolve Descendants and Ancestors Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/DAG.java | 21 ++- src/main/java/fonda/scheduler/dag/Edge.java | 2 +- .../java/fonda/scheduler/dag/InputEdge.java | 10 ++ src/main/java/fonda/scheduler/dag/Vertex.java | 110 ++++++++++-- .../java/fonda/scheduler/dag/DAGTest.java | 158 ++++++++++++++++++ 5 files changed, 284 insertions(+), 17 deletions(-) create mode 100644 src/main/test/java/fonda/scheduler/dag/DAGTest.java diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/fonda/scheduler/dag/DAG.java index 3eddcccb..c01a2707 100644 --- a/src/main/java/fonda/scheduler/dag/DAG.java +++ b/src/main/java/fonda/scheduler/dag/DAG.java @@ -7,26 +7,37 @@ public class DAG { private Map vertices = new HashMap<>(); + private Map processes = new HashMap<>(); - private Vertex getByUid( int uid ){ + Vertex getByUid( int uid ){ final Vertex vertex = vertices.get(uid); if ( vertex == null ) throw new IllegalStateException( "Cannot find vertex with id " + uid ); return vertex; } + public Vertex getByProcess( String process ){ + final Vertex vertex = processes.get( process ); + return vertex; + } + public void registerVertices( List vertices ){ for (Vertex vertex : vertices) { synchronized ( this.vertices ) { this.vertices.put( vertex.getUid(), vertex ); } + synchronized ( this.processes ) { + this.processes.put( vertex.getLabel(), vertex ); + } } } public void registerEdges( List edges ){ - for (InputEdge edge : edges) { - final Edge edgeNew = new Edge(edge.getLabel(), getByUid(edge.getFrom()), getByUid(edge.getTo())); - edgeNew.getFrom().addOutbound( edgeNew ); - edgeNew.getTo().addInbound( edgeNew ); + synchronized ( this.vertices ) { + for (InputEdge edge : edges) { + final Edge edgeNew = new Edge(edge.getLabel(), getByUid(edge.getFrom()), getByUid(edge.getTo())); + edgeNew.getFrom().addOutbound(edgeNew); + edgeNew.getTo().addInbound(edgeNew); + } } } diff --git a/src/main/java/fonda/scheduler/dag/Edge.java b/src/main/java/fonda/scheduler/dag/Edge.java index 3dcb92a5..8cb8fb3a 100644 --- a/src/main/java/fonda/scheduler/dag/Edge.java +++ b/src/main/java/fonda/scheduler/dag/Edge.java @@ -11,7 +11,7 @@ public class Edge { private final Vertex from; private final Vertex to; - public Edge(String label, Vertex from, Vertex to) { + Edge(String label, Vertex from, Vertex to) { this.label = label; this.from = from; this.to = to; diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/fonda/scheduler/dag/InputEdge.java index 0cf2cf6e..81290d94 100644 --- a/src/main/java/fonda/scheduler/dag/InputEdge.java +++ b/src/main/java/fonda/scheduler/dag/InputEdge.java @@ -17,4 +17,14 @@ private InputEdge() { this.to = -1; } + /** + * Just for testing + * @param from + * @param to + */ + InputEdge(int from, int to) { + label = null; + this.from = from; + this.to = to; + } } diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index 876a2f85..a22469e2 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -1,35 +1,123 @@ package fonda.scheduler.dag; import lombok.Getter; -import lombok.ToString; -import java.util.LinkedList; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; @Getter -@ToString public class Vertex { private final String label; private final Type type; private final int uid; - private List in = new LinkedList<>(); - private List out = new LinkedList<>(); + private final List in = new LinkedList<>(); + private final List out = new LinkedList<>(); + private final Set descendants; + private final Set ancestors; private Vertex() { - label = null; - type = null; - uid = -1; + this( null, null, -1 ); + } + + /** + * Just for testing + * @param label + * @param type + * @param uid + */ + Vertex(String label, Type type, int uid) { + this.label = label; + this.type = type; + this.uid = uid; + if ( type == Type.PROCESS ){ + descendants = new HashSet<>(); + ancestors = new HashSet<>(); + } else { + descendants = null; + ancestors = null; + } } public void addInbound( Edge e ) { - synchronized ( in) { - in.add(e); + in.add( e ); + final Vertex from = e.getFrom(); + final Set ancestors = from.getAncestors(); + + if ( from.type == Type.PROCESS ) ancestors.add( from ); + + if ( type == Type.PROCESS ) { + this.ancestors.addAll(ancestors); + if ( from.type == Type.PROCESS ) from.descendants.add( this ); } + + final Set descendants = this.getDescendants(); + ancestors.forEach( v -> { + v.descendants.addAll( descendants ); + if ( type == Type.PROCESS ) v.descendants.add( this ); + }); } public void addOutbound( Edge e ) { out.add( e ); + final Vertex to = e.getTo(); + final Set descendants = to.getDescendants(); + + if ( to.type == Type.PROCESS ) descendants.add(to); + + if ( type == Type.PROCESS ) { + this.descendants.addAll(descendants); + if ( to.type == Type.PROCESS ) to.ancestors.add( this ); + } + + final Set ancestors = this.getAncestors(); + descendants.forEach( v -> { + v.ancestors.addAll( ancestors ); + if ( type == Type.PROCESS ) v.ancestors.add( this ); + }); } + public Set getDescendants() { + final HashSet results = new HashSet<>(); + if ( this.type == Type.PROCESS ) { + results.addAll( descendants ); + } else if ( !out.isEmpty() ) { + for ( Edge edge : out ) { + final Vertex to = edge.getTo(); + if ( to.getType() == Type.PROCESS ) results.add( to ); + results.addAll( to.getDescendants() ); + } + } + return results; + } + + public Set getAncestors() { + final HashSet results = new HashSet<>(); + if ( this.type == Type.PROCESS ) { + results.addAll( ancestors ); + } else if ( !in.isEmpty() && type != Type.ORIGIN ){ + for ( Edge edge : in ) { + final Vertex from = edge.getFrom(); + if ( from.getType() == Type.PROCESS ) results.add( from ); + results.addAll( from.getAncestors() ); + } + } + return results; + } + + private String collectionToString( Collection v ){ + return v.stream().map( Vertex::getUid ).sorted().map( c -> c.toString() ) + .collect(Collectors.joining(",")); + } + + @Override + public String toString() { + return "Vertex{" + + "label='" + label + '\'' + + ", type=" + type + + ", uid=" + uid + + ", descendants=" + collectionToString( getDescendants() ) + + ", ancestors=" + collectionToString( getAncestors() ) + + '}'; + } } diff --git a/src/main/test/java/fonda/scheduler/dag/DAGTest.java b/src/main/test/java/fonda/scheduler/dag/DAGTest.java new file mode 100644 index 00000000..38663696 --- /dev/null +++ b/src/main/test/java/fonda/scheduler/dag/DAGTest.java @@ -0,0 +1,158 @@ +package fonda.scheduler.dag; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +@Slf4j +public class DAGTest { + + private void compare(int uid, List vertices, int[] ancestorIds, int[] descedantsIds ){ + final Vertex vertex = vertices.stream().filter(v -> v.getUid() == uid).findFirst().get(); + + if( vertex.getAncestors() == null ){ + assertNull( ancestorIds ); + } else { + final int[] a = vertex.getAncestors().stream().mapToInt( Vertex::getUid ).sorted().toArray(); + assertArrayEquals("Compare Ancestors for uid: " + uid, ancestorIds, a); + } + if( vertex.getDescendants() == null ){ + assertNull( descedantsIds ); + } else { + final int[] d = vertex.getDescendants().stream().mapToInt(Vertex::getUid).sorted().toArray(); + assertArrayEquals("Compare Descendants for uid: " + uid, descedantsIds, d); + } + } + + public void expectedResult( List vertexList ) { + compare( 1, vertexList, new int[]{}, new int[]{2,3,4,5,6,7,8,9,10,11,12} ); + compare( 2, vertexList, new int[]{1} , new int[]{6,8,10,12} ); + compare( 3, vertexList, new int[]{1}, new int[]{7,8,10,12} ); + compare( 4, vertexList, new int[]{1}, new int[]{7,8,9,10,11,12} ); + compare( 5, vertexList, new int[]{1}, new int[]{8,9,10,11,12} ); + compare( 6, vertexList, new int[]{1,2}, new int[]{8,10,12} ); + compare( 7, vertexList, new int[]{1,3,4}, new int[]{8,10,12} ); + compare( 8, vertexList, new int[]{1,2,3,4,5,6,7,9}, new int[]{10,12} ); + compare( 9, vertexList, new int[]{1,4,5}, new int[]{8,10,11,12} ); + compare( 10, vertexList, new int[]{1,2,3,4,5,6,7,8,9}, new int[]{12} ); + compare( 11, vertexList, new int[]{1,4,5,9}, new int[]{12} ); + compare( 12, vertexList, new int[]{1,2,3,4,5,6,7,8,9,10,11}, new int[]{} ); + } + + public List genVertexList(){ + List vertexList = new LinkedList<>(); + for (int i = 1; i <= 12; i++) { + vertexList.add( new Vertex("" + (char) ('a' + i), Type.PROCESS, i) ); + } + return vertexList; + } + + public List genEdgeList(){ + List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(1,2) ); + inputEdges.add( new InputEdge(1,3) ); + inputEdges.add( new InputEdge(1,4) ); + inputEdges.add( new InputEdge(1,5) ); + inputEdges.add( new InputEdge(2,6) ); + inputEdges.add( new InputEdge(3,7) ); + inputEdges.add( new InputEdge(4,7) ); + inputEdges.add( new InputEdge(4,9) ); + inputEdges.add( new InputEdge(5,9) ); + inputEdges.add( new InputEdge(6,8) ); + inputEdges.add( new InputEdge(7,8) ); + inputEdges.add( new InputEdge(9,8) ); + inputEdges.add( new InputEdge(8,10) ); + inputEdges.add( new InputEdge(9,11) ); + inputEdges.add( new InputEdge(10,12) ); + inputEdges.add( new InputEdge(11,12) ); + + Collections.shuffle( inputEdges ); + return inputEdges; + } + + public void debug( DAG dag, int length ){ + for (int i = 1; i <= length; i++) { + final Vertex process = dag.getByUid( i ); + log.info( process.toString() ); + } + } + + @Test + public void testRelations() { + for (int q = 0; q < 500 ; q++) { + final DAG dag = new DAG(); + List vertexList = genVertexList(); + dag.registerVertices( vertexList ); + List inputEdges = genEdgeList(); + dag.registerEdges( inputEdges ); + debug( dag, 12 ); + expectedResult ( vertexList ); + } + } + + @Test + public void testRelations2() { + for (int q = 0; q < 500 ; q++) { + final DAG dag = new DAG(); + List vertexList = genVertexList(); + List inputEdges = genEdgeList(); + for (int i = 0; i < 500; i++) { + int index = new Random().nextInt( inputEdges.size() ); + final InputEdge remove = inputEdges.remove(index); + final Vertex operator = new Vertex("Operator", Type.OPERATOR, vertexList.size() + 1 ); + vertexList.add( operator ); + inputEdges.add( new InputEdge( remove.getFrom(), operator.getUid() ) ); + inputEdges.add( new InputEdge( operator.getUid(), remove.getTo() ) ); + } + dag.registerVertices( vertexList ); + dag.registerEdges( inputEdges ); + debug( dag, 12 ); + expectedResult ( vertexList ); + } + } + + @Test + public void smallTest(){ + + final DAG dag = new DAG(); + + final Vertex a = new Vertex("a", Type.PROCESS, 1); + final Vertex filter = new Vertex("filter", Type.OPERATOR, 2); + final Vertex b = new Vertex("b", Type.PROCESS, 3); + + List vertexList = new LinkedList<>(); + vertexList.add(a); + vertexList.add(filter); + vertexList.add(b); + + List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(1,2) ); + inputEdges.add( new InputEdge(2,3) ); + + dag.registerVertices( vertexList ); + dag.registerEdges( inputEdges ); + + debug( dag, vertexList.size() ); + + assertEquals( new HashSet<>(), a.getAncestors() ); + final HashSet descA = new HashSet<>(); + descA.add( b ); + assertEquals( descA, a.getDescendants() ); + + + final HashSet ancB = new HashSet<>(); + ancB.add( a ); + assertEquals( new HashSet<>(), b.getDescendants() ); + assertEquals( ancB, b.getAncestors() ); + + assertEquals( ancB, filter.getAncestors() ); + assertEquals( descA, filter.getDescendants() ); + + } + + +} \ No newline at end of file From c3c8db5b88febc8f7792a4f81ce9e6acde8a5065 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 11:36:36 +0100 Subject: [PATCH 072/443] Introduced class hierarchy Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/DAG.java | 17 +-- .../java/fonda/scheduler/dag/NotProcess.java | 33 ++++++ .../java/fonda/scheduler/dag/Operator.java | 38 +++++++ src/main/java/fonda/scheduler/dag/Origin.java | 26 +++++ .../java/fonda/scheduler/dag/Process.java | 65 +++++++++++ src/main/java/fonda/scheduler/dag/Vertex.java | 106 +++--------------- .../scheduler/dag/VertexDeserializer.java | 37 ++++++ .../rest/SchedulerRestController.java | 1 - .../java/fonda/scheduler/dag/DAGTest.java | 39 ++++--- .../scheduler/dag/VertexDeserializerTest.java | 31 +++++ 10 files changed, 279 insertions(+), 114 deletions(-) create mode 100644 src/main/java/fonda/scheduler/dag/NotProcess.java create mode 100644 src/main/java/fonda/scheduler/dag/Operator.java create mode 100644 src/main/java/fonda/scheduler/dag/Origin.java create mode 100644 src/main/java/fonda/scheduler/dag/Process.java create mode 100644 src/main/java/fonda/scheduler/dag/VertexDeserializer.java create mode 100644 src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/fonda/scheduler/dag/DAG.java index c01a2707..3b5e5935 100644 --- a/src/main/java/fonda/scheduler/dag/DAG.java +++ b/src/main/java/fonda/scheduler/dag/DAG.java @@ -6,18 +6,17 @@ public class DAG { - private Map vertices = new HashMap<>(); - private Map processes = new HashMap<>(); + private final Map vertices = new HashMap<>(); + private final Map processes = new HashMap<>(); - Vertex getByUid( int uid ){ + private Vertex getByUid( int uid ){ final Vertex vertex = vertices.get(uid); if ( vertex == null ) throw new IllegalStateException( "Cannot find vertex with id " + uid ); return vertex; } - public Vertex getByProcess( String process ){ - final Vertex vertex = processes.get( process ); - return vertex; + public Process getByProcess( String s ){ + return processes.get( s ); } public void registerVertices( List vertices ){ @@ -25,8 +24,10 @@ public void registerVertices( List vertices ){ synchronized ( this.vertices ) { this.vertices.put( vertex.getUid(), vertex ); } - synchronized ( this.processes ) { - this.processes.put( vertex.getLabel(), vertex ); + if ( vertex.getType() == Type.PROCESS ) { + synchronized ( this.processes ) { + this.processes.put( vertex.getLabel(), (Process) vertex ); + } } } } diff --git a/src/main/java/fonda/scheduler/dag/NotProcess.java b/src/main/java/fonda/scheduler/dag/NotProcess.java new file mode 100644 index 00000000..e64d7ec1 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/NotProcess.java @@ -0,0 +1,33 @@ +package fonda.scheduler.dag; + +import java.util.HashSet; +import java.util.Set; + +public abstract class NotProcess extends Vertex { + + NotProcess(String label, int uid) { + super(label, uid); + } + + public Set getDescendants() { + final HashSet results = new HashSet<>(); + if ( !out.isEmpty() ) { + for ( Edge edge : out ) { + final Vertex to = edge.getTo(); + if ( to.getType() == Type.PROCESS ) results.add((Process) to); + results.addAll( to.getDescendants() ); + } + } + return results; + } + + public void addOutbound( Edge e ) { + out.add( e ); + final Vertex to = e.getTo(); + final Set descendants = to.getDescendants(); + if ( to.getType() == Type.PROCESS ) descendants.add((Process) to); + final Set ancestors = this.getAncestors(); + descendants.forEach( v -> v.ancestors.addAll( ancestors ) ); + } + +} diff --git a/src/main/java/fonda/scheduler/dag/Operator.java b/src/main/java/fonda/scheduler/dag/Operator.java new file mode 100644 index 00000000..e502cb70 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Operator.java @@ -0,0 +1,38 @@ +package fonda.scheduler.dag; + +import java.util.HashSet; +import java.util.Set; + +public class Operator extends NotProcess { + + Operator( String label, int uid ) { + super(label,uid); + } + + @Override + public Type getType() { + return Type.OPERATOR; + } + + public Set getAncestors() { + final HashSet results = new HashSet<>(); + if ( !in.isEmpty() ){ + for ( Edge edge : in ) { + final Vertex from = edge.getFrom(); + if ( from.getType() == Type.PROCESS ) results.add((Process) from); + results.addAll( from.getAncestors() ); + } + } + return results; + } + + public void addInbound( Edge e ) { + in.add( e ); + final Vertex from = e.getFrom(); + final Set ancestors = from.getAncestors(); + if ( from.getType() == Type.PROCESS ) ancestors.add((Process) from); + final Set descendants = this.getDescendants(); + ancestors.forEach( v -> v.descendants.addAll( descendants )); + } + +} diff --git a/src/main/java/fonda/scheduler/dag/Origin.java b/src/main/java/fonda/scheduler/dag/Origin.java new file mode 100644 index 00000000..b9b1e5c2 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Origin.java @@ -0,0 +1,26 @@ +package fonda.scheduler.dag; + +import java.util.HashSet; +import java.util.Set; + +public class Origin extends NotProcess { + + Origin(String label, int uid) { + super(label, uid); + } + + @Override + public Type getType() { + return Type.ORIGIN; + } + + @Override + public void addInbound(Edge e) { + throw new IllegalStateException("Cannot add an inbound to an Origin"); + } + + public Set getAncestors() { + return new HashSet<>(); + } + +} diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/fonda/scheduler/dag/Process.java new file mode 100644 index 00000000..38d528cf --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Process.java @@ -0,0 +1,65 @@ +package fonda.scheduler.dag; + +import java.util.HashSet; +import java.util.Set; + +public class Process extends Vertex { + + + final Set descendants; + final Set ancestors; + + Process( String label, int uid ) { + super(label, uid); + descendants = new HashSet<>(); + ancestors = new HashSet<>(); + } + + @Override + public Type getType() { + return Type.PROCESS; + } + + public Set getDescendants() { + return new HashSet<>( descendants ); + } + + public Set getAncestors() { + return new HashSet<>( ancestors ); + } + + public void addInbound( Edge e ) { + in.add( e ); + final Vertex from = e.getFrom(); + final Set ancestors = from.getAncestors(); + + if ( from.getType() == Type.PROCESS ) ancestors.add((Process) from); + + this.ancestors.addAll(ancestors); + if ( from.getType() == Type.PROCESS ) ((Process) from).descendants.add( this ); + + final Set descendants = this.getDescendants(); + ancestors.forEach( v -> { + v.descendants.addAll( descendants ); + v.descendants.add( this ); + }); + } + + public void addOutbound( Edge e ) { + out.add( e ); + final Vertex to = e.getTo(); + final Set descendants = to.getDescendants(); + + if ( to.getType() == Type.PROCESS ) descendants.add((Process) to); + + this.descendants.addAll(descendants); + if ( to.getType() == Type.PROCESS ) ((Process)to).ancestors.add( this ); + + final Set ancestors = this.getAncestors(); + descendants.forEach( v -> { + v.ancestors.addAll( ancestors ); + v.ancestors.add( this ); + }); + } + +} diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index a22469e2..96ffed9a 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -6,116 +6,38 @@ import java.util.stream.Collectors; @Getter -public class Vertex { +public abstract class Vertex { private final String label; - private final Type type; private final int uid; - private final List in = new LinkedList<>(); - private final List out = new LinkedList<>(); - private final Set descendants; - private final Set ancestors; + final List in = new LinkedList<>(); + final List out = new LinkedList<>(); - private Vertex() { - this( null, null, -1 ); - } - - /** - * Just for testing - * @param label - * @param type - * @param uid - */ - Vertex(String label, Type type, int uid) { + Vertex( String label, int uid ) { this.label = label; - this.type = type; this.uid = uid; - if ( type == Type.PROCESS ){ - descendants = new HashSet<>(); - ancestors = new HashSet<>(); - } else { - descendants = null; - ancestors = null; - } } - public void addInbound( Edge e ) { - in.add( e ); - final Vertex from = e.getFrom(); - final Set ancestors = from.getAncestors(); - - if ( from.type == Type.PROCESS ) ancestors.add( from ); - - if ( type == Type.PROCESS ) { - this.ancestors.addAll(ancestors); - if ( from.type == Type.PROCESS ) from.descendants.add( this ); - } - - final Set descendants = this.getDescendants(); - ancestors.forEach( v -> { - v.descendants.addAll( descendants ); - if ( type == Type.PROCESS ) v.descendants.add( this ); - }); - } + public abstract Type getType(); - public void addOutbound( Edge e ) { - out.add( e ); - final Vertex to = e.getTo(); - final Set descendants = to.getDescendants(); + public abstract void addInbound( Edge e ); - if ( to.type == Type.PROCESS ) descendants.add(to); + public abstract void addOutbound( Edge e ); - if ( type == Type.PROCESS ) { - this.descendants.addAll(descendants); - if ( to.type == Type.PROCESS ) to.ancestors.add( this ); - } + public abstract Set getDescendants(); - final Set ancestors = this.getAncestors(); - descendants.forEach( v -> { - v.ancestors.addAll( ancestors ); - if ( type == Type.PROCESS ) v.ancestors.add( this ); - }); - } - - public Set getDescendants() { - final HashSet results = new HashSet<>(); - if ( this.type == Type.PROCESS ) { - results.addAll( descendants ); - } else if ( !out.isEmpty() ) { - for ( Edge edge : out ) { - final Vertex to = edge.getTo(); - if ( to.getType() == Type.PROCESS ) results.add( to ); - results.addAll( to.getDescendants() ); - } - } - return results; - } - - public Set getAncestors() { - final HashSet results = new HashSet<>(); - if ( this.type == Type.PROCESS ) { - results.addAll( ancestors ); - } else if ( !in.isEmpty() && type != Type.ORIGIN ){ - for ( Edge edge : in ) { - final Vertex from = edge.getFrom(); - if ( from.getType() == Type.PROCESS ) results.add( from ); - results.addAll( from.getAncestors() ); - } - } - return results; - } + public abstract Set getAncestors(); - private String collectionToString( Collection v ){ - return v.stream().map( Vertex::getUid ).sorted().map( c -> c.toString() ) + private String collectionToString( Collection v ){ + return v.stream().map( Process::getUid ).sorted().map( Object::toString ) .collect(Collectors.joining(",")); } @Override public String toString() { - return "Vertex{" + - "label='" + label + '\'' + - ", type=" + type + - ", uid=" + uid + + return getType() + "{" + + label != null ? ("label='" + label + "', '") : "" + + "uid=" + uid + ", descendants=" + collectionToString( getDescendants() ) + ", ancestors=" + collectionToString( getAncestors() ) + '}'; diff --git a/src/main/java/fonda/scheduler/dag/VertexDeserializer.java b/src/main/java/fonda/scheduler/dag/VertexDeserializer.java new file mode 100644 index 00000000..80a877a2 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/VertexDeserializer.java @@ -0,0 +1,37 @@ +package fonda.scheduler.dag; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.springframework.boot.jackson.JsonComponent; + +import java.io.IOException; + +@JsonComponent +public class VertexDeserializer extends JsonDeserializer { + + @Override + public Vertex deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException { + + final TreeNode treeNode = jsonParser.getCodec().readTree(jsonParser); + + final Type type = Type.valueOf(((TextNode) treeNode.get("type")).asText()); + final String label = ((TextNode) treeNode.get("label")).asText(); + final int uid = ((IntNode) treeNode.get("uid")).asInt(); + + if ( Type.PROCESS == type ) { + return new Process( label, uid ); + } else if ( Type.OPERATOR == type ) { + return new Operator( label, uid ); + } else if ( Type.ORIGIN == type ) { + return new Origin( label, uid ); + } else { + throw new IllegalArgumentException( "No implementation for type: " + type ); + } + + } + +} diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 6ad1977e..1382a800 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -2,7 +2,6 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.Edge; import fonda.scheduler.dag.InputEdge; import fonda.scheduler.dag.Vertex; import fonda.scheduler.rest.exceptions.NotARealFileException; diff --git a/src/main/test/java/fonda/scheduler/dag/DAGTest.java b/src/main/test/java/fonda/scheduler/dag/DAGTest.java index 38663696..bec5da29 100644 --- a/src/main/test/java/fonda/scheduler/dag/DAGTest.java +++ b/src/main/test/java/fonda/scheduler/dag/DAGTest.java @@ -4,7 +4,6 @@ import org.junit.Test; import java.util.*; -import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -45,14 +44,17 @@ public void expectedResult( List vertexList ) { public List genVertexList(){ List vertexList = new LinkedList<>(); - for (int i = 1; i <= 12; i++) { - vertexList.add( new Vertex("" + (char) ('a' + i), Type.PROCESS, i) ); + int i = 1; + for (; i <= 12; i++) { + vertexList.add( new Process("" + (char) ('a' + i), i) ); } + vertexList.add( new Origin("Origin", 13 ) ); return vertexList; } public List genEdgeList(){ List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(13,1) ); inputEdges.add( new InputEdge(1,2) ); inputEdges.add( new InputEdge(1,3) ); inputEdges.add( new InputEdge(1,4) ); @@ -74,9 +76,9 @@ public List genEdgeList(){ return inputEdges; } - public void debug( DAG dag, int length ){ - for (int i = 1; i <= length; i++) { - final Vertex process = dag.getByUid( i ); + public void debug( List vertices, int start, int end ){ + for (int i = start; i <= end; i++) { + final Vertex process = vertices.get( i ) ; log.info( process.toString() ); } } @@ -89,7 +91,7 @@ public void testRelations() { dag.registerVertices( vertexList ); List inputEdges = genEdgeList(); dag.registerEdges( inputEdges ); - debug( dag, 12 ); + debug( vertexList, 0, 11 ); expectedResult ( vertexList ); } } @@ -103,14 +105,14 @@ public void testRelations2() { for (int i = 0; i < 500; i++) { int index = new Random().nextInt( inputEdges.size() ); final InputEdge remove = inputEdges.remove(index); - final Vertex operator = new Vertex("Operator", Type.OPERATOR, vertexList.size() + 1 ); + final Vertex operator = new Operator("Operator", vertexList.size() + 1 ); vertexList.add( operator ); inputEdges.add( new InputEdge( remove.getFrom(), operator.getUid() ) ); inputEdges.add( new InputEdge( operator.getUid(), remove.getTo() ) ); } dag.registerVertices( vertexList ); dag.registerEdges( inputEdges ); - debug( dag, 12 ); + debug( vertexList, 0, 11 ); expectedResult ( vertexList ); } } @@ -120,11 +122,14 @@ public void smallTest(){ final DAG dag = new DAG(); - final Vertex a = new Vertex("a", Type.PROCESS, 1); - final Vertex filter = new Vertex("filter", Type.OPERATOR, 2); - final Vertex b = new Vertex("b", Type.PROCESS, 3); + + final Origin o = new Origin("o", 1); + final Process a = new Process("a", 2); + final Operator filter = new Operator("filter", 3); + final Process b = new Process("b", 4); List vertexList = new LinkedList<>(); + vertexList.add(o); vertexList.add(a); vertexList.add(filter); vertexList.add(b); @@ -132,11 +137,12 @@ public void smallTest(){ List inputEdges = new LinkedList<>(); inputEdges.add( new InputEdge(1,2) ); inputEdges.add( new InputEdge(2,3) ); + inputEdges.add( new InputEdge(3,4) ); dag.registerVertices( vertexList ); dag.registerEdges( inputEdges ); - debug( dag, vertexList.size() ); + debug( vertexList, 0, 3 ); assertEquals( new HashSet<>(), a.getAncestors() ); final HashSet descA = new HashSet<>(); @@ -152,6 +158,13 @@ public void smallTest(){ assertEquals( ancB, filter.getAncestors() ); assertEquals( descA, filter.getDescendants() ); + assertEquals( new HashSet<>(),o.getAncestors() ); + final HashSet descO = new HashSet<>(); + + descO.add( a ); + descO.add( b ); + assertEquals( descO, o.getDescendants() ); + } diff --git a/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java b/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java new file mode 100644 index 00000000..8e6ae3da --- /dev/null +++ b/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java @@ -0,0 +1,31 @@ +package fonda.scheduler.dag; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.json.JsonTest; +import org.springframework.boot.test.json.JacksonTester; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.IOException; + +@JsonTest +public class VertexDeserializerTest { + + + @Test + public void testDeserialize() throws IOException { + String json = "{\"label\":\"a\", \"type\":\"PROCESS\", \"uid\":0}"; + final ObjectMapper objectMapper = new ObjectMapper(); + SimpleModule module = new SimpleModule(); + module.addDeserializer(Vertex.class, new VertexDeserializer()); + objectMapper.registerModule(module); + final Vertex process = objectMapper.readValue(json, Vertex.class); + System.out.println( process ); + } + +} \ No newline at end of file From b8b9ae207a85f62e054ec2ac0067b0a7e19a3eae Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 11:37:12 +0100 Subject: [PATCH 073/443] updated junit version to 4.13 Signed-off-by: Lehmann_Fabian --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index baed7ec8..3ed65af2 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ junit junit - 4.11 + 4.13 From c0340828901c5a1cc95c1aa671f6fb48e2526bda Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 15:40:30 +0100 Subject: [PATCH 074/443] changed fabric8 version to 5.12.1 Signed-off-by: Lehmann_Fabian --- pom.xml | 2 +- .../scheduler/client/KubernetesClient.java | 42 +++---------------- 2 files changed, 7 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index 3ed65af2..2c0f375d 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ io.fabric8 kubernetes-client - 5.1.1 + 5.12.1 diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index fbcc98c2..dcd0df44 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -3,19 +3,16 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; import fonda.scheduler.scheduler.Scheduler; -import io.fabric8.kubernetes.api.model.*; +import io.fabric8.kubernetes.api.model.ContainerStatus; +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.DefaultKubernetesClient; -import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; import io.fabric8.kubernetes.client.dsl.base.OperationContext; -import io.fabric8.kubernetes.client.informers.ListerWatcher; -import io.fabric8.kubernetes.client.informers.SharedInformerEventListener; -import io.fabric8.kubernetes.client.informers.impl.DefaultSharedIndexInformer; import lombok.extern.slf4j.Slf4j; import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; @Slf4j public class KubernetesClient extends DefaultKubernetesClient { @@ -23,27 +20,26 @@ public class KubernetesClient extends DefaultKubernetesClient { private final Map nodeHolder= new HashMap<>(); private final List schedulerList = new LinkedList<>(); private OperationContext operationContext; - private DefaultSharedIndexInformer defaultSharedIndexInformerNode; - private ConcurrentLinkedQueue workerQueueNode; public KubernetesClient(){ this.operationContext = new OperationContext(); - this.workerQueueNode = new ConcurrentLinkedQueue<>(); - setUpIndexInformerNode(); for( Node node : this.nodes().list().getItems() ){ nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node) ); } this.pods().inAnyNamespace().watch( new PodWatcher( this ) ); + this.nodes().watch( new NodeWatcher( this ) ); } public void addScheduler( Scheduler scheduler ){ + log.info("Added scheduler {}", scheduler.getName()); synchronized ( schedulerList ){ schedulerList.add( scheduler ); } } public void removeScheduler( Scheduler scheduler ){ + log.info("Removed scheduler {}", scheduler.getName()); synchronized ( schedulerList ){ schedulerList.remove( scheduler ); } @@ -77,32 +73,6 @@ public List getAllNodes(){ return new ArrayList<>(this.nodeHolder.values()); } - /** - * - */ - private void setUpIndexInformerNode() { - ListerWatcher listerWatcher = new ListerWatcher() { - @Override - public Watch watch(ListOptions params, String namespace, OperationContext context, Watcher watcher) { - return KubernetesClient.this.nodes().watch(new NodeWatcher( KubernetesClient.this )); - } - - @Override - public Object list(ListOptions params, String namespace, OperationContext context) { - return KubernetesClient.this.nodes().list(); - } - }; - - defaultSharedIndexInformerNode = new DefaultSharedIndexInformer(Node.class, listerWatcher, 6000, this.operationContext, this.workerQueueNode); - - try { - defaultSharedIndexInformerNode.run(); - } catch (Exception e) { - e.printStackTrace(); - } - - } - class NodeWatcher implements Watcher{ private final KubernetesClient kubernetesClient; From 2d987a6f95144193b209d4ca2740a379ebb92450 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 15:41:05 +0100 Subject: [PATCH 075/443] call add and remove Scheduler in KubernetesScheduler Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/rest/SchedulerRestController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 1382a800..73ba3e03 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -71,6 +71,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable S final Pair key = getKey( namespace, execution ); schedulerHolder.put( key, scheduler ); + client.addScheduler( scheduler ); return new ResponseEntity( HttpStatus.OK ); @@ -164,6 +165,7 @@ ResponseEntity delete(@PathVariable String namespace, @PathVariable String exec return new ResponseEntity( "No scheduler for: " + execution, HttpStatus.NOT_FOUND ); } schedulerHolder.remove( key ); + client.removeScheduler( scheduler ); scheduler.close(); return new ResponseEntity( HttpStatus.OK ); } From d29e83db525d4ff7088f6aa79e9d41bf9704c82d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 15:41:34 +0100 Subject: [PATCH 076/443] Lombok to 1.18.22 Signed-off-by: Lehmann_Fabian --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2c0f375d..89a38c5a 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ org.projectlombok lombok - 1.18.4 + 1.18.22 provided From 157c04055645e1f2a8ce0d6d7d7da6daad46ea07 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 15:41:42 +0100 Subject: [PATCH 077/443] Null check Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/location/NodeLocation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java index 62b8de37..766fe682 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -23,6 +23,7 @@ public static NodeLocation getLocation( Node node ){ } public static NodeLocation getLocation( String node ){ + if ( node == null ) throw new IllegalArgumentException("Node cannot be null"); final NodeLocation nodeLocation = locationHolder.get(node); if ( nodeLocation == null ){ locationHolder.putIfAbsent( node, new NodeLocation( node ) ); From 2d5317237ae5845ebd4274a855213fd93e742186 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 17:41:58 +0100 Subject: [PATCH 078/443] Use DAG process Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/Origin.java | 7 +++- .../java/fonda/scheduler/dag/Process.java | 7 +++- .../java/fonda/scheduler/model/Process.java | 41 ------------------- src/main/java/fonda/scheduler/model/Task.java | 8 ++-- .../scheduler/model/TaskResultParser.java | 1 + .../location/hierachy/LocationWrapper.java | 2 +- .../model/location/hierachy/RealFile.java | 2 +- .../fonda/scheduler/scheduler/Scheduler.java | 2 +- .../scheduler/model/TaskResultParserTest.java | 20 +++++++-- .../hierachy/HierarchyWrapperTest.java | 17 ++++++-- .../model/location/hierachy/RealFileTest.java | 24 ++++++++--- 11 files changed, 70 insertions(+), 61 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/model/Process.java diff --git a/src/main/java/fonda/scheduler/dag/Origin.java b/src/main/java/fonda/scheduler/dag/Origin.java index b9b1e5c2..21be9be2 100644 --- a/src/main/java/fonda/scheduler/dag/Origin.java +++ b/src/main/java/fonda/scheduler/dag/Origin.java @@ -5,7 +5,12 @@ public class Origin extends NotProcess { - Origin(String label, int uid) { + /** + * Only public for tests + * @param label + * @param uid + */ + public Origin(String label, int uid) { super(label, uid); } diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/fonda/scheduler/dag/Process.java index 38d528cf..4c80e5ac 100644 --- a/src/main/java/fonda/scheduler/dag/Process.java +++ b/src/main/java/fonda/scheduler/dag/Process.java @@ -9,7 +9,12 @@ public class Process extends Vertex { final Set descendants; final Set ancestors; - Process( String label, int uid ) { + /** + * Only public for tests + * @param label + * @param uid + */ + public Process(String label, int uid) { super(label, uid); descendants = new HashSet<>(); ancestors = new HashSet<>(); diff --git a/src/main/java/fonda/scheduler/model/Process.java b/src/main/java/fonda/scheduler/model/Process.java deleted file mode 100644 index b87ee692..00000000 --- a/src/main/java/fonda/scheduler/model/Process.java +++ /dev/null @@ -1,41 +0,0 @@ -package fonda.scheduler.model; - -import lombok.Getter; - -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -public class Process { - - static final private ConcurrentMap< String, Process > processHolder = new ConcurrentHashMap(); - - @Getter - private final String name; - - private Process(String name) { - this.name = name; - } - - public static Process getProcess ( final String process ){ - final Process processFound = processHolder.get( process ); - if ( processFound == null ){ - processHolder.putIfAbsent( process, new Process( process ) ); - return processHolder.get( process ); - } - return processFound; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Process)) return false; - Process process = (Process) o; - return getName().equals(process.getName()); - } - - @Override - public int hashCode() { - return Objects.hash(getName()); - } -} diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index b470ad04..aabdbb99 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -1,8 +1,8 @@ package fonda.scheduler.model; +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.Process; import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealFile; import fonda.scheduler.util.Batch; import lombok.Getter; import lombok.Setter; @@ -35,9 +35,9 @@ public class Task { @Setter private Batch batch; - public Task(TaskConfig config) { + public Task( TaskConfig config, DAG dag ) { this.config = config; - this.process = Process.getProcess( config.getTask() ); + this.process = dag.getByProcess( config.getTask() ); } public String getWorkingDir(){ diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 9d351f56..35dbfe2c 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -6,6 +6,7 @@ import fonda.scheduler.model.outfiles.PathLocationWrapperPair; import fonda.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; +import fonda.scheduler.dag.Process; import java.io.File; import java.io.FileNotFoundException; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 89b49e03..8b0c725e 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -1,6 +1,6 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.Process; +import fonda.scheduler.dag.Process; import fonda.scheduler.model.location.Location; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 2b16abd1..ee27bf46 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -1,6 +1,6 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.Process; +import fonda.scheduler.dag.Process; import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 4944a0d3..19f429d4 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -194,7 +194,7 @@ public void markPodAsDeleted( PodWithAge pod ) { /* External access to Tasks */ public void addTask( TaskConfig conf ) { - final Task task = new Task(conf); + final Task task = new Task( conf, dag ); synchronized (tasksByHash) { if( ! tasksByHash.containsKey( conf.getHash() ) ){ tasksByHash.put( conf.getHash(), task ); diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java index 7f303c96..5ec8d06d 100644 --- a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -1,5 +1,8 @@ package fonda.scheduler.model; +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.Process; +import fonda.scheduler.dag.Vertex; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.outfiles.OutputFile; @@ -40,6 +43,17 @@ private Path storeData( String[] inputdata, String[] outputdata ){ return tmpDir; } + private DAG dag; + + @Before + public void before(){ + dag = new DAG(); + List vertexList = new LinkedList<>(); + final Process a = new Process("P1", 2); + vertexList.add(a); + dag.registerVertices(vertexList); + } + @Test public void test1(){ @@ -60,7 +74,7 @@ public void test1(){ final Path path = storeData(infiles, outfiles); final TaskResultParser taskResultParser = new TaskResultParser(); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), Process.getProcess("P1"), false); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), dag.getByProcess("P1"), false); log.info("{}", newAndUpdatedFiles); @@ -95,7 +109,7 @@ public void test2(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); - final Process p1 = Process.getProcess("P1"); + final Process p1 = dag.getByProcess("P1"); final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); log.info("{}", newAndUpdatedFiles); @@ -139,7 +153,7 @@ public void test3(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); - final Process p1 = Process.getProcess("P1"); + final Process p1 = dag.getByProcess("P1"); final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); log.info("{}", newAndUpdatedFiles); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index d1591c2d..e689e527 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -1,10 +1,12 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.Process; +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.Vertex; import fonda.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; import java.nio.file.Path; import java.nio.file.Paths; @@ -21,14 +23,23 @@ public class HierarchyWrapperTest { List files; Collection result; String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; - LocationWrapper node1 = getLocationWrapper("Node1"); + LocationWrapper node1; + DAG dag; private LocationWrapper getLocationWrapper( String location ){ - return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, Process.getProcess("processA") ); + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, dag.getByProcess("processA") ); } @Before public void init() { + dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new fonda.scheduler.dag.Process("processA", 1)); + vertexList.add(new fonda.scheduler.dag.Process("processB", 2)); + dag.registerVertices(vertexList); + + node1 = getLocationWrapper("Node1"); + hw = new HierarchyWrapper(workdir); files = new LinkedList(); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index c09e0af5..f547f002 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -1,7 +1,10 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.model.Process; +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.Process; +import fonda.scheduler.dag.Vertex; import fonda.scheduler.model.location.NodeLocation; +import org.junit.Before; import org.junit.Test; import java.util.*; @@ -12,8 +15,19 @@ public class RealFileTest { + private DAG dag; + + @Before + public void before(){ + dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + dag.registerVertices(vertexList); + } + private LocationWrapper getLocationWrapper( String location ){ - return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, Process.getProcess("processA") ); + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, dag.getByProcess("processA") ); } @@ -88,7 +102,7 @@ public void changeFile() { final LocationWrapper node3 = getLocationWrapper("Node3"); realFile.addOrUpdateLocation( false, node3); - final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, Process.getProcess("processB") ); + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, dag.getByProcess("processB") ); realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); @@ -100,12 +114,12 @@ public void changeFileOnExistingLocation() { final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); - final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, Process.getProcess("processA") ); + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, dag.getByProcess("processA") ); realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); - final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, Process.getProcess("processB") ); + final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, dag.getByProcess("processB") ); realFile.addOrUpdateLocation( false, nodeNew2 ); LocationWrapper[] expected2 = { nodeNew2 }; assertArrayEquals( expected2, realFile.getLocations() ); From 7fb255d91de80f20b30c176786853d9c31a9d833 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 17:42:24 +0100 Subject: [PATCH 079/443] Fix TaskResultParser tests Signed-off-by: Lehmann_Fabian --- .../scheduler/model/TaskResultParserTest.java | 106 +++++++++--------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java index 5ec8d06d..a8cc4e18 100644 --- a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -7,14 +7,17 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.outfiles.OutputFile; import fonda.scheduler.model.outfiles.PathLocationWrapperPair; +import fonda.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; +import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Set; +import java.util.*; + +import static org.junit.Assert.*; @Slf4j public class TaskResultParserTest { @@ -59,16 +62,16 @@ public void before(){ public void test1(){ String infiles[] = { - "/tmp/nxf.3iuGDWr6Id;;4096;directory;-;2021-11-10 12:58:11.210414589 +0000;2021-11-10 12:58:11.222414603 +0000", - "/tmp/nxf.3iuGDWr6Id/file.txt;/pvcdata/testfile.txt;6;regular file;-;2021-11-10 12:58:07.485035700 +0000;2021-11-10 12:58:07.219065500 +0000", - "/tmp/nxf.3iuGDWr6Id/.command.err;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000", - "/tmp/nxf.3iuGDWr6Id/.command.out;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000" + "/tmp/nxf.3iuGDWr6Id;0;;4096;directory;-;2021-11-10 12:58:11.210414589 +0000;2021-11-10 12:58:11.222414603 +0000", + "/tmp/nxf.3iuGDWr6Id/file.txt;1;/pvcdata/testfile.txt;6;regular file;-;2021-11-10 12:58:07.485035700 +0000;2021-11-10 12:58:07.219065500 +0000", + "/tmp/nxf.3iuGDWr6Id/.command.err;1;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000", + "/tmp/nxf.3iuGDWr6Id/.command.out;1;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000" }; String[] outfiles = { - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650;;4096;directory;-;2021-11-10 12:58:11.278414667 +0000;2021-11-10 12:58:11.278414667 +0000", - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000", - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000" + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650;0;;4096;directory;-;2021-11-10 12:58:11.278414667 +0000;2021-11-10 12:58:11.278414667 +0000", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;1;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;1;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000" }; final Path path = storeData(infiles, outfiles); @@ -78,11 +81,12 @@ public void test1(){ log.info("{}", newAndUpdatedFiles); - final PathLocationWrapperPair[] expected = { - new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230l, 13, Process.getProcess("P1"))) - }; + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230l, 13, dag.getByProcess("P1"))) ); + expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt", "/pvcdata/testfile.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt", "/pvcdata/testfile.txt") ); - assertArrayEquals( expected, newAndUpdatedFiles.toArray() ); + assertEquals( expected, newAndUpdatedFiles ); } @@ -90,19 +94,19 @@ public void test1(){ public void test2(){ String infiles[] = { - "/tmp/nxf.IANFIlM3Kv;;4096;directory;-;2021-11-12 12:42:42.155614026 +0000;2021-11-12 12:42:42.171614019 +0000", - "/tmp/nxf.IANFIlM3Kv/file.txt;/pvcdata/testfile.txt;0;regular empty file;-;2021-11-12 12:42:29.000000000 +0000;2021-11-12 12:42:29.000000000 +0000", - "/tmp/nxf.IANFIlM3Kv/.command.err;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000", - "/tmp/nxf.IANFIlM3Kv/.command.out;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000" + "/tmp/nxf.IANFIlM3Kv;0;;4096;directory;-;2021-11-12 12:42:42.155614026 +0000;2021-11-12 12:42:42.171614019 +0000", + "/tmp/nxf.IANFIlM3Kv/file.txt;1;/pvcdata/testfile.txt;0;regular empty file;-;2021-11-12 12:42:29.000000000 +0000;2021-11-12 12:42:29.000000000 +0000", + "/tmp/nxf.IANFIlM3Kv/.command.err;1;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000", + "/tmp/nxf.IANFIlM3Kv/.command.out;1;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000" }; String[] outfiles = { - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa;;4096;directory;-;2021-11-12 12:42:42.239613991 +0000;2021-11-12 12:42:42.243613989 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t;;4096;directory;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt;;0;regular empty file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000" + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa;0;;4096;directory;-;2021-11-12 12:42:42.239613991 +0000;2021-11-12 12:42:42.243613989 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t;1;;4096;directory;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt;1;;0;regular empty file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt;1;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt;1;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt;1;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000" }; final Path path = storeData(infiles, outfiles); @@ -112,17 +116,15 @@ public void test2(){ final Process p1 = dag.getByProcess("P1"); final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); - log.info("{}", newAndUpdatedFiles); + log.info("{}", newAndUpdatedFiles.toArray()); - final PathLocationWrapperPair[] expected = { - new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223l, 0, p1)), - new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)), - new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)), - new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223l, 0, p1)) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) ); - }; - - assertArrayEquals( expected, newAndUpdatedFiles.toArray() ); + assertEquals( expected, newAndUpdatedFiles ); } @@ -130,23 +132,23 @@ public void test2(){ public void test3(){ String infiles[] = { - "/tmp/nxf.9J6Y5mcXRD;;4096;directory;-;2021-11-12 14:42:20.941053482 +0000;2021-11-12 14:42:20.937053480 +0000", - "/tmp/nxf.9J6Y5mcXRD/t;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/b.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/c.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/a.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/.command.err;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000", - "/tmp/nxf.9J6Y5mcXRD/.command.out;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000" + "/tmp/nxf.9J6Y5mcXRD;0;;4096;directory;-;2021-11-12 14:42:20.941053482 +0000;2021-11-12 14:42:20.937053480 +0000", + "/tmp/nxf.9J6Y5mcXRD/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/tmp/nxf.9J6Y5mcXRD/.command.err;1;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000", + "/tmp/nxf.9J6Y5mcXRD/.command.out;1;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000" }; String[] outfiles = { - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6;;4096;directory;-;2021-11-12 14:42:21.005053535 +0000;2021-11-12 14:42:21.009053539 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;4;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:20.993053525 +0000" + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6;0;;4096;directory;-;2021-11-12 14:42:21.005053535 +0000;2021-11-12 14:42:21.009053539 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;4;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:20.993053525 +0000" }; final Path path = storeData(infiles, outfiles); @@ -158,12 +160,14 @@ public void test3(){ log.info("{}", newAndUpdatedFiles); - final PathLocationWrapperPair[] expected = { - new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper(node1, 1636728140993l, 4, p1)) - - }; + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper(node1, 1636728140993l, 4, p1)) ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt") ); - assertArrayEquals( expected, newAndUpdatedFiles.toArray() ); + assertEquals( expected, newAndUpdatedFiles ); } From f5892d070b1b066da564310e283e49b3ee796711 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 8 Feb 2022 17:42:41 +0100 Subject: [PATCH 080/443] SymlinkOutput toString Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/outfiles/SymlinkOutput.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java b/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java index 0aef4c5a..dfbfd673 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java +++ b/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java @@ -29,4 +29,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(super.hashCode(), dst); } + + @Override + public String toString() { + return "SymlinkOutput{" + super.getPath() + " -> " + getDst() + "}"; + } } From 0319e6427a11cf6890a5cecd89578a8ba37e687d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Feb 2022 10:26:37 +0100 Subject: [PATCH 081/443] Save LocationWrapper's lineage Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 2 +- .../fonda/scheduler/model/TaskConfig.java | 10 ++++++- .../scheduler/model/TaskResultParser.java | 6 ++-- .../location/hierachy/LocationWrapper.java | 29 ++++++++++--------- .../scheduler/SchedulerWithDaemonSet.java | 10 ++----- .../scheduler/model/TaskResultParserTest.java | 22 +++++++------- .../hierachy/HierarchyWrapperTest.java | 5 ++-- .../model/location/hierachy/RealFileTest.java | 14 ++++++--- 8 files changed, 57 insertions(+), 41 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index aabdbb99..181472b3 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -52,7 +52,7 @@ public boolean wasSuccessfullyExecuted(){ public String toString() { return "Task{" + "state=" + state + - ", pod=" + pod.getMetadata().getName() + + ", pod=" + pod == null ? "--" : pod.getMetadata().getName() + ", node='" + (node != null ? node.getIdentifier() : "--") + '\'' + ", workDir='" + getWorkingDir() + '\'' + '}'; diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 77319c67..1d37e6f9 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -19,11 +19,19 @@ public class TaskConfig { private final long memoryInBytes; private TaskConfig() { + this( null ); + } + + /** + * Only for testing + * @param task + */ + public TaskConfig(String task) { + this.task = task; this.name = null; this.schedulerParams = null; this.inputs = null; this.hash = null; - this.task = null; this.cpus = 0; this.memoryInBytes = 0; } diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 35dbfe2c..654d9c32 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -45,13 +45,15 @@ private String getRootDir( File file ){ * @param workdir * @param location * @param process + * @param finishedTask * @return A list of all new or updated files */ public Set getNewAndUpdatedFiles( final Path workdir, final Location location, final Process process, - final boolean onlyUpdated + final boolean onlyUpdated, + Task finishedTask ){ final Path infile = workdir.resolve(".command.infiles"); @@ -97,7 +99,7 @@ public Set getNewAndUpdatedFiles( location, fileTimeFromString(modificationDate), Long.parseLong(data[ SIZE ]), - process + finishedTask ); newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 8b0c725e..b037b1d7 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -1,8 +1,10 @@ package fonda.scheduler.model.location.hierachy; import fonda.scheduler.dag.Process; +import fonda.scheduler.model.Task; import fonda.scheduler.model.location.Location; import lombok.Getter; +import lombok.ToString; import java.util.Objects; @@ -12,24 +14,24 @@ public class LocationWrapper { private final Location location; private final long timestamp; private final long sizeInBytes; - private final Process process; private final long createTime = System.currentTimeMillis(); + private final Task createdByTask; + private final LocationWrapper copyOf; - public LocationWrapper(Location location, long timestamp, long sizeInBytes, Process process) { + public LocationWrapper(Location location, long timestamp, long sizeInBytes, Task task) { + this( location, timestamp, sizeInBytes ,task, null ); + } + + private LocationWrapper(Location location, long timestamp, long sizeInBytes, Task createdByTask, LocationWrapper copyOf) { this.location = location; this.timestamp = timestamp; this.sizeInBytes = sizeInBytes; - this.process = process; + this.createdByTask = createdByTask; + this.copyOf = copyOf; } - @Override - public String toString() { - return "LocationWrapper{" + - "location=" + location + - ", timestamp=" + timestamp + - ", sizeInBytes=" + sizeInBytes + - ", process='" + process + '\'' + - '}'; + public LocationWrapper getCopyOf( Location location ) { + return new LocationWrapper( location, timestamp, sizeInBytes, createdByTask, copyOf == null ? this : copyOf ); } @Override @@ -40,11 +42,12 @@ public boolean equals(Object o) { return getTimestamp() == that.getTimestamp() && getSizeInBytes() == that.getSizeInBytes() && Objects.equals(getLocation(), that.getLocation()) - && Objects.equals(getProcess(), that.getProcess()); + && Objects.equals(getCopyOf(), that.getCopyOf()) + && Objects.equals(getCreatedByTask(), that.getCreatedByTask()); } @Override public int hashCode() { - return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getProcess()); + return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getCreatedByTask()); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 9e81805a..427f9449 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -88,7 +88,8 @@ int terminateTasks(List finishedTasks) { Paths.get(finishedTask.getWorkingDir()), finishedTask.getNode(), finishedTask.getProcess(), - !finishedTask.wasSuccessfullyExecuted() + !finishedTask.wasSuccessfullyExecuted(), + finishedTask ); for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { @@ -133,12 +134,7 @@ List writeInitConfig( NodeTaskFilesAlignment align inputFiles.add( new TaskInputFileLocationWrapper( filePath.file, - new LocationWrapper( - location, - locationWrapper.getTimestamp(), - locationWrapper.getSizeInBytes(), - locationWrapper.getProcess() - ) + locationWrapper.getCopyOf( location ) ) ); } diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java index a8cc4e18..72bd197e 100644 --- a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -77,12 +77,12 @@ public void test1(){ final Path path = storeData(infiles, outfiles); final TaskResultParser taskResultParser = new TaskResultParser(); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), dag.getByProcess("P1"), false); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), dag.getByProcess("P1"), false, null); log.info("{}", newAndUpdatedFiles); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230l, 13, dag.getByProcess("P1"))) ); + expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230l, 13, new Task( new TaskConfig("P1"), dag) )) ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt", "/pvcdata/testfile.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt", "/pvcdata/testfile.txt") ); @@ -113,16 +113,16 @@ public void test2(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); - final Process p1 = dag.getByProcess("P1"); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); + final Task task = new Task( new TaskConfig("P1"), dag); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, null, false, task); log.info("{}", newAndUpdatedFiles.toArray()); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223l, 0, p1)) ); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) ); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) ); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223l, 2, p1)) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223l, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223l, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223l, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223l, 2, task )) ); assertEquals( expected, newAndUpdatedFiles ); @@ -155,13 +155,13 @@ public void test3(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); - final Process p1 = dag.getByProcess("P1"); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, p1, false); + final Task task = new Task( new TaskConfig("P1"), dag ); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, null, false, task); log.info("{}", newAndUpdatedFiles); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper(node1, 1636728140993l, 4, p1)) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper( node1, 1636728140993l, 4, task )) ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt") ); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index e689e527..af9256a8 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -2,11 +2,12 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Vertex; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.TaskConfig; import fonda.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; -import org.junit.jupiter.api.BeforeEach; import java.nio.file.Path; import java.nio.file.Paths; @@ -27,7 +28,7 @@ public class HierarchyWrapperTest { DAG dag; private LocationWrapper getLocationWrapper( String location ){ - return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, dag.getByProcess("processA") ); + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, new Task( new TaskConfig("processA"), dag) ); } @Before diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index f547f002..3dc3a952 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -3,6 +3,8 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Process; import fonda.scheduler.dag.Vertex; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.TaskConfig; import fonda.scheduler.model.location.NodeLocation; import org.junit.Before; import org.junit.Test; @@ -16,6 +18,8 @@ public class RealFileTest { private DAG dag; + private Task processA; + private Task processB; @Before public void before(){ @@ -24,10 +28,12 @@ public void before(){ vertexList.add(new Process("processA", 1)); vertexList.add(new Process("processB", 2)); dag.registerVertices(vertexList); + processA = new Task( new TaskConfig("procesA"), dag ); + processB = new Task( new TaskConfig("procesB"), dag ); } private LocationWrapper getLocationWrapper( String location ){ - return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, dag.getByProcess("processA") ); + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, processA ); } @@ -102,7 +108,7 @@ public void changeFile() { final LocationWrapper node3 = getLocationWrapper("Node3"); realFile.addOrUpdateLocation( false, node3); - final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, dag.getByProcess("processB") ); + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, processB ); realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); @@ -114,12 +120,12 @@ public void changeFileOnExistingLocation() { final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); - final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, dag.getByProcess("processA") ); + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, processA ); realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { nodeNew }; assertArrayEquals( expected, realFile.getLocations() ); - final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, dag.getByProcess("processB") ); + final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, processB ); realFile.addOrUpdateLocation( false, nodeNew2 ); LocationWrapper[] expected2 = { nodeNew2 }; assertArrayEquals( expected2, realFile.getLocations() ); From ae9e84783d6a3a7c78f502d9753d73ec3db6e01b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Feb 2022 12:34:06 +0100 Subject: [PATCH 082/443] Refactoring Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/Main.java | 2 +- .../scheduler/client/KubernetesClient.java | 7 +- .../java/fonda/scheduler/dag/InputEdge.java | 1 + src/main/java/fonda/scheduler/dag/Vertex.java | 2 +- .../fonda/scheduler/model/NodeWithAlloc.java | 4 +- .../scheduler/model/PodListWithIndex.java | 4 +- .../java/fonda/scheduler/model/State.java | 2 +- src/main/java/fonda/scheduler/model/Task.java | 2 +- .../scheduler/model/TaskResultParser.java | 2 + .../scheduler/model/location/Location.java | 4 +- .../model/location/NodeLocation.java | 3 +- .../model/location/hierachy/Folder.java | 2 - .../location/hierachy/LocationWrapper.java | 2 - .../model/location/hierachy/RealFile.java | 3 +- .../rest/SchedulerRestController.java | 101 +++++++++--------- .../scheduler/scheduler/RandomScheduler.java | 2 +- .../fonda/scheduler/scheduler/Scheduler.java | 13 ++- .../scheduler/SchedulerWithDaemonSet.java | 12 +-- .../scheduler/TaskprocessingThread.java | 2 +- .../scheduler/copystrategy/CopyStrategy.java | 22 ++-- .../schedulingstrategy/InputEntry.java | 1 - src/main/java/fonda/scheduler/util/Batch.java | 3 + .../util/NodeTaskFilesAlignment.java | 3 - .../java/fonda/scheduler/dag/DAGTest.java | 7 +- .../scheduler/dag/VertexDeserializerTest.java | 6 -- .../scheduler/model/TaskResultParserTest.java | 20 ++-- .../hierachy/HierarchyWrapperTest.java | 14 +-- .../model/location/hierachy/RealFileTest.java | 9 +- 28 files changed, 126 insertions(+), 129 deletions(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 77802858..3bf4c7ea 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -34,7 +34,7 @@ public void afterStart(){ try{ log.info( "Started with namespace: {}", client.getNamespace() ); final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null, null, "/localwork/", null, "ftp")); - final Pair key = new Pair( "default", "test-run" ); + final Pair key = new Pair<>("default", "test-run"); SchedulerRestController.addScheduler( key, randomScheduler ); } catch (Exception e){ e.printStackTrace(); diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index dcd0df44..e54794a5 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -19,11 +19,10 @@ public class KubernetesClient extends DefaultKubernetesClient { private final Map nodeHolder= new HashMap<>(); private final List schedulerList = new LinkedList<>(); - private OperationContext operationContext; public KubernetesClient(){ - this.operationContext = new OperationContext(); + OperationContext operationContext = new OperationContext(); for( Node node : this.nodes().list().getItems() ){ nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node) ); } @@ -73,7 +72,7 @@ public List getAllNodes(){ return new ArrayList<>(this.nodeHolder.values()); } - class NodeWatcher implements Watcher{ + static class NodeWatcher implements Watcher{ private final KubernetesClient kubernetesClient; @@ -124,7 +123,7 @@ public void onClose(WatcherException cause) { } } - class PodWatcher implements Watcher { + static class PodWatcher implements Watcher { private final KubernetesClient kubernetesClient; diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/fonda/scheduler/dag/InputEdge.java index 81290d94..2be13ef3 100644 --- a/src/main/java/fonda/scheduler/dag/InputEdge.java +++ b/src/main/java/fonda/scheduler/dag/InputEdge.java @@ -11,6 +11,7 @@ public class InputEdge { private final int from; private final int to; + @SuppressWarnings("unused") private InputEdge() { this.label = null; this.from = -1; diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index 96ffed9a..80b15a6e 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -36,7 +36,7 @@ private String collectionToString( Collection v ){ @Override public String toString() { return getType() + "{" + - label != null ? ("label='" + label + "', '") : "" + + (label != null ? ("label='" + label + "', '") : "") + "uid=" + uid + ", descendants=" + collectionToString( getDescendants() ) + ", ancestors=" + collectionToString( getAncestors() ) + diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 7a470d99..744512ff 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -17,7 +17,7 @@ public class NodeWithAlloc extends Node implements Comparable { private final PodRequirements max_resources; - Map assignedPods; + final Map assignedPods; @Getter private final NodeLocation nodeLocation; @@ -48,7 +48,7 @@ public NodeWithAlloc(Node node) { public void addPod( PodWithAge pod ){ PodRequirements request = pod.getRequest(); synchronized (assignedPods) { - assignedPods.put( pod.getMetadata().getUid(), request );; + assignedPods.put( pod.getMetadata().getUid(), request ); } } diff --git a/src/main/java/fonda/scheduler/model/PodListWithIndex.java b/src/main/java/fonda/scheduler/model/PodListWithIndex.java index 742789a6..b697b9ad 100644 --- a/src/main/java/fonda/scheduler/model/PodListWithIndex.java +++ b/src/main/java/fonda/scheduler/model/PodListWithIndex.java @@ -10,12 +10,12 @@ public class PodListWithIndex extends PodList { - private Map nameIndexMap = new HashMap<>(); + private final Map nameIndexMap = new HashMap<>(); public PodListWithIndex() {} public PodListWithIndex(List pods) { - pods.forEach( pod -> addPodToList(pod) ); + pods.forEach(this::addPodToList); } /** diff --git a/src/main/java/fonda/scheduler/model/State.java b/src/main/java/fonda/scheduler/model/State.java index c1068e73..fe7db5ea 100644 --- a/src/main/java/fonda/scheduler/model/State.java +++ b/src/main/java/fonda/scheduler/model/State.java @@ -2,6 +2,6 @@ public enum State { - RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, PREPARED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, DELETED; + RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, PREPARED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, DELETED } diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 181472b3..d3499cec 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -52,7 +52,7 @@ public boolean wasSuccessfullyExecuted(){ public String toString() { return "Task{" + "state=" + state + - ", pod=" + pod == null ? "--" : pod.getMetadata().getName() + + ", pod=" + (pod == null ? "--" : pod.getMetadata().getName()) + ", node='" + (node != null ? node.getIdentifier() : "--") + '\'' + ", workDir='" + getWorkingDir() + '\'' + '}'; diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 654d9c32..f9327452 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -60,7 +60,9 @@ public Set getNewAndUpdatedFiles( final Path outfile = workdir.resolve(".command.outfiles"); String taskRootDir = getRootDir( infile.toFile() ); + if( taskRootDir == null ) throw new IllegalStateException("taskRootDir is null"); String outputRootDir = getRootDir( outfile.toFile() ); + if( outputRootDir == null ) throw new IllegalStateException("outputRootDir is null"); final Map inputdata = new HashMap<>(); diff --git a/src/main/java/fonda/scheduler/model/location/Location.java b/src/main/java/fonda/scheduler/model/location/Location.java index e671f84d..ff737a37 100644 --- a/src/main/java/fonda/scheduler/model/location/Location.java +++ b/src/main/java/fonda/scheduler/model/location/Location.java @@ -1,7 +1,5 @@ package fonda.scheduler.model.location; -import lombok.Getter; - import java.util.Objects; public abstract class Location { @@ -13,7 +11,7 @@ public abstract class Location { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || !(o instanceof Location) ) return false; + if (!(o instanceof Location)) return false; Location that = (Location) o; return ( getType() == that.getType() ) && ( getIdentifier().equals( that.getIdentifier() )); } diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java index 766fe682..54d0b3fa 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -3,13 +3,12 @@ import io.fabric8.kubernetes.api.model.Node; import lombok.Getter; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; public class NodeLocation extends Location { - static final private ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap(); + static final private ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap<>(); @Getter private final String identifier; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index f6e87794..8052cf80 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -3,8 +3,6 @@ import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index b037b1d7..fe9a32d4 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -1,10 +1,8 @@ package fonda.scheduler.model.location.hierachy; -import fonda.scheduler.dag.Process; import fonda.scheduler.model.Task; import fonda.scheduler.model.location.Location; import lombok.Getter; -import lombok.ToString; import java.util.Objects; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index ee27bf46..9d9ecb3b 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -6,6 +6,7 @@ import lombok.Getter; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; public class RealFile extends AbstractFile { @@ -75,7 +76,7 @@ public List getFilesForProcess( Process process ){ lastUpdated = location; } } - return List.of( lastUpdated ); + return lastUpdated == null ? new LinkedList<>() : List.of( lastUpdated ); } public LocationWrapper getLastUpdate( LocationType type ){ diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 73ba3e03..db834fc4 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -52,28 +52,29 @@ public static void addScheduler(Pair key, Scheduler scheduler ){ * @return */ @PutMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") - ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { + ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { log.trace("Register execution: {} strategy: {} config: {}", execution, strategy, config); - Scheduler scheduler = null; + Scheduler scheduler; - if( schedulerHolder.containsKey( execution.toLowerCase() ) ) { - return new ResponseEntity( "There is already a scheduler for " + execution, HttpStatus.BAD_REQUEST ); + final Pair key = getKey( namespace, execution ); + + if( schedulerHolder.containsKey( key ) ) { + return new ResponseEntity<>( "There is already a scheduler for " + execution, HttpStatus.BAD_REQUEST ); } switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace, config ); break; - default: return new ResponseEntity( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); + default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } - final Pair key = getKey( namespace, execution ); schedulerHolder.put( key, scheduler ); client.addScheduler( scheduler ); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } @@ -85,46 +86,46 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable S * @return Parameters the scheduler suggests for the task */ @PutMapping("/scheduler/registerTask/{namespace}/{execution}") - ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody(required = true) TaskConfig config ) { + ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody TaskConfig config ) { log.trace( execution + " " + config.getTask() + " got: " + config ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } scheduler.addTask( config ); Map schedulerParams = scheduler.getSchedulerParams(config.getTask(), config.getName()); - return new ResponseEntity( schedulerParams, HttpStatus.OK ); + return new ResponseEntity<>( schedulerParams, HttpStatus.OK ); } @PostMapping("/scheduler/startBatch/{namespace}/{execution}") - ResponseEntity startBatch(@PathVariable String namespace, @PathVariable String execution ) { + ResponseEntity startBatch(@PathVariable String namespace, @PathVariable String execution ) { final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } scheduler.startBatch(); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } @PostMapping("/scheduler/endBatch/{namespace}/{execution}") - ResponseEntity endBatch(@PathVariable String namespace, @PathVariable String execution, @RequestBody(required = true) int tasksInBatch ) { + ResponseEntity endBatch(@PathVariable String namespace, @PathVariable String execution, @RequestBody int tasksInBatch ) { final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } scheduler.endBatch( tasksInBatch ); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } @@ -136,15 +137,15 @@ ResponseEntity endBatch(@PathVariable String namespace, @PathVariable String exe * @return boolean */ @GetMapping("/scheduler/taskstate/{namespace}/{execution}/{taskid}") - ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { + ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } - return new ResponseEntity( scheduler.getTaskState( taskid ), HttpStatus.OK ); + return new ResponseEntity<>( scheduler.getTaskState( taskid ), HttpStatus.OK ); } @@ -155,140 +156,140 @@ ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String * @return */ @DeleteMapping ("/scheduler/{namespace}/{execution}") - ResponseEntity delete(@PathVariable String namespace, @PathVariable String execution) { + ResponseEntity delete(@PathVariable String namespace, @PathVariable String execution) { log.info("Delete name: " + execution); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution, HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution, HttpStatus.NOT_FOUND ); } schedulerHolder.remove( key ); client.removeScheduler( scheduler ); scheduler.close(); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } @GetMapping("/daemon/{namespace}/{execution}/{node}") - ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable String execution, @PathVariable String node ) { + ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable String execution, @PathVariable String node ) { log.info( "Got request: {}{}{}", namespace, execution, node ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + if(!(scheduler instanceof SchedulerWithDaemonSet)){ + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } String daemon = ((SchedulerWithDaemonSet) scheduler).getDaemonOnNode( node ); if ( daemon == null ){ - return new ResponseEntity( "No daemon for node found: " + node , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No daemon for node found: " + node , HttpStatus.NOT_FOUND ); } - return new ResponseEntity( daemon, HttpStatus.OK ); + return new ResponseEntity<>( daemon, HttpStatus.OK ); } @GetMapping("/file/{namespace}/{execution}") - ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestParam String path ) { + ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestParam String path ) { log.info( "Get file location request: {} {} {}", namespace, execution, path ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + if(!(scheduler instanceof SchedulerWithDaemonSet)){ + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } - FileResponse fileResponse = null; + FileResponse fileResponse; try { fileResponse = ((SchedulerWithDaemonSet) scheduler).nodeOfLastFileVersion( path ); log.info(fileResponse.toString()); } catch (NotARealFileException e) { - return new ResponseEntity( "Requested path is not a real file: " + path , HttpStatus.BAD_REQUEST ); + return new ResponseEntity<>( "Requested path is not a real file: " + path , HttpStatus.BAD_REQUEST ); } if ( fileResponse == null ){ - return new ResponseEntity( "No node for file found: " + path , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No node for file found: " + path , HttpStatus.NOT_FOUND ); } - return new ResponseEntity( fileResponse, HttpStatus.OK ); + return new ResponseEntity<>( fileResponse, HttpStatus.OK ); } @PostMapping("/file/location/{method}/{namespace}/{execution}") - ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @RequestBody PathAttributes pa ) { + ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @RequestBody PathAttributes pa ) { return changeLocationForFile(method,namespace,execution,null,pa); } @PostMapping("/file/location/{method}/{namespace}/{execution}/{node}") - ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @PathVariable String node, @RequestBody PathAttributes pa ) { + ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @PathVariable String node, @RequestBody PathAttributes pa ) { log.info( "Change file location request: {} {} {} {}", method, namespace, execution, pa ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null || !(scheduler instanceof SchedulerWithDaemonSet) ){ + if(!(scheduler instanceof SchedulerWithDaemonSet)){ log.info("No scheduler for: " + execution); - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } if ( !method.equals("add") && !method.equals("overwrite") ) { log.info("Method not found: " + method); - return new ResponseEntity( "Method not found: " + method , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "Method not found: " + method , HttpStatus.NOT_FOUND ); } boolean overwrite = method.equals("overwrite"); ((SchedulerWithDaemonSet) scheduler).addFile( pa.getPath(), pa.getSize(), pa.getTimestamp(), overwrite, node ); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } @GetMapping ("/health") - ResponseEntity checkHealth() { - return new ResponseEntity( HttpStatus.OK ); + ResponseEntity checkHealth() { + return new ResponseEntity<>( HttpStatus.OK ); } - private Pair getKey(String namespace, String execution ){ - return new Pair(namespace.toLowerCase(), execution.toLowerCase()); + private Pair getKey(String namespace, String execution ){ + return new Pair<>(namespace.toLowerCase(), execution.toLowerCase()); } @PutMapping("/scheduler/DAG/addVertices/{namespace}/{execution}") - ResponseEntity addVertices(@PathVariable String namespace, @PathVariable String execution, @RequestBody List vertices ) { + ResponseEntity addVertices(@PathVariable String namespace, @PathVariable String execution, @RequestBody List vertices ) { log.trace( "submit vertices: {}", vertices ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } scheduler.getDag().registerVertices( vertices ); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } @PutMapping("/scheduler/DAG/addEdges/{namespace}/{execution}") - ResponseEntity addEdges(@PathVariable String namespace, @PathVariable String execution, @RequestBody List edges ) { + ResponseEntity addEdges(@PathVariable String namespace, @PathVariable String execution, @RequestBody List edges ) { log.trace( "submit edges: {}", edges ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); } final DAG dag = scheduler.getDag(); dag.registerEdges( edges ); - return new ResponseEntity( HttpStatus.OK ); + return new ResponseEntity<>( HttpStatus.OK ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index a15c77c8..f4849d20 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -29,7 +29,7 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ Map availableByNode = new HashMap<>(); - List logInfo = new LinkedList(); + List logInfo = new LinkedList<>(); logInfo.add("------------------------------------"); for (NodeWithAlloc item : items) { final PodRequirements availableResources = item.getAvailableResources(); diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 19f429d4..ff9d86d0 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -36,7 +36,7 @@ public abstract class Scheduler { @Getter private boolean close; @Getter - private DAG dag; + private final DAG dag; private final Object batchHelper = new Object(); private int currentBatch = 0; @@ -169,7 +169,7 @@ private void tryToScheduleBatch( Batch batch ){ unscheduledTasks.addAll(tasksToScheduleAndDestroy); unscheduledTasks.notifyAll(); synchronized ( upcomingTasks ){ - upcomingTasks.removeAll( tasksToScheduleAndDestroy ); + tasksToScheduleAndDestroy.forEach(upcomingTasks::remove); } } } @@ -217,9 +217,14 @@ TaskConfig getConfigFor( String hash ){ return null; } + /** + * Chooses best param for a task + * @param taskname + * @param name + * @return + */ public Map getSchedulerParams( String taskname, String name ){ - Map result = new HashMap(); - return result; + return new HashMap<>(); } public TaskState getTaskState(String taskid) { diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 427f9449..c2331a24 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; @@ -115,7 +114,7 @@ int terminateTasks(List finishedTasks) { List writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); - LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList(); + LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); try { final Inputs inputs = new Inputs( this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/" @@ -140,9 +139,7 @@ List writeInitConfig( NodeTaskFilesAlignment align } } - for (SymlinkInput symlink : alignment.fileAlignment.symlinks) { - inputs.symlinks.add( symlink ); - } + inputs.symlinks.addAll(alignment.fileAlignment.symlinks); new ObjectMapper().writeValue( config, inputs ); return inputFiles; @@ -199,7 +196,7 @@ List getInputsOfTask( Task task ){ .fileInputs .parallelStream() .map( x -> Path.of(x.value.sourceObj) ) - .filter( x -> this.hierarchyWrapper.isInScope(x) ) + .filter(this.hierarchyWrapper::isInScope) .flatMap( sourcePath -> streamFile( hierarchyWrapper.getFile(sourcePath), task, sourcePath )) .collect(Collectors.toList()); } @@ -255,11 +252,12 @@ void podEventReceived(Watcher.Action action, Pod pod){ this.workflowEngineNode = pod.getSpec().getNodeName(); log.info( "WorkflowEngineNode was set to {}", workflowEngineNode ); } + //noinspection LoopConditionNotUpdatedInsideLoop while ( daemonByNode == null ){ //The Watcher can be started before the class is initialized try { Thread.sleep(20); - } catch (InterruptedException e) {} + } catch (InterruptedException ignored) {} } if( ( "mount-" + this.getExecution().replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ final String nodeName = pod.getSpec().getNodeName(); diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java index bf9b4981..af21c1bb 100644 --- a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java @@ -21,7 +21,7 @@ public void run() { int unscheduled = 0; while(!Thread.interrupted()){ try{ - LinkedList tasks; + LinkedList tasks; synchronized (unprocessedTasks) { do { if (unscheduled == unprocessedTasks.size()) { diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java index 42d63780..3cacaf19 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -19,17 +19,19 @@ public void generateCopyScript( Task task ){ ClassLoader classLoader = getClass().getClassLoader(); - try (InputStream inputStream = classLoader.getResourceAsStream( getResource() ); - InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(streamReader)) { - - String line; - while ((line = reader.readLine()) != null) { - pw.println(line); + try (InputStream inputStream = classLoader.getResourceAsStream( getResource() )) { + assert inputStream != null; + try (InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(streamReader)) { + + String line; + while ((line = reader.readLine()) != null) { + pw.println(line); + } + + Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); + Files.setPosixFilePermissions( file.toPath(), executable ); } - - Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); - Files.setPosixFilePermissions( file.toPath(), executable ); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java index 9205c196..04dc903f 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java @@ -1,6 +1,5 @@ package fonda.scheduler.scheduler.schedulingstrategy; -import java.util.LinkedList; import java.util.List; public class InputEntry { diff --git a/src/main/java/fonda/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/util/Batch.java index 50136dbd..3c9978c6 100644 --- a/src/main/java/fonda/scheduler/util/Batch.java +++ b/src/main/java/fonda/scheduler/util/Batch.java @@ -28,6 +28,9 @@ public void close( int tasksInBatch ){ } public void registerTask( Task task ){ + assert ready != null; + assert unready != null; + synchronized ( unready ){ if ( closed && ready.size() + unready.size() >= tasksInBatch ) { throw new IllegalStateException("Batch was closed!"); diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java index e8e8a201..a171a218 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java @@ -3,9 +3,6 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; -import java.util.List; -import java.util.Map; - public class NodeTaskFilesAlignment extends NodeTaskAlignment { diff --git a/src/main/test/java/fonda/scheduler/dag/DAGTest.java b/src/main/test/java/fonda/scheduler/dag/DAGTest.java index bec5da29..766b287d 100644 --- a/src/main/test/java/fonda/scheduler/dag/DAGTest.java +++ b/src/main/test/java/fonda/scheduler/dag/DAGTest.java @@ -11,6 +11,7 @@ public class DAGTest { private void compare(int uid, List vertices, int[] ancestorIds, int[] descedantsIds ){ + //noinspection OptionalGetWithoutIsPresent final Vertex vertex = vertices.stream().filter(v -> v.getUid() == uid).findFirst().get(); if( vertex.getAncestors() == null ){ @@ -145,12 +146,12 @@ public void smallTest(){ debug( vertexList, 0, 3 ); assertEquals( new HashSet<>(), a.getAncestors() ); - final HashSet descA = new HashSet<>(); + final HashSet descA = new HashSet<>(); descA.add( b ); assertEquals( descA, a.getDescendants() ); - final HashSet ancB = new HashSet<>(); + final HashSet ancB = new HashSet<>(); ancB.add( a ); assertEquals( new HashSet<>(), b.getDescendants() ); assertEquals( ancB, b.getAncestors() ); @@ -159,7 +160,7 @@ public void smallTest(){ assertEquals( descA, filter.getDescendants() ); assertEquals( new HashSet<>(),o.getAncestors() ); - final HashSet descO = new HashSet<>(); + final HashSet descO = new HashSet<>(); descO.add( a ); descO.add( b ); diff --git a/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java b/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java index 8e6ae3da..f16d2ab6 100644 --- a/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java +++ b/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java @@ -3,13 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.json.JsonTest; -import org.springframework.boot.test.json.JacksonTester; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.junit4.SpringRunner; import java.io.IOException; diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java index 72bd197e..8fad7c9a 100644 --- a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -42,7 +42,7 @@ private Path storeData( String[] inputdata, String[] outputdata ){ pw.println( s ); } pw.close(); - } catch ( Exception e ){} + } catch ( Exception ignored ){} return tmpDir; } @@ -61,7 +61,7 @@ public void before(){ @Test public void test1(){ - String infiles[] = { + String[] infiles = { "/tmp/nxf.3iuGDWr6Id;0;;4096;directory;-;2021-11-10 12:58:11.210414589 +0000;2021-11-10 12:58:11.222414603 +0000", "/tmp/nxf.3iuGDWr6Id/file.txt;1;/pvcdata/testfile.txt;6;regular file;-;2021-11-10 12:58:07.485035700 +0000;2021-11-10 12:58:07.219065500 +0000", "/tmp/nxf.3iuGDWr6Id/.command.err;1;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000", @@ -82,7 +82,7 @@ public void test1(){ log.info("{}", newAndUpdatedFiles); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230l, 13, new Task( new TaskConfig("P1"), dag) )) ); + expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230L, 13, new Task( new TaskConfig("P1"), dag) )) ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt", "/pvcdata/testfile.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt", "/pvcdata/testfile.txt") ); @@ -93,7 +93,7 @@ public void test1(){ @Test public void test2(){ - String infiles[] = { + String[] infiles = { "/tmp/nxf.IANFIlM3Kv;0;;4096;directory;-;2021-11-12 12:42:42.155614026 +0000;2021-11-12 12:42:42.171614019 +0000", "/tmp/nxf.IANFIlM3Kv/file.txt;1;/pvcdata/testfile.txt;0;regular empty file;-;2021-11-12 12:42:29.000000000 +0000;2021-11-12 12:42:29.000000000 +0000", "/tmp/nxf.IANFIlM3Kv/.command.err;1;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000", @@ -119,10 +119,10 @@ public void test2(){ log.info("{}", newAndUpdatedFiles.toArray()); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223l, 0, task )) ); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223l, 2, task )) ); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223l, 2, task )) ); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223l, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); assertEquals( expected, newAndUpdatedFiles ); @@ -131,7 +131,7 @@ public void test2(){ @Test public void test3(){ - String infiles[] = { + String[] infiles = { "/tmp/nxf.9J6Y5mcXRD;0;;4096;directory;-;2021-11-12 14:42:20.941053482 +0000;2021-11-12 14:42:20.937053480 +0000", "/tmp/nxf.9J6Y5mcXRD/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", @@ -161,7 +161,7 @@ public void test3(){ log.info("{}", newAndUpdatedFiles); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper( node1, 1636728140993l, 4, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper( node1, 1636728140993L, 4, task )) ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt") ); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index af9256a8..f04bc663 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -23,7 +23,7 @@ public class HierarchyWrapperTest { HierarchyWrapper hw; List files; Collection result; - String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; + final String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; LocationWrapper node1; DAG dag; @@ -42,7 +42,7 @@ public void init() { node1 = getLocationWrapper("Node1"); hw = new HierarchyWrapper(workdir); - files = new LinkedList(); + files = new LinkedList<>(); files.add( Paths.get(temporaryDir + "test" )); files.add( Paths.get(temporaryDir + "file.abc" )); @@ -60,7 +60,7 @@ public void init() { private void compare( List a, Collection b){ assertEquals( - new HashSet<>(a.stream().map(x -> x.normalize()).collect(Collectors.toList())), + new HashSet<>(a.stream().map(Path::normalize).collect(Collectors.toList())), new HashSet<>(b) ); assertEquals(a.size(),b.size()); @@ -155,9 +155,9 @@ public void testParallelAdd() { System.gc(); long intialMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - List files = new LinkedList(); + List files = new LinkedList<>(); String wd = workdir + "ab/abcdefg/"; - Map> m = new HashMap(); + Map> m = new HashMap<>(); int iters = 1_000_000; @@ -165,7 +165,7 @@ public void testParallelAdd() { String p = wd + (i % 3) + "/" + (i % 4); Path file = Paths.get(p + "/" + "file-" + i); files.add( file ); - final List currentData = m.getOrDefault(p, new LinkedList()); + final List currentData = m.getOrDefault(p, new LinkedList<>()); currentData.add(file); m.put(p, currentData); } @@ -200,7 +200,7 @@ public void testParallelAdd() { public void testGetFile() { hw = new HierarchyWrapper(workdir); - files = new LinkedList(); + files = new LinkedList<>(); files.add( Paths.get( temporaryDir + "test" ) ); files.add( Paths.get( temporaryDir + "file.abc" ) ); diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 3dc3a952..33c0fc02 100644 --- a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -17,19 +17,18 @@ public class RealFileTest { - private DAG dag; private Task processA; private Task processB; @Before public void before(){ - dag = new DAG(); + DAG dag = new DAG(); List vertexList = new LinkedList<>(); vertexList.add(new Process("processA", 1)); vertexList.add(new Process("processB", 2)); dag.registerVertices(vertexList); - processA = new Task( new TaskConfig("procesA"), dag ); - processB = new Task( new TaskConfig("procesB"), dag ); + processA = new Task( new TaskConfig("procesA"), dag); + processB = new Task( new TaskConfig("procesB"), dag); } private LocationWrapper getLocationWrapper( String location ){ @@ -67,8 +66,10 @@ public void addLocation() { @Test public void addEmptyLocation() { final RealFile realFile = new RealFile( getLocationWrapper("node1") ); + //noinspection ConfusingArgumentToVarargsMethod assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false )); + //noinspection RedundantArrayCreation assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, new LocationWrapper[0] )); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, new LocationWrapper[1] )); } From 615e343b62a780d3dfff8c641b19070080a79be8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Feb 2022 12:34:53 +0100 Subject: [PATCH 083/443] Updated dependencies' version + removed unused dependencies Signed-off-by: Lehmann_Fabian --- pom.xml | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index 89a38c5a..518df9c4 100644 --- a/pom.xml +++ b/pom.xml @@ -35,18 +35,6 @@ 1.2 - - org.postgresql - postgresql - 42.2.18 - - - - com.zaxxer - HikariCP - 3.4.5 - - org.projectlombok lombok @@ -57,13 +45,13 @@ junit junit - 4.13 + 4.13.2 org.mockito mockito-inline - 3.4.4 + 4.3.1 test @@ -85,16 +73,10 @@ compile - - commons-codec - commons-codec - 1.15 - - org.apache.commons commons-compress - 1.20 + 1.21 From ae2e3acff289ab4a5846d215ac200fd6dfbf86c3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Feb 2022 16:25:16 +0100 Subject: [PATCH 084/443] Store lineage of added files Signed-off-by: Lehmann_Fabian --- .../location/hierachy/LocationWrapper.java | 4 ++++ .../fonda/scheduler/rest/PathAttributes.java | 2 ++ .../rest/SchedulerRestController.java | 2 +- .../rest/response/getfile/FileResponse.java | 5 ++++- .../scheduler/SchedulerWithDaemonSet.java | 18 ++++++++++++++---- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index fe9a32d4..4bdcafc2 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -5,10 +5,14 @@ import lombok.Getter; import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; @Getter public class LocationWrapper { + static private final AtomicLong nextID = new AtomicLong(0); + + private final long id = nextID.getAndIncrement(); private final Location location; private final long timestamp; private final long sizeInBytes; diff --git a/src/main/java/fonda/scheduler/rest/PathAttributes.java b/src/main/java/fonda/scheduler/rest/PathAttributes.java index 17398f09..d99dd472 100644 --- a/src/main/java/fonda/scheduler/rest/PathAttributes.java +++ b/src/main/java/fonda/scheduler/rest/PathAttributes.java @@ -10,10 +10,12 @@ public class PathAttributes { private final String path; private final long size; private final long timestamp; + private final long locationWrapperID; private PathAttributes() { this.path = null; this.size = -1; this.timestamp = -1; + this.locationWrapperID = -1; } } diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index db834fc4..f10e61d4 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -243,7 +243,7 @@ ResponseEntity changeLocationForFile(@PathVariable String method, @PathV boolean overwrite = method.equals("overwrite"); - ((SchedulerWithDaemonSet) scheduler).addFile( pa.getPath(), pa.getSize(), pa.getTimestamp(), overwrite, node ); + ((SchedulerWithDaemonSet) scheduler).addFile( pa.getPath(), pa.getSize(), pa.getTimestamp(), pa.getLocationWrapperID(), overwrite, node ); return new ResponseEntity<>( HttpStatus.OK ); diff --git a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java index a83f1dba..24aae732 100644 --- a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java +++ b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java @@ -12,14 +12,16 @@ public class FileResponse { public final String daemon; public final List symlinks; public final boolean notInContext; + public final long locationWrapperID; - public FileResponse( String path, String node, String daemon, boolean sameAsEngine, List symlinks) { + public FileResponse( String path, String node, String daemon, boolean sameAsEngine, List symlinks, long locationWrapperID) { this.path = path; this.sameAsEngine = sameAsEngine; this.node = node; this.daemon = daemon; this.symlinks = symlinks; notInContext = false; + this.locationWrapperID = locationWrapperID; } public FileResponse( String path, List symlinks) { @@ -29,6 +31,7 @@ public FileResponse( String path, List symlinks) { this.daemon = null; this.symlinks = symlinks; notInContext = true; + locationWrapperID = -1; } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index c2331a24..d9bad727 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -33,6 +33,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -45,6 +46,7 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { @Getter private final CopyStrategy copyStrategy; final HierarchyWrapper hierarchyWrapper; + private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); @@ -201,7 +203,7 @@ List getInputsOfTask( Task task ){ .collect(Collectors.toList()); } - public FileResponse nodeOfLastFileVersion(String path ) throws NotARealFileException { + public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileException { LinkedList symlinks = new LinkedList<>(); Path currentPath = Paths.get(path); fonda.scheduler.model.location.hierachy.File currentFile = hierarchyWrapper.getFile( currentPath ); @@ -223,13 +225,21 @@ public FileResponse nodeOfLastFileVersion(String path ) throws NotARealFileExcep final RealFile file = (RealFile) currentFile; final LocationWrapper lastUpdate = file.getLastUpdate(LocationType.NODE); if( lastUpdate == null ) return null; + requestedLocations.put( lastUpdate.getId(), lastUpdate ); String node = lastUpdate.getLocation().getIdentifier(); - return new FileResponse( currentPath.toString(), node, getDaemonOnNode(node), node.equals(workflowEngineNode), symlinks ); + return new FileResponse( currentPath.toString(), node, getDaemonOnNode(node), node.equals(workflowEngineNode), symlinks, lastUpdate.getId() ); } - public void addFile( String path, long size, long timestamp, boolean overwrite, String node ){ + public void addFile( String path, long size, long timestamp, long locationWrapperID, boolean overwrite, String node ){ final NodeLocation location = NodeLocation.getLocation( node == null ? workflowEngineNode : node ); - final LocationWrapper locationWrapper = new LocationWrapper( location, timestamp, size, null); + + LocationWrapper locationWrapper; + if( !overwrite && locationWrapperID != -1 ){ + locationWrapper = requestedLocations.get( locationWrapperID ).getCopyOf( location ); + } else { + locationWrapper = new LocationWrapper( location, timestamp, size, null); + } + hierarchyWrapper.addFile( Paths.get( path ), overwrite, locationWrapper ); } From d3399b13cc91edfd22632981f330cc4318c3bedc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 10 Feb 2022 11:21:50 +0100 Subject: [PATCH 085/443] No need for Process to Parse Output Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/TaskResultParser.java | 4 ---- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 3 +-- .../fonda/scheduler/model/TaskResultParserTest.java | 10 ++++++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index f9327452..f9908fe3 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -6,7 +6,6 @@ import fonda.scheduler.model.outfiles.PathLocationWrapperPair; import fonda.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; -import fonda.scheduler.dag.Process; import java.io.File; import java.io.FileNotFoundException; @@ -14,7 +13,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Stream; @@ -44,14 +42,12 @@ private String getRootDir( File file ){ * * @param workdir * @param location - * @param process * @param finishedTask * @return A list of all new or updated files */ public Set getNewAndUpdatedFiles( final Path workdir, final Location location, - final Process process, final boolean onlyUpdated, Task finishedTask ){ diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index d9bad727..651b81f7 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -88,7 +88,6 @@ int terminateTasks(List finishedTasks) { final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( Paths.get(finishedTask.getWorkingDir()), finishedTask.getNode(), - finishedTask.getProcess(), !finishedTask.wasSuccessfullyExecuted(), finishedTask ); @@ -106,7 +105,7 @@ int terminateTasks(List finishedTasks) { } } } catch ( Exception e ){ - log.info( "Problem while finishing task: " + finishedTask.getConfig().getName(), e ); + log.info( "Problem while finishing task: " + finishedTask.getConfig().getHash() + " (" + finishedTask.getConfig().getName() + ")", e ); } super.taskWasFinished( finishedTask ); }); diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java index 8fad7c9a..ed94c599 100644 --- a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -76,13 +76,15 @@ public void test1(){ final Path path = storeData(infiles, outfiles); + final Task task = new Task(new TaskConfig("P1"), dag); + final TaskResultParser taskResultParser = new TaskResultParser(); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), dag.getByProcess("P1"), false, null); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), false, task); log.info("{}", newAndUpdatedFiles); final HashSet expected = new HashSet<>(); - expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230L, 13, new Task( new TaskConfig("P1"), dag) )) ); + expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230L, 13, task)) ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt", "/pvcdata/testfile.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt", "/pvcdata/testfile.txt") ); @@ -114,7 +116,7 @@ public void test2(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); final Task task = new Task( new TaskConfig("P1"), dag); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, null, false, task); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); log.info("{}", newAndUpdatedFiles.toArray()); @@ -156,7 +158,7 @@ public void test3(){ final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); final Task task = new Task( new TaskConfig("P1"), dag ); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, null, false, task); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); log.info("{}", newAndUpdatedFiles); From b53750db30264faf769e812df06cbf70ff6c1432 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 10 Feb 2022 11:26:05 +0100 Subject: [PATCH 086/443] Parse also dates with format MMM dd HH:mm:ss yyyy Signed-off-by: Lehmann_Fabian --- .../scheduler/model/TaskResultParser.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index f9908fe3..1abca2d1 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -115,17 +115,23 @@ public Set getNewAndUpdatedFiles( private long fileTimeFromString(String date) { if( date == null || date.equals("-")) { - return -1; + throw new RuntimeException("Date was: " + date); } - String[] parts = date.split(" "); - parts[1] = parts[1].substring(0, 12); - String shortenedDate = String.join(" ", parts); try { - return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); - } catch (ParseException e) { - e.printStackTrace(); + if (Character.isLetter(date.charAt(0))) { + // if ls was used, date has the format "Nov 2 08:49:30 2021" + return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime(); + } else { + // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" + String[] parts = date.split(" "); + parts[1] = parts[1].substring(0, 12); + // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" + String shortenedDate = String.join(" ", parts); + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); + } + } catch ( Exception e ){ + throw new RuntimeException( "Cannot parse date: " + date, e ); } - return -1; } } From 495675252baccff9012fe651f0d05f7697867a82 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 10 Feb 2022 12:40:17 +0100 Subject: [PATCH 087/443] Also parse NodeVertices Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/Node.java | 13 +++++++++++++ .../fonda/scheduler/dag/VertexDeserializer.java | 8 ++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 src/main/java/fonda/scheduler/dag/Node.java diff --git a/src/main/java/fonda/scheduler/dag/Node.java b/src/main/java/fonda/scheduler/dag/Node.java new file mode 100644 index 00000000..be3a0f57 --- /dev/null +++ b/src/main/java/fonda/scheduler/dag/Node.java @@ -0,0 +1,13 @@ +package fonda.scheduler.dag; + +public class Node extends Operator { + + Node(String label, int uid) { + super(label, uid); + } + + @Override + public Type getType() { + return Type.NODE; + } +} diff --git a/src/main/java/fonda/scheduler/dag/VertexDeserializer.java b/src/main/java/fonda/scheduler/dag/VertexDeserializer.java index 80a877a2..dd12183e 100644 --- a/src/main/java/fonda/scheduler/dag/VertexDeserializer.java +++ b/src/main/java/fonda/scheduler/dag/VertexDeserializer.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.TextNode; import org.springframework.boot.jackson.JsonComponent; @@ -19,7 +20,8 @@ public Vertex deserialize(JsonParser jsonParser, DeserializationContext ctxt) th final TreeNode treeNode = jsonParser.getCodec().readTree(jsonParser); final Type type = Type.valueOf(((TextNode) treeNode.get("type")).asText()); - final String label = ((TextNode) treeNode.get("label")).asText(); + final TreeNode labelNode = treeNode.get("label"); + final String label = labelNode instanceof NullNode ? null : ((TextNode) labelNode).asText(); final int uid = ((IntNode) treeNode.get("uid")).asInt(); if ( Type.PROCESS == type ) { @@ -28,8 +30,10 @@ public Vertex deserialize(JsonParser jsonParser, DeserializationContext ctxt) th return new Operator( label, uid ); } else if ( Type.ORIGIN == type ) { return new Origin( label, uid ); + } else if ( Type.NODE == type ) { + return new Node( label, uid ); } else { - throw new IllegalArgumentException( "No implementation for type: " + type ); + throw new IllegalArgumentException( "No implementation for type: " + type + "(" + treeNode + ")" ); } } From 1efbdbef2eb0e5a6eb5af507d84edad12bb4bcc1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 10 Feb 2022 16:15:53 +0100 Subject: [PATCH 088/443] Logic to select file for process Signed-off-by: Lehmann_Fabian --- .../model/location/hierachy/RealFile.java | 79 +++++++++++++++++-- .../scheduler/SchedulerWithDaemonSet.java | 4 +- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 9d9ecb3b..42b5fdca 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -1,13 +1,12 @@ package fonda.scheduler.model.location.hierachy; import fonda.scheduler.dag.Process; +import fonda.scheduler.model.Task; import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import lombok.Getter; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; +import java.util.*; public class RealFile extends AbstractFile { @@ -69,14 +68,78 @@ public void addOrUpdateLocation( boolean overwrite, LocationWrapper... location } } - public List getFilesForProcess( Process process ){ - LocationWrapper lastUpdated = null; + public List getFilesForTask( Task task ){ + LocationWrapper[] locations = this.locations; + + LinkedList results = new LinkedList<>(); + LinkedList ancestors = null; + LinkedList descendants = null; + LinkedList unrelated = null; + LinkedList initial = null; + for (LocationWrapper location : locations) { - if ( lastUpdated == null || lastUpdated.getCreateTime() < location.getCreateTime() ) { - lastUpdated = location; + + LinkedList listToFilter; + Set locationsAncestors; + Process locationProcess; + + if ( location.getCreatedByTask() == null ) { + //File was modified by an operator (no relation known) + if ( initial == null ) initial = new LinkedList<>(); + initial.add( location ); + continue; + } else if ( (locationProcess = location.getCreatedByTask().getProcess() ) == task.getProcess() ) { + //Location was created by the same process == does definitely fit. + results.add( location ); + continue; + } else if ( (locationsAncestors = locationProcess.getAncestors()).contains( task.getProcess() ) ) { + // location is a direct ancestor + if ( ancestors == null ) ancestors = new LinkedList<>(); + listToFilter = ancestors; + } else if ( locationProcess.getDescendants().contains( task.getProcess() ) ) { + // location is a direct descendant + if ( descendants == null ) descendants = new LinkedList<>(); + descendants.add( location ); + continue; + } else { + // location was possibly generated in parallel + if ( unrelated == null ) unrelated = new LinkedList<>(); + listToFilter = unrelated; } + + //Add location to list if it could be the last version + if ( listToFilter.isEmpty() ) listToFilter.add( location ); + else { + final Iterator iterator = listToFilter.iterator(); + final Set locationsDescendants = locationProcess.getDescendants(); + boolean isAncestor = false; + while (iterator.hasNext()) { + + final LocationWrapper next = iterator.next(); + final Process currentProcess = next.getCreatedByTask().getProcess(); + if (locationProcess == currentProcess) { + break; + } else if (locationsAncestors.contains(currentProcess)) { + iterator.remove(); + } else if (locationsDescendants.contains(currentProcess)) { + isAncestor = true; + break; + } + + } + if (!isAncestor) listToFilter.add(location); + } + + } + + if ( ancestors == null && unrelated == null && descendants == null ){ + results.addAll( initial ); + } else { + if ( ancestors != null ) results.addAll( ancestors ); + if ( unrelated != null ) results.addAll( unrelated ); + if ( descendants != null ) results.addAll( descendants ); } - return lastUpdated == null ? new LinkedList<>() : List.of( lastUpdated ); + return results; } public LocationWrapper getLastUpdate( LocationType type ){ diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 651b81f7..d01aed41 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -174,7 +174,7 @@ private Stream streamFile( return Stream.of(new PathFileLocationTriple( y.getKey(), ((RealFile) y.getValue()), - ((RealFile) y.getValue()).getFilesForProcess(task.getProcess()) + ((RealFile) y.getValue()).getFilesForTask( task ) )); else { return streamFile( y.getValue(), task, y.getKey() ); @@ -186,7 +186,7 @@ private Stream streamFile( return Stream.of( new PathFileLocationTriple( sourcePath, realFile, - realFile.getFilesForProcess( task.getProcess() ) + realFile.getFilesForTask( task ) ) ); } From 2755b125ecf92d2ba8d13a692b96eb9564c3a926 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 22 Feb 2022 15:10:30 +0100 Subject: [PATCH 089/443] Move test folder into src/ Signed-off-by: Lehmann_Fabian --- src/{main => }/test/java/fonda/scheduler/dag/DAGTest.java | 0 .../test/java/fonda/scheduler/dag/VertexDeserializerTest.java | 0 .../test/java/fonda/scheduler/model/TaskResultParserTest.java | 0 .../scheduler/model/location/hierachy/HierarchyWrapperTest.java | 0 .../fonda/scheduler/model/location/hierachy/RealFileTest.java | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/{main => }/test/java/fonda/scheduler/dag/DAGTest.java (100%) rename src/{main => }/test/java/fonda/scheduler/dag/VertexDeserializerTest.java (100%) rename src/{main => }/test/java/fonda/scheduler/model/TaskResultParserTest.java (100%) rename src/{main => }/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java (100%) rename src/{main => }/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java (100%) diff --git a/src/main/test/java/fonda/scheduler/dag/DAGTest.java b/src/test/java/fonda/scheduler/dag/DAGTest.java similarity index 100% rename from src/main/test/java/fonda/scheduler/dag/DAGTest.java rename to src/test/java/fonda/scheduler/dag/DAGTest.java diff --git a/src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java b/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java similarity index 100% rename from src/main/test/java/fonda/scheduler/dag/VertexDeserializerTest.java rename to src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java diff --git a/src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java similarity index 100% rename from src/main/test/java/fonda/scheduler/model/TaskResultParserTest.java rename to src/test/java/fonda/scheduler/model/TaskResultParserTest.java diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java similarity index 100% rename from src/main/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java rename to src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java diff --git a/src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java similarity index 100% rename from src/main/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java rename to src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java From c08a14d2fed6192bcf0a734b160c26f048806203 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 22 Feb 2022 15:11:10 +0100 Subject: [PATCH 090/443] added junit-vintage-engine Signed-off-by: Lehmann_Fabian --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 518df9c4..b34a9c21 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,13 @@ 1.21 + + org.junit.vintage + junit-vintage-engine + test + + + From 11c379baa6922780fac0d05292e30327285a5fe5 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 16:59:40 +0100 Subject: [PATCH 091/443] Add surefire and jacoco mvn plugins --- pom.xml | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pom.xml b/pom.xml index b34a9c21..4205bfbf 100644 --- a/pom.xml +++ b/pom.xml @@ -124,6 +124,51 @@ + + org.jacoco + jacoco-maven-plugin + 0.8.7 + + ${project.build.directory}/coverage-reports/jacoco-unit.exec + ${project.build.directory}/coverage-reports/jacoco-unit.exec + ${project.reporting.outputDirectory}/jacoco-report + surefireArgLine + + + + pre-unit-test + process-test-classes + + prepare-agent + + + + post-unit-test + test + + report + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + ${surefireArgLine} --illegal-access=permit + + + + default-test + test + + test + + + + From 856bd6e87e3a2096b6b845c5c145d8725322899d Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 17:00:06 +0100 Subject: [PATCH 092/443] Add Jenkinsfile and pod decleration --- Jenkinsfile | 134 +++++++++++++++++++++++++++++++++++++++++++++++ jenkins-pod.yaml | 32 +++++++++++ 2 files changed, 166 insertions(+) create mode 100644 Jenkinsfile create mode 100644 jenkins-pod.yaml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..8b83e217 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,134 @@ +pipeline { + agent { + kubernetes { + yamlFile 'jenkins-pod.yaml' + } + } + environment { + // creates DOCKERHUB_USR and DOCKERHUB_PSW env variables + DOCKERHUB = credentials('fondahub-dockerhub') + } + + stages { + stage('Build') { + steps { + container('maven') { + // run a clean build without tests to see if the project compiles + sh 'mvn clean test-compile -DskipTests=true -Dmaven.javadoc.skip=true -B -V' + } + } + } + + stage('Test') { + steps { + container('maven') { + // run JUnit tests + sh 'mvn test -B -V' + } + } + post { + // collect test results + always { + junit 'target/surefire-reports/TEST-*.xml' + jacoco classPattern: 'target/classes,target/test-classes', execPattern: 'target/coverage-reports/*.exec', inclusionPattern: '**/*.class', sourcePattern: 'src/main/java,src/test/java' + archiveArtifacts 'target/surefire-reports/TEST-*.xml' + archiveArtifacts 'target/coverage-reports/*.exec' + } + } + } + + stage('Package') { + steps { + container('maven') { + sh 'mvn package -DskipTests=true -Dmaven.javadoc.skip=true -B -V' + } + } + post { + success { + archiveArtifacts 'target/*.jar' + } + } + } + + stage('Static Code Analysis') { + steps { + script { + def scannerHome = tool 'SonarScanner'; + withSonarQubeEnv() { + sh ''' + ${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ + -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ + -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) + ''' + } + } + } + } + + stage('Build and push Docker') { + // agents are specified per stage to enable real parallel execution + parallel { + stage('workflow-k8s-scheduler') { + agent { + kubernetes { + yamlFile 'jenkins-pod.yaml' + } + } + steps { + container('hadolint') { + sh "hadolint --format json Dockerfile | tee -a hadolint_scheduler.json" + } + // build and push image to fondahub/workflow-k8s-scheduler + container('docker') { + sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin" + sh "docker build . -t fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}" + sh "docker tag fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]} fondahub/workflow-k8s-scheduler:latest" + sh "docker push fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}" + sh "docker push fondahub/workflow-k8s-scheduler:latest" + } + } + post { + always { + archiveArtifacts "hadolint_scheduler.json" + recordIssues( + aggregatingResults: true, + tools: [hadoLint(pattern: "hadolint_scheduler.json")] + ) + } + } + } + stage('vsftpd') { + agent { + kubernetes { + yamlFile 'jenkins-pod.yaml' + } + } + steps { + container('hadolint') { + sh "hadolint --format json daemons/ftp/Dockerfile | tee -a hadolint_vsftpd.json" + } + // build and push image to fondahub/vsftpd + container('docker') { + sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin" + sh "docker build daemons/ftp/ -t fondahub/vsftpd:${GIT_COMMIT[0..7]}" + sh "docker tag fondahub/vsftpd:${GIT_COMMIT[0..7]} fondahub/vsftpd:latest" + sh "docker push fondahub/vsftpd:${GIT_COMMIT[0..7]}" + sh "docker push fondahub/vsftpd:latest" + } + } + post { + always { + archiveArtifacts "hadolint_vsftpd.json" + recordIssues( + aggregatingResults: true, + tools: [hadoLint(pattern: "hadolint_vsftpd.json")] + ) + } + } + } + } + } + } +} \ No newline at end of file diff --git a/jenkins-pod.yaml b/jenkins-pod.yaml new file mode 100644 index 00000000..71628a4d --- /dev/null +++ b/jenkins-pod.yaml @@ -0,0 +1,32 @@ + +apiVersion: v1 +kind: Pod +spec: + containers: + - name: maven + image: maven:3.8.3-jdk-11-slim + command: ['sleep', '99d'] + imagePullPolicy: Always + tty: true + + - name: hadolint + image: hadolint/hadolint:latest-debian + imagePullPolicy: Always + command: + - cat + tty: true + + - name: docker + image: docker:20.10.12 + command: ['sleep', '99d'] + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + + - name: docker-daemon + image: docker:20.10.12-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" From c116683dae81e2dc45d0b948a402d0652c8daf43 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 17:58:41 +0100 Subject: [PATCH 093/443] Run sonarqube in jnlp container --- Jenkinsfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8b83e217..5a3b1439 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,16 +52,18 @@ pipeline { stage('Static Code Analysis') { steps { - script { - def scannerHome = tool 'SonarScanner'; - withSonarQubeEnv() { - sh ''' - ${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ - -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ - -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ - -Dsonar.junit.reportPaths=target/surefire-reports \ - -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) - ''' + container('jnlp') { + script { + def scannerHome = tool 'SonarScanner'; + withSonarQubeEnv() { + sh ''' + ${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ + -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ + -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) + ''' + } } } } From c8be617138ddf7064276bda09e7847dfd011a584 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 18:08:16 +0100 Subject: [PATCH 094/443] Exchange ticks to enable variable usage --- Jenkinsfile | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5a3b1439..94b3406f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,18 +52,16 @@ pipeline { stage('Static Code Analysis') { steps { - container('jnlp') { - script { - def scannerHome = tool 'SonarScanner'; - withSonarQubeEnv() { - sh ''' - ${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ - -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ - -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ - -Dsonar.junit.reportPaths=target/surefire-reports \ - -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) - ''' - } + script { + def scannerHome = tool 'SonarScanner'; + withSonarQubeEnv() { + sh """ + ${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ + -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ + -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) + """ } } } From 41ae52af9593abe39344eb5db551f797ad4711bc Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 18:14:52 +0100 Subject: [PATCH 095/443] Fix typo --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 94b3406f..f55eff95 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -55,13 +55,13 @@ pipeline { script { def scannerHome = tool 'SonarScanner'; withSonarQubeEnv() { - sh """ - ${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ + sh ''' + "${scannerHome}"/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ -Dsonar.junit.reportPaths=target/surefire-reports \ -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) - """ + ''' } } } From 17ab1e6810480eb85f266850408bee39218fedad Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 18:27:07 +0100 Subject: [PATCH 096/443] Move to declerative syntax --- Jenkinsfile | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f55eff95..b801caee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,17 +52,14 @@ pipeline { stage('Static Code Analysis') { steps { - script { - def scannerHome = tool 'SonarScanner'; - withSonarQubeEnv() { - sh ''' - "${scannerHome}"/bin/sonar-scanner -Dsonar.projectKey=workflow_k8s_scheduler \ - -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ - -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ - -Dsonar.junit.reportPaths=target/surefire-reports \ - -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) - ''' - } + withSonarQubeEnv('SonarScanner') { + sh ''' + mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ + -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ + -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) + ''' } } } From 8729834bdeea1c55a3dd82af2a76b009f33da372 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 18:33:54 +0100 Subject: [PATCH 097/443] Specify correct sonarqube server --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b801caee..a33cc09c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,7 +52,7 @@ pipeline { stage('Static Code Analysis') { steps { - withSonarQubeEnv('SonarScanner') { + withSonarQubeEnv('fonda-sonarqube') { sh ''' mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ From 44a3b7e5a9804d6af0cb7edd97b8e24812a2bea1 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Tue, 22 Feb 2022 18:41:49 +0100 Subject: [PATCH 098/443] Run sonar scanner in maven container --- Jenkinsfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a33cc09c..e169ee58 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,14 +52,16 @@ pipeline { stage('Static Code Analysis') { steps { - withSonarQubeEnv('fonda-sonarqube') { - sh ''' - mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ - -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ - -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ - -Dsonar.junit.reportPaths=target/surefire-reports \ - -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) - ''' + container('maven') { + withSonarQubeEnv('fonda-sonarqube') { + sh ''' + mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ + -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ + -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) + ''' + } } } } From 6f0cdaa233f222e94e38a813b1db7695505881d8 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Wed, 23 Feb 2022 09:22:34 +0100 Subject: [PATCH 099/443] Add debug step --- Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e169ee58..39c170f7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,6 +54,10 @@ pipeline { steps { container('maven') { withSonarQubeEnv('fonda-sonarqube') { + sh """ + println ${env.SONAR_HOST_URL} + println ${SONAR_AUTH_TOKEN} + """ sh ''' mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ From 64e6770d37b5de3238cf8fdfd3c1e6dd58c16799 Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Wed, 23 Feb 2022 09:45:32 +0100 Subject: [PATCH 100/443] Remove debug step --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39c170f7..e169ee58 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,10 +54,6 @@ pipeline { steps { container('maven') { withSonarQubeEnv('fonda-sonarqube') { - sh """ - println ${env.SONAR_HOST_URL} - println ${SONAR_AUTH_TOKEN} - """ sh ''' mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ From 9140aae2cd8216af83c9f2621d6d2ee6e9432dac Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Wed, 23 Feb 2022 10:24:04 +0100 Subject: [PATCH 101/443] Add id for hadolint task --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e169ee58..a8cc3d80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,6 +93,7 @@ pipeline { archiveArtifacts "hadolint_scheduler.json" recordIssues( aggregatingResults: true, + id: "scheduler-hadolint", tools: [hadoLint(pattern: "hadolint_scheduler.json")] ) } @@ -122,6 +123,7 @@ pipeline { archiveArtifacts "hadolint_vsftpd.json" recordIssues( aggregatingResults: true, + id: "vsfptd-hadolint", tools: [hadoLint(pattern: "hadolint_vsftpd.json")] ) } From 17a24661ff9864ab40c2359ffc46fca7d967892f Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Wed, 23 Feb 2022 10:38:17 +0100 Subject: [PATCH 102/443] Move id to hadolint stage --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a8cc3d80..d25f777a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -67,6 +67,10 @@ pipeline { } stage('Build and push Docker') { + // only push images from the master branch + when { + branch "master" + } // agents are specified per stage to enable real parallel execution parallel { stage('workflow-k8s-scheduler') { @@ -93,8 +97,7 @@ pipeline { archiveArtifacts "hadolint_scheduler.json" recordIssues( aggregatingResults: true, - id: "scheduler-hadolint", - tools: [hadoLint(pattern: "hadolint_scheduler.json")] + tools: [hadoLint(pattern: "hadolint_scheduler.json", id: "scheduler")] ) } } @@ -123,8 +126,7 @@ pipeline { archiveArtifacts "hadolint_vsftpd.json" recordIssues( aggregatingResults: true, - id: "vsfptd-hadolint", - tools: [hadoLint(pattern: "hadolint_vsftpd.json")] + tools: [hadoLint(pattern: "hadolint_vsftpd.json", id: "vsfptd")] ) } } From a900acff4c91b9dfc545c1f92f41f6287ea81856 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Feb 2022 17:35:35 +0100 Subject: [PATCH 103/443] Refactoring basaed on SonarQube Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 8 +- .../java/fonda/scheduler/config/Beans.java | 4 +- .../java/fonda/scheduler/dag/Process.java | 24 +-- .../fonda/scheduler/model/NodeWithAlloc.java | 29 ++- .../scheduler/model/PodListWithIndex.java | 39 ---- .../fonda/scheduler/model/PodWithAge.java | 16 ++ .../scheduler/model/SchedulerConfig.java | 24 +-- .../fonda/scheduler/model/TaskConfig.java | 2 +- .../model/TaskInputFileLocationWrapper.java | 4 +- .../model/location/NodeLocation.java | 19 +- .../location/hierachy/LocationWrapper.java | 2 +- .../model/location/hierachy/RealFile.java | 166 ++++++++++++------ .../scheduler/scheduler/RandomScheduler.java | 19 +- .../fonda/scheduler/scheduler/Scheduler.java | 14 +- .../scheduler/SchedulerWithDaemonSet.java | 16 +- src/main/java/fonda/scheduler/util/Batch.java | 4 +- .../scheduler/dag/VertexDeserializerTest.java | 5 + 17 files changed, 233 insertions(+), 162 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/model/PodListWithIndex.java diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index e54794a5..3cfd5667 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -22,7 +22,6 @@ public class KubernetesClient extends DefaultKubernetesClient { public KubernetesClient(){ - OperationContext operationContext = new OperationContext(); for( Node node : this.nodes().list().getItems() ){ nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node) ); } @@ -112,8 +111,9 @@ public void eventReceived(Action action, Node node) { break; case MODIFIED: log.info("Node {} was an modified", node.getMetadata().getName()); - //todo deal with changed stae + //todo deal with changed state break; + default: log.warn("No implementation for {}", action); } } @@ -141,11 +141,12 @@ public void eventReceived(Action action, Pod pod) { node.addPod( new PodWithAge(pod) ); break; case MODIFIED: final List containerStatuses = pod.getStatus().getContainerStatuses(); - if ( containerStatuses.size() == 0 + if ( containerStatuses.isEmpty() || containerStatuses.get(0).getState().getTerminated() == null ) { break; } + //Pod is finished case DELETED: case ERROR: log.info("Pod has released its resources: {}", pod.getMetadata().getName()); @@ -153,6 +154,7 @@ public void eventReceived(Action action, Pod pod) { node.removePod( pod ); kubernetesClient.informAllScheduler(); break; + default: log.warn("No implementation for {}", action); } } diff --git a/src/main/java/fonda/scheduler/config/Beans.java b/src/main/java/fonda/scheduler/config/Beans.java index 1eca2cda..718e4700 100644 --- a/src/main/java/fonda/scheduler/config/Beans.java +++ b/src/main/java/fonda/scheduler/config/Beans.java @@ -9,8 +9,10 @@ public class Beans { private static KubernetesClient kubernetesClient; + private Beans() {} + @Bean - KubernetesClient getClient(){ + static KubernetesClient getClient(){ if(kubernetesClient == null) { kubernetesClient = new KubernetesClient(); kubernetesClient.getConfiguration().setNamespace(null); diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/fonda/scheduler/dag/Process.java index 4c80e5ac..bec8da9d 100644 --- a/src/main/java/fonda/scheduler/dag/Process.java +++ b/src/main/java/fonda/scheduler/dag/Process.java @@ -36,16 +36,16 @@ public Set getAncestors() { public void addInbound( Edge e ) { in.add( e ); final Vertex from = e.getFrom(); - final Set ancestors = from.getAncestors(); + final Set fromAncestors = from.getAncestors(); - if ( from.getType() == Type.PROCESS ) ancestors.add((Process) from); + if ( from.getType() == Type.PROCESS ) fromAncestors.add((Process) from); - this.ancestors.addAll(ancestors); + this.ancestors.addAll(fromAncestors); if ( from.getType() == Type.PROCESS ) ((Process) from).descendants.add( this ); - final Set descendants = this.getDescendants(); - ancestors.forEach( v -> { - v.descendants.addAll( descendants ); + final Set descendantsCopy = this.getDescendants(); + fromAncestors.forEach( v -> { + v.descendants.addAll( descendantsCopy ); v.descendants.add( this ); }); } @@ -53,16 +53,16 @@ public void addInbound( Edge e ) { public void addOutbound( Edge e ) { out.add( e ); final Vertex to = e.getTo(); - final Set descendants = to.getDescendants(); + final Set toDescendants = to.getDescendants(); - if ( to.getType() == Type.PROCESS ) descendants.add((Process) to); + if ( to.getType() == Type.PROCESS ) toDescendants.add((Process) to); - this.descendants.addAll(descendants); + this.descendants.addAll(toDescendants); if ( to.getType() == Type.PROCESS ) ((Process)to).ancestors.add( this ); - final Set ancestors = this.getAncestors(); - descendants.forEach( v -> { - v.ancestors.addAll( ancestors ); + final Set ancestorsCopy = this.getAncestors(); + toDescendants.forEach( v -> { + v.ancestors.addAll( ancestorsCopy ); v.ancestors.add( this ); }); } diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 744512ff..f674077c 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -15,7 +15,7 @@ @Slf4j public class NodeWithAlloc extends Node implements Comparable { - private final PodRequirements max_resources; + private final PodRequirements maxResources; final Map assignedPods; @@ -33,16 +33,16 @@ public NodeWithAlloc(Node node) { this.setAdditionalProperty( e.getKey(), e.getValue() ); } - BigDecimal max_cpu = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("cpu")); - BigDecimal max_ram = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("memory")); + BigDecimal maxCpu = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("cpu")); + BigDecimal maxRam = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("memory")); - max_resources = new PodRequirements( max_cpu, max_ram); + maxResources = new PodRequirements( maxCpu, maxRam); assignedPods = new HashMap<>(); this.nodeLocation = NodeLocation.getLocation( node ); - log.info("Node {} has RAM: {} and CPU: {}", node.getMetadata().getName(), max_ram, max_cpu); + log.info("Node {} has RAM: {} and CPU: {}", node.getMetadata().getName(), maxRam, maxCpu); } public void addPod( PodWithAge pod ){ @@ -66,7 +66,7 @@ public PodRequirements getRequestedResources(){ } public PodRequirements getAvailableResources(){ - return max_resources.sub(getRequestedResources()); + return maxResources.sub(getRequestedResources()); } public boolean canSchedule( PodWithAge pod ){ @@ -89,4 +89,21 @@ public int compareTo(NodeWithAlloc o) { } } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NodeWithAlloc)) return false; + if (!super.equals(o)) return false; + NodeWithAlloc that = (NodeWithAlloc) o; + return getMetadata().getName() != null ? getMetadata().getName().equals(that.getMetadata().getName()) : that.getMetadata().getName() == null; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getMaxResources() != null ? getMaxResources().hashCode() : 0); + result = 31 * result + (getAssignedPods() != null ? getAssignedPods().hashCode() : 0); + result = 31 * result + (getNodeLocation() != null ? getNodeLocation().hashCode() : 0); + return result; + } } diff --git a/src/main/java/fonda/scheduler/model/PodListWithIndex.java b/src/main/java/fonda/scheduler/model/PodListWithIndex.java deleted file mode 100644 index b697b9ad..00000000 --- a/src/main/java/fonda/scheduler/model/PodListWithIndex.java +++ /dev/null @@ -1,39 +0,0 @@ -package fonda.scheduler.model; - -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodList; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class PodListWithIndex extends PodList { - - private final Map nameIndexMap = new HashMap<>(); - - public PodListWithIndex() {} - - public PodListWithIndex(List pods) { - pods.forEach(this::addPodToList); - } - - /** - * Overrides existing pod if pod is already in list - * - * @param pod - */ - public void addPodToList(Pod pod) { - nameIndexMap.put(pod.getMetadata().getName(), pod); - } - - public void removePodFromList(Pod pod) { - nameIndexMap.remove(pod.getMetadata().getName()); - } - - - @Override - public List getItems() { - return new ArrayList<>(nameIndexMap.values()); - } -} diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index 04b9b75d..3ba6e6d2 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -42,5 +42,21 @@ public String getName(){ return this.getMetadata().getName(); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof PodWithAge)) return false; + if (!super.equals(o)) return false; + PodWithAge that = (PodWithAge) o; + + return getName() != null ? getName().equals(that.getName()) : that.getName() == null; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getAge() != null ? getAge().hashCode() : 0); + return result; + } } diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 8890c0b8..92ee4685 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -4,11 +4,11 @@ public class SchedulerConfig { - final public List localClaims; - final public List volumeClaims; - final public String workDir; - final public String dns; - final public String copyStrategy; + public final List localClaims; + public final List volumeClaims; + public final String workDir; + public final String dns; + public final String copyStrategy; public SchedulerConfig(List localClaims, List volumeClaims, @@ -30,9 +30,9 @@ private SchedulerConfig(){ this.copyStrategy = null; } - static public class LocalClaim { - final public String mountPath; - final public String hostPath; + public static class LocalClaim { + public final String mountPath; + public final String hostPath; private LocalClaim(){ this.mountPath = null; @@ -48,10 +48,10 @@ public String toString() { } } - static public class VolumeClaim { - final public String mountPath; - final public String claimName; - final public String subPath; + public static class VolumeClaim { + public final String mountPath; + public final String claimName; + public final String subPath; private VolumeClaim(){ this.mountPath = null; diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 1d37e6f9..c8062a8f 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -38,7 +38,7 @@ public TaskConfig(String task) { @Getter @ToString - static public class Input { + public static class Input { private final String name; private final Object value; diff --git a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java index fd745e71..da9ea166 100644 --- a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java @@ -7,8 +7,8 @@ @Getter public class TaskInputFileLocationWrapper { - final private RealFile file; - final private LocationWrapper wrapper; + private final RealFile file; + private final LocationWrapper wrapper; public TaskInputFileLocationWrapper(RealFile file, LocationWrapper wrapper) { this.file = file; diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java index 54d0b3fa..162ff888 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -8,7 +8,7 @@ public class NodeLocation extends Location { - static final private ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap<>(); + private static final ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap<>(); @Getter private final String identifier; @@ -41,4 +41,21 @@ public String toString() { return "Node(" + identifier + ")"; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NodeLocation)) return false; + if (!super.equals(o)) return false; + + NodeLocation that = (NodeLocation) o; + + return getIdentifier() != null ? getIdentifier().equals(that.getIdentifier()) : that.getIdentifier() == null; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getIdentifier() != null ? getIdentifier().hashCode() : 0); + return result; + } } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 4bdcafc2..01b55c7f 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -10,7 +10,7 @@ @Getter public class LocationWrapper { - static private final AtomicLong nextID = new AtomicLong(0); + private static final AtomicLong nextID = new AtomicLong(0); private final long id = nextID.getAndIncrement(); private final Location location; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 42b5fdca..6039def3 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -68,78 +68,136 @@ public void addOrUpdateLocation( boolean overwrite, LocationWrapper... location } } + private List combineResultsWithInitial ( + LinkedList current, + LinkedList ancestors, + LinkedList descendants, + LinkedList unrelated, + LinkedList initial + ) { + LinkedList result; + long time = 0; + //Only keep last update + if ( initial.size() > 1 ) { + result = new LinkedList<>(); + for (LocationWrapper locationWrapper : initial) { + if ( locationWrapper.getCreateTime() > time ) { + result.clear(); + result.add( locationWrapper ); + } else if ( locationWrapper.getCreateTime() == time ) { + result.add( locationWrapper ); + } + } + } else { + time = initial.get(0).getCreateTime(); + result = initial; + } + addAllLaterLocationsToResult( current, result, time ); + if( current == null ) addAllLaterLocationsToResult( ancestors, result, time ); + addAllLaterLocationsToResult( unrelated, result, time ); + addAllLaterLocationsToResult( descendants, result, time ); + return result; + } + + private List combineResultsEmptyInitial ( + LinkedList current, + LinkedList ancestors, + LinkedList descendants, + LinkedList unrelated + ) { + LinkedList result = null; + if ( current != null ) result = current; + if ( current == null && ancestors != null ) result = ancestors; + if ( unrelated != null ) { + if ( result == null ) result = unrelated; + else result.addAll( unrelated ); + } + if ( descendants != null ) { + if ( result == null ) result = descendants; + else result.addAll( descendants ); + } + return result; + } + + private void addToAncestors(LinkedList ancestors, LocationWrapper location, Process locationProcess) { + //Add location to list if it could be the last version + final Iterator iterator = ancestors.iterator(); + Set locationAncestors = null; + while (iterator.hasNext()) { + final LocationWrapper next = iterator.next(); + final Process currentProcess = next.getCreatedByTask().getProcess(); + if (locationProcess == currentProcess) { + break; + } else { + if( locationAncestors == null ) locationAncestors = locationProcess.getAncestors(); + if ( locationAncestors.contains(currentProcess) ) { + iterator.remove(); + } else if (locationProcess.getDescendants().contains(currentProcess)) { + return; + } + } + } + ancestors.add(location); + } + + private LinkedList addAndCreateList(LinkedList list, LocationWrapper toAdd ){ + if ( list == null ) list = new LinkedList<>(); + list.add( toAdd ); + return list; + } + public List getFilesForTask( Task task ){ - LocationWrapper[] locations = this.locations; + LocationWrapper[] locationsRef = this.locations; - LinkedList results = new LinkedList<>(); + LinkedList current = null; LinkedList ancestors = null; LinkedList descendants = null; LinkedList unrelated = null; LinkedList initial = null; - for (LocationWrapper location : locations) { - - LinkedList listToFilter; - Set locationsAncestors; - Process locationProcess; + final Process taskProcess = task.getProcess(); + final Set taskAncestors = taskProcess.getAncestors(); + final Set taskDescendants = taskProcess.getDescendants(); + for ( LocationWrapper location : locationsRef ) { + //File was modified by an operator (no relation known) if ( location.getCreatedByTask() == null ) { - //File was modified by an operator (no relation known) - if ( initial == null ) initial = new LinkedList<>(); - initial.add( location ); + initial = addAndCreateList( initial, location ); continue; - } else if ( (locationProcess = location.getCreatedByTask().getProcess() ) == task.getProcess() ) { + } + + final Process locationProcess = location.getCreatedByTask().getProcess(); + if ( locationProcess == taskProcess) //Location was created by the same process == does definitely fit. - results.add( location ); - continue; - } else if ( (locationsAncestors = locationProcess.getAncestors()).contains( task.getProcess() ) ) { + current = addAndCreateList( current, location ); + else if ( taskAncestors.contains(locationProcess) ) { // location is a direct ancestor - if ( ancestors == null ) ancestors = new LinkedList<>(); - listToFilter = ancestors; - } else if ( locationProcess.getDescendants().contains( task.getProcess() ) ) { + if ( ancestors == null ) { + ancestors = new LinkedList<>(); + ancestors.add( location ); + } else { + addToAncestors( ancestors, location, locationProcess ); + } + } + else if ( taskDescendants.contains(locationProcess) ) // location is a direct descendant - if ( descendants == null ) descendants = new LinkedList<>(); - descendants.add( location ); - continue; - } else { + descendants = addAndCreateList( descendants, location ); + else // location was possibly generated in parallel - if ( unrelated == null ) unrelated = new LinkedList<>(); - listToFilter = unrelated; - } + unrelated = addAndCreateList( unrelated, location ); + } - //Add location to list if it could be the last version - if ( listToFilter.isEmpty() ) listToFilter.add( location ); - else { - final Iterator iterator = listToFilter.iterator(); - final Set locationsDescendants = locationProcess.getDescendants(); - boolean isAncestor = false; - while (iterator.hasNext()) { - - final LocationWrapper next = iterator.next(); - final Process currentProcess = next.getCreatedByTask().getProcess(); - if (locationProcess == currentProcess) { - break; - } else if (locationsAncestors.contains(currentProcess)) { - iterator.remove(); - } else if (locationsDescendants.contains(currentProcess)) { - isAncestor = true; - break; - } + return ( initial == null ) + ? combineResultsEmptyInitial( current, ancestors, descendants, unrelated ) + : combineResultsWithInitial( current, ancestors, descendants, unrelated, initial ); + } - } - if (!isAncestor) listToFilter.add(location); + private void addAllLaterLocationsToResult( List list, List result, long time ){ + if( list != null ) { + for ( LocationWrapper l : list ) { + if( l.getCreateTime() >= time ) result.add( l ); } - - } - - if ( ancestors == null && unrelated == null && descendants == null ){ - results.addAll( initial ); - } else { - if ( ancestors != null ) results.addAll( ancestors ); - if ( unrelated != null ) results.addAll( unrelated ); - if ( descendants != null ) results.addAll( descendants ); } - return results; } public LocationWrapper getLastUpdate( LocationType type ){ diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index f4849d20..9e64dd94 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -18,6 +18,8 @@ @Slf4j public class RandomScheduler extends SchedulerWithDaemonSet { + private final Random random = new Random(); + public RandomScheduler(String name, KubernetesClient client, String namespace, SchedulerConfig config) { super(name, client, namespace, config); } @@ -37,8 +39,7 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ logInfo.add("Node: " + item.getName() + " " + availableResources); } logInfo.add("------------------------------------"); - System.out.println(String.join("\n", logInfo)); - + log.info(String.join("\n", logInfo)); for ( final Task task : unscheduledTasks) { final PodWithAge pod = task.getPod(); @@ -50,22 +51,20 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ //&& !x.getName().equals(this.getWorkflowEngineNode()) ) .collect(Collectors.toList()); - System.out.println("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); Optional node = matchingNodes.isEmpty() ? Optional.empty() - : Optional.of(matchingNodes.get(new Random().nextInt(matchingNodes.size()))); + : Optional.of(matchingNodes.get(random.nextInt(matchingNodes.size()))); if( node.isPresent() ){ - //log.info("Task needs: " + task.getConfig().getInputs().toString()); final List inputsOfTask = getInputsOfTask(task); final FileAlignment fileAlignment = scheduleFiles(task, inputsOfTask, node.get()); alignment.add( new NodeTaskFilesAlignment(node.get(),task, fileAlignment ) ); availableByNode.get(node.get().getName()).subFromThis(pod.getRequest()); - System.out.println("--> " + node.get().getName()); + log.info("--> " + node.get().getName()); } else { log.info( "No node with enough resources for {}", pod.getName() ); } } - System.out.flush(); final ScheduleObject scheduleObject = new ScheduleObject(alignment); scheduleObject.setCheckStillPossible( true ); return scheduleObject; @@ -78,12 +77,10 @@ FileAlignment scheduleFiles(Task task, List inputsOfTask, NodeWithAlloc n if( entry instanceof PathFileLocationTriple ){ final PathFileLocationTriple pathFileLocationTriple = (PathFileLocationTriple) entry; final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( - new Random().nextInt( pathFileLocationTriple.locations.size() ) + random.nextInt( pathFileLocationTriple.locations.size() ) ); final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - if ( !map.containsKey( nodeIdentifier )){ - map.put( nodeIdentifier, new LinkedList<>() ); - } + map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); final List pathsOfNode = map.get( nodeIdentifier ); pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file ) ); } else if ( entry instanceof SymlinkInput ){ diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index ff9d86d0..0ffe505c 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -43,9 +43,9 @@ public abstract class Scheduler { private Batch currentBatchInstance = null; final KubernetesClient client; - final private Set upcomingTasks = new HashSet<>(); - final private List unscheduledTasks = new ArrayList<>(100); - final private List unfinishedTasks = new ArrayList<>(100); + private final Set upcomingTasks = new HashSet<>(); + private final List unscheduledTasks = new ArrayList<>(100); + private final List unfinishedTasks = new ArrayList<>(100); final Map tasksByHash = new HashMap<>(); private final Watch watcher; private final TaskprocessingThread schedulingThread; @@ -334,11 +334,6 @@ Task changeStateOfTask(Pod pod, State state){ return null; } - @Deprecated - PodResource findPodByName(String name ){ - return client.pods().withName( name ); - } - public void informResourceChange() { synchronized (unscheduledTasks){ unscheduledTasks.notifyAll(); @@ -410,7 +405,7 @@ public void eventReceived(Action action, Pod pod) { } break; case MODIFIED: - if (pod.getStatus().getContainerStatuses().size() > 0 && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { + if (!pod.getStatus().getContainerStatuses().isEmpty() && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { scheduler.onPodTermination(pwa); } else { final Task task = scheduler.getTaskByPod(pwa); @@ -419,6 +414,7 @@ public void eventReceived(Action action, Pod pod) { break; case DELETED: scheduler.markPodAsDeleted(pwa); + default: log.info( "No implementation for {}", action ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index d01aed41..1b13efa5 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -251,6 +251,7 @@ private void podWasInitialized( Pod pod ){ * Since task was not yet initialized: set scheduled * @param task */ + @Override void taskWasScheduledSetState( Task task ){ task.getState().setState( State.SCHEDULED ); } @@ -266,7 +267,9 @@ void podEventReceived(Watcher.Action action, Pod pod){ //The Watcher can be started before the class is initialized try { Thread.sleep(20); - } catch (InterruptedException ignored) {} + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } } if( ( "mount-" + this.getExecution().replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ final String nodeName = pod.getSpec().getNodeName(); @@ -290,13 +293,10 @@ void podEventReceived(Watcher.Action action, Pod pod){ } } } else if ( this.getName().equals(pod.getSpec().getSchedulerName())) { - if ( action == Watcher.Action.MODIFIED ){ - Task t = getTaskByPod( pod ); - if ( t.getState().getState() == State.SCHEDULED ) { - final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); - if ( ! initContainerStatuses.isEmpty() && initContainerStatuses.get(0).getState().getTerminated() != null ) { - podWasInitialized( pod ); - } + if ( action == Watcher.Action.MODIFIED && getTaskByPod( pod ).getState().getState() == State.SCHEDULED ) { + final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); + if ( ! initContainerStatuses.isEmpty() && initContainerStatuses.get(0).getState().getTerminated() != null ) { + podWasInitialized( pod ); } } } diff --git a/src/main/java/fonda/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/util/Batch.java index 3c9978c6..b8aafd5e 100644 --- a/src/main/java/fonda/scheduler/util/Batch.java +++ b/src/main/java/fonda/scheduler/util/Batch.java @@ -53,10 +53,10 @@ public boolean canSchedule(){ public List getTasksToScheduleAndDestroy(){ if ( !closed ) throw new IllegalStateException("Batch was not yet closed!"); - final List ready = this.ready; + final List readyList = this.ready; this.ready = null; this.unready = null; - return ready; + return readyList; } } diff --git a/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java b/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java index f16d2ab6..133c3092 100644 --- a/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java +++ b/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java @@ -7,6 +7,8 @@ import java.io.IOException; +import static org.junit.Assert.assertEquals; + @JsonTest public class VertexDeserializerTest { @@ -19,6 +21,9 @@ public void testDeserialize() throws IOException { module.addDeserializer(Vertex.class, new VertexDeserializer()); objectMapper.registerModule(module); final Vertex process = objectMapper.readValue(json, Vertex.class); + assertEquals( "a", process.getLabel() ); + assertEquals( Type.PROCESS, process.getType() ); + assertEquals( 0, process.getUid() ); System.out.println( process ); } From c5d3539367dcd7a1848dec7b1360d2ff3070b361 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Feb 2022 17:36:33 +0100 Subject: [PATCH 104/443] Check for null Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index d3499cec..846dd894 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -41,6 +41,7 @@ public Task( TaskConfig config, DAG dag ) { } public String getWorkingDir(){ + if ( this.pod == null ) return null; return pod.getSpec().getContainers().get(0).getWorkingDir(); } From 5126b9177943f236283752becf0b87cde985da82 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Feb 2022 17:37:09 +0100 Subject: [PATCH 105/443] New LocationWrapper Constructor Signed-off-by: Lehmann_Fabian --- .../location/hierachy/LocationWrapper.java | 23 +++++++++++++++---- .../scheduler/SchedulerWithDaemonSet.java | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 01b55c7f..66f3b19c 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -20,6 +20,10 @@ public class LocationWrapper { private final Task createdByTask; private final LocationWrapper copyOf; + public LocationWrapper(Location location, long timestamp, long sizeInBytes) { + this( location, timestamp, sizeInBytes ,null); + } + public LocationWrapper(Location location, long timestamp, long sizeInBytes, Task task) { this( location, timestamp, sizeInBytes ,task, null ); } @@ -41,15 +45,24 @@ public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof LocationWrapper)) return false; LocationWrapper that = (LocationWrapper) o; - return getTimestamp() == that.getTimestamp() - && getSizeInBytes() == that.getSizeInBytes() - && Objects.equals(getLocation(), that.getLocation()) - && Objects.equals(getCopyOf(), that.getCopyOf()) - && Objects.equals(getCreatedByTask(), that.getCreatedByTask()); + return getId() == that.getId(); } @Override public int hashCode() { return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getCreatedByTask()); } + + @Override + public String toString() { + return "LocationWrapper{" + + "id=" + id + + ", location=" + location.getIdentifier() + + ", timestamp=" + timestamp + + ", sizeInBytes=" + sizeInBytes + + ", createTime=" + createTime + + ", createdByTask=" + createdByTask + + ", copyOf=" + copyOf + + '}'; + } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 1b13efa5..85b83abd 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -236,7 +236,7 @@ public void addFile( String path, long size, long timestamp, long locationWrappe if( !overwrite && locationWrapperID != -1 ){ locationWrapper = requestedLocations.get( locationWrapperID ).getCopyOf( location ); } else { - locationWrapper = new LocationWrapper( location, timestamp, size, null); + locationWrapper = new LocationWrapper( location, timestamp, size ); } hierarchyWrapper.addFile( Paths.get( path ), overwrite, locationWrapper ); From 3790e53e9e28eb0819a033fe040126c5dbf9089a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Feb 2022 17:43:50 +0100 Subject: [PATCH 106/443] Made Serializable Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 4 +++- src/main/java/fonda/scheduler/model/PodRequirements.java | 5 ++++- src/main/java/fonda/scheduler/model/location/Location.java | 3 ++- .../java/fonda/scheduler/model/location/NodeLocation.java | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index f674077c..d1399e61 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -15,9 +15,11 @@ @Slf4j public class NodeWithAlloc extends Node implements Comparable { + private static final long serialVersionUID = 1L; + private final PodRequirements maxResources; - final Map assignedPods; + private final Map assignedPods; @Getter private final NodeLocation nodeLocation; diff --git a/src/main/java/fonda/scheduler/model/PodRequirements.java b/src/main/java/fonda/scheduler/model/PodRequirements.java index b6b56fe2..1e1ade2d 100644 --- a/src/main/java/fonda/scheduler/model/PodRequirements.java +++ b/src/main/java/fonda/scheduler/model/PodRequirements.java @@ -2,9 +2,12 @@ import lombok.Getter; +import java.io.Serializable; import java.math.BigDecimal; -public class PodRequirements { +public class PodRequirements implements Serializable { + + private static final long serialVersionUID = 1L; public static final PodRequirements ZERO = new PodRequirements(); diff --git a/src/main/java/fonda/scheduler/model/location/Location.java b/src/main/java/fonda/scheduler/model/location/Location.java index ff737a37..cf4f8b7b 100644 --- a/src/main/java/fonda/scheduler/model/location/Location.java +++ b/src/main/java/fonda/scheduler/model/location/Location.java @@ -1,8 +1,9 @@ package fonda.scheduler.model.location; +import java.io.Serializable; import java.util.Objects; -public abstract class Location { +public abstract class Location implements Serializable { public abstract String getIdentifier(); diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java index 162ff888..64eda2f1 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -8,6 +8,8 @@ public class NodeLocation extends Location { + private static final long serialVersionUID = 1L; + private static final ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap<>(); @Getter From 43cf6b78a1365e0c24f1d79f7f751e97d282af3f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 11:20:58 +0100 Subject: [PATCH 107/443] Fixed equals Method Signed-off-by: Lehmann_Fabian --- .../model/location/hierachy/LocationWrapper.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 66f3b19c..7d9e4459 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -44,8 +44,15 @@ public LocationWrapper getCopyOf( Location location ) { public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof LocationWrapper)) return false; + LocationWrapper that = (LocationWrapper) o; - return getId() == that.getId(); + + if (getTimestamp() != that.getTimestamp()) return false; + if (getSizeInBytes() != that.getSizeInBytes()) return false; + if (!getLocation().equals(that.getLocation())) return false; + if (getCreatedByTask() != null ? !getCreatedByTask().equals(that.getCreatedByTask()) : that.getCreatedByTask() != null) + return false; + return getCopyOf() != null ? getCopyOf().equals(that.getCopyOf()) : that.getCopyOf() == null; } @Override From c30604823bf18eedf69f6d97edef46915e8b5bfd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 16:34:22 +0100 Subject: [PATCH 108/443] Simplify Logic Signed-off-by: Lehmann_Fabian --- .../model/location/hierachy/Folder.java | 6 ++-- .../location/hierachy/HierarchyWrapper.java | 10 +++---- .../model/location/hierachy/RealFile.java | 28 +++++++------------ .../model/location/hierachy/RealFileTest.java | 5 +--- 4 files changed, 19 insertions(+), 30 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 8052cf80..e3c9d518 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -60,12 +60,12 @@ private void getAllChildren( final Map result, Path currentPa } } - public boolean addOrUpdateFile( final String name, boolean overwrite, final LocationWrapper... locations ) { + public boolean addOrUpdateFile( final String name, boolean overwrite, final LocationWrapper location ) { children.compute( name, (k,v) -> { if (v == null || v.isDirectory() || v.isSymlink() ) - return new RealFile( locations ); + return new RealFile( location ); final RealFile file = (RealFile) v; - file.addOrUpdateLocation( overwrite, locations ); + file.addOrUpdateLocation( overwrite, location ); return v; } ); return true; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index cf305ae3..97d451a4 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -66,19 +66,19 @@ public Map getAllFilesInDir( final Path path ){ /** * * @param path file to add (absolute path) - * @param locations locations where the file is located + * @param location location where the file is located * @return false if file can not be created */ - public boolean addFile( final Path path, final LocationWrapper... locations ){ - return addFile( path, false, locations ); + public boolean addFile( final Path path, final LocationWrapper location ){ + return addFile( path, false, location ); } - public boolean addFile( final Path path, boolean overwrite, final LocationWrapper... locations ){ + public boolean addFile( final Path path, boolean overwrite, final LocationWrapper location ){ final Folder folderToInsert = findFolderToInsert(path); if( folderToInsert == null ) return false; - else return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, locations ); + else return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, location ); } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 6039def3..ad519532 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -39,31 +39,23 @@ private void checkIfValidInput( LocationWrapper[] location ){ } } - public void addOrUpdateLocation( boolean overwrite, LocationWrapper... location ){ - checkIfValidInput( location ); + public void addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ + if ( location == null ) throw new IllegalArgumentException( "location contains null value" ); if ( overwrite ){ - this.locations = location; + this.locations = new LocationWrapper[]{ location }; return; } synchronized ( this ){ - int index = 0; - LocationWrapper[] newLocationsTmp = new LocationWrapper[location.length]; - for (LocationWrapper newLoc : location) { - boolean foundEqual = false; - for (int i = 0; i < locations.length; i++) { - if ( newLoc.getLocation().equals( locations[i].getLocation() ) ) { - foundEqual = true; - if ( newLoc.getTimestamp() > locations[i].getTimestamp() ) - locations[i] = newLoc; - break; + for (int i = 0; i < locations.length; i++) { + if ( location.getLocation().equals( locations[i].getLocation() ) ) { + if ( location.getTimestamp() > locations[i].getTimestamp() ) { + locations[i] = location; } - } - if ( !foundEqual ){ - newLocationsTmp[index++] = newLoc; + return; } } - final LocationWrapper[] newLocation = Arrays.copyOf(locations, locations.length + index ); - System.arraycopy( newLocationsTmp, 0, newLocation, locations.length, index ); + final LocationWrapper[] newLocation = Arrays.copyOf(locations, locations.length + 1); + newLocation[ locations.length ] = location; locations = newLocation; } } diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 33c0fc02..4207f7a6 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -68,10 +68,7 @@ public void addEmptyLocation() { final RealFile realFile = new RealFile( getLocationWrapper("node1") ); //noinspection ConfusingArgumentToVarargsMethod assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false )); - //noinspection RedundantArrayCreation - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, new LocationWrapper[0] )); - assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, new LocationWrapper[1] )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( true, null )); } @Test From da53efbb104c257e0ec0f01a6525b64c239f8828 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 16:40:11 +0100 Subject: [PATCH 109/443] Simplify logic 2 Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/location/hierachy/RealFile.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index ad519532..26f039da 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -16,9 +16,9 @@ public class RealFile extends AbstractFile { @Getter private LocationWrapper[] locations; - public RealFile( LocationWrapper... locations ) { - checkIfValidInput( locations ); - this.locations = locations; + public RealFile( LocationWrapper location ) { + if ( location == null ) throw new IllegalArgumentException( "location is null" ); + this.locations = new LocationWrapper[]{ location }; } @Override @@ -40,7 +40,7 @@ private void checkIfValidInput( LocationWrapper[] location ){ } public void addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ - if ( location == null ) throw new IllegalArgumentException( "location contains null value" ); + if ( location == null ) throw new IllegalArgumentException( "location is null" ); if ( overwrite ){ this.locations = new LocationWrapper[]{ location }; return; From 2d5aa9cc0e05f79b2423000aaa9d09349a925763 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 16:40:26 +0100 Subject: [PATCH 110/443] Fix time was not set Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/location/hierachy/RealFile.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index 26f039da..e52af78a 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -76,6 +76,7 @@ private List combineResultsWithInitial ( if ( locationWrapper.getCreateTime() > time ) { result.clear(); result.add( locationWrapper ); + time = locationWrapper.getCreateTime(); } else if ( locationWrapper.getCreateTime() == time ) { result.add( locationWrapper ); } From 8effb3d27413efe21e79d30facd5eec30eb2166f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 17:30:11 +0100 Subject: [PATCH 111/443] Removed unused method Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/location/hierachy/RealFile.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index e52af78a..bf254b21 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -31,14 +31,6 @@ public boolean isSymlink() { return false; } - private void checkIfValidInput( LocationWrapper[] location ){ - if ( location == null || location.length == 0 ) - throw new IllegalArgumentException( "location was null or empty" ); - for (LocationWrapper loc : location) { - if ( loc == null ) throw new IllegalArgumentException( "location contains null value" ); - } - } - public void addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ if ( location == null ) throw new IllegalArgumentException( "location is null" ); if ( overwrite ){ From 52f295245ec230f902829913accce3d314b4f336 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 17:30:39 +0100 Subject: [PATCH 112/443] Test RealFile Signed-off-by: Lehmann_Fabian --- pom.xml | 11 +- .../java/fonda/scheduler/dag/InputEdge.java | 2 +- .../model/location/hierachy/RealFileTest.java | 281 +++++++++++++++++- 3 files changed, 283 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 4205bfbf..bf6a0ae0 100644 --- a/pom.xml +++ b/pom.xml @@ -74,14 +74,15 @@ - org.apache.commons - commons-compress - 1.21 + org.junit.vintage + junit-vintage-engine + test - org.junit.vintage - junit-vintage-engine + org.apache.commons + commons-collections4 + 4.1 test diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/fonda/scheduler/dag/InputEdge.java index 2be13ef3..1ca4b8b1 100644 --- a/src/main/java/fonda/scheduler/dag/InputEdge.java +++ b/src/main/java/fonda/scheduler/dag/InputEdge.java @@ -23,7 +23,7 @@ private InputEdge() { * @param from * @param to */ - InputEdge(int from, int to) { + public InputEdge(int from, int to) { label = null; this.from = from; this.to = to; diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 4207f7a6..397dac9d 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -1,24 +1,31 @@ package fonda.scheduler.model.location.hierachy; import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.InputEdge; import fonda.scheduler.dag.Process; import fonda.scheduler.dag.Vertex; import fonda.scheduler.model.Task; import fonda.scheduler.model.TaskConfig; +import fonda.scheduler.model.location.LocationType; import fonda.scheduler.model.location.NodeLocation; +import org.apache.commons.collections4.iterators.PermutationIterator; import org.junit.Before; import org.junit.Test; import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertThrows; public class RealFileTest { private Task processA; private Task processB; + private Task processC; + private Task processD; + private Task processE; @Before public void before(){ @@ -26,9 +33,21 @@ public void before(){ List vertexList = new LinkedList<>(); vertexList.add(new Process("processA", 1)); vertexList.add(new Process("processB", 2)); + vertexList.add(new Process("processC", 3)); + vertexList.add(new Process("processD", 4)); + vertexList.add(new Process("processE", 5)); dag.registerVertices(vertexList); - processA = new Task( new TaskConfig("procesA"), dag); - processB = new Task( new TaskConfig("procesB"), dag); + List edgeList = new LinkedList<>(); + edgeList.add( new InputEdge(1,2) ); + edgeList.add( new InputEdge(2,4) ); + edgeList.add( new InputEdge(4,5) ); + edgeList.add( new InputEdge(1,3) ); + dag.registerEdges(edgeList); + processA = new Task( new TaskConfig("processA"), dag); + processB = new Task( new TaskConfig("processB"), dag); + processC = new Task( new TaskConfig("processC"), dag); + processD = new Task( new TaskConfig("processD"), dag); + processE = new Task( new TaskConfig("processE"), dag); } private LocationWrapper getLocationWrapper( String location ){ @@ -66,7 +85,6 @@ public void addLocation() { @Test public void addEmptyLocation() { final RealFile realFile = new RealFile( getLocationWrapper("node1") ); - //noinspection ConfusingArgumentToVarargsMethod assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( true, null )); } @@ -130,4 +148,257 @@ public void changeFileOnExistingLocation() { } + @Test + public void isFile() { + final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); + assertFalse( realFile.isDirectory() ); + assertFalse( realFile.isSymlink() ); + } + + + @Test + public void getFilesForTaskTest() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealFile realFile = new RealFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + + assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA))); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB))); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC))); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD))); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE))); + } + + locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealFile realFile = new RealFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + realFile.addOrUpdateLocation(false, next.get(4)); + + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processA))); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processB))); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processC))); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processD))); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processE))); + } + } + + @Test + public void getFilesForTaskTestInitFiles() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealFile realFile = new RealFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + realFile.addOrUpdateLocation(false, next.get(4)); + + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE))); + } + + } + + @Test + public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); + + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealFile realFile = new RealFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + realFile.addOrUpdateLocation(false, next.get(4)); + + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE))); + } + } + + @Test + public void getFilesForTaskTestDifferentAncestors() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null ); + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealFile realFile = new RealFile( next.get(0) ); + realFile.addOrUpdateLocation( false, next.get(1) ); + realFile.addOrUpdateLocation( false, next.get(2) ); + realFile.addOrUpdateLocation( false, next.get(3) ); + realFile.addOrUpdateLocation( false, next.get(4) ); + + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ) ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ) ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ) ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ) ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ) ) ); + } + + + while( loc4.getCreateTime() != loc5.getCreateTime() ){ + loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, null ); + loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null ); + } + + locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealFile realFile = new RealFile( next.get(0) ); + realFile.addOrUpdateLocation( false, next.get(1) ); + realFile.addOrUpdateLocation( false, next.get(2) ); + realFile.addOrUpdateLocation( false, next.get(3) ); + realFile.addOrUpdateLocation( false, next.get(4) ); + + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ) ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ) ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ) ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ) ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ) ) ); + } + + + } + + @Test + public void getLastLocationTest() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 2, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + + RealFile realFile = new RealFile( loc1 ); + assertEquals( loc1, realFile.getLastUpdate( LocationType.NODE )); + realFile.addOrUpdateLocation(false, loc4 ); + assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); + realFile.addOrUpdateLocation(false, loc2 ); + assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); + realFile.addOrUpdateLocation(false, loc3 ); + assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); + realFile.addOrUpdateLocation(false, loc5 ); + assertEquals( loc5, realFile.getLastUpdate( LocationType.NODE )); + + realFile.addOrUpdateLocation(true, loc2 ); + assertEquals( loc2, realFile.getLastUpdate( LocationType.NODE )); + + } + + @Test + public void getLocationWrapperForNodeTest() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 2, 2, null ); + assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + + RealFile realFile = new RealFile(loc1); + assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); + realFile.addOrUpdateLocation(false, loc4 ); + assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + realFile.addOrUpdateLocation(false, loc3 ); + assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + realFile.addOrUpdateLocation(false, loc2 ); + assertEquals( loc2, realFile.getLocationWrapper( NodeLocation.getLocation("Node2") ) ); + realFile.addOrUpdateLocation(false, loc5 ); + assertEquals( loc5, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); + + assertThrows( RuntimeException.class, () -> realFile.getLocationWrapper( NodeLocation.getLocation("Node99") ) ); + + } + + + } \ No newline at end of file From 83ea91e819c41f00865620f0e8e1e37531478c5b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Feb 2022 17:44:27 +0100 Subject: [PATCH 113/443] Set timestamp to current time Signed-off-by: Lehmann_Fabian --- .../model/location/hierachy/RealFileTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 397dac9d..2cc57e5a 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -342,15 +342,15 @@ public void getLastLocationTest() throws InterruptedException { final CountDownLatch waiter = new CountDownLatch(1); - LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, null ); + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 2, 2, null ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); RealFile realFile = new RealFile( loc1 ); assertEquals( loc1, realFile.getLastUpdate( LocationType.NODE )); @@ -373,15 +373,15 @@ public void getLocationWrapperForNodeTest() throws InterruptedException { final CountDownLatch waiter = new CountDownLatch(1); - LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, null ); + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 2, 2, null ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); RealFile realFile = new RealFile(loc1); assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); From 0653dedbcd0e03ecc89d72692622f2a746770568 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 09:36:40 +0100 Subject: [PATCH 114/443] Combine conditions Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 85b83abd..420e85c8 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -292,12 +292,13 @@ void podEventReceived(Watcher.Action action, Pod pod){ } } } - } else if ( this.getName().equals(pod.getSpec().getSchedulerName())) { - if ( action == Watcher.Action.MODIFIED && getTaskByPod( pod ).getState().getState() == State.SCHEDULED ) { - final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); - if ( ! initContainerStatuses.isEmpty() && initContainerStatuses.get(0).getState().getTerminated() != null ) { - podWasInitialized( pod ); - } + } else if ( this.getName().equals(pod.getSpec().getSchedulerName()) + && action == Watcher.Action.MODIFIED + && getTaskByPod( pod ).getState().getState() == State.SCHEDULED ) + { + final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); + if ( ! initContainerStatuses.isEmpty() && initContainerStatuses.get(0).getState().getTerminated() != null ) { + podWasInitialized( pod ); } } } From da1b1c80949c5bf9411066ea898e3400b9ed4dbb Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 09:37:16 +0100 Subject: [PATCH 115/443] Only one call in Closure Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/location/hierachy/RealFileTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 2cc57e5a..c9df1a51 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -395,7 +395,8 @@ public void getLocationWrapperForNodeTest() throws InterruptedException { assertEquals( loc5, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); - assertThrows( RuntimeException.class, () -> realFile.getLocationWrapper( NodeLocation.getLocation("Node99") ) ); + final NodeLocation node99 = NodeLocation.getLocation("Node99"); + assertThrows( RuntimeException.class, () -> realFile.getLocationWrapper( node99 ) ); } From ab7fc69481b60d6c9fd6f1bb4b119b0bf1cc26be Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 09:41:22 +0100 Subject: [PATCH 116/443] Same String into Method Signed-off-by: Lehmann_Fabian --- .../rest/SchedulerRestController.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index f10e61d4..780e260d 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -43,6 +43,10 @@ public static void addScheduler(Pair key, Scheduler scheduler ){ schedulerHolder.put( key, scheduler ); } + private ResponseEntity noSchedulerFor( String execution ){ + return new ResponseEntity<>( "There is already a scheduler for " + execution, HttpStatus.BAD_REQUEST ); + } + /** * Register a sheduler for a workflow execution * @param namespace namespace where the workflow runs @@ -61,7 +65,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa final Pair key = getKey( namespace, execution ); if( schedulerHolder.containsKey( key ) ) { - return new ResponseEntity<>( "There is already a scheduler for " + execution, HttpStatus.BAD_REQUEST ); + return noSchedulerFor( execution ); } switch ( strategy.toLowerCase() ){ @@ -86,14 +90,14 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa * @return Parameters the scheduler suggests for the task */ @PutMapping("/scheduler/registerTask/{namespace}/{execution}") - ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody TaskConfig config ) { + ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody TaskConfig config ) { log.trace( execution + " " + config.getTask() + " got: " + config ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } scheduler.addTask( config ); @@ -109,7 +113,7 @@ ResponseEntity startBatch(@PathVariable String namespace, @PathVariable final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } scheduler.startBatch(); return new ResponseEntity<>( HttpStatus.OK ); @@ -122,7 +126,7 @@ ResponseEntity endBatch(@PathVariable String namespace, @PathVariable St final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } scheduler.endBatch( tasksInBatch ); return new ResponseEntity<>( HttpStatus.OK ); @@ -137,12 +141,12 @@ ResponseEntity endBatch(@PathVariable String namespace, @PathVariable St * @return boolean */ @GetMapping("/scheduler/taskstate/{namespace}/{execution}/{taskid}") - ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { + ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } return new ResponseEntity<>( scheduler.getTaskState( taskid ), HttpStatus.OK ); @@ -163,7 +167,7 @@ ResponseEntity delete(@PathVariable String namespace, @PathVariable Str final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution, HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } schedulerHolder.remove( key ); client.removeScheduler( scheduler ); @@ -179,7 +183,7 @@ ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariab final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if(!(scheduler instanceof SchedulerWithDaemonSet)){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } String daemon = ((SchedulerWithDaemonSet) scheduler).getDaemonOnNode( node ); @@ -193,14 +197,14 @@ ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariab } @GetMapping("/file/{namespace}/{execution}") - ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestParam String path ) { + ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestParam String path ) { log.info( "Get file location request: {} {} {}", namespace, execution, path ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if(!(scheduler instanceof SchedulerWithDaemonSet)){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } FileResponse fileResponse; @@ -233,7 +237,7 @@ ResponseEntity changeLocationForFile(@PathVariable String method, @PathV final Scheduler scheduler = schedulerHolder.get( key ); if(!(scheduler instanceof SchedulerWithDaemonSet)){ log.info("No scheduler for: " + execution); - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } if ( !method.equals("add") && !method.equals("overwrite") ) { @@ -266,7 +270,7 @@ ResponseEntity addVertices(@PathVariable String namespace, @PathVariable final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } scheduler.getDag().registerVertices( vertices ); @@ -283,7 +287,7 @@ ResponseEntity addEdges(@PathVariable String namespace, @PathVariable St final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); if( scheduler == null ){ - return new ResponseEntity<>( "No scheduler for: " + execution , HttpStatus.NOT_FOUND ); + return noSchedulerFor( execution ); } final DAG dag = scheduler.getDag(); From b7335dd0170d1e2401d2bdc55937b915df0a17dc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 10:22:16 +0100 Subject: [PATCH 117/443] Make constructor available again Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/config/Beans.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/config/Beans.java b/src/main/java/fonda/scheduler/config/Beans.java index 718e4700..9bc53e73 100644 --- a/src/main/java/fonda/scheduler/config/Beans.java +++ b/src/main/java/fonda/scheduler/config/Beans.java @@ -9,10 +9,10 @@ public class Beans { private static KubernetesClient kubernetesClient; - private Beans() {} + Beans() {} @Bean - static KubernetesClient getClient(){ + KubernetesClient getClient(){ if(kubernetesClient == null) { kubernetesClient = new KubernetesClient(); kubernetesClient.getConfiguration().setNamespace(null); From 42295b6dd69437c29c53016e35e61794b1762210 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 10:22:59 +0100 Subject: [PATCH 118/443] Date parser in own class Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/DateParser.java | 34 +++++++++++++++++++ .../scheduler/model/TaskResultParser.java | 23 +------------ 2 files changed, 35 insertions(+), 22 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/DateParser.java diff --git a/src/main/java/fonda/scheduler/model/DateParser.java b/src/main/java/fonda/scheduler/model/DateParser.java new file mode 100644 index 00000000..e032c174 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/DateParser.java @@ -0,0 +1,34 @@ +package fonda.scheduler.model; + +import java.nio.file.attribute.FileTime; +import java.text.SimpleDateFormat; + +public class DateParser { + + public static Long millisFromString( String date ) { + if( date == null || date.isEmpty() || date.equals("-") || date.equals("w") ) { + return null; + } + try { + if (Character.isLetter(date.charAt(0))) { + // if ls was used, date has the format "Nov 2 08:49:30 2021" + return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime(); + } else { + // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" + String[] parts = date.split(" "); + parts[1] = parts[1].substring(0, 12); + // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" + String shortenedDate = String.join(" ", parts); + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); + } + } catch ( Exception e ){ + return null; + } + } + + public static FileTime fileTimeFromString(String date ) { + final Long millisFromSring = millisFromString( date ); + return millisFromSring == null ? null : FileTime.fromMillis( millisFromSring ); + } + +} diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 1abca2d1..67d9e513 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -95,7 +95,7 @@ public Set getNewAndUpdatedFiles( { final LocationWrapper locationWrapper = new LocationWrapper( location, - fileTimeFromString(modificationDate), + DateParser.millisFromString(modificationDate), Long.parseLong(data[ SIZE ]), finishedTask ); @@ -113,25 +113,4 @@ public Set getNewAndUpdatedFiles( } - private long fileTimeFromString(String date) { - if( date == null || date.equals("-")) { - throw new RuntimeException("Date was: " + date); - } - try { - if (Character.isLetter(date.charAt(0))) { - // if ls was used, date has the format "Nov 2 08:49:30 2021" - return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime(); - } else { - // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" - String[] parts = date.split(" "); - parts[1] = parts[1].substring(0, 12); - // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" - String shortenedDate = String.join(" ", parts); - return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); - } - } catch ( Exception e ){ - throw new RuntimeException( "Cannot parse date: " + date, e ); - } - } - } From 9255a0dc3824657665bf1a45aa7d395a98bd93d9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 11:33:30 +0100 Subject: [PATCH 119/443] Do not fail if a task has no outputs. Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/TaskResultParser.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 67d9e513..0aea3ce7 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -56,13 +56,19 @@ public Set getNewAndUpdatedFiles( final Path outfile = workdir.resolve(".command.outfiles"); String taskRootDir = getRootDir( infile.toFile() ); - if( taskRootDir == null ) throw new IllegalStateException("taskRootDir is null"); + if( taskRootDir == null + && (finishedTask.getInputFiles() == null || finishedTask.getInputFiles().isEmpty()) ) + throw new IllegalStateException("taskRootDir is null"); + + + final Set newOrUpdated = new HashSet<>(); + String outputRootDir = getRootDir( outfile.toFile() ); - if( outputRootDir == null ) throw new IllegalStateException("outputRootDir is null"); + //No outputs defined / found + if( outputRootDir == null ) return newOrUpdated; final Map inputdata = new HashMap<>(); - Set newOrUpdated = new HashSet<>(); try ( Stream in = Files.lines(infile); From 6e80d9ddd5f801ecafcad4e0d2e1eb624270d2c2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Mar 2022 11:36:53 +0100 Subject: [PATCH 120/443] Remove unused import Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/TaskResultParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 0aea3ce7..4817972c 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -13,7 +13,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Stream; From b8c0491ba1863a1bb4e20101ee8df77be9c86261 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 2 Mar 2022 17:46:58 +0100 Subject: [PATCH 121/443] Deactivate and not delete LocationWrapper, mark LocationWrapper as inUse Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 7 +- .../model/location/hierachy/Folder.java | 8 +- .../location/hierachy/HierarchyWrapper.java | 6 +- .../location/hierachy/LocationWrapper.java | 86 ++++++++++++++----- .../model/location/hierachy/RealFile.java | 25 +++--- .../scheduler/SchedulerWithDaemonSet.java | 19 +++- .../fonda/scheduler/util/FileAlignment.java | 14 +++ .../hierachy/HierarchyWrapperTest.java | 20 ++--- .../model/location/hierachy/RealFileTest.java | 24 ++++++ 9 files changed, 155 insertions(+), 54 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 846dd894..e5a0bd79 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -3,6 +3,7 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Process; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.util.Batch; import lombok.Getter; import lombok.Setter; @@ -21,7 +22,11 @@ public class Task { @Getter @Setter - private List< TaskInputFileLocationWrapper > inputFiles; + private List inputFiles; + + @Getter + @Setter + private List< TaskInputFileLocationWrapper > copiedFiles; @Getter @Setter diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index e3c9d518..d2a746a4 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -60,15 +60,13 @@ private void getAllChildren( final Map result, Path currentPa } } - public boolean addOrUpdateFile( final String name, boolean overwrite, final LocationWrapper location ) { - children.compute( name, (k,v) -> { + public LocationWrapper addOrUpdateFile(final String name, boolean overwrite, final LocationWrapper location ) { + final RealFile file = (RealFile) children.compute( name, (k,v) -> { if (v == null || v.isDirectory() || v.isSymlink() ) return new RealFile( location ); - final RealFile file = (RealFile) v; - file.addOrUpdateLocation( overwrite, location ); return v; } ); - return true; + return file.addOrUpdateLocation( overwrite, location ); } public boolean addSymlink( final String name, final Path dst ){ diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 97d451a4..1bce7e2c 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -69,15 +69,15 @@ public Map getAllFilesInDir( final Path path ){ * @param location location where the file is located * @return false if file can not be created */ - public boolean addFile( final Path path, final LocationWrapper location ){ + public LocationWrapper addFile(final Path path, final LocationWrapper location ){ return addFile( path, false, location ); } - public boolean addFile( final Path path, boolean overwrite, final LocationWrapper location ){ + public LocationWrapper addFile(final Path path, boolean overwrite, final LocationWrapper location ){ final Folder folderToInsert = findFolderToInsert(path); - if( folderToInsert == null ) return false; + if( folderToInsert == null ) return null; else return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, location ); } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 7d9e4459..763c9cd6 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -14,11 +14,13 @@ public class LocationWrapper { private final long id = nextID.getAndIncrement(); private final Location location; - private final long timestamp; - private final long sizeInBytes; - private final long createTime = System.currentTimeMillis(); - private final Task createdByTask; - private final LocationWrapper copyOf; + private long timestamp; + private long sizeInBytes; + private long createTime = System.currentTimeMillis(); + private Task createdByTask; + private LocationWrapper copyOf; + private boolean active = true; + private int inUse = 0; public LocationWrapper(Location location, long timestamp, long sizeInBytes) { this( location, timestamp, sizeInBytes ,null); @@ -36,8 +38,42 @@ private LocationWrapper(Location location, long timestamp, long sizeInBytes, Tas this.copyOf = copyOf; } + public void update( LocationWrapper update ){ + assert location == update.location; + synchronized ( this ) { + this.timestamp = update.timestamp; + this.sizeInBytes = update.sizeInBytes; + this.createTime = update.createTime; + this.createdByTask = update.createdByTask; + this.copyOf = update.copyOf; + this.active = update.active; + } + } + + public void deactivate(){ + this.active = false; + } + + public void use(){ + synchronized ( this ) { + inUse++; + } + } + + public void free(){ + synchronized ( this ) { + inUse--; + } + } + + public boolean isInUse(){ + return inUse == 0; + } + public LocationWrapper getCopyOf( Location location ) { - return new LocationWrapper( location, timestamp, sizeInBytes, createdByTask, copyOf == null ? this : copyOf ); + synchronized ( this ) { + return new LocationWrapper(location, timestamp, sizeInBytes, createdByTask, copyOf == null ? this : copyOf); + } } @Override @@ -47,29 +83,35 @@ public boolean equals(Object o) { LocationWrapper that = (LocationWrapper) o; - if (getTimestamp() != that.getTimestamp()) return false; - if (getSizeInBytes() != that.getSizeInBytes()) return false; - if (!getLocation().equals(that.getLocation())) return false; - if (getCreatedByTask() != null ? !getCreatedByTask().equals(that.getCreatedByTask()) : that.getCreatedByTask() != null) - return false; - return getCopyOf() != null ? getCopyOf().equals(that.getCopyOf()) : that.getCopyOf() == null; + synchronized ( this ) { + if (getTimestamp() != that.getTimestamp()) return false; + if (getSizeInBytes() != that.getSizeInBytes()) return false; + if (!getLocation().equals(that.getLocation())) return false; + if (getCreatedByTask() != null ? !getCreatedByTask().equals(that.getCreatedByTask()) : that.getCreatedByTask() != null) + return false; + return getCopyOf() != null ? getCopyOf().equals(that.getCopyOf()) : that.getCopyOf() == null; + } } @Override public int hashCode() { - return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getCreatedByTask()); + synchronized ( this ) { + return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getCreatedByTask()); + } } @Override public String toString() { - return "LocationWrapper{" + - "id=" + id + - ", location=" + location.getIdentifier() + - ", timestamp=" + timestamp + - ", sizeInBytes=" + sizeInBytes + - ", createTime=" + createTime + - ", createdByTask=" + createdByTask + - ", copyOf=" + copyOf + - '}'; + synchronized ( this ) { + return "LocationWrapper{" + + "id=" + id + + ", location=" + location.getIdentifier() + + ", timestamp=" + timestamp + + ", sizeInBytes=" + sizeInBytes + + ", createTime=" + createTime + + ", createdByTask=" + createdByTask + + ", copyOf=" + copyOf + + '}'; + } } } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index bf254b21..f99b1f99 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -31,25 +31,27 @@ public boolean isSymlink() { return false; } - public void addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ + public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ if ( location == null ) throw new IllegalArgumentException( "location is null" ); - if ( overwrite ){ - this.locations = new LocationWrapper[]{ location }; - return; - } synchronized ( this ){ + LocationWrapper locationWrapperToUpdate = null; for (int i = 0; i < locations.length; i++) { if ( location.getLocation().equals( locations[i].getLocation() ) ) { - if ( location.getTimestamp() > locations[i].getTimestamp() ) { - locations[i] = location; + locationWrapperToUpdate = locations[i]; + if ( overwrite || location.getTimestamp() > locations[i].getTimestamp() ) { + locationWrapperToUpdate.update( location ); } - return; + if ( !overwrite ) return locationWrapperToUpdate; + } else if ( overwrite ){ + locations[i].deactivate(); } } + if ( overwrite && locationWrapperToUpdate != null ) return locationWrapperToUpdate; final LocationWrapper[] newLocation = Arrays.copyOf(locations, locations.length + 1); newLocation[ locations.length ] = location; locations = newLocation; } + return location; } private List combineResultsWithInitial ( @@ -145,6 +147,9 @@ public List getFilesForTask( Task task ){ final Set taskDescendants = taskProcess.getDescendants(); for ( LocationWrapper location : locationsRef ) { + + if ( !location.isActive() ) continue; + //File was modified by an operator (no relation known) if ( location.getCreatedByTask() == null ) { initial = addAndCreateList( initial, location ); @@ -188,7 +193,7 @@ private void addAllLaterLocationsToResult( List list, List locationWrappers ){ + locationWrappers.parallelStream().forEach( LocationWrapper::use ); + } + + private void freeLocations( List locationWrappers ){ + locationWrappers.parallelStream().forEach( LocationWrapper::free ); + } @Override void assignTaskToNode( NodeTaskAlignment alignment ) { - final List< TaskInputFileLocationWrapper > locationWrappers = writeInitConfig((NodeTaskFilesAlignment) alignment); - alignment.task.setInputFiles( locationWrappers ); + final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; + final List< TaskInputFileLocationWrapper > locationWrappers = writeInitConfig( nodeTaskFilesAlignment ); + alignment.task.setCopiedFiles( locationWrappers ); getCopyStrategy().generateCopyScript( alignment.task ); + final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); + alignment.task.setInputFiles( allLocationWrappers ); + useLocations( allLocationWrappers ); super.assignTaskToNode( alignment ); } @@ -85,6 +97,7 @@ int terminateTasks(List finishedTasks) { final TaskResultParser taskResultParser = new TaskResultParser(); finishedTasks.parallelStream().forEach( finishedTask -> { try{ + freeLocations( finishedTask.getInputFiles() ); final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( Paths.get(finishedTask.getWorkingDir()), finishedTask.getNode(), @@ -244,7 +257,7 @@ public void addFile( String path, long size, long timestamp, long locationWrappe private void podWasInitialized( Pod pod ){ final Task task = changeStateOfTask(pod, State.PREPARED); - task.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); + task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); } /** diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 71a57c98..d5858326 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -1,9 +1,11 @@ package fonda.scheduler.util; +import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.util.inputs.SymlinkInput; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class FileAlignment { @@ -18,4 +20,16 @@ public FileAlignment(Map> nodeFileAlignment, List getAllLocationWrappers(){ + return nodeFileAlignment + .entrySet() + .parallelStream() + .flatMap( l -> l + .getValue() + .parallelStream() + .map( p -> p.locationWrapper ) + ) + .collect(Collectors.toList()); + } } diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index f04bc663..5fa00a37 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -53,7 +53,7 @@ public void init() { files.add( Paths.get(temporaryDir + "b/c/test.abc" )); files.add( Paths.get(temporaryDir + "bc/file.abc" )); - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, node1))); + files.parallelStream().forEach( x -> assertNotNull(hw.addFile(x, node1))); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @@ -104,7 +104,7 @@ public void getAllFilesinAllWorkdirs() { @Test public void createFileinFile() { - assertTrue(hw.addFile( Paths.get(temporaryDir + "test/b.txt"), node1)); + assertNotNull(hw.addFile( Paths.get(temporaryDir + "test/b.txt"), node1)); files.remove( Paths.get(temporaryDir + "test" )); files.add( Paths.get(temporaryDir + "test/b.txt") ); @@ -115,7 +115,7 @@ public void createFileinFile() { @Test public void createFileinWorkdir() { - assertFalse(hw.addFile(Paths.get(workdir + "ab/b.txt"), node1)); + assertNull(hw.addFile(Paths.get(workdir + "ab/b.txt"), node1)); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); @@ -123,8 +123,8 @@ public void createFileinWorkdir() { @Test public void createFileOutOfScope() { - assertFalse(hw.addFile(Paths.get("/somewhere/test.txt"), node1)); - assertFalse(hw.addFile(Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/test.txt"), node1)); + assertNull(hw.addFile(Paths.get("/somewhere/test.txt"), node1)); + assertNull(hw.addFile(Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/test.txt"), node1)); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); @@ -132,7 +132,7 @@ public void createFileOutOfScope() { @Test public void createFileTwice() { - assertTrue(hw.addFile(Paths.get(temporaryDir + "bc/file.abc"), node1)); + assertNotNull(hw.addFile(Paths.get(temporaryDir + "bc/file.abc"), node1)); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); @@ -140,7 +140,7 @@ public void createFileTwice() { @Test public void createFileButWasFolder() { - assertTrue(hw.addFile(Paths.get(temporaryDir + "bc"),node1)); + assertNotNull(hw.addFile(Paths.get(temporaryDir + "bc"),node1)); files.remove( Paths.get(temporaryDir + "bc/file.abc") ); files.add( Paths.get(temporaryDir + "bc") ); @@ -178,7 +178,7 @@ public void testParallelAdd() { intialMem = finalMem; - files.parallelStream().forEach( x -> assertTrue(hw.addFile(x, getLocationWrapper("Node1")))); + files.parallelStream().forEach( x -> assertNotNull(hw.addFile(x, getLocationWrapper("Node1")))); finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); @@ -214,7 +214,7 @@ public void testGetFile() { LocationWrapper[] lw = new LocationWrapper[ files.size() ]; for ( Path file : files) { lw[index] = getLocationWrapper( "Node" + index ); - assertTrue(hw.addFile(file,lw[index++])); + assertNotNull(hw.addFile(file,lw[index++])); } result = hw.getAllFilesInDir( Paths.get(temporaryDir) ).keySet(); @@ -250,7 +250,7 @@ public void testGetFileButIsDir() { @Test public void testFileIsNowDir(){ - assertTrue(hw.addFile( Paths.get(temporaryDir + "d"), getLocationWrapper("nodeXY") )); + assertNotNull(hw.addFile( Paths.get(temporaryDir + "d"), getLocationWrapper("nodeXY") )); result = hw.getAllFilesInDir( Paths.get(temporaryDir ) ).keySet(); assertNotNull( hw.getFile( Paths.get(temporaryDir + "d" )) ); assertNull( hw.getFile( Paths.get(temporaryDir + "d/e/file.txt" )) ); diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index c9df1a51..0e197e35 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -131,6 +131,30 @@ public void changeFile() { } + @Test + public void overwriteFile() { + final LinkedList results = new LinkedList<>(); + final LocationWrapper node0 = getLocationWrapper("Node0"); + results.add( node0 ); + assertTrue(node0.isActive()); + final RealFile realFile = new RealFile(node0); + assertArrayEquals( results.toArray(), realFile.getLocations() ); + assertTrue(node0.isActive()); + final LocationWrapper node1 = getLocationWrapper("Node1"); + results.add( node1 ); + realFile.addOrUpdateLocation( true, node1); + assertArrayEquals( results.toArray(), realFile.getLocations() ); + assertFalse(node0.isActive()); + assertTrue(node1.isActive()); + final LocationWrapper node2 = getLocationWrapper("Node2"); + assertArrayEquals( results.toArray(), realFile.getLocations() ); + results.add( node2 ); + realFile.addOrUpdateLocation( true, node2); + assertFalse(node0.isActive()); + assertFalse(node1.isActive()); + assertTrue(node2.isActive()); + } + @Test public void changeFileOnExistingLocation() { From e823e3d6e0762dd8cf91f2009a4ae0bff26e8cb2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 4 Mar 2022 15:22:55 +0100 Subject: [PATCH 122/443] Enable NodeSelector Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/RandomScheduler.java | 6 +----- .../java/fonda/scheduler/scheduler/Scheduler.java | 15 +++++++++++++++ .../scheduler/SchedulerWithDaemonSet.java | 4 ++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 9e64dd94..a3b1974c 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -45,11 +45,7 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ final PodWithAge pod = task.getPod(); final List matchingNodes = items .stream() - .filter( - x -> availableByNode.get(x.getName()).higherOrEquals(pod.getRequest()) - && this.getDaemonOnNode(x) != null - //&& !x.getName().equals(this.getWorkflowEngineNode()) - ) + .filter( x -> this.canSchedulePodOnNode( availableByNode, task.getPod(), x ) ) .collect(Collectors.toList()); log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); Optional node = matchingNodes.isEmpty() diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 0ffe505c..ec7a95c6 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -238,6 +238,21 @@ public TaskState getTaskState(String taskid) { /* Nodes */ + boolean canSchedulePodOnNode( Map availableByNode, PodWithAge pod, NodeWithAlloc node ) { + return availableByNode.get( node.getName() ).higherOrEquals( pod.getRequest() ) + && affinitiesMatch( pod, node ); + } + + boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ + final Map podsNodeSelector = pod.getSpec().getNodeSelector(); + final Map nodesLabels = node.getMetadata().getLabels(); + if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) return true; + //cannot be fulfilled if podsNodeSelector is not empty + if ( nodesLabels == null || nodesLabels.isEmpty() ) return false; + + return nodesLabels.entrySet().containsAll( podsNodeSelector.entrySet() ); + } + public void newNode(NodeWithAlloc node) { informResourceChange(); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 420e85c8..c2f1fc06 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -247,6 +247,10 @@ private void podWasInitialized( Pod pod ){ task.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); } + boolean canSchedulePodOnNode( Map availableByNode, PodWithAge pod, NodeWithAlloc node ) { + return this.getDaemonOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); + } + /** * Since task was not yet initialized: set scheduled * @param task From ac570b9fc314776955e164e854a60a5b39563291 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 4 Mar 2022 16:37:52 +0100 Subject: [PATCH 123/443] moved input classes Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/{util/inputs => model/taskinputs}/Input.java | 0 .../{util/inputs => model/taskinputs}/PathFileLocationTriple.java | 0 .../scheduler/{util/inputs => model/taskinputs}/SymlinkInput.java | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename src/main/java/fonda/scheduler/{util/inputs => model/taskinputs}/Input.java (100%) rename src/main/java/fonda/scheduler/{util/inputs => model/taskinputs}/PathFileLocationTriple.java (100%) rename src/main/java/fonda/scheduler/{util/inputs => model/taskinputs}/SymlinkInput.java (100%) diff --git a/src/main/java/fonda/scheduler/util/inputs/Input.java b/src/main/java/fonda/scheduler/model/taskinputs/Input.java similarity index 100% rename from src/main/java/fonda/scheduler/util/inputs/Input.java rename to src/main/java/fonda/scheduler/model/taskinputs/Input.java diff --git a/src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java similarity index 100% rename from src/main/java/fonda/scheduler/util/inputs/PathFileLocationTriple.java rename to src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java diff --git a/src/main/java/fonda/scheduler/util/inputs/SymlinkInput.java b/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java similarity index 100% rename from src/main/java/fonda/scheduler/util/inputs/SymlinkInput.java rename to src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java From 6cf7d69e4d4c627c3c7cd4d2dc3d85e9029ea342 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 4 Mar 2022 16:38:39 +0100 Subject: [PATCH 124/443] moved input classes and changed package Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/taskinputs/Input.java | 2 +- .../scheduler/model/taskinputs/PathFileLocationTriple.java | 2 +- .../java/fonda/scheduler/model/taskinputs/SymlinkInput.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/Input.java b/src/main/java/fonda/scheduler/model/taskinputs/Input.java index 35c3e48c..36779441 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/Input.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/Input.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util.inputs; +package fonda.scheduler.model.taskinputs; public interface Input { } diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index a96354f2..f39acf60 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util.inputs; +package fonda.scheduler.model.taskinputs; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealFile; diff --git a/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java b/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java index 4a6f9bcf..05594f96 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util.inputs; +package fonda.scheduler.model.taskinputs; import lombok.Getter; From f5ff27772539c9d1c083aafff4a420228a06cc63 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 10:50:09 +0100 Subject: [PATCH 125/443] moved input classes: changed package reference Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/rest/response/getfile/FileResponse.java | 2 +- .../java/fonda/scheduler/scheduler/RandomScheduler.java | 6 +++--- .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 6 +++--- .../scheduler/scheduler/schedulingstrategy/Inputs.java | 2 +- src/main/java/fonda/scheduler/util/FileAlignment.java | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java index 24aae732..eb29bfd2 100644 --- a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java +++ b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java @@ -1,6 +1,6 @@ package fonda.scheduler.rest.response.getfile; -import fonda.scheduler.util.inputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.SymlinkInput; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index a3b1974c..94b6b7c8 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -6,10 +6,10 @@ import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.inputs.Input; +import fonda.scheduler.model.taskinputs.Input; import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.inputs.PathFileLocationTriple; -import fonda.scheduler.util.inputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.SymlinkInput; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 92d0c77d..43a9604d 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -17,9 +17,9 @@ import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.inputs.Input; -import fonda.scheduler.util.inputs.PathFileLocationTriple; -import fonda.scheduler.util.inputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.Input; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.util.FilePath; import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index f4fe317e..b71a5aa2 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -1,6 +1,6 @@ package fonda.scheduler.scheduler.schedulingstrategy; -import fonda.scheduler.util.inputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.SymlinkInput; import java.util.LinkedList; import java.util.List; diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index d5858326..36f6b4bf 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -1,7 +1,7 @@ package fonda.scheduler.util; import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.util.inputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.SymlinkInput; import java.util.List; import java.util.Map; From 06aea731661624152c9ef594bb417d1fbeb7114d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 11:57:27 +0100 Subject: [PATCH 126/443] Mark as overwritten Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 43a9604d..0a0e7ad5 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -260,6 +260,7 @@ private void podWasInitialized( Pod pod ){ task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); } + @Override boolean canSchedulePodOnNode( Map availableByNode, PodWithAge pod, NodeWithAlloc node ) { return this.getDaemonOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); } From d9c0790ae6c57918d89f8c75065afecd7f4cc1d4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 11:58:04 +0100 Subject: [PATCH 127/443] added description Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 0a0e7ad5..8fc47ee2 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -267,7 +267,7 @@ boolean canSchedulePodOnNode( Map availableByNode, PodWi /** * Since task was not yet initialized: set scheduled - * @param task + * @param task task that was scheduled */ @Override void taskWasScheduledSetState( Task task ){ From 10050b30c75c7786ad4f28401f093b7b8363de6d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 12:02:14 +0100 Subject: [PATCH 128/443] Taskinputs not as a list Signed-off-by: Lehmann_Fabian --- .../model/taskinputs/TaskInputs.java | 21 +++++ .../scheduler/scheduler/RandomScheduler.java | 29 ++++--- .../scheduler/SchedulerWithDaemonSet.java | 84 ++++++++----------- src/main/java/fonda/scheduler/util/Tuple.java | 15 ++++ 4 files changed, 85 insertions(+), 64 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java create mode 100644 src/main/java/fonda/scheduler/util/Tuple.java diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java new file mode 100644 index 00000000..d30d23b6 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -0,0 +1,21 @@ +package fonda.scheduler.model.taskinputs; + +import fonda.scheduler.model.location.NodeLocation; +import lombok.Getter; + +import java.util.List; +import java.util.Set; + +@Getter +public class TaskInputs { + + private final List symliks; + private final List files; + private final Set excludedNodes; + + public TaskInputs(List symliks, List files, Set excludedNodes) { + this.symliks = symliks; + this.files = files; + this.excludedNodes = excludedNodes; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 94b6b7c8..95bf00e4 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -3,6 +3,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; @@ -52,7 +53,7 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ ? Optional.empty() : Optional.of(matchingNodes.get(random.nextInt(matchingNodes.size()))); if( node.isPresent() ){ - final List inputsOfTask = getInputsOfTask(task); + final TaskInputs inputsOfTask = getInputsOfTask(task); final FileAlignment fileAlignment = scheduleFiles(task, inputsOfTask, node.get()); alignment.add( new NodeTaskFilesAlignment(node.get(),task, fileAlignment ) ); availableByNode.get(node.get().getName()).subFromThis(pod.getRequest()); @@ -66,22 +67,20 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ return scheduleObject; } - FileAlignment scheduleFiles(Task task, List inputsOfTask, NodeWithAlloc node) { + FileAlignment scheduleFiles(Task task, TaskInputs inputsOfTask, NodeWithAlloc node) { final HashMap> map = new HashMap<>(); final List symlinkInputs = new LinkedList<>(); - for ( Input entry : inputsOfTask ) { - if( entry instanceof PathFileLocationTriple ){ - final PathFileLocationTriple pathFileLocationTriple = (PathFileLocationTriple) entry; - final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( - random.nextInt( pathFileLocationTriple.locations.size() ) - ); - final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); - final List pathsOfNode = map.get( nodeIdentifier ); - pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file ) ); - } else if ( entry instanceof SymlinkInput ){ - symlinkInputs.add( (SymlinkInput) entry ); - } + for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { + final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( + random.nextInt( pathFileLocationTriple.locations.size() ) + ); + final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); + map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); + final List pathsOfNode = map.get( nodeIdentifier ); + pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file ) ); + } + for (SymlinkInput symlik : inputsOfTask.getSymliks()) { + symlinkInputs.add( symlik ); } return new FileAlignment( map, symlinkInputs ); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 8fc47ee2..047ed4c4 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -4,6 +4,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.location.LocationType; +import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.model.location.NodeLocation; @@ -17,10 +18,10 @@ import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.model.taskinputs.Input; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.util.FilePath; +import fonda.scheduler.util.Tuple; import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; @@ -35,7 +36,6 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import java.util.stream.Stream; @Slf4j public abstract class SchedulerWithDaemonSet extends Scheduler { @@ -163,56 +163,42 @@ List writeInitConfig( NodeTaskFilesAlignment align return null; } - private Stream streamFile( - final fonda.scheduler.model.location.hierachy.File file, - final Task task, - final Path sourcePath ) - { - if( file == null ){ - log.info( "File to stream was null: {}", sourcePath ); - return Stream.empty(); - } - if ( file.isSymlink() ){ - final Path linkTo = ((LinkFile) file).getDst(); - final fonda.scheduler.model.location.hierachy.File destFile = hierarchyWrapper.getFile(linkTo); - final Stream inputStream = streamFile(destFile, task, linkTo); - return Stream.concat(Stream.of( new SymlinkInput( sourcePath, linkTo ) ), inputStream); + TaskInputs getInputsOfTask( Task task ){ + + final List> fileInputs = task.getConfig().getInputs().fileInputs; + final LinkedList> toProcess = new LinkedList<>(); + for ( InputParam fileInput : fileInputs) { + final Path path = Path.of(fileInput.value.sourceObj); + if ( this.hierarchyWrapper.isInScope( path ) ){ + toProcess.add( new Tuple<>(hierarchyWrapper.getFile( path ), path) ); + } } - if( file.isDirectory() ){ - return ((Folder) file).getAllChildren( sourcePath ) - .entrySet() - .stream() - .flatMap( y -> { - if ( !y.getValue().isSymlink() ) - return Stream.of(new PathFileLocationTriple( - y.getKey(), - ((RealFile) y.getValue()), - ((RealFile) y.getValue()).getFilesForTask( task ) - )); - else { - return streamFile( y.getValue(), task, y.getKey() ); - } - } - ); + + List symlinks = new ArrayList<>( fileInputs.size() ); + List files = new ArrayList<>( fileInputs.size() ); + + while ( !toProcess.isEmpty() ){ + final Tuple pop = toProcess.removeLast(); + final fonda.scheduler.model.location.hierachy.File file = pop.getA(); + if( file == null ) continue; + final Path path = pop.getB(); + if ( file.isSymlink() ){ + final Path linkTo = ((LinkFile) file).getDst(); + symlinks.add( new SymlinkInput( path, linkTo ) ); + toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); + } else if( file.isDirectory() ){ + final Map allChildren = ((Folder) file).getAllChildren(path); + for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { + toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); + } + } else { + final RealFile realFile = (RealFile) file; + files.add( new PathFileLocationTriple( path, realFile, realFile.getFilesForTask(task) ) ); + } } - final RealFile realFile = (RealFile) file; - return Stream.of( new PathFileLocationTriple( - sourcePath, - realFile, - realFile.getFilesForTask( task ) - ) - ); - } - List getInputsOfTask( Task task ){ - return task.getConfig() - .getInputs() - .fileInputs - .parallelStream() - .map( x -> Path.of(x.value.sourceObj) ) - .filter(this.hierarchyWrapper::isInScope) - .flatMap( sourcePath -> streamFile( hierarchyWrapper.getFile(sourcePath), task, sourcePath )) - .collect(Collectors.toList()); + return new TaskInputs( symlinks, files,null ); + } public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileException { diff --git a/src/main/java/fonda/scheduler/util/Tuple.java b/src/main/java/fonda/scheduler/util/Tuple.java new file mode 100644 index 00000000..ec67bc01 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/Tuple.java @@ -0,0 +1,15 @@ +package fonda.scheduler.util; + +import lombok.Getter; + +@Getter +public class Tuple { + + private final S a; + private final T b; + + public Tuple( S a, T b ) { + this.a = a; + this.b = b; + } +} From b2a3ed915fe3bd735ec8d946499d35f2add67c01 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 12:08:57 +0100 Subject: [PATCH 129/443] Dateparser private constructor and final class Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/DateParser.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/DateParser.java b/src/main/java/fonda/scheduler/model/DateParser.java index e032c174..a1a8cc80 100644 --- a/src/main/java/fonda/scheduler/model/DateParser.java +++ b/src/main/java/fonda/scheduler/model/DateParser.java @@ -3,7 +3,9 @@ import java.nio.file.attribute.FileTime; import java.text.SimpleDateFormat; -public class DateParser { +public final class DateParser { + + private DateParser(){} public static Long millisFromString( String date ) { if( date == null || date.isEmpty() || date.equals("-") || date.equals("w") ) { From a2e1c432b6fd577f204fdf6d9d73567ce1bd172e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 12:09:25 +0100 Subject: [PATCH 130/443] assert to if + throw exception Signed-off-by: Lehmann_Fabian --- .../scheduler/model/location/hierachy/LocationWrapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 763c9cd6..5aa258e2 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -39,7 +39,8 @@ private LocationWrapper(Location location, long timestamp, long sizeInBytes, Tas } public void update( LocationWrapper update ){ - assert location == update.location; + if (location != update.location) + throw new IllegalArgumentException( "Can only update LocationWrapper with the same location." ); synchronized ( this ) { this.timestamp = update.timestamp; this.sizeInBytes = update.sizeInBytes; From c81608846a6970cc4b5efd16500fb90b6086932b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 7 Mar 2022 12:20:54 +0100 Subject: [PATCH 131/443] Refactored to reduce complexity Signed-off-by: Lehmann_Fabian --- .../scheduler/model/TaskResultParser.java | 96 +++++++++++-------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 4817972c..b397e43f 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -37,10 +37,59 @@ private String getRootDir( File file ){ return null; } + private void processInput( Stream in, final Map inputdata, final String taskRootDir ){ + in.skip( 1 ) + .forEach( line -> { + String[] data = line.split(";"); + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; + String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ].substring( taskRootDir.length() + 1 ) : data[ REAL_PATH ]; + String modificationDate = data[ MODIFICATION_DATE ]; + inputdata.put( path , modificationDate ); + }); + } + + private Set processOutput( + final Stream out, + final Map inputdata, + final Location location, + final boolean onlyUpdated, + final Task finishedTask, + final String outputRootDir + ){ + final Set newOrUpdated = new HashSet<>(); + out.skip( 1 ) + .forEach( line -> { + String[] data = line.split(";"); + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; + boolean realFile = data[ REAL_PATH ].equals(""); + String path = realFile ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; + String modificationDate = data[ MODIFICATION_DATE ]; + if ( "directory".equals(data[ FILE_TYPE ]) ) return; + String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; + if ( ( !inputdata.containsKey(lockupPath) && !onlyUpdated ) + || + !modificationDate.equals( inputdata.get( lockupPath ) )) + { + final LocationWrapper locationWrapper = new LocationWrapper( + location, + DateParser.millisFromString(modificationDate), + Long.parseLong(data[ SIZE ]), + finishedTask + ); + newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); + } + if( !realFile ){ + newOrUpdated.add( new SymlinkOutput( data[ VIRTUAL_PATH ], data[ REAL_PATH ])); + } + }); + return newOrUpdated; + } + /** * * @param workdir * @param location + * @param onlyUpdated * @param finishedTask * @return A list of all new or updated files */ @@ -54,17 +103,15 @@ public Set getNewAndUpdatedFiles( final Path infile = workdir.resolve(".command.infiles"); final Path outfile = workdir.resolve(".command.outfiles"); - String taskRootDir = getRootDir( infile.toFile() ); + final String taskRootDir = getRootDir( infile.toFile() ); if( taskRootDir == null && (finishedTask.getInputFiles() == null || finishedTask.getInputFiles().isEmpty()) ) throw new IllegalStateException("taskRootDir is null"); - final Set newOrUpdated = new HashSet<>(); - - String outputRootDir = getRootDir( outfile.toFile() ); + final String outputRootDir = getRootDir( outfile.toFile() ); //No outputs defined / found - if( outputRootDir == null ) return newOrUpdated; + if( outputRootDir == null ) return new HashSet<>(); final Map inputdata = new HashMap<>(); @@ -74,47 +121,14 @@ public Set getNewAndUpdatedFiles( Stream out = Files.lines(outfile) ) { - in.skip( 1 ) - .forEach( line -> { - String[] data = line.split(";"); - if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; - String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ].substring( taskRootDir.length() + 1 ) : data[ REAL_PATH ]; - String modificationDate = data[ MODIFICATION_DATE ]; - inputdata.put( path , modificationDate ); - }); - + processInput( in, inputdata, taskRootDir ); log.trace( "{}", inputdata ); - - out.skip( 1 ) - .forEach( line -> { - String[] data = line.split(";"); - if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; - boolean realFile = data[ REAL_PATH ].equals(""); - String path = realFile ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; - String modificationDate = data[ MODIFICATION_DATE ]; - if ( "directory".equals(data[ FILE_TYPE ]) ) return; - String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; - if ( ( !inputdata.containsKey(lockupPath) && !onlyUpdated ) - || - !modificationDate.equals( inputdata.get( lockupPath ) )) - { - final LocationWrapper locationWrapper = new LocationWrapper( - location, - DateParser.millisFromString(modificationDate), - Long.parseLong(data[ SIZE ]), - finishedTask - ); - newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); - } - if( !realFile ){ - newOrUpdated.add( new SymlinkOutput( data[ VIRTUAL_PATH ], data[ REAL_PATH ])); - } - }); + return processOutput( out, inputdata, location, onlyUpdated, finishedTask, outputRootDir ); } catch (IOException e) { e.printStackTrace(); } - return newOrUpdated; + return new HashSet<>(); } From 40d4411ba8757ccaa8b0355f39c2254f2459d004 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 11:40:35 +0100 Subject: [PATCH 132/443] Exclude Nodes if file is in use. Refactor Random Scheduler Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/Main.java | 4 +- .../model/location/hierachy/RealFile.java | 37 +++++++- .../model/taskinputs/TaskInputs.java | 6 +- .../rest/SchedulerRestController.java | 3 +- .../scheduler/scheduler/RandomScheduler.java | 94 +++++++++---------- .../fonda/scheduler/scheduler/Scheduler.java | 38 ++++++-- .../scheduler/SchedulerWithDaemonSet.java | 28 +++++- .../filealignment/InputAlignment.java | 13 +++ .../filealignment/RandomAlignment.java | 36 +++++++ .../java/fonda/scheduler/util/FilePath.java | 5 +- .../model/location/hierachy/RealFileTest.java | 60 ++++++------ 11 files changed, 225 insertions(+), 99 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 3bf4c7ea..48e7afc9 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -4,6 +4,7 @@ import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.rest.SchedulerRestController; import fonda.scheduler.scheduler.RandomScheduler; +import fonda.scheduler.scheduler.filealignment.RandomAlignment; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +34,8 @@ public static void main(String[] args) { public void afterStart(){ try{ log.info( "Started with namespace: {}", client.getNamespace() ); - final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", new SchedulerConfig(null, null, "/localwork/", null, "ftp")); + final SchedulerConfig schedlerConfig = new SchedulerConfig(null, null, "/localwork/", null, "ftp"); + final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", schedlerConfig, new RandomAlignment()); final Pair key = new Pair<>("default", "test-run"); SchedulerRestController.addScheduler( key, randomScheduler ); } catch (Exception e){ diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java index f99b1f99..8fb93c3f 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java @@ -133,7 +133,7 @@ private LinkedList addAndCreateList(LinkedList return list; } - public List getFilesForTask( Task task ){ + public MatchingLocationsPair getFilesForTask( Task task ){ LocationWrapper[] locationsRef = this.locations; LinkedList current = null; @@ -146,8 +146,15 @@ public List getFilesForTask( Task task ){ final Set taskAncestors = taskProcess.getAncestors(); final Set taskDescendants = taskProcess.getDescendants(); + Set inUse = null; + for ( LocationWrapper location : locationsRef ) { + if( location.isInUse() ) { + if ( inUse == null ) inUse = new HashSet<>(); + inUse.add(location.getLocation()); + } + if ( !location.isActive() ) continue; //File was modified by an operator (no relation known) @@ -177,9 +184,35 @@ else if ( taskDescendants.contains(locationProcess) ) unrelated = addAndCreateList( unrelated, location ); } - return ( initial == null ) + final List matchingLocations = ( initial == null ) ? combineResultsEmptyInitial( current, ancestors, descendants, unrelated ) : combineResultsWithInitial( current, ancestors, descendants, unrelated, initial ); + + removeMatchingLocations( matchingLocations, inUse ); + + return new MatchingLocationsPair( matchingLocations, inUse ); + } + + private void removeMatchingLocations( List matchingLocations, Set locations ){ + if ( locations == null ) return; + for ( LocationWrapper matchingLocation : matchingLocations ) { + if( matchingLocation.isInUse() ) { + locations.remove(matchingLocation.getLocation()); + } + } + } + + @Getter + public class MatchingLocationsPair { + + private final List matchingLocations; + private final Set excludedNodes; + + private MatchingLocationsPair(List matchingLocations, Set excludedNodes) { + this.matchingLocations = matchingLocations; + this.excludedNodes = excludedNodes; + } + } private void addAllLaterLocationsToResult( List list, List result, long time ){ diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java index d30d23b6..86cc4b52 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -1,6 +1,6 @@ package fonda.scheduler.model.taskinputs; -import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.Location; import lombok.Getter; import java.util.List; @@ -11,9 +11,9 @@ public class TaskInputs { private final List symliks; private final List files; - private final Set excludedNodes; + private final Set excludedNodes; - public TaskInputs(List symliks, List files, Set excludedNodes) { + public TaskInputs(List symliks, List files, Set excludedNodes) { this.symliks = symliks; this.files = files; this.excludedNodes = excludedNodes; diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 780e260d..ce4a6528 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -11,6 +11,7 @@ import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.TaskConfig; import fonda.scheduler.scheduler.SchedulerWithDaemonSet; +import fonda.scheduler.scheduler.filealignment.RandomAlignment; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; @@ -71,7 +72,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : - case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace, config ); break; + case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace, config, new RandomAlignment() ); break; default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 95bf00e4..f41226f7 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -2,15 +2,12 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.model.taskinputs.Input; -import fonda.scheduler.util.FilePath; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.SymlinkInput; import lombok.extern.slf4j.Slf4j; import java.util.*; @@ -19,45 +16,56 @@ @Slf4j public class RandomScheduler extends SchedulerWithDaemonSet { + private final InputAlignment inputAlignment; private final Random random = new Random(); - public RandomScheduler(String name, KubernetesClient client, String namespace, SchedulerConfig config) { + public RandomScheduler( + String name, + KubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment + ) { super(name, client, namespace, config); + this.inputAlignment = inputAlignment; + } + + private Optional selectNode( Set matchingNodes, Task task ){ + return matchingNodes.isEmpty() + ? Optional.empty() + : Optional.of( new LinkedList<>(matchingNodes).get(random.nextInt(matchingNodes.size()))); } @Override - public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ - List items = getNodeList(); + public ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ){ List alignment = new LinkedList<>(); - Map availableByNode = new HashMap<>(); - - List logInfo = new LinkedList<>(); - logInfo.add("------------------------------------"); - for (NodeWithAlloc item : items) { - final PodRequirements availableResources = item.getAvailableResources(); - availableByNode.put(item.getName(), availableResources); - logInfo.add("Node: " + item.getName() + " " + availableResources); - } - logInfo.add("------------------------------------"); - log.info(String.join("\n", logInfo)); - - for ( final Task task : unscheduledTasks) { + for ( final Task task : unscheduledTasks ) { final PodWithAge pod = task.getPod(); - final List matchingNodes = items - .stream() - .filter( x -> this.canSchedulePodOnNode( availableByNode, task.getPod(), x ) ) - .collect(Collectors.toList()); - log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); - Optional node = matchingNodes.isEmpty() - ? Optional.empty() - : Optional.of(matchingNodes.get(random.nextInt(matchingNodes.size()))); - if( node.isPresent() ){ + final Set matchingNodes = getMatchingNodesForTask(availableByNode,task); + if( !matchingNodes.isEmpty() ) { + + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); + final TaskInputs inputsOfTask = getInputsOfTask(task); - final FileAlignment fileAlignment = scheduleFiles(task, inputsOfTask, node.get()); - alignment.add( new NodeTaskFilesAlignment(node.get(),task, fileAlignment ) ); - availableByNode.get(node.get().getName()).subFromThis(pod.getRequest()); - log.info("--> " + node.get().getName()); + filterMatchingNodesForTask( matchingNodes, inputsOfTask ); + + if( matchingNodes.isEmpty() ) { + log.info( "No node which fulfills all requirements {}", pod.getName() ); + continue; + } + + Optional node = selectNode( matchingNodes, task ); + if( node.isPresent() ) { + final FileAlignment fileAlignment = inputAlignment.getInputAlignment( task, inputsOfTask, node.get() ); + alignment.add(new NodeTaskFilesAlignment(node.get(), task, fileAlignment)); + availableByNode.get(node.get()).subFromThis(pod.getRequest()); + log.info("--> " + node.get().getName()); + } + } else { log.info( "No node with enough resources for {}", pod.getName() ); } @@ -67,22 +75,4 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ){ return scheduleObject; } - FileAlignment scheduleFiles(Task task, TaskInputs inputsOfTask, NodeWithAlloc node) { - final HashMap> map = new HashMap<>(); - final List symlinkInputs = new LinkedList<>(); - for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { - final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( - random.nextInt( pathFileLocationTriple.locations.size() ) - ); - final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); - final List pathsOfNode = map.get( nodeIdentifier ); - pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file ) ); - } - for (SymlinkInput symlik : inputsOfTask.getSymliks()) { - symlinkInputs.add( symlik ); - } - return new FileAlignment( map, symlinkInputs ); - } - } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index ec7a95c6..5c1b4f7c 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -12,7 +12,6 @@ import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; -import io.fabric8.kubernetes.client.dsl.PodResource; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -20,6 +19,7 @@ import java.io.FileNotFoundException; import java.io.PrintWriter; import java.util.*; +import java.util.stream.Collectors; @Slf4j public abstract class Scheduler { @@ -81,7 +81,7 @@ public abstract class Scheduler { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { - final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks); + final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode()); final List taskNodeAlignment = scheduleObject.getTaskAlignments(); //check if still possible... @@ -114,7 +114,10 @@ public boolean validSchedulePlan( List taskNodeAlignment ){ return true; } - abstract ScheduleObject getTaskNodeAlignment( final List unscheduledTasks ); + abstract ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ); abstract int terminateTasks( final List finishedTasks ); @@ -238,9 +241,8 @@ public TaskState getTaskState(String taskid) { /* Nodes */ - boolean canSchedulePodOnNode( Map availableByNode, PodWithAge pod, NodeWithAlloc node ) { - return availableByNode.get( node.getName() ).higherOrEquals( pod.getRequest() ) - && affinitiesMatch( pod, node ); + boolean canSchedulePodOnNode( PodRequirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + return availableByNode.higherOrEquals( pod.getRequest() ) && affinitiesMatch( pod, node ); } boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ @@ -370,6 +372,30 @@ Task getTaskByPod( Pod pod ) { return t; } + Map getAvailableByNode(){ + Map availableByNode = new HashMap<>(); + List logInfo = new LinkedList<>(); + logInfo.add("------------------------------------"); + for (NodeWithAlloc item : getNodeList()) { + final PodRequirements availableResources = item.getAvailableResources(); + availableByNode.put(item, availableResources); + logInfo.add("Node: " + item.getName() + " " + availableResources); + } + logInfo.add("------------------------------------"); + log.info(String.join("\n", logInfo)); + return availableByNode; + } + + Set getMatchingNodesForTask( Map availableByNode, Task task ){ + Set result = new HashSet<>(); + for (Map.Entry entry : availableByNode.entrySet()) { + if ( this.canSchedulePodOnNode( entry.getValue(), task.getPod(), entry.getKey() ) ){ + result.add( entry.getKey() ); + } + } + return result; + } + LinkedList getUpcomingTasksCopy() { return new LinkedList<>( upcomingTasks ); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 047ed4c4..8c6e545d 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.rest.exceptions.NotARealFileException; @@ -176,6 +177,7 @@ TaskInputs getInputsOfTask( Task task ){ List symlinks = new ArrayList<>( fileInputs.size() ); List files = new ArrayList<>( fileInputs.size() ); + Set excludedLocations = new HashSet<>(); while ( !toProcess.isEmpty() ){ final Tuple pop = toProcess.removeLast(); @@ -193,11 +195,15 @@ TaskInputs getInputsOfTask( Task task ){ } } else { final RealFile realFile = (RealFile) file; - files.add( new PathFileLocationTriple( path, realFile, realFile.getFilesForTask(task) ) ); + final RealFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); + if ( filesForTask.getExcludedNodes() != null ) { + excludedLocations.addAll(filesForTask.getExcludedNodes()); + } + files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); } } - return new TaskInputs( symlinks, files,null ); + return new TaskInputs( symlinks, files, excludedLocations ); } @@ -247,10 +253,26 @@ private void podWasInitialized( Pod pod ){ } @Override - boolean canSchedulePodOnNode( Map availableByNode, PodWithAge pod, NodeWithAlloc node ) { + boolean canSchedulePodOnNode( PodRequirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { return this.getDaemonOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); } + /** + * Remove all Nodes with a location contained in taskInputs.excludedNodes + * @param matchingNodes + * @param taskInputs + */ + void filterMatchingNodesForTask( Set matchingNodes, TaskInputs taskInputs ){ + final Iterator iterator = matchingNodes.iterator(); + final Set excludedNodes = taskInputs.getExcludedNodes(); + while ( iterator.hasNext() ){ + final NodeWithAlloc next = iterator.next(); + if( excludedNodes.contains( next.getNodeLocation() ) ){ + iterator.remove(); + } + } + } + /** * Since task was not yet initialized: set scheduled * @param task task that was scheduled diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java new file mode 100644 index 00000000..2da23a19 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java @@ -0,0 +1,13 @@ +package fonda.scheduler.scheduler.filealignment; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.util.FileAlignment; +import org.jetbrains.annotations.NotNull; + +public interface InputAlignment { + + FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node); + +} diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java new file mode 100644 index 00000000..a9632f13 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -0,0 +1,36 @@ +package fonda.scheduler.scheduler.filealignment; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.FilePath; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +public class RandomAlignment implements InputAlignment { + + private final Random random = new Random(); + + @Override + public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node) { + final HashMap> map = new HashMap<>(); + for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { + final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( + random.nextInt( pathFileLocationTriple.locations.size() ) + ); + final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); + map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); + final List pathsOfNode = map.get( nodeIdentifier ); + pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file, locationWrapper ) ); + } + return new FileAlignment( map, inputsOfTask.getSymliks() ); + } + +} diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/fonda/scheduler/util/FilePath.java index f4e3e65d..b0123ded 100644 --- a/src/main/java/fonda/scheduler/util/FilePath.java +++ b/src/main/java/fonda/scheduler/util/FilePath.java @@ -1,14 +1,17 @@ package fonda.scheduler.util; +import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealFile; public class FilePath { public final String path; public final RealFile file; + public final LocationWrapper locationWrapper; - public FilePath(String path, RealFile file) { + public FilePath(String path, RealFile file, LocationWrapper locationWrapper ) { this.path = path; this.file = file; + this.locationWrapper = locationWrapper; } } diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java index 0e197e35..b8dc938f 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java @@ -205,11 +205,11 @@ public void getFilesForTaskTest() throws InterruptedException { realFile.addOrUpdateLocation(false, next.get(2)); realFile.addOrUpdateLocation(false, next.get(3)); - assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA))); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB))); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC))); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD))); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE))); + assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); } locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); @@ -223,11 +223,11 @@ public void getFilesForTaskTest() throws InterruptedException { realFile.addOrUpdateLocation(false, next.get(3)); realFile.addOrUpdateLocation(false, next.get(4)); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processA))); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processB))); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processC))); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processD))); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processE))); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); + assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); } } @@ -257,11 +257,11 @@ public void getFilesForTaskTestInitFiles() throws InterruptedException { realFile.addOrUpdateLocation(false, next.get(3)); realFile.addOrUpdateLocation(false, next.get(4)); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); } } @@ -293,11 +293,11 @@ public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException { realFile.addOrUpdateLocation(false, next.get(3)); realFile.addOrUpdateLocation(false, next.get(4)); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD))); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE))); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); } } @@ -327,11 +327,11 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException realFile.addOrUpdateLocation( false, next.get(3) ); realFile.addOrUpdateLocation( false, next.get(4) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ) ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ) ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ) ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ) ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ) ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ).getMatchingLocations() ) ); } @@ -351,11 +351,11 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException realFile.addOrUpdateLocation( false, next.get(3) ); realFile.addOrUpdateLocation( false, next.get(4) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ) ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ) ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ) ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ) ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ) ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ).getMatchingLocations() ) ); + assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ).getMatchingLocations() ) ); } From c9e112dced6c7852cb5ed0ea247d74ccee6bd4c5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 11:41:14 +0100 Subject: [PATCH 133/443] Run on java 17 Signed-off-by: Lehmann_Fabian --- Dockerfile | 4 ++-- Dockerfile-development | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd3f75e2..bfe8e90c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,12 @@ WORKDIR /build COPY pom.xml pom.xml RUN mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository COPY src/ src/ -RUN mvn package -Dmaven.repo.local=/mvn/.m2nrepo/repository +RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository # # Package stage # -FROM openjdk:11-jre-slim +FROM openjdk:17-jre-slim WORKDIR /app RUN groupadd -r javauser && useradd --no-log-init -r -g javauser javauser && mkdir data COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar diff --git a/Dockerfile-development b/Dockerfile-development index 08cf9ce8..40960266 100644 --- a/Dockerfile-development +++ b/Dockerfile-development @@ -1,6 +1,6 @@ -FROM maven:3.8.3-jdk-11-slim AS build +FROM maven:3-openjdk-17-slim AS build WORKDIR /build COPY pom.xml pom.xml RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository COPY src/ src/ -RUN mvn package -Dmaven.repo.local=/mvn/.m2nrepo/repository +RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository From 1cf285bb0c0b6fb8471bb4287b9c733e7916f37c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 12:00:14 +0100 Subject: [PATCH 134/443] refactored getInputsOfTask Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 68 +++++++++++-------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 8c6e545d..f19deb0d 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -164,9 +164,7 @@ List writeInitConfig( NodeTaskFilesAlignment align return null; } - TaskInputs getInputsOfTask( Task task ){ - - final List> fileInputs = task.getConfig().getInputs().fileInputs; + private LinkedList> filterFilesToProcess( List> fileInputs ){ final LinkedList> toProcess = new LinkedList<>(); for ( InputParam fileInput : fileInputs) { final Path path = Path.of(fileInput.value.sourceObj); @@ -174,33 +172,49 @@ TaskInputs getInputsOfTask( Task task ){ toProcess.add( new Tuple<>(hierarchyWrapper.getFile( path ), path) ); } } + return toProcess; + } + private void processNext( + final LinkedList> toProcess, + final List symlinks, + final List files, + final Set excludedLocations, + final Task task + ){ + final Tuple tuple = toProcess.removeLast(); + final fonda.scheduler.model.location.hierachy.File file = tuple.getA(); + if( file == null ) return; + final Path path = tuple.getB(); + if ( file.isSymlink() ){ + final Path linkTo = ((LinkFile) file).getDst(); + symlinks.add( new SymlinkInput( path, linkTo ) ); + toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); + } else if( file.isDirectory() ){ + final Map allChildren = ((Folder) file).getAllChildren(path); + for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { + toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); + } + } else { + final RealFile realFile = (RealFile) file; + final RealFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); + if ( filesForTask.getExcludedNodes() != null ) { + excludedLocations.addAll(filesForTask.getExcludedNodes()); + } + files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); + } + } + + TaskInputs getInputsOfTask( Task task ){ + + final List> fileInputs = task.getConfig().getInputs().fileInputs; + final LinkedList> toProcess = filterFilesToProcess( fileInputs ); - List symlinks = new ArrayList<>( fileInputs.size() ); - List files = new ArrayList<>( fileInputs.size() ); - Set excludedLocations = new HashSet<>(); + final List symlinks = new ArrayList<>( fileInputs.size() ); + final List files = new ArrayList<>( fileInputs.size() ); + final Set excludedLocations = new HashSet<>(); while ( !toProcess.isEmpty() ){ - final Tuple pop = toProcess.removeLast(); - final fonda.scheduler.model.location.hierachy.File file = pop.getA(); - if( file == null ) continue; - final Path path = pop.getB(); - if ( file.isSymlink() ){ - final Path linkTo = ((LinkFile) file).getDst(); - symlinks.add( new SymlinkInput( path, linkTo ) ); - toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); - } else if( file.isDirectory() ){ - final Map allChildren = ((Folder) file).getAllChildren(path); - for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { - toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); - } - } else { - final RealFile realFile = (RealFile) file; - final RealFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); - if ( filesForTask.getExcludedNodes() != null ) { - excludedLocations.addAll(filesForTask.getExcludedNodes()); - } - files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); - } + processNext( toProcess, symlinks, files, excludedLocations, task ); } return new TaskInputs( symlinks, files, excludedLocations ); From 47119f4d0767d462f3ec304a7fc698b2c68235f2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 12:00:33 +0100 Subject: [PATCH 135/443] added java docs Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/Scheduler.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 5c1b4f7c..4db5c362 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -241,6 +241,15 @@ public TaskState getTaskState(String taskid) { /* Nodes */ + /** + * Checks if a node fulfills all requirements for a pod. This means:
+ * - enough resources available
+ * - Affinities match + * @param availableByNode + * @param pod + * @param node + * @return + */ boolean canSchedulePodOnNode( PodRequirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { return availableByNode.higherOrEquals( pod.getRequest() ) && affinitiesMatch( pod, node ); } @@ -386,6 +395,12 @@ Map getAvailableByNode(){ return availableByNode; } + /** + * Filters all nodes, that have enough resources and fulfill the affinities + * @param availableByNode + * @param task + * @return + */ Set getMatchingNodesForTask( Map availableByNode, Task task ){ Set result = new HashSet<>(); for (Map.Entry entry : availableByNode.entrySet()) { From 3dbf1936209c9c9ef85ce14adf36341b0274e5a0 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 12:00:55 +0100 Subject: [PATCH 136/443] Changed order of commands Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/RandomScheduler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index f41226f7..df682c98 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -45,11 +45,11 @@ public ScheduleObject getTaskNodeAlignment( for ( final Task task : unscheduledTasks ) { final PodWithAge pod = task.getPod(); + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); + final Set matchingNodes = getMatchingNodesForTask(availableByNode,task); if( !matchingNodes.isEmpty() ) { - log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); - final TaskInputs inputsOfTask = getInputsOfTask(task); filterMatchingNodesForTask( matchingNodes, inputsOfTask ); From 3919ef98112c1d0f5d6998c45e743dba59fd5d84 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 12:10:01 +0100 Subject: [PATCH 137/443] Renamed File to HierarchyFile to avoid name collision with JAVA's File Signed-off-by: Lehmann_Fabian --- .../model/TaskInputFileLocationWrapper.java | 6 ++-- .../model/location/hierachy/AbstractFile.java | 5 --- .../hierachy/AbstractHierarchyFile.java | 5 +++ .../model/location/hierachy/Folder.java | 26 +++++++-------- .../{File.java => HierarchyFile.java} | 2 +- .../location/hierachy/HierarchyWrapper.java | 8 ++--- .../{LinkFile.java => LinkHierarchyFile.java} | 4 +-- .../{RealFile.java => RealHierarchyFile.java} | 4 +-- .../taskinputs/PathFileLocationTriple.java | 6 ++-- .../scheduler/SchedulerWithDaemonSet.java | 32 +++++++++---------- .../java/fonda/scheduler/util/FilePath.java | 6 ++-- .../hierachy/HierarchyWrapperTest.java | 4 +-- ...leTest.java => RealHierarchyFileTest.java} | 32 +++++++++---------- 13 files changed, 70 insertions(+), 70 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java rename src/main/java/fonda/scheduler/model/location/hierachy/{File.java => HierarchyFile.java} (74%) rename src/main/java/fonda/scheduler/model/location/hierachy/{LinkFile.java => LinkHierarchyFile.java} (77%) rename src/main/java/fonda/scheduler/model/location/hierachy/{RealFile.java => RealHierarchyFile.java} (98%) rename src/test/java/fonda/scheduler/model/location/hierachy/{RealFileTest.java => RealHierarchyFileTest.java} (94%) diff --git a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java index da9ea166..c445eb3a 100644 --- a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java @@ -1,16 +1,16 @@ package fonda.scheduler.model; import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealFile; +import fonda.scheduler.model.location.hierachy.RealHierarchyFile; import lombok.Getter; @Getter public class TaskInputFileLocationWrapper { - private final RealFile file; + private final RealHierarchyFile file; private final LocationWrapper wrapper; - public TaskInputFileLocationWrapper(RealFile file, LocationWrapper wrapper) { + public TaskInputFileLocationWrapper(RealHierarchyFile file, LocationWrapper wrapper) { this.file = file; this.wrapper = wrapper; } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java deleted file mode 100644 index 212c4f63..00000000 --- a/src/main/java/fonda/scheduler/model/location/hierachy/AbstractFile.java +++ /dev/null @@ -1,5 +0,0 @@ -package fonda.scheduler.model.location.hierachy; - -public abstract class AbstractFile extends File { - -} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java new file mode 100644 index 00000000..ab6abc30 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java @@ -0,0 +1,5 @@ +package fonda.scheduler.model.location.hierachy; + +public abstract class AbstractHierarchyFile extends HierarchyFile { + +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index d2a746a4..8756df2e 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -9,9 +9,9 @@ import java.util.concurrent.ConcurrentMap; @Slf4j -public class Folder extends File { +public class Folder extends HierarchyFile { - private final ConcurrentMap children = new ConcurrentHashMap<>(); + private final ConcurrentMap children = new ConcurrentHashMap<>(); Folder(){} @@ -25,7 +25,7 @@ public boolean isSymlink() { return false; } - public File get( String name ){ + public HierarchyFile get(String name ){ return children.get( name ); } @@ -35,35 +35,35 @@ public File get( String name ){ * @return the file with the name, or the new created folder */ public Folder getOrCreateFolder(String name ){ - final File file = children.get(name); + final HierarchyFile file = children.get(name); if( file == null || !file.isDirectory() ){ return (Folder) children.compute( name, (key,value) -> (value == null || !value.isDirectory()) ? new Folder() : value ); } return (Folder) file; } - public Map getAllChildren( Path currentPath ){ - Map result = new TreeMap<>(); + public Map getAllChildren(Path currentPath ){ + Map result = new TreeMap<>(); getAllChildren( result, currentPath ); return result; } - private void getAllChildren( final Map result, Path currentPath ){ - for (Map.Entry entry : children.entrySet()) { + private void getAllChildren(final Map result, Path currentPath ){ + for (Map.Entry entry : children.entrySet()) { Path resolve = currentPath.resolve(entry.getKey()); if ( !entry.getValue().isSymlink() && entry.getValue().isDirectory() ){ final Folder value = (Folder) entry.getValue(); value.getAllChildren( result, resolve ); } else { - result.put( resolve, (AbstractFile) entry.getValue()); + result.put( resolve, (AbstractHierarchyFile) entry.getValue()); } } } public LocationWrapper addOrUpdateFile(final String name, boolean overwrite, final LocationWrapper location ) { - final RealFile file = (RealFile) children.compute( name, (k,v) -> { + final RealHierarchyFile file = (RealHierarchyFile) children.compute( name, (k, v) -> { if (v == null || v.isDirectory() || v.isSymlink() ) - return new RealFile( location ); + return new RealHierarchyFile( location ); return v; } ); return file.addOrUpdateLocation( overwrite, location ); @@ -71,8 +71,8 @@ public LocationWrapper addOrUpdateFile(final String name, boolean overwrite, fin public boolean addSymlink( final String name, final Path dst ){ children.compute( name, (k,v) -> { - if ( v == null || !v.isSymlink() || !((LinkFile) v).getDst().equals(dst) ) - return new LinkFile( dst ); + if ( v == null || !v.isSymlink() || !((LinkHierarchyFile) v).getDst().equals(dst) ) + return new LinkHierarchyFile( dst ); return v; } ); return true; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/File.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyFile.java similarity index 74% rename from src/main/java/fonda/scheduler/model/location/hierachy/File.java rename to src/main/java/fonda/scheduler/model/location/hierachy/HierarchyFile.java index 79de9243..32090ef8 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/File.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyFile.java @@ -1,6 +1,6 @@ package fonda.scheduler.model.location.hierachy; -public abstract class File { +public abstract class HierarchyFile { public abstract boolean isDirectory(); diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 1bce7e2c..26b53f64 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -44,10 +44,10 @@ private Folder getWorkdir( Iterator iterator, boolean create ){ * @param path get all files recursively in this folder (absolute path) * @return Null if folder is empty, or not found */ - public Map getAllFilesInDir( final Path path ){ + public Map getAllFilesInDir(final Path path ){ final Path relativePath = relativize( path ); Iterator iterator = relativePath.iterator(); - File current = getWorkdir( iterator, false ); + HierarchyFile current = getWorkdir( iterator, false ); if( current == null ) return null; while(iterator.hasNext()){ Path p = iterator.next(); @@ -118,7 +118,7 @@ private Folder findFolderToInsert( final Path path ){ * @param path file to get (absolute path) * @return File or null if file does not exist */ - public File getFile( Path path ){ + public HierarchyFile getFile(Path path ){ final Path relativePath = relativize( path ); if (relativePath.startsWith("..")){ return null; @@ -128,7 +128,7 @@ public File getFile( Path path ){ if( current == null ) return null; while(iterator.hasNext()) { Path p = iterator.next(); - final File file = current.get( p.toString() ); + final HierarchyFile file = current.get( p.toString() ); if( iterator.hasNext() && file.isDirectory() ){ //folder current = (Folder) file; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java similarity index 77% rename from src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java rename to src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java index db5e04f4..3d3ddd17 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LinkFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java @@ -4,12 +4,12 @@ import java.nio.file.Path; -public class LinkFile extends AbstractFile { +public class LinkHierarchyFile extends AbstractHierarchyFile { @Getter private final Path dst; - public LinkFile( Path dst ) { + public LinkHierarchyFile(Path dst ) { this.dst = dst; } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java similarity index 98% rename from src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java rename to src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index 8fb93c3f..ef3d9e21 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -8,7 +8,7 @@ import java.util.*; -public class RealFile extends AbstractFile { +public class RealHierarchyFile extends AbstractHierarchyFile { /** * This field contains the newest LocationWrapper of one file for each node. @@ -16,7 +16,7 @@ public class RealFile extends AbstractFile { @Getter private LocationWrapper[] locations; - public RealFile( LocationWrapper location ) { + public RealHierarchyFile(LocationWrapper location ) { if ( location == null ) throw new IllegalArgumentException( "location is null" ); this.locations = new LocationWrapper[]{ location }; } diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index f39acf60..d0445aff 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -1,7 +1,7 @@ package fonda.scheduler.model.taskinputs; import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealFile; +import fonda.scheduler.model.location.hierachy.RealHierarchyFile; import java.nio.file.Path; import java.util.List; @@ -9,10 +9,10 @@ public class PathFileLocationTriple implements Input { public final Path path; - public final RealFile file; + public final RealHierarchyFile file; public final List locations; - public PathFileLocationTriple(Path path, RealFile file, List locations) { + public PathFileLocationTriple(Path path, RealHierarchyFile file, List locations) { this.path = path; this.file = file; this.locations = locations; diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index f19deb0d..7b44a5ad 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -164,8 +164,8 @@ List writeInitConfig( NodeTaskFilesAlignment align return null; } - private LinkedList> filterFilesToProcess( List> fileInputs ){ - final LinkedList> toProcess = new LinkedList<>(); + private LinkedList> filterFilesToProcess(List> fileInputs ){ + final LinkedList> toProcess = new LinkedList<>(); for ( InputParam fileInput : fileInputs) { final Path path = Path.of(fileInput.value.sourceObj); if ( this.hierarchyWrapper.isInScope( path ) ){ @@ -175,28 +175,28 @@ private LinkedList> fil return toProcess; } private void processNext( - final LinkedList> toProcess, + final LinkedList> toProcess, final List symlinks, final List files, final Set excludedLocations, final Task task ){ - final Tuple tuple = toProcess.removeLast(); - final fonda.scheduler.model.location.hierachy.File file = tuple.getA(); + final Tuple tuple = toProcess.removeLast(); + final HierarchyFile file = tuple.getA(); if( file == null ) return; final Path path = tuple.getB(); if ( file.isSymlink() ){ - final Path linkTo = ((LinkFile) file).getDst(); + final Path linkTo = ((LinkHierarchyFile) file).getDst(); symlinks.add( new SymlinkInput( path, linkTo ) ); toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); } else if( file.isDirectory() ){ - final Map allChildren = ((Folder) file).getAllChildren(path); - for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { + final Map allChildren = ((Folder) file).getAllChildren(path); + for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); } } else { - final RealFile realFile = (RealFile) file; - final RealFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); + final RealHierarchyFile realFile = (RealHierarchyFile) file; + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); if ( filesForTask.getExcludedNodes() != null ) { excludedLocations.addAll(filesForTask.getExcludedNodes()); } @@ -207,7 +207,7 @@ private void processNext( TaskInputs getInputsOfTask( Task task ){ final List> fileInputs = task.getConfig().getInputs().fileInputs; - final LinkedList> toProcess = filterFilesToProcess( fileInputs ); + final LinkedList> toProcess = filterFilesToProcess( fileInputs ); final List symlinks = new ArrayList<>( fileInputs.size() ); final List files = new ArrayList<>( fileInputs.size() ); @@ -224,9 +224,9 @@ TaskInputs getInputsOfTask( Task task ){ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileException { LinkedList symlinks = new LinkedList<>(); Path currentPath = Paths.get(path); - fonda.scheduler.model.location.hierachy.File currentFile = hierarchyWrapper.getFile( currentPath ); - while ( currentFile instanceof LinkFile ){ - final LinkFile linkFile = (LinkFile) currentFile; + HierarchyFile currentFile = hierarchyWrapper.getFile( currentPath ); + while ( currentFile instanceof LinkHierarchyFile){ + final LinkHierarchyFile linkFile = (LinkHierarchyFile) currentFile; symlinks.add( new SymlinkInput( currentPath, linkFile.getDst() ) ); currentPath = linkFile.getDst(); currentFile = hierarchyWrapper.getFile( currentPath ); @@ -236,11 +236,11 @@ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileExce if ( currentFile == null ) { return new FileResponse( currentPath.toString(), symlinks ); } - if ( ! (currentFile instanceof RealFile) ){ + if ( ! (currentFile instanceof RealHierarchyFile) ){ log.info( "File was: {}", currentFile ); throw new NotARealFileException(); } - final RealFile file = (RealFile) currentFile; + final RealHierarchyFile file = (RealHierarchyFile) currentFile; final LocationWrapper lastUpdate = file.getLastUpdate(LocationType.NODE); if( lastUpdate == null ) return null; requestedLocations.put( lastUpdate.getId(), lastUpdate ); diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/fonda/scheduler/util/FilePath.java index b0123ded..542498b1 100644 --- a/src/main/java/fonda/scheduler/util/FilePath.java +++ b/src/main/java/fonda/scheduler/util/FilePath.java @@ -1,15 +1,15 @@ package fonda.scheduler.util; import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealFile; +import fonda.scheduler.model.location.hierachy.RealHierarchyFile; public class FilePath { public final String path; - public final RealFile file; + public final RealHierarchyFile file; public final LocationWrapper locationWrapper; - public FilePath(String path, RealFile file, LocationWrapper locationWrapper ) { + public FilePath(String path, RealHierarchyFile file, LocationWrapper locationWrapper ) { this.path = path; this.file = file; this.locationWrapper = locationWrapper; diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java index 5fa00a37..7abdd7d2 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -222,7 +222,7 @@ public void testGetFile() { index = 0; for ( Path file : files) { - RealFile realFile = (RealFile) hw.getFile(file); + RealHierarchyFile realFile = (RealHierarchyFile) hw.getFile(file); final LocationWrapper[] locations = realFile.getLocations(); assertEquals(lw[index++], locations[0]); @@ -243,7 +243,7 @@ public void testGetFileWorkdir() { @Test public void testGetFileButIsDir() { - final File file = hw.getFile(Paths.get(temporaryDir + "d")); + final HierarchyFile file = hw.getFile(Paths.get(temporaryDir + "d")); assertNotNull( file ); assertTrue( file.isDirectory() ); } diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java similarity index 94% rename from src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java rename to src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java index b8dc938f..ea61bab4 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertThrows; -public class RealFileTest { +public class RealHierarchyFileTest { private Task processA; private Task processB; @@ -63,7 +63,7 @@ public void addLocation() { final LocationWrapper node1 = getLocationWrapper("Node1"); locations.add(node1); - final RealFile realFile = new RealFile(node1); + final RealHierarchyFile realFile = new RealHierarchyFile(node1); assertArrayEquals( locations.toArray(), realFile.getLocations() ); final LocationWrapper node2 = getLocationWrapper("Node2"); @@ -84,7 +84,7 @@ public void addLocation() { @Test public void addEmptyLocation() { - final RealFile realFile = new RealFile( getLocationWrapper("node1") ); + final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("node1") ); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( true, null )); } @@ -92,7 +92,7 @@ public void addEmptyLocation() { @Test public void addInParallel() { final LocationWrapper node0 = getLocationWrapper("Node0"); - final RealFile realFile = new RealFile(node0); + final RealHierarchyFile realFile = new RealHierarchyFile(node0); List locations = new LinkedList<>(); @@ -116,7 +116,7 @@ public void addInParallel() { public void changeFile() { final LocationWrapper node0 = getLocationWrapper("Node0"); - final RealFile realFile = new RealFile(node0); + final RealHierarchyFile realFile = new RealHierarchyFile(node0); final LocationWrapper node1 = getLocationWrapper("Node1"); realFile.addOrUpdateLocation( false, node1); final LocationWrapper node2 = getLocationWrapper("Node2"); @@ -137,7 +137,7 @@ public void overwriteFile() { final LocationWrapper node0 = getLocationWrapper("Node0"); results.add( node0 ); assertTrue(node0.isActive()); - final RealFile realFile = new RealFile(node0); + final RealHierarchyFile realFile = new RealHierarchyFile(node0); assertArrayEquals( results.toArray(), realFile.getLocations() ); assertTrue(node0.isActive()); final LocationWrapper node1 = getLocationWrapper("Node1"); @@ -158,7 +158,7 @@ public void overwriteFile() { @Test public void changeFileOnExistingLocation() { - final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); + final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("Node0") ); final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, processA ); realFile.addOrUpdateLocation( false, nodeNew ); @@ -174,7 +174,7 @@ public void changeFileOnExistingLocation() { @Test public void isFile() { - final RealFile realFile = new RealFile( getLocationWrapper("Node0") ); + final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("Node0") ); assertFalse( realFile.isDirectory() ); assertFalse( realFile.isSymlink() ); } @@ -200,7 +200,7 @@ public void getFilesForTaskTest() throws InterruptedException { PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); while ( permutationIterator.hasNext() ) { final List next = permutationIterator.next(); - RealFile realFile = new RealFile(next.get(0)); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); realFile.addOrUpdateLocation(false, next.get(1)); realFile.addOrUpdateLocation(false, next.get(2)); realFile.addOrUpdateLocation(false, next.get(3)); @@ -217,7 +217,7 @@ public void getFilesForTaskTest() throws InterruptedException { permutationIterator = new PermutationIterator<>(locationWrapperList); while ( permutationIterator.hasNext() ) { final List next = permutationIterator.next(); - RealFile realFile = new RealFile(next.get(0)); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); realFile.addOrUpdateLocation(false, next.get(1)); realFile.addOrUpdateLocation(false, next.get(2)); realFile.addOrUpdateLocation(false, next.get(3)); @@ -251,7 +251,7 @@ public void getFilesForTaskTestInitFiles() throws InterruptedException { PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); while ( permutationIterator.hasNext() ) { final List next = permutationIterator.next(); - RealFile realFile = new RealFile(next.get(0)); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); realFile.addOrUpdateLocation(false, next.get(1)); realFile.addOrUpdateLocation(false, next.get(2)); realFile.addOrUpdateLocation(false, next.get(3)); @@ -287,7 +287,7 @@ public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException { PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); while ( permutationIterator.hasNext() ) { final List next = permutationIterator.next(); - RealFile realFile = new RealFile(next.get(0)); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); realFile.addOrUpdateLocation(false, next.get(1)); realFile.addOrUpdateLocation(false, next.get(2)); realFile.addOrUpdateLocation(false, next.get(3)); @@ -321,7 +321,7 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); while ( permutationIterator.hasNext() ) { final List next = permutationIterator.next(); - RealFile realFile = new RealFile( next.get(0) ); + RealHierarchyFile realFile = new RealHierarchyFile( next.get(0) ); realFile.addOrUpdateLocation( false, next.get(1) ); realFile.addOrUpdateLocation( false, next.get(2) ); realFile.addOrUpdateLocation( false, next.get(3) ); @@ -345,7 +345,7 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException permutationIterator = new PermutationIterator<>(locationWrapperList); while ( permutationIterator.hasNext() ) { final List next = permutationIterator.next(); - RealFile realFile = new RealFile( next.get(0) ); + RealHierarchyFile realFile = new RealHierarchyFile( next.get(0) ); realFile.addOrUpdateLocation( false, next.get(1) ); realFile.addOrUpdateLocation( false, next.get(2) ); realFile.addOrUpdateLocation( false, next.get(3) ); @@ -376,7 +376,7 @@ public void getLastLocationTest() throws InterruptedException { assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); - RealFile realFile = new RealFile( loc1 ); + RealHierarchyFile realFile = new RealHierarchyFile( loc1 ); assertEquals( loc1, realFile.getLastUpdate( LocationType.NODE )); realFile.addOrUpdateLocation(false, loc4 ); assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); @@ -407,7 +407,7 @@ public void getLocationWrapperForNodeTest() throws InterruptedException { assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); - RealFile realFile = new RealFile(loc1); + RealHierarchyFile realFile = new RealHierarchyFile(loc1); assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); realFile.addOrUpdateLocation(false, loc4 ); assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); From e39054bf28e7bbb0627bae50f993b87f6d432423 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 13:09:56 +0100 Subject: [PATCH 138/443] Fixed FROM Signed-off-by: Lehmann_Fabian --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bfe8e90c..12afa1ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository # # Package stage # -FROM openjdk:17-jre-slim +FROM openjdk:17-alpine WORKDIR /app RUN groupadd -r javauser && useradd --no-log-init -r -g javauser javauser && mkdir data COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar From 54e6445a67f016eff24e365084c94f163285fd2a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 13:10:26 +0100 Subject: [PATCH 139/443] toString also logs inUse Signed-off-by: Lehmann_Fabian --- .../scheduler/model/location/hierachy/LocationWrapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 5aa258e2..fc0f3324 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -68,7 +68,7 @@ public void free(){ } public boolean isInUse(){ - return inUse == 0; + return inUse > 0; } public LocationWrapper getCopyOf( Location location ) { @@ -108,6 +108,7 @@ public String toString() { "id=" + id + ", location=" + location.getIdentifier() + ", timestamp=" + timestamp + + ", inUse=" + inUse + "x" + ", sizeInBytes=" + sizeInBytes + ", createTime=" + createTime + ", createdByTask=" + createdByTask + From 65d7b3c86a8d61636ed7ab6b058d9b0c0065cee6 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 13:14:30 +0100 Subject: [PATCH 140/443] Check excluded Nodes Signed-off-by: Lehmann_Fabian --- .../hierachy/RealHierarchyFileTest.java | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java index ea61bab4..4f0066f1 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -306,15 +306,22 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException final CountDownLatch waiter = new CountDownLatch(1); - LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, null ); + final NodeLocation location1 = NodeLocation.getLocation("Node1"); + LocationWrapper loc1 = new LocationWrapper( location1, System.currentTimeMillis() - 2, 2, null ); + loc1.use(); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, null ); + final NodeLocation location3 = NodeLocation.getLocation("Node3"); + LocationWrapper loc3 = new LocationWrapper( location3, System.currentTimeMillis() - 5, 2, null ); + loc3.use(); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, null ); + final NodeLocation location4 = NodeLocation.getLocation("Node4"); + LocationWrapper loc4 = new LocationWrapper( location4, System.currentTimeMillis() - 2, 2, null ); assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); - LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null ); + final NodeLocation location5 = NodeLocation.getLocation("Node5"); + LocationWrapper loc5 = new LocationWrapper( location5, System.currentTimeMillis() - 5, 2, null ); + loc5.use(); List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); @@ -327,17 +334,18 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException realFile.addOrUpdateLocation( false, next.get(3) ); realFile.addOrUpdateLocation( false, next.get(4) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ).getMatchingLocations() ) ); + for (Task task : List.of(processA, processB, processC, processD, processE)) { + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask( task ); + assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); + assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); + } + } while( loc4.getCreateTime() != loc5.getCreateTime() ){ - loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, null ); - loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null ); + loc4 = new LocationWrapper( location4, System.currentTimeMillis() - 2, 2, null ); + loc5 = new LocationWrapper( location5, System.currentTimeMillis() - 5, 2, null ); } locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); @@ -351,13 +359,13 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException realFile.addOrUpdateLocation( false, next.get(3) ); realFile.addOrUpdateLocation( false, next.get(4) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processA ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processB ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processC ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processD ).getMatchingLocations() ) ); - assertEquals( new HashSet<>( Arrays.asList( loc4, loc5 ) ), new HashSet<>( realFile.getFilesForTask( processE ).getMatchingLocations() ) ); - } + for (Task task : List.of(processA, processB, processC, processD, processE)) { + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask( task ); + assertEquals( new HashSet<>( List.of( loc4, loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); + assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); + } + } } From c5d852bea2cbd8e2a2d0391f2914cd3de9fe0a21 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 9 Mar 2022 13:39:02 +0100 Subject: [PATCH 141/443] Fixed Dockfile Signed-off-by: Lehmann_Fabian --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 12afa1ba..cf952254 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,9 @@ RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository # FROM openjdk:17-alpine WORKDIR /app -RUN groupadd -r javauser && useradd --no-log-init -r -g javauser javauser && mkdir data +RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar -RUN chown -R javauser:javauser /app +RUN chown -R javauser:javagroup /app USER javauser EXPOSE 8080 ENTRYPOINT ["java","-jar","k8s-scheduler.jar"] \ No newline at end of file From e557df8066ab9aa63bdfbd3efd33660a2fcb0a8b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Mar 2022 12:45:39 +0100 Subject: [PATCH 142/443] Removed unused imports Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/client/KubernetesClient.java | 1 - src/main/java/fonda/scheduler/scheduler/Scheduler.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 3cfd5667..ff8d7aff 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -9,7 +9,6 @@ import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; -import io.fabric8.kubernetes.client.dsl.base.OperationContext; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 4db5c362..e6c3fa8d 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -19,7 +19,6 @@ import java.io.FileNotFoundException; import java.io.PrintWriter; import java.util.*; -import java.util.stream.Collectors; @Slf4j public abstract class Scheduler { From e5fcc157bc94dbc40f6d80df0db953bec2ad03f3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Mar 2022 12:51:07 +0100 Subject: [PATCH 143/443] Collect Input Files: moved logic into separate class. Signed-off-by: Lehmann_Fabian --- .../scheduler/model/InputFileCollector.java | 79 +++++++++++++++++++ .../scheduler/SchedulerWithDaemonSet.java | 58 +------------- 2 files changed, 83 insertions(+), 54 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/InputFileCollector.java diff --git a/src/main/java/fonda/scheduler/model/InputFileCollector.java b/src/main/java/fonda/scheduler/model/InputFileCollector.java new file mode 100644 index 00000000..2cb151bd --- /dev/null +++ b/src/main/java/fonda/scheduler/model/InputFileCollector.java @@ -0,0 +1,79 @@ +package fonda.scheduler.model; + +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.hierachy.*; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.util.Tuple; + +import java.nio.file.Path; +import java.util.*; + +public class InputFileCollector { + + private final HierarchyWrapper hierarchyWrapper; + + public InputFileCollector( HierarchyWrapper hierarchyWrapper ) { + this.hierarchyWrapper = hierarchyWrapper; + } + + private void processNext( + final LinkedList> toProcess, + final List symlinks, + final List files, + final Set excludedLocations, + final Task task + ){ + final Tuple tuple = toProcess.removeLast(); + final HierarchyFile file = tuple.getA(); + if( file == null ) return; + final Path path = tuple.getB(); + if ( file.isSymlink() ){ + final Path linkTo = ((LinkHierarchyFile) file).getDst(); + symlinks.add( new SymlinkInput( path, linkTo ) ); + toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); + } else if( file.isDirectory() ){ + final Map allChildren = ((Folder) file).getAllChildren(path); + for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { + toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); + } + } else { + final RealHierarchyFile realFile = (RealHierarchyFile) file; + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); + if ( filesForTask.getExcludedNodes() != null ) { + excludedLocations.addAll(filesForTask.getExcludedNodes()); + } + files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); + } + } + + public TaskInputs getInputsOfTask(Task task ){ + + final List> fileInputs = task.getConfig().getInputs().fileInputs; + final LinkedList> toProcess = filterFilesToProcess( fileInputs ); + + final List symlinks = new ArrayList<>( fileInputs.size() ); + final List files = new ArrayList<>( fileInputs.size() ); + final Set excludedLocations = new HashSet<>(); + + while ( !toProcess.isEmpty() ){ + processNext( toProcess, symlinks, files, excludedLocations, task ); + } + + return new TaskInputs( symlinks, files, excludedLocations ); + + } + + private LinkedList> filterFilesToProcess(List> fileInputs ){ + final LinkedList> toProcess = new LinkedList<>(); + for ( InputParam fileInput : fileInputs) { + final Path path = Path.of(fileInput.value.sourceObj); + if ( this.hierarchyWrapper.isInScope( path ) ){ + toProcess.add( new Tuple<>(hierarchyWrapper.getFile( path ), path) ); + } + } + return toProcess; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 7b44a5ad..99da44c6 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -47,11 +47,13 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { @Getter private final CopyStrategy copyStrategy; final HierarchyWrapper hierarchyWrapper; + private final InputFileCollector inputFileCollector; private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); + this.inputFileCollector = new InputFileCollector( hierarchyWrapper ); if ( config.copyStrategy == null ) throw new IllegalArgumentException( "Copy strategy is null" ); switch ( config.copyStrategy ){ case "ftp": @@ -164,62 +166,10 @@ List writeInitConfig( NodeTaskFilesAlignment align return null; } - private LinkedList> filterFilesToProcess(List> fileInputs ){ - final LinkedList> toProcess = new LinkedList<>(); - for ( InputParam fileInput : fileInputs) { - final Path path = Path.of(fileInput.value.sourceObj); - if ( this.hierarchyWrapper.isInScope( path ) ){ - toProcess.add( new Tuple<>(hierarchyWrapper.getFile( path ), path) ); - } - } - return toProcess; - } - private void processNext( - final LinkedList> toProcess, - final List symlinks, - final List files, - final Set excludedLocations, - final Task task - ){ - final Tuple tuple = toProcess.removeLast(); - final HierarchyFile file = tuple.getA(); - if( file == null ) return; - final Path path = tuple.getB(); - if ( file.isSymlink() ){ - final Path linkTo = ((LinkHierarchyFile) file).getDst(); - symlinks.add( new SymlinkInput( path, linkTo ) ); - toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); - } else if( file.isDirectory() ){ - final Map allChildren = ((Folder) file).getAllChildren(path); - for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { - toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); - } - } else { - final RealHierarchyFile realFile = (RealHierarchyFile) file; - final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); - if ( filesForTask.getExcludedNodes() != null ) { - excludedLocations.addAll(filesForTask.getExcludedNodes()); - } - files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); - } + TaskInputs getInputsOfTask(Task task ){ + return inputFileCollector.getInputsOfTask( task ); } - TaskInputs getInputsOfTask( Task task ){ - - final List> fileInputs = task.getConfig().getInputs().fileInputs; - final LinkedList> toProcess = filterFilesToProcess( fileInputs ); - - final List symlinks = new ArrayList<>( fileInputs.size() ); - final List files = new ArrayList<>( fileInputs.size() ); - final Set excludedLocations = new HashSet<>(); - - while ( !toProcess.isEmpty() ){ - processNext( toProcess, symlinks, files, excludedLocations, task ); - } - - return new TaskInputs( symlinks, files, excludedLocations ); - - } public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileException { LinkedList symlinks = new LinkedList<>(); From 582f0b3c8bfd2e1d107ae604657adba0ec6510d5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Mar 2022 14:15:16 +0100 Subject: [PATCH 144/443] removed unused imports Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/Vertex.java | 5 ++++- .../scheduler/rest/SchedulerRestController.java | 4 ++-- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 12 +++++------- .../scheduler/scheduler/TaskprocessingThread.java | 4 +++- .../fonda/scheduler/model/TaskResultParserTest.java | 7 +++++-- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index 80b15a6e..83dbfe9a 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -2,7 +2,10 @@ import lombok.Getter; -import java.util.*; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; @Getter diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index ce4a6528..37e2f8f6 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -4,12 +4,12 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.InputEdge; import fonda.scheduler.dag.Vertex; +import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.model.TaskConfig; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.RandomScheduler; import fonda.scheduler.scheduler.Scheduler; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.model.TaskConfig; import fonda.scheduler.scheduler.SchedulerWithDaemonSet; import fonda.scheduler.scheduler.filealignment.RandomAlignment; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 99da44c6..01e7abde 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -5,24 +5,22 @@ import fonda.scheduler.model.*; import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.rest.exceptions.NotARealFileException; -import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.*; import fonda.scheduler.model.outfiles.OutputFile; import fonda.scheduler.model.outfiles.PathLocationWrapperPair; import fonda.scheduler.model.outfiles.SymlinkOutput; +import fonda.scheduler.model.taskinputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.rest.exceptions.NotARealFileException; +import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; +import fonda.scheduler.util.FilePath; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.SymlinkInput; -import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.Tuple; import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java index af21c1bb..89429d31 100644 --- a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java @@ -2,7 +2,9 @@ import fonda.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; -import java.util.*; + +import java.util.LinkedList; +import java.util.List; import java.util.function.Function; @Slf4j diff --git a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java index ed94c599..4f928400 100644 --- a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -15,9 +15,12 @@ import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; @Slf4j public class TaskResultParserTest { From 9526643d96abedabf58f6aff408720bbe619f623 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Mar 2022 15:02:53 +0100 Subject: [PATCH 145/443] Spelling Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/taskinputs/TaskInputs.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java index 86cc4b52..c6fab5e8 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -9,12 +9,12 @@ @Getter public class TaskInputs { - private final List symliks; + private final List symlinks; private final List files; private final Set excludedNodes; - public TaskInputs(List symliks, List files, Set excludedNodes) { - this.symliks = symliks; + public TaskInputs(List symlinks, List files, Set excludedNodes) { + this.symlinks = symlinks; this.files = files; this.excludedNodes = excludedNodes; } From f5f1772bf9f66cb56e875e050099dabf2b07a290 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Mar 2022 15:28:27 +0100 Subject: [PATCH 146/443] Fix root folder of a task was not found Signed-off-by: Lehmann_Fabian --- .../model/location/hierachy/HierarchyWrapper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index 26b53f64..de3a0da9 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -67,7 +67,7 @@ public Map getAllFilesInDir(final Path path ){ * * @param path file to add (absolute path) * @param location location where the file is located - * @return false if file can not be created + * @return null if file can not be created */ public LocationWrapper addFile(final Path path, final LocationWrapper location ){ return addFile( path, false, location ); @@ -124,9 +124,10 @@ public HierarchyFile getFile(Path path ){ return null; } Iterator iterator = relativePath.iterator(); - Folder current = getWorkdir( iterator, true ); + Folder current = getWorkdir( iterator, false ); if( current == null ) return null; - while(iterator.hasNext()) { + if( !iterator.hasNext() ) return current; + while( iterator.hasNext() ) { Path p = iterator.next(); final HierarchyFile file = current.get( p.toString() ); if( iterator.hasNext() && file.isDirectory() ){ From 1e1dc23f51a5438e3d94ac3a736573f17736b050 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Mar 2022 14:37:48 +0200 Subject: [PATCH 147/443] Also consider direct ancestor if current exists Signed-off-by: Lehmann_Fabian --- .../scheduler/model/location/hierachy/RealHierarchyFile.java | 5 ++++- .../model/location/hierachy/RealHierarchyFileTest.java | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index ef3d9e21..4631e6d2 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -94,7 +94,10 @@ private List combineResultsEmptyInitial ( ) { LinkedList result = null; if ( current != null ) result = current; - if ( current == null && ancestors != null ) result = ancestors; + if ( ancestors != null ) { + if (current == null ) result = ancestors; + else result.addAll( ancestors ); + } if ( unrelated != null ) { if ( result == null ) result = unrelated; else result.addAll( unrelated ); diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java index 4f0066f1..6765e094 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -206,8 +206,8 @@ public void getFilesForTaskTest() throws InterruptedException { realFile.addOrUpdateLocation(false, next.get(3)); assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); + assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); } From 326ef8b95a4a11a9aeaf7cc10e3d4399a2827925 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Mar 2022 14:52:06 +0200 Subject: [PATCH 148/443] Replaced toString methods with Lombok Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/FileHolder.java | 10 ++----- .../fonda/scheduler/model/InputParam.java | 10 ++----- .../scheduler/model/PodRequirements.java | 9 ++---- .../scheduler/model/SchedulerConfig.java | 28 ++++--------------- .../java/fonda/scheduler/model/TaskInput.java | 24 ++++++++++------ .../taskinputs/PathFileLocationTriple.java | 5 ++++ .../model/taskinputs/TaskInputs.java | 3 ++ .../rest/response/getfile/FileResponse.java | 13 ++------- 8 files changed, 39 insertions(+), 63 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/FileHolder.java b/src/main/java/fonda/scheduler/model/FileHolder.java index 66ef7089..e1cf17e3 100644 --- a/src/main/java/fonda/scheduler/model/FileHolder.java +++ b/src/main/java/fonda/scheduler/model/FileHolder.java @@ -1,5 +1,8 @@ package fonda.scheduler.model; +import lombok.ToString; + +@ToString( exclude = {"stageName", "storePath"}) public class FileHolder { public final String storePath; @@ -12,11 +15,4 @@ private FileHolder() { this.stageName = null; } - @Override - public String toString() { - return "FileHolder{" + - "sourceObj='" + sourceObj + '\'' + - '}'; - } - } diff --git a/src/main/java/fonda/scheduler/model/InputParam.java b/src/main/java/fonda/scheduler/model/InputParam.java index b3bef8a9..4b342826 100644 --- a/src/main/java/fonda/scheduler/model/InputParam.java +++ b/src/main/java/fonda/scheduler/model/InputParam.java @@ -1,5 +1,8 @@ package fonda.scheduler.model; +import lombok.ToString; + +@ToString public class InputParam { public final String name; @@ -10,11 +13,4 @@ private InputParam() { this.value = null; } - @Override - public String toString() { - return "InputParam{" + - "name='" + name + '\'' + - ", value=" + value + - '}'; - } } diff --git a/src/main/java/fonda/scheduler/model/PodRequirements.java b/src/main/java/fonda/scheduler/model/PodRequirements.java index 1e1ade2d..280fca68 100644 --- a/src/main/java/fonda/scheduler/model/PodRequirements.java +++ b/src/main/java/fonda/scheduler/model/PodRequirements.java @@ -1,10 +1,12 @@ package fonda.scheduler.model; import lombok.Getter; +import lombok.ToString; import java.io.Serializable; import java.math.BigDecimal; +@ToString public class PodRequirements implements Serializable { private static final long serialVersionUID = 1L; @@ -59,11 +61,4 @@ public boolean higherOrEquals( PodRequirements podRequirements ){ && this.ram.compareTo( podRequirements.ram ) >= 0; } - @Override - public String toString() { - return "PodRequirements{" + - "cpu=" + cpu + - ", ram=" + ram + - '}'; - } } diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 92ee4685..2c0433c3 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -1,7 +1,10 @@ package fonda.scheduler.model; +import lombok.ToString; + import java.util.List; +@ToString public class SchedulerConfig { public final List localClaims; @@ -30,6 +33,7 @@ private SchedulerConfig(){ this.copyStrategy = null; } + @ToString public static class LocalClaim { public final String mountPath; public final String hostPath; @@ -39,15 +43,9 @@ private LocalClaim(){ this.hostPath = null; } - @Override - public String toString() { - return "LocalClaim{" + - "mountPath='" + mountPath + '\'' + - ", hostPath='" + hostPath + '\'' + - '}'; - } } + @ToString public static class VolumeClaim { public final String mountPath; public final String claimName; @@ -59,22 +57,6 @@ private VolumeClaim(){ this.subPath = null; } - @Override - public String toString() { - return "VolumeClaim{" + - "mountPath='" + mountPath + '\'' + - ", claimName='" + claimName + '\'' + - ", subPath='" + subPath + '\'' + - '}'; - } } - @Override - public String toString() { - return "SchedulerConfig{" + - "localClaims=" + localClaims + - ", volumeClaims=" + volumeClaims + - ", workDir='" + workDir + '\'' + - '}'; - } } diff --git a/src/main/java/fonda/scheduler/model/TaskInput.java b/src/main/java/fonda/scheduler/model/TaskInput.java index 79558a9b..f4eaffea 100644 --- a/src/main/java/fonda/scheduler/model/TaskInput.java +++ b/src/main/java/fonda/scheduler/model/TaskInput.java @@ -1,7 +1,10 @@ package fonda.scheduler.model; +import lombok.ToString; + import java.util.List; +@ToString public class TaskInput { public final List> booleanInputs; @@ -16,13 +19,18 @@ private TaskInput() { this.fileInputs = null; } - @Override - public String toString() { - return "TaskInput{" + - "booleanInputs=" + booleanInputs + - ", numberInputs=" + numberInputs + - ", stringInputs=" + stringInputs + - ", fileInputs=" + fileInputs + - '}'; + /** + * Only for testing + * @param booleanInputs + * @param numberInputs + * @param stringInputs + * @param fileInputs + */ + TaskInput(List> booleanInputs, List> numberInputs, List> stringInputs, List> fileInputs) { + this.booleanInputs = booleanInputs; + this.numberInputs = numberInputs; + this.stringInputs = stringInputs; + this.fileInputs = fileInputs; } + } diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index d0445aff..98f1d0b4 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -2,10 +2,14 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealHierarchyFile; +import lombok.EqualsAndHashCode; +import lombok.ToString; import java.nio.file.Path; import java.util.List; +@ToString( exclude = "file" ) +@EqualsAndHashCode public class PathFileLocationTriple implements Input { public final Path path; @@ -17,4 +21,5 @@ public PathFileLocationTriple(Path path, RealHierarchyFile file, List symlinks, List file this.files = files; this.excludedNodes = excludedNodes; } + } diff --git a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java index eb29bfd2..f499278f 100644 --- a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java +++ b/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java @@ -1,9 +1,11 @@ package fonda.scheduler.rest.response.getfile; import fonda.scheduler.model.taskinputs.SymlinkInput; +import lombok.ToString; import java.util.List; +@ToString( exclude = "locationWrapperID" ) public class FileResponse { public final String path; @@ -34,15 +36,4 @@ public FileResponse( String path, List symlinks) { locationWrapperID = -1; } - @Override - public String toString() { - return "FileResponse{" + - "path='" + path + '\'' + - ", sameAsEngine=" + sameAsEngine + - ", node='" + node + '\'' + - ", daemon='" + daemon + '\'' + - ", symlinks=" + symlinks + - ", notInContext=" + notInContext + - '}'; - } } From d10f6ae9c9645d1d546dcb6994da74f71c865aa7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Mar 2022 14:59:17 +0200 Subject: [PATCH 149/443] Tests for getInputsOfTask Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/FileHolder.java | 12 + .../fonda/scheduler/model/InputParam.java | 10 + .../fonda/scheduler/model/TaskConfig.java | 3 +- .../model/taskinputs/SymlinkInput.java | 3 + .../model/InputFileCollectorTest.java | 238 ++++++++++++++++++ 5 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 src/test/java/fonda/scheduler/model/InputFileCollectorTest.java diff --git a/src/main/java/fonda/scheduler/model/FileHolder.java b/src/main/java/fonda/scheduler/model/FileHolder.java index e1cf17e3..305cf2fd 100644 --- a/src/main/java/fonda/scheduler/model/FileHolder.java +++ b/src/main/java/fonda/scheduler/model/FileHolder.java @@ -15,4 +15,16 @@ private FileHolder() { this.stageName = null; } + /** + * Only for testing + * @param storePath + * @param sourceObj + * @param stageName + */ + public FileHolder( String storePath, String sourceObj, String stageName ) { + this.storePath = storePath; + this.sourceObj = sourceObj; + this.stageName = stageName; + } + } diff --git a/src/main/java/fonda/scheduler/model/InputParam.java b/src/main/java/fonda/scheduler/model/InputParam.java index 4b342826..3b638ed5 100644 --- a/src/main/java/fonda/scheduler/model/InputParam.java +++ b/src/main/java/fonda/scheduler/model/InputParam.java @@ -13,4 +13,14 @@ private InputParam() { this.value = null; } + /** + * Only for testing + * @param name + * @param value + */ + public InputParam( String name, T value ) { + this.name = name; + this.value = value; + } + } diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index c8062a8f..0e08cef5 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -3,6 +3,7 @@ import lombok.Getter; import lombok.ToString; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -30,7 +31,7 @@ public TaskConfig(String task) { this.task = task; this.name = null; this.schedulerParams = null; - this.inputs = null; + this.inputs = new TaskInput( null, null, null, new LinkedList<>() ); this.hash = null; this.cpus = 0; this.memoryInBytes = 0; diff --git a/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java b/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java index 05594f96..10dcf887 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java @@ -1,10 +1,12 @@ package fonda.scheduler.model.taskinputs; +import lombok.EqualsAndHashCode; import lombok.Getter; import java.nio.file.Path; @Getter +@EqualsAndHashCode public class SymlinkInput implements Input { private final String src; @@ -14,4 +16,5 @@ public SymlinkInput(Path src, Path dst) { this.src = src.toAbsolutePath().toString(); this.dst = dst.toAbsolutePath().toString(); } + } diff --git a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java new file mode 100644 index 00000000..f1337c28 --- /dev/null +++ b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java @@ -0,0 +1,238 @@ +package fonda.scheduler.model; + +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.InputEdge; +import fonda.scheduler.dag.Process; +import fonda.scheduler.dag.Vertex; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.HierarchyFile; +import fonda.scheduler.model.location.hierachy.HierarchyWrapper; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.RealHierarchyFile; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.TaskInputs; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.*; + +@Slf4j +public class InputFileCollectorTest { + + private LocationWrapper location11; + private LocationWrapper location12; + private LocationWrapper location21; + private LocationWrapper location23; + private LocationWrapper location34; + private LocationWrapper location35; + private LocationWrapper location44; + private LocationWrapper location45; + + @Before + public void init(){ + location11 = new LocationWrapper(NodeLocation.getLocation("Node1"), 1, 100); + location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 1, 101); + location21 = new LocationWrapper(NodeLocation.getLocation("Node1"), 1, 102); + location23 = new LocationWrapper(NodeLocation.getLocation("Node3"), 1, 103); + location34 = new LocationWrapper(NodeLocation.getLocation("Node4"), 1, 104); + location35 = new LocationWrapper(NodeLocation.getLocation("Node5"), 1, 105); + location44 = new LocationWrapper(NodeLocation.getLocation("Node4"), 1, 106); + location45 = new LocationWrapper(NodeLocation.getLocation("Node5"), 1, 107); + } + + @Test + public void getInputsOfTaskTest() { + + final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; + final String root2 = "/workdir/01/db62d739d658b839f07a1a77d877d1/"; + final String root3 = "/workdir/02/db62d739d658b839f07a1a77d877d2/"; + + final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); + + final Path path1 = Paths.get(root + "a.txt"); + assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) hierarchyWrapper.getFile(path1), List.of(location11, location12)); + + final Path path2 = Paths.get(root + "a/b.txt"); + assertNotNull( hierarchyWrapper.addFile(path2, location21) ); + assertNotNull( hierarchyWrapper.addFile(path2, location23) ); + final PathFileLocationTriple file2 = new PathFileLocationTriple(path2, (RealHierarchyFile) hierarchyWrapper.getFile(path2), List.of(location21, location23)); + + final Path path3 = Paths.get(root2 + "a/b.txt"); + assertNotNull( hierarchyWrapper.addFile(path3, location34) ); + assertNotNull( hierarchyWrapper.addFile(path3, location35) ); + final PathFileLocationTriple file3 = new PathFileLocationTriple(path3, (RealHierarchyFile) hierarchyWrapper.getFile(path3), List.of(location34, location35)); + + final Path path4 = Paths.get(root3 + "b/c.txt"); + assertNotNull( hierarchyWrapper.addFile(path4, location44) ); + assertNotNull( hierarchyWrapper.addFile(path4, location45) ); + final PathFileLocationTriple file4 = new PathFileLocationTriple(path4, (RealHierarchyFile) hierarchyWrapper.getFile(path4), List.of(location44, location45)); + + final Path path5 = Paths.get(root2 + "b/c.txt"); + assertTrue(hierarchyWrapper.addSymlink( path5, path4 ) ); + final SymlinkInput symlink = new SymlinkInput( path5, path4 ); + + + InputFileCollector inputFileCollector = new InputFileCollector( hierarchyWrapper ); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + dag.registerVertices(vertexList); + + final TaskConfig taskConfig = new TaskConfig("processA"); + final List> fileInputs = taskConfig.getInputs().fileInputs; + final InputParam a = new InputParam<>("a", new FileHolder(null, root, null)); + final InputParam b = new InputParam<>("b", new FileHolder(null, root2 + "a/b.txt", null)); + + fileInputs.add( a ); + fileInputs.add( b ); + final Task task = new Task(taskConfig, dag); + + final Set expected = new HashSet( List.of( file1, file2, file3 ) ); + + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(task); + List inputsOfTaskFiles = inputsOfTask.getFiles(); + assertEquals( 3, inputsOfTaskFiles.size() ); + assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + + //Not existing file + final InputParam c = new InputParam<>("c", new FileHolder(null, root2 + "a.txt", null)); + fileInputs.add( c ); + + inputsOfTask = inputFileCollector.getInputsOfTask(task); + inputsOfTaskFiles = inputsOfTask.getFiles(); + assertEquals( 3, inputsOfTaskFiles.size() ); + assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + + //Symlink + final InputParam d = new InputParam<>("d", new FileHolder(null, root2 + "b/c.txt", null)); + fileInputs.add( d ); + expected.add( file4 ); + + inputsOfTask = inputFileCollector.getInputsOfTask(task); + inputsOfTaskFiles = inputsOfTask.getFiles(); + assertEquals( 4, inputsOfTaskFiles.size() ); + assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + assertEquals( Set.of( symlink ), new HashSet<>(inputsOfTask.getSymlinks()) ); + assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + + + } + + @Test + public void getInputsOfTaskTestExcludeNodes() { + + final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; + + final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); + + InputFileCollector inputFileCollector = new InputFileCollector( hierarchyWrapper ); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + vertexList.add(new Process("processC", 3)); + dag.registerVertices(vertexList); + dag.registerEdges( List.of( new InputEdge(1,2),new InputEdge(2,3) ) ); + + final TaskConfig taskConfigA = new TaskConfig("processA"); + final Task taskA = new Task(taskConfigA, dag); + + final TaskConfig taskConfigB = new TaskConfig("processB"); + final Task taskB = new Task(taskConfigB, dag); + + final TaskConfig taskConfigC = new TaskConfig("processC"); + final List> fileInputs = taskConfigC.getInputs().fileInputs; + final InputParam a = new InputParam<>("a.txt", new FileHolder(null, root, null)); + fileInputs.add( a ); + final Task taskC = new Task(taskConfigC, dag); + + LocationWrapper location13 = new LocationWrapper(NodeLocation.getLocation("Node3"), 2, 102, taskB); + LocationWrapper location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 2, 102, taskC); + LocationWrapper location11 = new LocationWrapper(NodeLocation.getLocation("Node1"), 2, 102, taskA); + location11.use(); + + final Path path1 = Paths.get(root + "a.txt"); + assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + assertNotNull( hierarchyWrapper.addFile(path1, location13) ); + final HierarchyFile file = hierarchyWrapper.getFile(path1); + assertNotNull( file ); + final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); + + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC); + List inputsOfTaskFiles = inputsOfTask.getFiles(); + assertEquals( 1, inputsOfTaskFiles.size() ); + assertEquals( file1, inputsOfTaskFiles.get(0) ); + assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + assertEquals( 1, inputsOfTask.getExcludedNodes().size() ); + assertEquals( NodeLocation.getLocation( "Node1" ), inputsOfTask.getExcludedNodes().iterator().next() ); + + } + + @Test + public void getInputsOfTaskTestNotExcludeNodes() { + + final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; + + final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); + + InputFileCollector inputFileCollector = new InputFileCollector( hierarchyWrapper ); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + vertexList.add(new Process("processC", 3)); + dag.registerVertices(vertexList); + dag.registerEdges( List.of( new InputEdge(1,2),new InputEdge(2,3) ) ); + + final TaskConfig taskConfigA = new TaskConfig("processA"); + final Task taskA = new Task(taskConfigA, dag); + + final TaskConfig taskConfigB = new TaskConfig("processB"); + final Task taskB = new Task(taskConfigB, dag); + + final TaskConfig taskConfigC = new TaskConfig("processC"); + final List> fileInputs = taskConfigC.getInputs().fileInputs; + final InputParam a = new InputParam<>("a.txt", new FileHolder(null, root, null)); + fileInputs.add( a ); + final Task taskC = new Task(taskConfigC, dag); + + LocationWrapper location13 = new LocationWrapper(NodeLocation.getLocation("Node3"), 2, 102, taskB); + LocationWrapper location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 2, 102, taskC); + LocationWrapper location11 = new LocationWrapper(NodeLocation.getLocation("Node1"), 2, 102, taskA); + location13.use(); + + final Path path1 = Paths.get(root + "a.txt"); + assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + assertNotNull( hierarchyWrapper.addFile(path1, location13) ); + final HierarchyFile file = hierarchyWrapper.getFile(path1); + assertNotNull( file ); + final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); + log.info(file1.toString()); + + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC); + List inputsOfTaskFiles = inputsOfTask.getFiles(); + assertEquals( 1, inputsOfTaskFiles.size() ); + assertEquals( file1, inputsOfTaskFiles.get(0) ); + assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + + } + +} \ No newline at end of file From 4e54cc019dfd61c37012f04eeb5dc5c2e62e71a3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Mar 2022 15:00:04 +0200 Subject: [PATCH 150/443] fixed method call Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/filealignment/RandomAlignment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index a9632f13..b6641d87 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -30,7 +30,7 @@ public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTa final List pathsOfNode = map.get( nodeIdentifier ); pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file, locationWrapper ) ); } - return new FileAlignment( map, inputsOfTask.getSymliks() ); + return new FileAlignment( map, inputsOfTask.getSymlinks() ); } } From d49b4014c95c4d3a40f873d623bc3d96885d063b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Mar 2022 16:24:09 +0200 Subject: [PATCH 151/443] Stop if all nodes are excluded Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/InputFileCollector.java | 6 ++++-- .../fonda/scheduler/scheduler/RandomScheduler.java | 6 ++++++ .../fonda/scheduler/model/InputFileCollectorTest.java | 10 +++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/InputFileCollector.java b/src/main/java/fonda/scheduler/model/InputFileCollector.java index 2cb151bd..8021ae42 100644 --- a/src/main/java/fonda/scheduler/model/InputFileCollector.java +++ b/src/main/java/fonda/scheduler/model/InputFileCollector.java @@ -48,7 +48,7 @@ private void processNext( } } - public TaskInputs getInputsOfTask(Task task ){ + public TaskInputs getInputsOfTask( Task task, int numberNode ){ final List> fileInputs = task.getConfig().getInputs().fileInputs; final LinkedList> toProcess = filterFilesToProcess( fileInputs ); @@ -57,10 +57,12 @@ public TaskInputs getInputsOfTask(Task task ){ final List files = new ArrayList<>( fileInputs.size() ); final Set excludedLocations = new HashSet<>(); - while ( !toProcess.isEmpty() ){ + while ( !toProcess.isEmpty() && excludedLocations.size() < numberNode ){ processNext( toProcess, symlinks, files, excludedLocations, task ); } + if( excludedLocations.size() == numberNode ) return null; + return new TaskInputs( symlinks, files, excludedLocations ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index df682c98..c17a940c 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -51,6 +51,12 @@ public ScheduleObject getTaskNodeAlignment( if( !matchingNodes.isEmpty() ) { final TaskInputs inputsOfTask = getInputsOfTask(task); + + if( inputsOfTask == null ) { + log.info( "No node, where the pod can start, pod: {}", pod.getName() ); + continue; + } + filterMatchingNodesForTask( matchingNodes, inputsOfTask ); if( matchingNodes.isEmpty() ) { diff --git a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java index f1337c28..5d615d4f 100644 --- a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java @@ -100,7 +100,7 @@ public void getInputsOfTaskTest() { final Set expected = new HashSet( List.of( file1, file2, file3 ) ); - TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(task); + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); assertEquals( 3, inputsOfTaskFiles.size() ); assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); @@ -111,7 +111,7 @@ public void getInputsOfTaskTest() { final InputParam c = new InputParam<>("c", new FileHolder(null, root2 + "a.txt", null)); fileInputs.add( c ); - inputsOfTask = inputFileCollector.getInputsOfTask(task); + inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); inputsOfTaskFiles = inputsOfTask.getFiles(); assertEquals( 3, inputsOfTaskFiles.size() ); assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); @@ -123,7 +123,7 @@ public void getInputsOfTaskTest() { fileInputs.add( d ); expected.add( file4 ); - inputsOfTask = inputFileCollector.getInputsOfTask(task); + inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); inputsOfTaskFiles = inputsOfTask.getFiles(); assertEquals( 4, inputsOfTaskFiles.size() ); assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); @@ -174,7 +174,7 @@ public void getInputsOfTaskTestExcludeNodes() { assertNotNull( file ); final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); - TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC); + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); assertEquals( 1, inputsOfTaskFiles.size() ); assertEquals( file1, inputsOfTaskFiles.get(0) ); @@ -226,7 +226,7 @@ public void getInputsOfTaskTestNotExcludeNodes() { final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); log.info(file1.toString()); - TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC); + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); assertEquals( 1, inputsOfTaskFiles.size() ); assertEquals( file1, inputsOfTaskFiles.get(0) ); From 9fc55e76efda3a7be5edf061bb86bed537ed93c9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 29 Mar 2022 14:38:57 +0200 Subject: [PATCH 152/443] removed comma Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/RandomScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index c17a940c..a77d65cb 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -53,7 +53,7 @@ public ScheduleObject getTaskNodeAlignment( final TaskInputs inputsOfTask = getInputsOfTask(task); if( inputsOfTask == null ) { - log.info( "No node, where the pod can start, pod: {}", pod.getName() ); + log.info( "No node where the pod can start, pod: {}", pod.getName() ); continue; } From 8daf988bd036ac08e1d95b9a8fbad0792df56f87 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 29 Mar 2022 14:39:49 +0200 Subject: [PATCH 153/443] KubernetesClient returns the number of Nodes Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/client/KubernetesClient.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index ff8d7aff..c3f70564 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -66,6 +66,10 @@ private void informAllSchedulersRemovedNode( NodeWithAlloc node ){ } } + public int getNumberOfNodes(){ + return this.nodeHolder.size(); + } + public List getAllNodes(){ return new ArrayList<>(this.nodeHolder.values()); } From 331f814bda9ff3098f4244088f6950bf3f12af0a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 29 Mar 2022 14:41:29 +0200 Subject: [PATCH 154/443] Call with number of nodes Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 01e7abde..378172c6 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -165,7 +165,7 @@ List writeInitConfig( NodeTaskFilesAlignment align } TaskInputs getInputsOfTask(Task task ){ - return inputFileCollector.getInputsOfTask( task ); + return inputFileCollector.getInputsOfTask( task, client.getNumberOfNodes() ); } From 8f8aa14a55f8849f35d58d9b27d1636d3cca836b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 29 Mar 2022 15:56:42 +0200 Subject: [PATCH 155/443] Undo state if scheduling fails Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/Scheduler.java | 26 ++++++++++++++++--- .../scheduler/SchedulerWithDaemonSet.java | 9 +++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index e6c3fa8d..162f01fa 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -89,14 +89,30 @@ public int schedule( final List unscheduledTasks ) { if (!possible) return taskNodeAlignment.size(); } + int failure = 0; for (NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment) { - if (isClose()) return -1; - assignTaskToNode( nodeTaskAlignment ); + try { + if (isClose()) return -1; + if ( !assignTaskToNode( nodeTaskAlignment ) ){ + failure++; + continue; + } + } catch ( Exception e ){ + undoTaskScheduling( nodeTaskAlignment.task ); + continue; + } taskWasScheduled(nodeTaskAlignment.task); } - return unscheduledTasks.size() - taskNodeAlignment.size(); + return unscheduledTasks.size() - taskNodeAlignment.size() + failure; } + /** + * Call this method in case of any scheduling problems + * @param task + */ + void undoTaskScheduling( Task task ){} + + public boolean validSchedulePlan( List taskNodeAlignment ){ List items = getNodeList(); Map< NodeWithAlloc,PodRequirements> availableByNode = new HashMap<>(); @@ -283,7 +299,7 @@ boolean canPodBeScheduled( PodWithAge pod, NodeWithAlloc node ){ return node.canSchedule( pod ); } - void assignTaskToNode( NodeTaskAlignment alignment ){ + boolean assignTaskToNode( NodeTaskAlignment alignment ){ final File nodeFile = new File(alignment.task.getWorkingDir() + '/' + ".command.node"); @@ -319,6 +335,8 @@ void assignTaskToNode( NodeTaskAlignment alignment ){ pod.getSpec().setNodeName( alignment.node.getMetadata().getName() ); log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); + + return true; } /* Helper */ diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 378172c6..7a8ce827 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -82,15 +82,20 @@ private void freeLocations( List locationWrappers ){ } @Override - void assignTaskToNode( NodeTaskAlignment alignment ) { + boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; final List< TaskInputFileLocationWrapper > locationWrappers = writeInitConfig( nodeTaskFilesAlignment ); + if ( locationWrappers == null ) return false; alignment.task.setCopiedFiles( locationWrappers ); getCopyStrategy().generateCopyScript( alignment.task ); final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); useLocations( allLocationWrappers ); - super.assignTaskToNode( alignment ); + return super.assignTaskToNode( alignment ); + } + + void undoTaskScheduling( Task task ){ + if ( task.getInputFiles() != null ) freeLocations( task.getInputFiles() ); } @Override From 448e1553c6a3ac2d3541963eeddd5223f255033b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 1 Apr 2022 15:48:46 +0200 Subject: [PATCH 156/443] Updated versions in pom Signed-off-by: Lehmann_Fabian --- pom.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index bf6a0ae0..84590e13 100644 --- a/pom.xml +++ b/pom.xml @@ -58,31 +58,34 @@ org.springframework.boot spring-boot-starter-web + 2.6.5 org.springframework.boot spring-boot-starter-test + 2.6.5 test org.jetbrains annotations - 22.0.0 + 23.0.0 compile org.junit.vintage junit-vintage-engine + 5.8.2 test org.apache.commons commons-collections4 - 4.1 + 4.4 test From e2d3202ba081384a2299d59f98a3595a5e00c763 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 1 Apr 2022 15:51:17 +0200 Subject: [PATCH 157/443] Don't log No implementation for DELETE Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 162f01fa..37148b17 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -487,6 +487,7 @@ public void eventReceived(Action action, Pod pod) { break; case DELETED: scheduler.markPodAsDeleted(pwa); + break; default: log.info( "No implementation for {}", action ); } From cd8b41b5a4fe11618f823f66fa9751ded1f19ac1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 1 Apr 2022 15:52:55 +0200 Subject: [PATCH 158/443] Wait for other tasks copying the same files Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 7 ++ .../scheduler/model/WriteConfigResult.java | 27 +++++ .../scheduler/SchedulerWithDaemonSet.java | 98 +++++++++++++++---- .../scheduler/schedulingstrategy/Inputs.java | 20 +++- src/main/resources/copystrategies/ftp.py | 82 +++++++++++++--- 5 files changed, 198 insertions(+), 36 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/WriteConfigResult.java diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index e5a0bd79..2e5dd06c 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -2,12 +2,15 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Process; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.util.Batch; +import fonda.scheduler.util.Tuple; import lombok.Getter; import lombok.Setter; +import java.util.HashMap; import java.util.List; public class Task { @@ -40,6 +43,10 @@ public class Task { @Setter private Batch batch; + @Getter + @Setter + private HashMap< String, Tuple> copyingToNode; + public Task( TaskConfig config, DAG dag ) { this.config = config; this.process = dag.getByProcess( config.getTask() ); diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java new file mode 100644 index 00000000..ffbdac20 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -0,0 +1,27 @@ +package fonda.scheduler.model; + +import fonda.scheduler.model.location.Location; +import fonda.scheduler.util.Tuple; +import lombok.Getter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Getter +public class WriteConfigResult { + + private final List inputFiles; + private final Map waitForTask; + private final HashMap< String, Tuple> copyingToNode; + + public WriteConfigResult( + List inputFiles, + Map waitForTask, + HashMap> copyingToNode + ) { + this.inputFiles = inputFiles; + this.waitForTask = waitForTask; + this.copyingToNode = copyingToNode; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 7a8ce827..d957664a 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -21,6 +21,7 @@ import fonda.scheduler.util.FilePath; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.Tuple; import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; @@ -34,7 +35,6 @@ import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; @Slf4j public abstract class SchedulerWithDaemonSet extends Scheduler { @@ -47,6 +47,7 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { final HierarchyWrapper hierarchyWrapper; private final InputFileCollector inputFileCollector; private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); + private final String localWorkDir; SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); @@ -61,6 +62,7 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { default: throw new IllegalArgumentException( "Copy strategy is unknown " + config.copyStrategy ); } + this.localWorkDir = config.workDir; } public String getDaemonOnNode( String node ){ @@ -84,9 +86,11 @@ private void freeLocations( List locationWrappers ){ @Override boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; - final List< TaskInputFileLocationWrapper > locationWrappers = writeInitConfig( nodeTaskFilesAlignment ); - if ( locationWrappers == null ) return false; - alignment.task.setCopiedFiles( locationWrappers ); + final WriteConfigResult writeConfigResult = writeInitConfig(nodeTaskFilesAlignment); + if ( writeConfigResult == null ) return false; + alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); + addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); + alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); getCopyStrategy().generateCopyScript( alignment.task ); final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); @@ -95,7 +99,16 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { } void undoTaskScheduling( Task task ){ - if ( task.getInputFiles() != null ) freeLocations( task.getInputFiles() ); + if ( task.getInputFiles() != null ) { + freeLocations( task.getInputFiles() ); + task.setInputFiles( null ); + } + if ( task.getCopyingToNode() != null ) { + removeFromCopyingToNode( task.getNode(), task.getCopyingToNode()); + task.setCopyingToNode( null ); + } + task.setCopiedFiles( null ); + task.setNode( null ); } @Override @@ -131,38 +144,78 @@ int terminateTasks(List finishedTasks) { return 0; } - List writeInitConfig( NodeTaskFilesAlignment alignment ) { + private final Map< NodeLocation, HashMap< String, Tuple> > copyingToNode = new HashMap<>(); + + private void addToCopyingToNode( NodeLocation nodeLocation, HashMap< String, Tuple > toAdd ){ + if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); + if ( copyingToNode.containsKey( nodeLocation ) ){ + final HashMap> stringTupleHashMap = copyingToNode.get( nodeLocation ); + stringTupleHashMap.putAll( toAdd ); + } else { + copyingToNode.put( nodeLocation, toAdd ); + } + } + + private void removeFromCopyingToNode(NodeLocation nodeLocation, HashMap< String, Tuple> toRemove ){ + if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); + copyingToNode.get( nodeLocation ).keySet().removeAll( toRemove.keySet() ); + } + + WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); + LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); + Map waitForTask = new HashMap<>(); + try { final Inputs inputs = new Inputs( - this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/" + this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/", + this.localWorkDir + "/sync/", + alignment.task.getConfig().getHash() ); + + + final HashMap> filesForCurrentNode = new HashMap<>(); + final NodeLocation currentNode = alignment.node.getNodeLocation(); + final HashMap> filesOnCurrentNode = copyingToNode.get(currentNode); + for (Map.Entry> entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) continue; - final List collect = entry .getValue() - .stream() - .map(x -> x.path) - .collect(Collectors.toList()); - inputs.data.add( new InputEntry( getDaemonOnNode( entry.getKey() ), entry.getKey(), collect ) ); + + final List collect = new LinkedList<>(); final NodeLocation location = NodeLocation.getLocation( entry.getKey() ); for (FilePath filePath : entry.getValue()) { - final LocationWrapper locationWrapper = filePath.file.getLocationWrapper(location); - inputFiles.add( - new TaskInputFileLocationWrapper( - filePath.file, - locationWrapper.getCopyOf( location ) - ) - ); + if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey( filePath.path ) ) { + //Node copies currently from somewhere else! + final Tuple taskLocationTuple = filesOnCurrentNode.get(filePath.path); + if ( taskLocationTuple.getB() != location ) return null; + else { + //May be problematic if the task depending on fails/is stopped before all files are downloaded + waitForTask.put( filePath.path, taskLocationTuple.getA() ); + } + } else { + final LocationWrapper locationWrapper = filePath.file.getLocationWrapper(location); + inputFiles.add( + new TaskInputFileLocationWrapper( + filePath.file, + locationWrapper.getCopyOf(location) + ) + ); + collect.add(filePath.path ); + filesForCurrentNode.put( filePath.path, new Tuple<>( alignment.task, location ) ); + } + } + if( !collect.isEmpty() ) { + inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey()), entry.getKey(), collect)); } - } + inputs.waitForTask( waitForTask ); inputs.symlinks.addAll(alignment.fileAlignment.symlinks); new ObjectMapper().writeValue( config, inputs ); - return inputFiles; + return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode ); } catch (IOException e) { e.printStackTrace(); } @@ -217,6 +270,9 @@ public void addFile( String path, long size, long timestamp, long locationWrappe private void podWasInitialized( Pod pod ){ final Task task = changeStateOfTask(pod, State.PREPARED); task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); + task.setCopiedFiles( null ); + removeFromCopyingToNode( task.getNode(), task.getCopyingToNode() ); + task.setCopyingToNode( null ); } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index b71a5aa2..e0151875 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -1,20 +1,38 @@ package fonda.scheduler.scheduler.schedulingstrategy; +import fonda.scheduler.model.Task; import fonda.scheduler.model.taskinputs.SymlinkInput; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class Inputs { public final String dns; + public final String hash; public final List data; public final List symlinks; + public final String syncDir; + public final Map> waitForFilesOfTask; - public Inputs( String dns ) { + public Inputs( String dns, String syncDir, String hash ) { this.dns = dns; + this.syncDir = syncDir; + this.hash = hash; this.data = new LinkedList<>(); this.symlinks = new LinkedList<>(); + waitForFilesOfTask = new ConcurrentHashMap<>(); + } + + public void waitForTask( Map waitForTask ){ + for (Map.Entry e : waitForTask.entrySet()) { + final String hash = e.getValue().getConfig().getHash(); + if ( !waitForFilesOfTask.containsValue( hash ) ) waitForFilesOfTask.put( hash, new LinkedList<>() ); + final List listOfPathes = waitForFilesOfTask.get(hash); + listOfPathes.add( e.getKey() ); + } } } diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index f5d05956..1d8968ed 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -14,6 +14,7 @@ def getIP( node ): def clearLocatation( path ): if os.path.exists( path ): + print("Delete",path) if os.path.islink( path ): os.unlink( path ) elif os.path.isdir( path ): @@ -22,7 +23,7 @@ def clearLocatation( path ): os.remove( path ) -def download( node, files ): +def download( node, files, syncFile ): ftp = None size = len(files) @@ -84,6 +85,7 @@ def download( node, files ): continue files.pop(0) + syncFile.write(filename + '\n') if ftp is not None: try: @@ -92,8 +94,43 @@ def download( node, files ): except BaseException as err: print(f"Unexpected {err=}, {type(err)=}") - - +def waitForFiles( syncFileTask, files, starttime ): + #wait max. 10 seconds + while True: + if starttime + 10 < time.time(): + return False + if os.path.isfile( syncFileTask ): + break + print("Wait for file creation") + time.sleep(0.1) + + #Read file live + with open( syncFileTask, 'r' ) as syncFileTask: + current = [] + while len(files) > 0: + data = syncFileTask.read() + if not data: + time.sleep(0.3) + else: + for d in data: + if d != "\n": + current.append(d) + else: + text = ''.join(current) + if text == "##FAILURE##": + print("Read FAILURE in " + syncFileTask) + sys.exit(1001) + if text == "##FINISHED##": + print("Read FINISHED in " + syncFileTask + " before all files were found") + sys.exit(1002) + if text in files: + files.remove( text ) + if len(files) == 0: + return True + current = [] + return True + +starttime = time.time() print("Start to setup the environment") exitIfFileWasNotFound = False @@ -104,21 +141,38 @@ def download( node, files ): print ("Config file not found:", configFilePath ) exit(102) -configFile = open(configFilePath) -config = json.load( configFile ) +with open(configFilePath,'r') as configFile: + config = json.load( configFile ) + print( config ) dns = config["dns"] data = config[ "data" ] symlinks = config[ "symlinks" ] -for d in data: - files = d["files"] - download( d["node"], files ) +os.makedirs( config[ "syncDir" ], exist_ok=True) +with open( config[ "syncDir" ] + config[ "hash" ], 'w' ) as syncFile: + + syncFile.write('##STARTED##\n') + syncFile.flush() + + for s in symlinks: + src = s["src"] + dst = s["dst"] + clearLocatation( src ) + Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) + os.symlink( dst, src ) + + syncFile.write('##SYMLINKS##\n') + + for d in data: + files = d["files"] + download( d["node"], files, syncFile ) + + syncFile.write('##FINISHED##\n') -for s in symlinks: - src = s["src"] - dst = s["dst"] - clearLocatation( src ) - Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) - os.symlink( dst, src ) \ No newline at end of file +#No check for files of other tasks +for waitForTask in config[ "waitForFilesOfTask" ]: + waitForFilesSet = set( config[ "waitForFilesOfTask" ][ waitForTask ] ) + if not waitForFiles( config[ "syncDir" ] + waitForTask, waitForFilesSet, starttime ): + sys.exit(2000) From 56b421e273b3787ded38ec8e382ef435902c1b00 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 4 Apr 2022 17:03:34 +0200 Subject: [PATCH 159/443] PodRequirements to Requirements Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/NodeWithAlloc.java | 18 +++--- .../scheduler/model/PodRequirements.java | 64 ------------------- .../fonda/scheduler/model/PodWithAge.java | 6 +- .../fonda/scheduler/model/Requirements.java | 64 +++++++++++++++++++ .../scheduler/scheduler/RandomScheduler.java | 2 +- .../fonda/scheduler/scheduler/Scheduler.java | 22 +++---- .../scheduler/SchedulerWithDaemonSet.java | 2 +- 7 files changed, 89 insertions(+), 89 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/model/PodRequirements.java create mode 100644 src/main/java/fonda/scheduler/model/Requirements.java diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index d1399e61..56c3a615 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -17,9 +17,9 @@ public class NodeWithAlloc extends Node implements Comparable { private static final long serialVersionUID = 1L; - private final PodRequirements maxResources; + private final Requirements maxResources; - private final Map assignedPods; + private final Map assignedPods; @Getter private final NodeLocation nodeLocation; @@ -38,7 +38,7 @@ public NodeWithAlloc(Node node) { BigDecimal maxCpu = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("cpu")); BigDecimal maxRam = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("memory")); - maxResources = new PodRequirements( maxCpu, maxRam); + maxResources = new Requirements( maxCpu, maxRam); assignedPods = new HashMap<>(); @@ -48,7 +48,7 @@ public NodeWithAlloc(Node node) { } public void addPod( PodWithAge pod ){ - PodRequirements request = pod.getRequest(); + Requirements request = pod.getRequest(); synchronized (assignedPods) { assignedPods.put( pod.getMetadata().getUid(), request ); } @@ -60,20 +60,20 @@ public void removePod( Pod pod ){ } } - public PodRequirements getRequestedResources(){ + public Requirements getRequestedResources(){ synchronized (assignedPods) { return assignedPods.values().stream() - .reduce( new PodRequirements(), PodRequirements::addToThis ); + .reduce( new Requirements(), Requirements::addToThis ); } } - public PodRequirements getAvailableResources(){ + public Requirements getAvailableResources(){ return maxResources.sub(getRequestedResources()); } public boolean canSchedule( PodWithAge pod ){ - final PodRequirements request = pod.getRequest(); - PodRequirements availableResources = getAvailableResources(); + final Requirements request = pod.getRequest(); + Requirements availableResources = getAvailableResources(); return request.getCpu().compareTo(availableResources.getCpu()) <= 0 && request.getRam().compareTo(availableResources.getRam()) <= 0; } diff --git a/src/main/java/fonda/scheduler/model/PodRequirements.java b/src/main/java/fonda/scheduler/model/PodRequirements.java deleted file mode 100644 index 280fca68..00000000 --- a/src/main/java/fonda/scheduler/model/PodRequirements.java +++ /dev/null @@ -1,64 +0,0 @@ -package fonda.scheduler.model; - -import lombok.Getter; -import lombok.ToString; - -import java.io.Serializable; -import java.math.BigDecimal; - -@ToString -public class PodRequirements implements Serializable { - - private static final long serialVersionUID = 1L; - - public static final PodRequirements ZERO = new PodRequirements(); - - @Getter - private BigDecimal cpu; - @Getter - private BigDecimal ram; - - public PodRequirements( BigDecimal cpu, BigDecimal ram ) { - this.cpu = cpu == null ? BigDecimal.ZERO : cpu; - this.ram = ram == null ? BigDecimal.ZERO : ram; - } - - public PodRequirements(){ - this( BigDecimal.ZERO, BigDecimal.ZERO ); - } - - public PodRequirements addToThis( PodRequirements podRequirements ){ - this.cpu = this.cpu.add(podRequirements.cpu); - this.ram = this.ram.add(podRequirements.ram); - return this; - } - - public PodRequirements addRAMtoThis( BigDecimal ram ){ - this.ram = this.ram.add( ram ); - return this; - } - - public PodRequirements addCPUtoThis( BigDecimal cpu ){ - this.cpu = this.cpu.add( cpu ); - return this; - } - - public PodRequirements subFromThis( PodRequirements podRequirements ){ - this.cpu = this.cpu.subtract(podRequirements.getCpu()); - this.ram = this.ram.subtract(podRequirements.getRam()); - return this; - } - - public PodRequirements sub( PodRequirements podRequirements ){ - return new PodRequirements( - this.cpu.subtract(podRequirements.getCpu()), - this.ram.subtract(podRequirements.getRam()) - ); - } - - public boolean higherOrEquals( PodRequirements podRequirements ){ - return this.cpu.compareTo( podRequirements.cpu ) >= 0 - && this.ram.compareTo( podRequirements.ram ) >= 0; - } - -} diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index 3ba6e6d2..bc68dcea 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -25,17 +25,17 @@ public PodWithAge(Pod pod) { this.age = BigDecimal.ZERO; } - public PodRequirements getRequest(){ + public Requirements getRequest(){ return this .getSpec().getContainers().stream() .filter( x -> x.getResources() != null && x.getResources().getRequests() != null ) .map( x -> - new PodRequirements( + new Requirements( x.getResources().getRequests().get("cpu") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("cpu")), x.getResources().getRequests().get("memory") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("memory")) ) - ).reduce( new PodRequirements(), PodRequirements::addToThis ); + ).reduce( new Requirements(), Requirements::addToThis ); } public String getName(){ diff --git a/src/main/java/fonda/scheduler/model/Requirements.java b/src/main/java/fonda/scheduler/model/Requirements.java new file mode 100644 index 00000000..3e6d95fc --- /dev/null +++ b/src/main/java/fonda/scheduler/model/Requirements.java @@ -0,0 +1,64 @@ +package fonda.scheduler.model; + +import lombok.Getter; +import lombok.ToString; + +import java.io.Serializable; +import java.math.BigDecimal; + +@ToString +public class Requirements implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final Requirements ZERO = new Requirements(); + + @Getter + private BigDecimal cpu; + @Getter + private BigDecimal ram; + + public Requirements( BigDecimal cpu, BigDecimal ram ) { + this.cpu = cpu == null ? BigDecimal.ZERO : cpu; + this.ram = ram == null ? BigDecimal.ZERO : ram; + } + + public Requirements(){ + this( BigDecimal.ZERO, BigDecimal.ZERO ); + } + + public Requirements addToThis( Requirements requirements ){ + this.cpu = this.cpu.add(requirements.cpu); + this.ram = this.ram.add(requirements.ram); + return this; + } + + public Requirements addRAMtoThis( BigDecimal ram ){ + this.ram = this.ram.add( ram ); + return this; + } + + public Requirements addCPUtoThis( BigDecimal cpu ){ + this.cpu = this.cpu.add( cpu ); + return this; + } + + public Requirements subFromThis( Requirements requirements ){ + this.cpu = this.cpu.subtract(requirements.getCpu()); + this.ram = this.ram.subtract(requirements.getRam()); + return this; + } + + public Requirements sub( Requirements requirements ){ + return new Requirements( + this.cpu.subtract(requirements.getCpu()), + this.ram.subtract(requirements.getRam()) + ); + } + + public boolean higherOrEquals( Requirements requirements ){ + return this.cpu.compareTo( requirements.cpu ) >= 0 + && this.ram.compareTo( requirements.ram ) >= 0; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index a77d65cb..029a8a64 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -39,7 +39,7 @@ private Optional selectNode( Set matchingNodes, Ta @Override public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks, - final Map availableByNode + final Map availableByNode ){ List alignment = new LinkedList<>(); diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 37148b17..193c6fd6 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -115,23 +115,23 @@ void undoTaskScheduling( Task task ){} public boolean validSchedulePlan( List taskNodeAlignment ){ List items = getNodeList(); - Map< NodeWithAlloc,PodRequirements> availableByNode = new HashMap<>(); + Map< NodeWithAlloc, Requirements> availableByNode = new HashMap<>(); for ( NodeWithAlloc item : items ) { - final PodRequirements availableResources = item.getAvailableResources(); + final Requirements availableResources = item.getAvailableResources(); availableByNode.put(item, availableResources); } for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { availableByNode.get(nodeTaskAlignment.node).subFromThis(nodeTaskAlignment.task.getPod().getRequest()); } - for ( Map.Entry e : availableByNode.entrySet() ) { - if ( ! e.getValue().higherOrEquals( PodRequirements.ZERO ) ) return false; + for ( Map.Entry e : availableByNode.entrySet() ) { + if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) return false; } return true; } abstract ScheduleObject getTaskNodeAlignment( final List unscheduledTasks, - final Map availableByNode + final Map availableByNode ); abstract int terminateTasks( final List finishedTasks ); @@ -265,7 +265,7 @@ public TaskState getTaskState(String taskid) { * @param node * @return */ - boolean canSchedulePodOnNode( PodRequirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { return availableByNode.higherOrEquals( pod.getRequest() ) && affinitiesMatch( pod, node ); } @@ -398,12 +398,12 @@ Task getTaskByPod( Pod pod ) { return t; } - Map getAvailableByNode(){ - Map availableByNode = new HashMap<>(); + Map getAvailableByNode(){ + Map availableByNode = new HashMap<>(); List logInfo = new LinkedList<>(); logInfo.add("------------------------------------"); for (NodeWithAlloc item : getNodeList()) { - final PodRequirements availableResources = item.getAvailableResources(); + final Requirements availableResources = item.getAvailableResources(); availableByNode.put(item, availableResources); logInfo.add("Node: " + item.getName() + " " + availableResources); } @@ -418,9 +418,9 @@ Map getAvailableByNode(){ * @param task * @return */ - Set getMatchingNodesForTask( Map availableByNode, Task task ){ + Set getMatchingNodesForTask(Map availableByNode, Task task ){ Set result = new HashSet<>(); - for (Map.Entry entry : availableByNode.entrySet()) { + for (Map.Entry entry : availableByNode.entrySet()) { if ( this.canSchedulePodOnNode( entry.getValue(), task.getPod(), entry.getKey() ) ){ result.add( entry.getKey() ); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index d957664a..5467c1b3 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -276,7 +276,7 @@ private void podWasInitialized( Pod pod ){ } @Override - boolean canSchedulePodOnNode( PodRequirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { return this.getDaemonOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); } From 8461a37f57b6807f63979cfe74c6a20b118177a7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 4 Apr 2022 17:24:38 +0200 Subject: [PATCH 160/443] Renamed method Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/RandomScheduler.java | 2 +- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 029a8a64..6a4cfc22 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -57,7 +57,7 @@ public ScheduleObject getTaskNodeAlignment( continue; } - filterMatchingNodesForTask( matchingNodes, inputsOfTask ); + filterNotMatchingNodesForTask( matchingNodes, inputsOfTask ); if( matchingNodes.isEmpty() ) { log.info( "No node which fulfills all requirements {}", pod.getName() ); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 5467c1b3..b8d879f6 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -285,7 +285,7 @@ boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeW * @param matchingNodes * @param taskInputs */ - void filterMatchingNodesForTask( Set matchingNodes, TaskInputs taskInputs ){ + void filterNotMatchingNodesForTask(Set matchingNodes, TaskInputs taskInputs ){ final Iterator iterator = matchingNodes.iterator(); final Set excludedNodes = taskInputs.getExcludedNodes(); while ( iterator.hasNext() ){ From bfb374e4d516e2bf3824344c41e392ed67dd7438 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 5 Apr 2022 13:18:33 +0200 Subject: [PATCH 161/443] More precise logging Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/rest/SchedulerRestController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 37e2f8f6..275eb344 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -179,7 +179,7 @@ ResponseEntity delete(@PathVariable String namespace, @PathVariable Str @GetMapping("/daemon/{namespace}/{execution}/{node}") ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable String execution, @PathVariable String node ) { - log.info( "Got request: {}{}{}", namespace, execution, node ); + log.info( "Asking for Daemon ns: {} exec: {} node: {}", namespace, execution, node ); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); From 1b04712157e6754a3f79546e311aad6bd725bda5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 5 Apr 2022 13:34:54 +0200 Subject: [PATCH 162/443] Deal with init failure Signed-off-by: Lehmann_Fabian --- pom.xml | 6 ++ .../model/TaskInputFileLocationWrapper.java | 10 +- .../location/hierachy/RealHierarchyFile.java | 11 +++ .../scheduler/SchedulerWithDaemonSet.java | 71 ++++++++++++- src/main/resources/copystrategies/ftp.py | 99 ++++++++++++------- 5 files changed, 158 insertions(+), 39 deletions(-) diff --git a/pom.xml b/pom.xml index 84590e13..56eea8ec 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,12 @@ test + + commons-net + commons-net + 3.6 + + diff --git a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java index c445eb3a..dc027654 100644 --- a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java @@ -7,16 +7,22 @@ @Getter public class TaskInputFileLocationWrapper { + private final String path; private final RealHierarchyFile file; private final LocationWrapper wrapper; - public TaskInputFileLocationWrapper(RealHierarchyFile file, LocationWrapper wrapper) { + public TaskInputFileLocationWrapper( String path, RealHierarchyFile file, LocationWrapper wrapper) { + this.path = path; this.file = file; this.wrapper = wrapper; } - public void apply(){ + public void success(){ file.addOrUpdateLocation( false, wrapper ); } + public void failure(){ + file.removeLocation( wrapper ); + } + } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index 4631e6d2..3e6c206d 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -31,6 +31,17 @@ public boolean isSymlink() { return false; } + public void removeLocation( LocationWrapper location ){ + if ( location == null ) throw new IllegalArgumentException( "location is null" ); + synchronized ( this ){ + for (int i = 0; i < locations.length; i++) { + if ( location.getLocation().equals( locations[i].getLocation() ) ) { + locations[i].deactivate(); + } + } + } + } + public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ if ( location == null ) throw new IllegalArgumentException( "location is null" ); synchronized ( this ){ diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index b8d879f6..fc9beebc 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -28,9 +28,9 @@ import io.fabric8.kubernetes.client.Watcher; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.net.ftp.FTPClient; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -199,6 +199,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final LocationWrapper locationWrapper = filePath.file.getLocationWrapper(location); inputFiles.add( new TaskInputFileLocationWrapper( + filePath.path, filePath.file, locationWrapper.getCopyOf(location) ) @@ -267,11 +268,73 @@ public void addFile( String path, long size, long timestamp, long locationWrappe hierarchyWrapper.addFile( Paths.get( path ), overwrite, locationWrapper ); } + private void handleProblematicInit( Task task ){ + String file = this.localWorkDir + "/sync/" + task.getConfig().getHash(); + try { + Map wrapperByPath = new HashMap<>(); + task.getCopiedFiles().stream().forEach( x -> wrapperByPath.put( x.getPath(), x )); + log.info( "Get daemon on node {}; daemons: {}", task.getNode().getIdentifier(), daemonByNode ); + final InputStream inputStream = getConnection(getDaemonOnNode(task.getNode().getIdentifier())).retrieveFileStream(file); + if (inputStream == null) { + //Init has not even started + return; + } + Scanner scanner = new Scanner(inputStream); + Set openedFiles = new HashSet(); + while( scanner.hasNext() ){ + String line = scanner.nextLine(); + if ( line.startsWith( "S-" ) ){ + openedFiles.add( line.substring( 2 ) ); + } else if ( line.startsWith( "F-" ) ){ + openedFiles.remove( line.substring( 2 ) ); + wrapperByPath.get( line.substring( 2 ) ).success(); + log.info("task {}, file: {} success", task.getConfig().getName(), line); + } + } + for ( String openedFile : openedFiles ) { + wrapperByPath.get( openedFile ).failure(); + log.info("task {}, file: {} success", task.getConfig().getName(), openedFile); + } + } catch ( Exception e ){ + log.info( "Can't handle failed init from pod " + task.getPod().getName()); + e.printStackTrace(); + } + } + + private FTPClient getConnection( String daemon ){ + int trial = 0; + while ( true ) { + try { + FTPClient f = new FTPClient(); + f.connect(daemon); + f.login("ftp", "nextflowClient"); + f.enterLocalPassiveMode(); + return f; + } catch ( IOException e ) { + if ( trial > 5 ) throw new RuntimeException(e); + log.error("Cannot create FTP client: {}", daemon); + try { + Thread.sleep((long) Math.pow(2, trial++)); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + } + } + private void podWasInitialized( Pod pod ){ final Task task = changeStateOfTask(pod, State.PREPARED); - task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::apply ); - task.setCopiedFiles( null ); + task.setPod( new PodWithAge( pod ) ); + final Integer exitCode = pod.getStatus().getInitContainerStatuses().get(0).getState().getTerminated().getExitCode(); + log.info( "Pod {}, Init Code: {}", pod.getMetadata().getName(), exitCode); removeFromCopyingToNode( task.getNode(), task.getCopyingToNode() ); + if( exitCode == 0 ){ + task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success); + } else { + handleProblematicInit( task ); + task.setInputFiles( null ); + } + task.setCopiedFiles( null ); task.setCopyingToNode( null ); } diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 1d8968ed..fbec66a9 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -4,9 +4,28 @@ import os import json import urllib.request -import sys from pathlib import Path import shutil +import signal +import logging as log + +CLOSE = False +EXIT = 0 +log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) + +def myExit( code ): + global EXIT + EXIT = code + exit(EXIT) + +def close( signalnum, syncfile ): + syncFile.write('##FAILURE##\n') + syncFile.flush() + syncFile.close() + global CLOSE + CLOSE = True + log.info( "Killed: %s",str(signalnum)) + myExit(50) def getIP( node ): ip = urllib.request.urlopen(dns + node).read() @@ -14,7 +33,7 @@ def getIP( node ): def clearLocatation( path ): if os.path.exists( path ): - print("Delete",path) + log.debug( "Delete",path ) if os.path.islink( path ): os.unlink( path ) elif os.path.isdir( path ): @@ -22,77 +41,77 @@ def clearLocatation( path ): else: os.remove( path ) - def download( node, files, syncFile ): ftp = None size = len(files) connectionProblem = 0 + global CLOSE - while len(files) > 0: + while not CLOSE and len(files) > 0: if connectionProblem > 1: ftp = None if connectionProblem == 8: - exit(8) - print( "Connection refused! Try again..." ) + myExit( 8 ) + log.warning( "Connection refused! Try again..." ) time.sleep(2**connectionProblem) - if ftp is None: try: connectionProblem += 1 ip = getIP( node ) - print( "Try to connect to", ip ) + log.info( "Try to connect to %s", ip ) ftp = ftplib.FTP( ip ) ftp.login("ftp", "pythonclient") ftp.set_pasv(True) ftp.encoding='utf-8' - print( "Connection established" ) + log.info( "Connection established" ) connectionProblem = 0 except ConnectionRefusedError as err: continue except BaseException as err: - print(f"Unexpected {err=}, {type(err)=}") + log.exception( "Unexpected error" ) continue filename = files[0] index = size - len(files) + 1 - print("Download", "[" + str( index ).rjust( len( str( size ) ) ) + "/" + str( size ) + "]" , filename) + log.info( "Download [%s/%s] - %s", str( index ).rjust( len( str( size ) ) ), str( size ), filename ) try: + syncFile.write("S-" + filename + '\n') clearLocatation( filename ) Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) except ftplib.error_perm as err: if( str(err) == "550 Failed to open file." ): - print( "File not found:", node + filename ) + log.warning( "File not found node: %s file: %s", node, filename ) if exitIfFileWasNotFound: - exit(550) + myExit( 40 ) except FileNotFoundError as err: - print( "File not found:", node + filename ) + log.warning( "File not found node: %s file: %s", node, filename ) if exitIfFileWasNotFound: - exit(404) + myExit( 41 ) except EOFError as err: - print( "It seems the connection was lost! Try again..." ) + log.warning( "It seems the connection was lost! Try again..." ) ftp = None continue except BaseException as err: - print(f"Unexpected {err=}, {type(err)=}") + log.exception( "Unexpected error" ) ftp = None continue files.pop(0) - syncFile.write(filename + '\n') + syncFile.write("F-" + filename + '\n') if ftp is not None: try: ftp.quit() ftp.close() except BaseException as err: - print(f"Unexpected {err=}, {type(err)=}") + log.exception( "Unexpected error" ) def waitForFiles( syncFileTask, files, starttime ): #wait max. 10 seconds @@ -101,7 +120,7 @@ def waitForFiles( syncFileTask, files, starttime ): return False if os.path.isfile( syncFileTask ): break - print("Wait for file creation") + log.debug( "Wait for file creation" ) time.sleep(0.1) #Read file live @@ -117,34 +136,37 @@ def waitForFiles( syncFileTask, files, starttime ): current.append(d) else: text = ''.join(current) + current = [] + if text.startswith( "S-" ): + continue if text == "##FAILURE##": - print("Read FAILURE in " + syncFileTask) - sys.exit(1001) + log.debug( "Read FAILURE in %s", syncFileTask ) + myExit( 51 ) if text == "##FINISHED##": - print("Read FINISHED in " + syncFileTask + " before all files were found") - sys.exit(1002) - if text in files: - files.remove( text ) + log.debug( "Read FINISHED in " + syncFileTask + " before all files were found" ) + myExit( 52 ) + log.debug( "Look for " + text[:2] + " with " + text[2:] + " in " + str(files) ) + if text[:2] == "F-" and text[2:] in files: + files.remove( text[2:] ) if len(files) == 0: return True - current = [] - return True + return len(files) == 0 starttime = time.time() -print("Start to setup the environment") +log.info( "Start to setup the environment" ) exitIfFileWasNotFound = False configFilePath = ".command.inputs.json" if not os.path.isfile( configFilePath ): - print ("Config file not found:", configFilePath ) - exit(102) + log.error( "Config file not found:", configFilePath ) + myExit( 102 ) with open(configFilePath,'r') as configFile: config = json.load( configFile ) -print( config ) +log.info( str(config) ) dns = config["dns"] data = config[ "data" ] @@ -153,6 +175,9 @@ def waitForFiles( syncFileTask, files, starttime ): os.makedirs( config[ "syncDir" ], exist_ok=True) with open( config[ "syncDir" ] + config[ "hash" ], 'w' ) as syncFile: + signal.signal( signal.SIGINT, lambda signalnum, handler: close( signalnum, syncFile ) ) + signal.signal( signal.SIGTERM, lambda signalnum, handler: close( signalnum, syncFile ) ) + syncFile.write('##STARTED##\n') syncFile.flush() @@ -169,10 +194,18 @@ def waitForFiles( syncFileTask, files, starttime ): files = d["files"] download( d["node"], files, syncFile ) + if CLOSE: + log.debug( "Close with code %s", str(EXIT) ) + exit( EXIT ) + syncFile.write('##FINISHED##\n') + signal.signal( signal.SIGINT, lambda signalnum, handler: myExit( 1 ) ) + signal.signal( signal.SIGTERM, lambda signalnum, handler: myExit( 1 ) ) + #No check for files of other tasks for waitForTask in config[ "waitForFilesOfTask" ]: waitForFilesSet = set( config[ "waitForFilesOfTask" ][ waitForTask ] ) if not waitForFiles( config[ "syncDir" ] + waitForTask, waitForFilesSet, starttime ): - sys.exit(2000) + log.error( config[ "syncDir" ] + waitForTask + " was not successful" ) + myExit( 200 ) From 9cd8f47ec3fe26cb83d7687735a98ae206e2343e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 11 Apr 2022 13:32:48 +0200 Subject: [PATCH 163/443] Set the right nodelocation Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index fc9beebc..05191aa3 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -201,7 +201,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { new TaskInputFileLocationWrapper( filePath.path, filePath.file, - locationWrapper.getCopyOf(location) + locationWrapper.getCopyOf( currentNode ) ) ); collect.add(filePath.path ); From 277893dae66afb63a16d650cb7be32be7f6043c2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 11 Apr 2022 13:43:50 +0200 Subject: [PATCH 164/443] Throw exception if no alignment can be found Signed-off-by: Lehmann_Fabian --- .../scheduler/model/InputFileCollector.java | 19 +++++++++++++------ .../hierachy/NoAlignmentFoundException.java | 4 ++++ .../location/hierachy/RealHierarchyFile.java | 7 +++++-- .../scheduler/scheduler/RandomScheduler.java | 8 +++++++- .../scheduler/SchedulerWithDaemonSet.java | 2 +- .../model/InputFileCollectorTest.java | 11 ++++------- .../hierachy/RealHierarchyFileTest.java | 8 ++++---- 7 files changed, 38 insertions(+), 21 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java diff --git a/src/main/java/fonda/scheduler/model/InputFileCollector.java b/src/main/java/fonda/scheduler/model/InputFileCollector.java index 8021ae42..41bca3b8 100644 --- a/src/main/java/fonda/scheduler/model/InputFileCollector.java +++ b/src/main/java/fonda/scheduler/model/InputFileCollector.java @@ -6,10 +6,12 @@ import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.Tuple; +import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; import java.util.*; +@Slf4j public class InputFileCollector { private final HierarchyWrapper hierarchyWrapper; @@ -24,7 +26,7 @@ private void processNext( final List files, final Set excludedLocations, final Task task - ){ + ) throws NoAlignmentFoundException { final Tuple tuple = toProcess.removeLast(); final HierarchyFile file = tuple.getA(); if( file == null ) return; @@ -40,15 +42,20 @@ private void processNext( } } else { final RealHierarchyFile realFile = (RealHierarchyFile) file; - final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); - if ( filesForTask.getExcludedNodes() != null ) { - excludedLocations.addAll(filesForTask.getExcludedNodes()); + try { + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); + if ( filesForTask.getExcludedNodes() != null ) { + excludedLocations.addAll(filesForTask.getExcludedNodes()); + } + files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); + } catch ( NoAlignmentFoundException e ){ + log.error( "No alignment for task: " + task.getConfig().getName() + " path: " + path, e ); + throw e; } - files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); } } - public TaskInputs getInputsOfTask( Task task, int numberNode ){ + public TaskInputs getInputsOfTask( Task task, int numberNode ) throws NoAlignmentFoundException { final List> fileInputs = task.getConfig().getInputs().fileInputs; final LinkedList> toProcess = filterFilesToProcess( fileInputs ); diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java b/src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java new file mode 100644 index 00000000..77e5b1b3 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java @@ -0,0 +1,4 @@ +package fonda.scheduler.model.location.hierachy; + +public class NoAlignmentFoundException extends Exception { +} diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index 3e6c206d..e4a94871 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -5,9 +5,11 @@ import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.LocationType; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.util.*; +@Slf4j public class RealHierarchyFile extends AbstractHierarchyFile { /** @@ -147,7 +149,7 @@ private LinkedList addAndCreateList(LinkedList return list; } - public MatchingLocationsPair getFilesForTask( Task task ){ + public MatchingLocationsPair getFilesForTask( Task task ) throws NoAlignmentFoundException { LocationWrapper[] locationsRef = this.locations; LinkedList current = null; @@ -202,13 +204,14 @@ else if ( taskDescendants.contains(locationProcess) ) ? combineResultsEmptyInitial( current, ancestors, descendants, unrelated ) : combineResultsWithInitial( current, ancestors, descendants, unrelated, initial ); + if ( matchingLocations == null ) throw new NoAlignmentFoundException(); removeMatchingLocations( matchingLocations, inUse ); return new MatchingLocationsPair( matchingLocations, inUse ); } private void removeMatchingLocations( List matchingLocations, Set locations ){ - if ( locations == null ) return; + if ( locations == null || matchingLocations == null ) return; for ( LocationWrapper matchingLocation : matchingLocations ) { if( matchingLocation.isInUse() ) { locations.remove(matchingLocation.getLocation()); diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 6a4cfc22..706d0272 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -3,6 +3,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.NoAlignmentFoundException; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.util.FileAlignment; @@ -50,7 +51,12 @@ public ScheduleObject getTaskNodeAlignment( final Set matchingNodes = getMatchingNodesForTask(availableByNode,task); if( !matchingNodes.isEmpty() ) { - final TaskInputs inputsOfTask = getInputsOfTask(task); + final TaskInputs inputsOfTask; + try { + inputsOfTask = getInputsOfTask(task); + } catch (NoAlignmentFoundException e) { + continue; + } if( inputsOfTask == null ) { log.info( "No node where the pod can start, pod: {}", pod.getName() ); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 05191aa3..2fc363e2 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -223,7 +223,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { return null; } - TaskInputs getInputsOfTask(Task task ){ + TaskInputs getInputsOfTask(Task task ) throws NoAlignmentFoundException { return inputFileCollector.getInputsOfTask( task, client.getNumberOfNodes() ); } diff --git a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java index 5d615d4f..5065e7bd 100644 --- a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java @@ -5,10 +5,7 @@ import fonda.scheduler.dag.Process; import fonda.scheduler.dag.Vertex; import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.HierarchyFile; -import fonda.scheduler.model.location.hierachy.HierarchyWrapper; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealHierarchyFile; +import fonda.scheduler.model.location.hierachy.*; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.model.taskinputs.TaskInputs; @@ -50,7 +47,7 @@ public void init(){ } @Test - public void getInputsOfTaskTest() { + public void getInputsOfTaskTest() throws NoAlignmentFoundException { final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; final String root2 = "/workdir/01/db62d739d658b839f07a1a77d877d1/"; @@ -134,7 +131,7 @@ public void getInputsOfTaskTest() { } @Test - public void getInputsOfTaskTestExcludeNodes() { + public void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; @@ -185,7 +182,7 @@ public void getInputsOfTaskTestExcludeNodes() { } @Test - public void getInputsOfTaskTestNotExcludeNodes() { + public void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundException { final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java index 6765e094..639e8d3b 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -181,7 +181,7 @@ public void isFile() { @Test - public void getFilesForTaskTest() throws InterruptedException { + public void getFilesForTaskTest() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); @@ -232,7 +232,7 @@ public void getFilesForTaskTest() throws InterruptedException { } @Test - public void getFilesForTaskTestInitFiles() throws InterruptedException { + public void getFilesForTaskTestInitFiles() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); @@ -267,7 +267,7 @@ public void getFilesForTaskTestInitFiles() throws InterruptedException { } @Test - public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException { + public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); @@ -302,7 +302,7 @@ public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException { } @Test - public void getFilesForTaskTestDifferentAncestors() throws InterruptedException { + public void getFilesForTaskTestDifferentAncestors() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); From 4b49c7fb336bcd93bbd46142df3097bdc856e4b9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 11 Apr 2022 13:45:56 +0200 Subject: [PATCH 165/443] Deal with failed initPod Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/State.java | 2 +- .../model/location/hierachy/LocationWrapper.java | 1 + .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/State.java b/src/main/java/fonda/scheduler/model/State.java index fe7db5ea..7e92ed77 100644 --- a/src/main/java/fonda/scheduler/model/State.java +++ b/src/main/java/fonda/scheduler/model/State.java @@ -2,6 +2,6 @@ public enum State { - RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, PREPARED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, DELETED + RECEIVED_CONFIG, UNSCHEDULED, SCHEDULED, PREPARED, ERROR, PROCESSING_FINISHED, FINISHED, FINISHED_WITH_ERROR, INIT_WITH_ERRORS, DELETED } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index fc0f3324..887990c5 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -106,6 +106,7 @@ public String toString() { synchronized ( this ) { return "LocationWrapper{" + "id=" + id + + ", active=" + active + ", location=" + location.getIdentifier() + ", timestamp=" + timestamp + ", inUse=" + inUse + "x" + diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 2fc363e2..1874e8d8 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -293,7 +293,7 @@ private void handleProblematicInit( Task task ){ } for ( String openedFile : openedFiles ) { wrapperByPath.get( openedFile ).failure(); - log.info("task {}, file: {} success", task.getConfig().getName(), openedFile); + log.info("task {}, file: {} deactivated on node {}", task.getConfig().getName(), openedFile, wrapperByPath.get( openedFile ).getWrapper().getLocation()); } } catch ( Exception e ){ log.info( "Can't handle failed init from pod " + task.getPod().getName()); @@ -323,13 +323,13 @@ private FTPClient getConnection( String daemon ){ } private void podWasInitialized( Pod pod ){ - final Task task = changeStateOfTask(pod, State.PREPARED); - task.setPod( new PodWithAge( pod ) ); final Integer exitCode = pod.getStatus().getInitContainerStatuses().get(0).getState().getTerminated().getExitCode(); + final Task task = changeStateOfTask(pod, exitCode == 0 ? State.PREPARED : State.INIT_WITH_ERRORS); + task.setPod( new PodWithAge( pod ) ); log.info( "Pod {}, Init Code: {}", pod.getMetadata().getName(), exitCode); removeFromCopyingToNode( task.getNode(), task.getCopyingToNode() ); if( exitCode == 0 ){ - task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success); + task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); } else { handleProblematicInit( task ); task.setInputFiles( null ); From 67b9f6de8d14cbfd7c3aaaf33443dd747e7b5f6b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 11 Apr 2022 13:46:22 +0200 Subject: [PATCH 166/443] Exit correctly Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index fbec66a9..3cbb9499 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -16,14 +16,14 @@ def myExit( code ): global EXIT EXIT = code + global CLOSE + CLOSE = True exit(EXIT) def close( signalnum, syncfile ): syncFile.write('##FAILURE##\n') syncFile.flush() syncFile.close() - global CLOSE - CLOSE = True log.info( "Killed: %s",str(signalnum)) myExit(50) From 9c5d5b736e1e29689f2f2b2c4d52efcde19d89a3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 12 Apr 2022 12:38:05 +0200 Subject: [PATCH 167/443] Release resources of failed pods Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 9 ++++---- .../fonda/scheduler/model/PodWithAge.java | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index c3f70564..6fc1f4e5 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -141,12 +141,13 @@ public void eventReceived(Action action, Pod pod) { NodeWithAlloc node = kubernetesClient.nodeHolder.get( pod.getSpec().getNodeName() ); switch ( action ){ case ADDED: - node.addPod( new PodWithAge(pod) ); break; + if ( !PodWithAge.hasFinishedOrFailed( pod ) ) { + node.addPod(new PodWithAge(pod)); + } + break; case MODIFIED: final List containerStatuses = pod.getStatus().getContainerStatuses(); - if ( containerStatuses.isEmpty() - || - containerStatuses.get(0).getState().getTerminated() == null ) { + if ( !PodWithAge.hasFinishedOrFailed( pod ) ) { break; } //Pod is finished diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index bc68dcea..2b7bacbe 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -5,6 +5,7 @@ import lombok.Setter; import java.math.BigDecimal; +import java.util.List; public class PodWithAge extends Pod { @@ -38,6 +39,10 @@ public Requirements getRequest(){ ).reduce( new Requirements(), Requirements::addToThis ); } + public boolean hasFinishedOrFailed(){ + return hasFinishedOrFailed( this ); + } + public String getName(){ return this.getMetadata().getName(); } @@ -59,4 +64,21 @@ public int hashCode() { result = 31 * result + (getAge() != null ? getAge().hashCode() : 0); return result; } + + public static boolean hasFinishedOrFailed( Pod pod ){ + //init Failure + final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); + for ( ContainerStatus containerStatus : initContainerStatuses ) { + final ContainerStateTerminated terminated = containerStatus.getState().getTerminated(); + if ( terminated != null && terminated.getExitCode() != 0 ){ + return true; + } + } + final List containerStatuses = pod.getStatus().getInitContainerStatuses(); + if ( containerStatuses.isEmpty() ) return false; + for ( ContainerStatus containerStatus : containerStatuses ) { + if ( containerStatus.getState().getTerminated() == null ) return false; + } + return true; + } } From 61de124ab81ed56e621ef51ea19cff9116df18a2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 20 Apr 2022 09:57:56 +0200 Subject: [PATCH 168/443] Warning if the whole scheduling plan cannot be fulfilled. Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 193c6fd6..a624f843 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -86,7 +86,10 @@ public int schedule( final List unscheduledTasks ) { //check if still possible... if ( scheduleObject.isCheckStillPossible() ) { boolean possible = validSchedulePlan ( taskNodeAlignment ); - if (!possible) return taskNodeAlignment.size(); + if (!possible) { + log.info("The whole scheduling plan is not possible anymore."); + return taskNodeAlignment.size(); + } } int failure = 0; From 5ae4f14dccdaed34dcce9a977b6c1ed59a1bc52a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 25 Apr 2022 15:13:49 +0200 Subject: [PATCH 169/443] Determine podFinishedOrFailed via Phase Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/PodWithAge.java | 16 ++-------- .../java/fonda/scheduler/util/PodPhase.java | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 src/main/java/fonda/scheduler/util/PodPhase.java diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index 2b7bacbe..39a1f079 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -1,5 +1,6 @@ package fonda.scheduler.model; +import fonda.scheduler.util.PodPhase; import io.fabric8.kubernetes.api.model.*; import lombok.Getter; import lombok.Setter; @@ -66,19 +67,6 @@ public int hashCode() { } public static boolean hasFinishedOrFailed( Pod pod ){ - //init Failure - final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); - for ( ContainerStatus containerStatus : initContainerStatuses ) { - final ContainerStateTerminated terminated = containerStatus.getState().getTerminated(); - if ( terminated != null && terminated.getExitCode() != 0 ){ - return true; - } - } - final List containerStatuses = pod.getStatus().getInitContainerStatuses(); - if ( containerStatuses.isEmpty() ) return false; - for ( ContainerStatus containerStatus : containerStatuses ) { - if ( containerStatus.getState().getTerminated() == null ) return false; - } - return true; + return PodPhase.get( pod.getStatus().getPhase() ).isFinished(); } } diff --git a/src/main/java/fonda/scheduler/util/PodPhase.java b/src/main/java/fonda/scheduler/util/PodPhase.java new file mode 100644 index 00000000..59f2d1c9 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/PodPhase.java @@ -0,0 +1,32 @@ +package fonda.scheduler.util; + +public enum PodPhase { + + PENDING( false ), + RUNNING( false ), + SUCCEEDED( true ), + FAILED( true ), + UNKNOWN( false ); + + private final boolean finished; + + PodPhase(boolean finished ){ + this.finished = finished; + } + + public boolean isFinished(){ + return finished; + } + + public static PodPhase get(String name){ + switch ( name.toUpperCase() ) { + case "PENDING": return PENDING; + case "RUNNING": return RUNNING; + case "SUCCEEDED": return SUCCEEDED; + case "FAILED": return FAILED; + case "UNKNOWN": return UNKNOWN; + default: throw new IllegalArgumentException( "No enum with name: " + name ); + } + } + +} From 01319bda2a6543ede1e53a61bfd0609fe0ac75fa Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 25 Apr 2022 15:20:01 +0200 Subject: [PATCH 170/443] Requested memory = max( Sum requested by all pods, actually used ) Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 16 +++++++++++++-- .../fonda/scheduler/model/NodeWithAlloc.java | 20 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 6fc1f4e5..866c6f5c 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -6,11 +6,13 @@ import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; import lombok.extern.slf4j.Slf4j; +import java.math.BigDecimal; import java.util.*; @Slf4j @@ -22,7 +24,7 @@ public class KubernetesClient extends DefaultKubernetesClient { public KubernetesClient(){ for( Node node : this.nodes().list().getItems() ){ - nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node) ); + nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node,this) ); } this.pods().inAnyNamespace().watch( new PodWatcher( this ) ); this.nodes().watch( new NodeWatcher( this ) ); @@ -74,6 +76,16 @@ public List getAllNodes(){ return new ArrayList<>(this.nodeHolder.values()); } + public BigDecimal getMemoryOfNode(NodeWithAlloc node ){ + final Quantity memory = this + .top() + .nodes() + .metrics(node.getName()) + .getUsage() + .get("memory"); + return Quantity.getAmountInBytes(memory); + } + static class NodeWatcher implements Watcher{ private final KubernetesClient kubernetesClient; @@ -91,7 +103,7 @@ public void eventReceived(Action action, Node node) { log.info("New Node {} was added", node.getMetadata().getName()); synchronized ( kubernetesClient.nodeHolder ){ if ( ! kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ - processedNode = new NodeWithAlloc(node); + processedNode = new NodeWithAlloc(node,kubernetesClient); kubernetesClient.nodeHolder.put( node.getMetadata().getName(), processedNode ); change = true; } diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 56c3a615..0e6e0f70 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -1,5 +1,6 @@ package fonda.scheduler.model; +import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; @@ -15,6 +16,8 @@ @Slf4j public class NodeWithAlloc extends Node implements Comparable { + private final KubernetesClient kubernetesClient; + private static final long serialVersionUID = 1L; private final Requirements maxResources; @@ -24,7 +27,9 @@ public class NodeWithAlloc extends Node implements Comparable { @Getter private final NodeLocation nodeLocation; - public NodeWithAlloc(Node node) { + public NodeWithAlloc( Node node, KubernetesClient kubernetesClient ) { + + this.kubernetesClient = kubernetesClient; this.setApiVersion( node.getApiVersion() ); this.setKind( node.getKind() ); @@ -60,11 +65,20 @@ public void removePod( Pod pod ){ } } + /** + * @return max(Requested by all and currently used ) + */ public Requirements getRequestedResources(){ + final Requirements requestedByPods; synchronized (assignedPods) { - return assignedPods.values().stream() - .reduce( new Requirements(), Requirements::addToThis ); + requestedByPods = assignedPods.values().stream() + .reduce(new Requirements(), Requirements::addToThis); + } + final BigDecimal currentMemoryOfNode = kubernetesClient.getMemoryOfNode( this ); + if ( requestedByPods.getRam().compareTo(currentMemoryOfNode) == -1 ){ + requestedByPods.addRAMtoThis(currentMemoryOfNode.subtract(requestedByPods.getRam())); } + return requestedByPods; } public Requirements getAvailableResources(){ From d29eadc1ac78bb42653645ff876844ab3b0ed08d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 26 Apr 2022 14:38:16 +0200 Subject: [PATCH 171/443] Renamed RandomScheduler to RandomLAScheduler Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/Main.java | 6 +++--- .../java/fonda/scheduler/rest/SchedulerRestController.java | 4 ++-- .../{RandomScheduler.java => RandomLAScheduler.java} | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename src/main/java/fonda/scheduler/scheduler/{RandomScheduler.java => RandomLAScheduler.java} (97%) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 48e7afc9..585e190f 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -3,7 +3,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.rest.SchedulerRestController; -import fonda.scheduler.scheduler.RandomScheduler; +import fonda.scheduler.scheduler.RandomLAScheduler; import fonda.scheduler.scheduler.filealignment.RandomAlignment; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; @@ -35,9 +35,9 @@ public void afterStart(){ try{ log.info( "Started with namespace: {}", client.getNamespace() ); final SchedulerConfig schedlerConfig = new SchedulerConfig(null, null, "/localwork/", null, "ftp"); - final RandomScheduler randomScheduler = new RandomScheduler("testscheduler", client, "default", schedlerConfig, new RandomAlignment()); + final RandomLAScheduler randomLAScheduler = new RandomLAScheduler("testscheduler", client, "default", schedlerConfig, new RandomAlignment()); final Pair key = new Pair<>("default", "test-run"); - SchedulerRestController.addScheduler( key, randomScheduler ); + SchedulerRestController.addScheduler( key, randomLAScheduler); } catch (Exception e){ e.printStackTrace(); System.exit(1); diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 275eb344..a41efc0b 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -8,7 +8,7 @@ import fonda.scheduler.model.TaskConfig; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; -import fonda.scheduler.scheduler.RandomScheduler; +import fonda.scheduler.scheduler.RandomLAScheduler; import fonda.scheduler.scheduler.Scheduler; import fonda.scheduler.scheduler.SchedulerWithDaemonSet; import fonda.scheduler.scheduler.filealignment.RandomAlignment; @@ -72,7 +72,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : - case "fifo-random" : scheduler = new RandomScheduler(execution, client, namespace, config, new RandomAlignment() ); break; + case "fifo-random" : scheduler = new RandomLAScheduler(execution, client, namespace, config, new RandomAlignment() ); break; default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java similarity index 97% rename from src/main/java/fonda/scheduler/scheduler/RandomScheduler.java rename to src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 706d0272..c5594642 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -15,12 +15,12 @@ import java.util.stream.Collectors; @Slf4j -public class RandomScheduler extends SchedulerWithDaemonSet { +public class RandomLAScheduler extends SchedulerWithDaemonSet { private final InputAlignment inputAlignment; private final Random random = new Random(); - public RandomScheduler( + public RandomLAScheduler( String name, KubernetesClient client, String namespace, From 22be56e272e02df9c2e36f925a45af19f9c405b9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 26 Apr 2022 14:46:24 +0200 Subject: [PATCH 172/443] Submit if locationAware Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/Main.java | 4 ++-- src/main/java/fonda/scheduler/model/SchedulerConfig.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 585e190f..762bbc64 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -34,8 +34,8 @@ public static void main(String[] args) { public void afterStart(){ try{ log.info( "Started with namespace: {}", client.getNamespace() ); - final SchedulerConfig schedlerConfig = new SchedulerConfig(null, null, "/localwork/", null, "ftp"); - final RandomLAScheduler randomLAScheduler = new RandomLAScheduler("testscheduler", client, "default", schedlerConfig, new RandomAlignment()); + final SchedulerConfig schedulerConfig = new SchedulerConfig(null, null, "/localwork/", null, "ftp", true); + final RandomLAScheduler randomLAScheduler = new RandomLAScheduler("testscheduler", client, "default", schedulerConfig, new RandomAlignment()); final Pair key = new Pair<>("default", "test-run"); SchedulerRestController.addScheduler( key, randomLAScheduler); } catch (Exception e){ diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 2c0433c3..e1778d8a 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -13,16 +13,20 @@ public class SchedulerConfig { public final String dns; public final String copyStrategy; + public final boolean locationAware; + public SchedulerConfig(List localClaims, List volumeClaims, String workDir, String dns, - String copyStrategy) { + String copyStrategy, + boolean locationAware) { this.localClaims = localClaims; this.volumeClaims = volumeClaims; this.workDir = workDir; this.dns = dns; this.copyStrategy = copyStrategy; + this.locationAware = locationAware; } private SchedulerConfig(){ @@ -31,6 +35,7 @@ private SchedulerConfig(){ this.workDir = null; this.dns = null; this.copyStrategy = null; + this.locationAware = false; } @ToString From 66ffab236619a646eb31dadfb32e12389172c884 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 26 Apr 2022 14:56:33 +0200 Subject: [PATCH 173/443] Terminate Task not abstact Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index a624f843..8cf1fb1c 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -137,7 +137,12 @@ abstract ScheduleObject getTaskNodeAlignment( final Map availableByNode ); - abstract int terminateTasks( final List finishedTasks ); + int terminateTasks( final List finishedTasks ) { + for (Task finishedTask : finishedTasks) { + taskWasFinished( finishedTask ); + } + return 0; + } /* Pod Event */ From f56c04da2a57b309f5d3b97e341f8db457d67b18 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 26 Apr 2022 16:42:10 +0200 Subject: [PATCH 174/443] Exception if process is not found Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/DAG.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/fonda/scheduler/dag/DAG.java index 3b5e5935..8644e97e 100644 --- a/src/main/java/fonda/scheduler/dag/DAG.java +++ b/src/main/java/fonda/scheduler/dag/DAG.java @@ -16,7 +16,11 @@ private Vertex getByUid( int uid ){ } public Process getByProcess( String s ){ - return processes.get( s ); + final Process process = processes.get(s); + if ( process == null ){ + throw new IllegalStateException("Process can not be found! Searching: " + s + " processes: " + processes ); + } + return process; } public void registerVertices( List vertices ){ From c0dfa8783d4467e443f1d89d4118241693001acd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 26 Apr 2022 16:42:30 +0200 Subject: [PATCH 175/443] Correct logging message Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/client/KubernetesClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 866c6f5c..cd0b2cc0 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -125,7 +125,7 @@ public void eventReceived(Action action, Node node) { //todo deal with error break; case MODIFIED: - log.info("Node {} was an modified", node.getMetadata().getName()); + log.info("Node {} was modified", node.getMetadata().getName()); //todo deal with changed state break; default: log.warn("No implementation for {}", action); From 6768c8750d629714beadf512fc8789a63178ee4e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 26 Apr 2022 16:43:55 +0200 Subject: [PATCH 176/443] Added a simple random scheduler without any location awareness Signed-off-by: Lehmann_Fabian --- .../rest/SchedulerRestController.java | 15 ++++-- .../scheduler/scheduler/RandomScheduler.java | 51 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/RandomScheduler.java diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index a41efc0b..35bff3fe 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -9,6 +9,7 @@ import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.RandomLAScheduler; +import fonda.scheduler.scheduler.RandomScheduler; import fonda.scheduler.scheduler.Scheduler; import fonda.scheduler.scheduler.SchedulerWithDaemonSet; import fonda.scheduler.scheduler.filealignment.RandomAlignment; @@ -45,7 +46,8 @@ public static void addScheduler(Pair key, Scheduler scheduler ){ } private ResponseEntity noSchedulerFor( String execution ){ - return new ResponseEntity<>( "There is already a scheduler for " + execution, HttpStatus.BAD_REQUEST ); + log.warn( "No scheduler for execution: {}", execution ); + return new ResponseEntity<>( "There is no scheduler for " + execution, HttpStatus.BAD_REQUEST ); } /** @@ -72,8 +74,13 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : - case "fifo-random" : scheduler = new RandomLAScheduler(execution, client, namespace, config, new RandomAlignment() ); break; - default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); + case "fifo-random" : + scheduler = config.locationAware + ? new RandomLAScheduler( execution, client, namespace, config, new RandomAlignment() ) + : new RandomScheduler( execution, client, namespace, config ); + break; + default: + return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } schedulerHolder.put( key, scheduler ); @@ -163,7 +170,7 @@ ResponseEntity getTaskState(@PathVariable String namespace, @P @DeleteMapping ("/scheduler/{namespace}/{execution}") ResponseEntity delete(@PathVariable String namespace, @PathVariable String execution) { - log.info("Delete name: " + execution); + log.info("Delete scheduler: " + execution); final Pair key = getKey( namespace, execution ); final Scheduler scheduler = schedulerHolder.get( key ); diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java new file mode 100644 index 00000000..04d9af71 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -0,0 +1,51 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.*; +import fonda.scheduler.util.NodeTaskAlignment; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class RandomScheduler extends Scheduler { + + public RandomScheduler( String execution, + KubernetesClient client, + String namespace, + SchedulerConfig config ) { + super(execution, client, namespace, config); + } + + @Override + public ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ){ + List alignment = new LinkedList<>(); + + for ( final Task task : unscheduledTasks ) { + final PodWithAge pod = task.getPod(); + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); + + final Set matchingNodes = getMatchingNodesForTask(availableByNode,task); + if( !matchingNodes.isEmpty() ) { + + Optional node = matchingNodes.stream().findAny(); + if( node.isPresent() ) { + alignment.add(new NodeTaskAlignment(node.get(), task)); + availableByNode.get(node.get()).subFromThis(pod.getRequest()); + log.info("--> " + node.get().getName()); + } + + } else { + log.info( "No node with enough resources for {}", pod.getName() ); + } + + } + final ScheduleObject scheduleObject = new ScheduleObject(alignment); + scheduleObject.setCheckStillPossible( true ); + return scheduleObject; + } + +} From 67ed766d002bcf2b29fe7a4370e3093b4cd1409d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 27 Apr 2022 13:02:01 +0200 Subject: [PATCH 177/443] Absolute pathes in dockerfile Signed-off-by: Lehmann_Fabian --- Dockerfile | 2 +- Dockerfile-development | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf952254..ee6d817b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,4 @@ COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar RUN chown -R javauser:javagroup /app USER javauser EXPOSE 8080 -ENTRYPOINT ["java","-jar","k8s-scheduler.jar"] \ No newline at end of file +ENTRYPOINT ["java","-jar","/app/k8s-scheduler.jar"] \ No newline at end of file diff --git a/Dockerfile-development b/Dockerfile-development index 40960266..445f1b5a 100644 --- a/Dockerfile-development +++ b/Dockerfile-development @@ -3,4 +3,4 @@ WORKDIR /build COPY pom.xml pom.xml RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository COPY src/ src/ -RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository +RUN mvn package -f /build/pom.xml -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository From 6f2af6390b97dcd62c875117d25b8446359902cb Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 28 Apr 2022 12:33:14 +0200 Subject: [PATCH 178/443] Do not always start a debug scheduler Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/Main.java | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 762bbc64..03ccf5ea 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -17,8 +17,6 @@ @Slf4j public class Main { - private final KubernetesClient client; - public static void main(String[] args) { if( System.getenv( "SCHEDULER_NAME" ) == null || System.getenv( "SCHEDULER_NAME" ).isEmpty() ){ throw new IllegalArgumentException( "Please define environment variable: SCHEDULER_NAME" ); @@ -26,22 +24,4 @@ public static void main(String[] args) { SpringApplication.run(Main.class, args); } - Main(@Autowired KubernetesClient client){ - this.client = client; - } - - @PostConstruct - public void afterStart(){ - try{ - log.info( "Started with namespace: {}", client.getNamespace() ); - final SchedulerConfig schedulerConfig = new SchedulerConfig(null, null, "/localwork/", null, "ftp", true); - final RandomLAScheduler randomLAScheduler = new RandomLAScheduler("testscheduler", client, "default", schedulerConfig, new RandomAlignment()); - final Pair key = new Pair<>("default", "test-run"); - SchedulerRestController.addScheduler( key, randomLAScheduler); - } catch (Exception e){ - e.printStackTrace(); - System.exit(1); - } - } - } From 58e2f5fe6ccbf1c16b79a1f88f14a38820e2f6e4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 28 Apr 2022 12:34:53 +0200 Subject: [PATCH 179/443] Close scheduler if autoClose is active Signed-off-by: Lehmann_Fabian --- .../rest/SchedulerRestController.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 35bff3fe..b721410c 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -16,8 +16,13 @@ import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ApplicationContext; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.bind.annotation.*; import java.util.HashMap; @@ -26,9 +31,13 @@ @RestController @Slf4j +@EnableScheduling public class SchedulerRestController { private final KubernetesClient client; + private final boolean autoClose; + private final ApplicationContext appContext; + private boolean closedOneScheduler = false; /** * Holds the scheduler for one execution @@ -37,8 +46,21 @@ public class SchedulerRestController { */ private static final Map< Pair, Scheduler > schedulerHolder = new HashMap<>(); - public SchedulerRestController( @Autowired KubernetesClient client ){ + public SchedulerRestController( + @Autowired KubernetesClient client, + @Value("#{environment.AUTOCLOSE}") String autoClose, + @Autowired ApplicationContext appContext ){ this.client = client; + this.autoClose = Boolean.parseBoolean(autoClose); + this.appContext = appContext; + } + + @Scheduled(fixedDelay = 5000) + public void close() throws InterruptedException { + if ( autoClose && closedOneScheduler && schedulerHolder.isEmpty() ) { + Thread.sleep( 1000 ); + SpringApplication.exit(appContext, () -> 0); + } } public static void addScheduler(Pair key, Scheduler scheduler ){ @@ -180,6 +202,7 @@ ResponseEntity delete(@PathVariable String namespace, @PathVariable Str schedulerHolder.remove( key ); client.removeScheduler( scheduler ); scheduler.close(); + closedOneScheduler = true; return new ResponseEntity<>( HttpStatus.OK ); } From 5661c39b0faac5eeabe8ed9cf9206ef412f84b7c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 28 Apr 2022 14:02:48 +0200 Subject: [PATCH 180/443] Removed unused method Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/rest/SchedulerRestController.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index b721410c..a84889db 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -63,10 +63,6 @@ public void close() throws InterruptedException { } } - public static void addScheduler(Pair key, Scheduler scheduler ){ - schedulerHolder.put( key, scheduler ); - } - private ResponseEntity noSchedulerFor( String execution ){ log.warn( "No scheduler for execution: {}", execution ); return new ResponseEntity<>( "There is no scheduler for " + execution, HttpStatus.BAD_REQUEST ); From 14c82126364ce779380d969433fb30c95abb43cf Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 28 Apr 2022 14:05:16 +0200 Subject: [PATCH 181/443] Wait for shutdown for 5 seconds after all schedulers are closed Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/rest/SchedulerRestController.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index a84889db..38b0bc15 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -37,7 +37,7 @@ public class SchedulerRestController { private final KubernetesClient client; private final boolean autoClose; private final ApplicationContext appContext; - private boolean closedOneScheduler = false; + private long closedLastScheduler = -1; /** * Holds the scheduler for one execution @@ -57,9 +57,9 @@ public SchedulerRestController( @Scheduled(fixedDelay = 5000) public void close() throws InterruptedException { - if ( autoClose && closedOneScheduler && schedulerHolder.isEmpty() ) { - Thread.sleep( 1000 ); - SpringApplication.exit(appContext, () -> 0); + if ( autoClose && schedulerHolder.isEmpty() && closedLastScheduler != -1 ) { + Thread.sleep( System.currentTimeMillis() - closedLastScheduler + 5000 ); + if ( schedulerHolder.isEmpty() ) SpringApplication.exit(appContext, () -> 0); } } @@ -198,7 +198,7 @@ ResponseEntity delete(@PathVariable String namespace, @PathVariable Str schedulerHolder.remove( key ); client.removeScheduler( scheduler ); scheduler.close(); - closedOneScheduler = true; + closedLastScheduler = System.currentTimeMillis(); return new ResponseEntity<>( HttpStatus.OK ); } From 11de5900bb91b228450d2a6cc276a7f15090bdc9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 28 Apr 2022 17:08:25 +0200 Subject: [PATCH 182/443] Submit to scheduler if tracing is enabled Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/SchedulerConfig.java | 13 ++++++------- .../java/fonda/scheduler/scheduler/Scheduler.java | 3 +++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index e1778d8a..14e620f3 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -15,27 +15,26 @@ public class SchedulerConfig { public final boolean locationAware; + public final boolean traceEnabled; + public SchedulerConfig(List localClaims, List volumeClaims, String workDir, String dns, String copyStrategy, - boolean locationAware) { + boolean locationAware, + boolean traceEnabled) { this.localClaims = localClaims; this.volumeClaims = volumeClaims; this.workDir = workDir; this.dns = dns; this.copyStrategy = copyStrategy; this.locationAware = locationAware; + this.traceEnabled = traceEnabled; } private SchedulerConfig(){ - this.localClaims = null; - this.volumeClaims = null; - this.workDir = null; - this.dns = null; - this.copyStrategy = null; - this.locationAware = false; + this(null,null,null,null,null,false,false); } @ToString diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 8cf1fb1c..0e5e9583 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -50,6 +50,8 @@ public abstract class Scheduler { private final TaskprocessingThread schedulingThread; private final TaskprocessingThread finishThread; + final boolean traceEnabled; + Scheduler(String execution, KubernetesClient client, String namespace, SchedulerConfig config){ this.execution = execution; this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + execution; @@ -58,6 +60,7 @@ public abstract class Scheduler { this.client = client; this.dns = config.dns; this.dag = new DAG(); + this.traceEnabled = config.traceEnabled; PodWatcher podWatcher = new PodWatcher(this); From e7bef6e8841cff75f8263981156f7549d3be06bd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 29 Apr 2022 11:21:25 +0200 Subject: [PATCH 183/443] PrintWriter to BufferedWriter Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/Scheduler.java | 11 ++++++----- .../scheduler/copystrategy/CopyStrategy.java | 7 +++++-- .../fonda/scheduler/model/TaskResultParserTest.java | 13 ++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 0e5e9583..cd431619 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -15,9 +15,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.PrintWriter; +import java.io.*; import java.util.*; @Slf4j @@ -314,10 +312,13 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ final File nodeFile = new File(alignment.task.getWorkingDir() + '/' + ".command.node"); - try(PrintWriter printWriter = new PrintWriter(nodeFile)){ - printWriter.println( alignment.node.getName() ); + try(BufferedWriter printWriter = new BufferedWriter( new FileWriter( nodeFile ))){ + printWriter.write( alignment.node.getName() ); + printWriter.write( '\n' ); } catch (FileNotFoundException e) { e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); } alignment.task.setNode( alignment.node.getNodeLocation() ); diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java index 3cacaf19..a508d60f 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -15,7 +15,7 @@ public void generateCopyScript( Task task ){ File file = new File(task.getWorkingDir() + '/' + ".command.init.run"); - try (PrintWriter pw = new PrintWriter(file)) { + try (BufferedWriter pw = new BufferedWriter( new FileWriter( file) ) ) { ClassLoader classLoader = getClass().getClassLoader(); @@ -26,7 +26,8 @@ public void generateCopyScript( Task task ){ String line; while ((line = reader.readLine()) != null) { - pw.println(line); + pw.write( line ); + pw.write( '\n' ); } Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); @@ -39,6 +40,8 @@ public void generateCopyScript( Task task ){ } catch (FileNotFoundException e) { e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); } } diff --git a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java index 4f928400..4e2efda1 100644 --- a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -12,7 +12,8 @@ import org.junit.Before; import org.junit.Test; -import java.io.PrintWriter; +import java.io.BufferedWriter; +import java.io.FileWriter; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; @@ -34,15 +35,17 @@ private Path storeData( String[] inputdata, String[] outputdata ){ tmpDir = Files.createTempDirectory( "results" ); log.info("Path: {}", tmpDir ); - PrintWriter pw = new PrintWriter( tmpDir.resolve(".command.infiles").toString() ); + BufferedWriter pw = new BufferedWriter( new FileWriter( tmpDir.resolve(".command.infiles").toString() ) ); for (String s : inputdata) { - pw.println( s ); + pw.write( s ); + pw.write( '\n' ); } pw.close(); - pw = new PrintWriter( tmpDir.resolve(".command.outfiles").toString() ); + pw = new BufferedWriter( new FileWriter( tmpDir.resolve(".command.outfiles").toString() ) ); for (String s : outputdata) { - pw.println( s ); + pw.write( s ); + pw.write( '\n' ); } pw.close(); } catch ( Exception ignored ){} From 0c8af81e2bbd27692b06e5599801b54f4550234c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 29 Apr 2022 11:24:31 +0200 Subject: [PATCH 184/443] Removed unnecessary import Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/Main.java | 9 --------- src/main/java/fonda/scheduler/model/PodWithAge.java | 1 - .../fonda/scheduler/rest/SchedulerRestController.java | 5 +---- .../fonda/scheduler/scheduler/RandomLAScheduler.java | 1 - .../scheduler/scheduler/SchedulerWithDaemonSet.java | 6 ++++-- 5 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/fonda/scheduler/Main.java index 03ccf5ea..fd72a1c6 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/fonda/scheduler/Main.java @@ -1,18 +1,9 @@ package fonda.scheduler; -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.rest.SchedulerRestController; -import fonda.scheduler.scheduler.RandomLAScheduler; -import fonda.scheduler.scheduler.filealignment.RandomAlignment; import lombok.extern.slf4j.Slf4j; -import org.javatuples.Pair; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import javax.annotation.PostConstruct; - @SpringBootApplication @Slf4j public class Main { diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index 39a1f079..a6187ffe 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -6,7 +6,6 @@ import lombok.Setter; import java.math.BigDecimal; -import java.util.List; public class PodWithAge extends Pod { diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 38b0bc15..afa19076 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -8,10 +8,7 @@ import fonda.scheduler.model.TaskConfig; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; -import fonda.scheduler.scheduler.RandomLAScheduler; -import fonda.scheduler.scheduler.RandomScheduler; -import fonda.scheduler.scheduler.Scheduler; -import fonda.scheduler.scheduler.SchedulerWithDaemonSet; +import fonda.scheduler.scheduler.*; import fonda.scheduler.scheduler.filealignment.RandomAlignment; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index c5594642..ac5033db 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -12,7 +12,6 @@ import lombok.extern.slf4j.Slf4j; import java.util.*; -import java.util.stream.Collectors; @Slf4j public class RandomLAScheduler extends SchedulerWithDaemonSet { diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 1874e8d8..fa919d87 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -30,7 +30,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.net.ftp.FTPClient; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -82,7 +84,7 @@ private void useLocations( List locationWrappers ){ private void freeLocations( List locationWrappers ){ locationWrappers.parallelStream().forEach( LocationWrapper::free ); } - + @Override boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; From adbf618c507b57fa0af2c09e53308aa6a62ec68f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 29 Apr 2022 17:00:29 +0200 Subject: [PATCH 185/443] Added scheduler tracing Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 27 ++++++ .../scheduler/model/tracing/TraceRecord.java | 92 +++++++++++++++++++ .../scheduler/RandomLAScheduler.java | 11 +++ .../fonda/scheduler/scheduler/Scheduler.java | 4 + .../scheduler/SchedulerWithDaemonSet.java | 51 +++++++++- src/main/resources/copystrategies/ftp.py | 5 + 6 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 src/main/java/fonda/scheduler/model/tracing/TraceRecord.java diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 2e5dd06c..50f1b1a9 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -5,14 +5,17 @@ import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.util.Batch; import fonda.scheduler.util.Tuple; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import java.util.HashMap; import java.util.List; +@Slf4j public class Task { @Getter @@ -47,6 +50,11 @@ public class Task { @Setter private HashMap< String, Tuple> copyingToNode; + @Getter + private final TraceRecord traceRecord = new TraceRecord(); + + private long timeAddedToQueue; + public Task( TaskConfig config, DAG dag ) { this.config = config; this.process = dag.getByProcess( config.getTask() ); @@ -61,6 +69,25 @@ public boolean wasSuccessfullyExecuted(){ return pod.getStatus().getContainerStatuses().get( 0 ).getState().getTerminated().getExitCode() == 0; } + public void writeTrace(){ + try { + final String tracePath = getWorkingDir() + '/' + ".command.scheduler.trace"; + traceRecord.writeRecord(tracePath); + } catch ( Exception e ){ + log.warn( "Cannot write trace of task: " + this.getConfig().getName(), e ); + } + } + + public void setPod(PodWithAge pod) { + if( pod == null ) + timeAddedToQueue = System.currentTimeMillis(); + this.pod = pod; + } + + public void submitted(){ + traceRecord.setSchedulerTimeInQueue( System.currentTimeMillis() - timeAddedToQueue ); + } + @Override public String toString() { return "Task{" + diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java new file mode 100644 index 00000000..113743ca --- /dev/null +++ b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java @@ -0,0 +1,92 @@ +package fonda.scheduler.model.tracing; + +import lombok.Getter; +import lombok.Setter; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; + + +public class TraceRecord { + + @Getter + @Setter + private Long schedulerFilesBytes = null; + + @Getter + @Setter + private Long schedulerFilesNodeBytes = null; + + @Getter + @Setter + private Long schedulerFilesNodeOtherTaskBytes = null; + + @Getter + @Setter + private Integer schedulerFiles = null; + + @Getter + @Setter + private Integer schedulerFilesNode = null; + + @Getter + @Setter + private Integer schedulerFilesNodeOtherTask = null; + + @Getter + @Setter + private Integer schedulerDependingTask = null; + + @Getter + @Setter + private long schedulerTimeInQueue = -1; + + @Getter + @Setter + private int schedulerPlaceInQueue = -1; + + @Getter + @Setter + private int schedulerLocationCount = -1; + + private int schedulerTriedToSchedule = 0; + + + public void writeRecord( String tracePath ) throws IOException { + + try ( BufferedWriter bw = new BufferedWriter( new FileWriter( tracePath ) ) ) { + bw.write("nextflow.scheduler.trace/v1\n"); + writeValue("scheduler_files_bytes", schedulerFilesBytes, bw); + writeValue("scheduler_files_node_bytes", schedulerFilesNodeBytes, bw); + writeValue("scheduler_files_node_other_task_bytes", schedulerFilesNodeOtherTaskBytes, bw); + writeValue("scheduler_files", schedulerFiles, bw); + writeValue("scheduler_files_node", schedulerFilesNode, bw); + writeValue("scheduler_files_node_other_task", schedulerFilesNodeOtherTask, bw); + writeValue("scheduler_depending_task", schedulerDependingTask, bw); + writeValue("scheduler_time_in_queue", schedulerTimeInQueue, bw); + writeValue("scheduler_place_in_queue", schedulerPlaceInQueue, bw); + writeValue("scheduler_location_count", schedulerLocationCount, bw); + writeValue("scheduler_tried_to_schedule", schedulerTriedToSchedule, bw); + } + + } + + private void writeValue( String name, Long value, BufferedWriter bw ) throws IOException { + if ( value != null ) { + bw.write( name + '=' + value + '\n' ); + } + } + + private void writeValue( String name, Integer value, BufferedWriter bw ) throws IOException { + if ( value != null ) { + bw.write( name + '=' + value + '\n' ); + } + } + + public void tryToSchedule(){ + schedulerTriedToSchedule++; + } + + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index ac5033db..af272ea4 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -43,7 +43,9 @@ public ScheduleObject getTaskNodeAlignment( ){ List alignment = new LinkedList<>(); + int index = 0; for ( final Task task : unscheduledTasks ) { + index++; final PodWithAge pod = task.getPod(); log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); @@ -75,6 +77,15 @@ public ScheduleObject getTaskNodeAlignment( alignment.add(new NodeTaskFilesAlignment(node.get(), task, fileAlignment)); availableByNode.get(node.get()).subFromThis(pod.getRequest()); log.info("--> " + node.get().getName()); + if ( traceEnabled ){ + task.getTraceRecord().setSchedulerPlaceInQueue( index ); + task.getTraceRecord().setSchedulerLocationCount( + inputsOfTask.getFiles() + .parallelStream() + .mapToInt( x -> x.locations.size() ) + .sum() + ); + } } } else { diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index cd431619..a0120cb8 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -81,6 +81,7 @@ public abstract class Scheduler { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { + if( traceEnabled ) unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule() ); final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode()); final List taskNodeAlignment = scheduleObject.getTaskAlignments(); @@ -348,6 +349,9 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ pod.getSpec().setNodeName( alignment.node.getMetadata().getName() ); log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); + alignment.task.submitted(); + if( traceEnabled ) alignment.task.writeTrace(); + return true; } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index fa919d87..f4040654 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -12,6 +12,7 @@ import fonda.scheduler.model.outfiles.SymlinkOutput; import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; @@ -85,6 +86,11 @@ private void freeLocations( List locationWrappers ){ locationWrappers.parallelStream().forEach( LocationWrapper::free ); } + private void calculateTrace( NodeTaskFilesAlignment nodeTaskFilesAlignment ){ + final Task task = nodeTaskFilesAlignment.task; + + } + @Override boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; @@ -97,6 +103,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); useLocations( allLocationWrappers ); + calculateTrace( nodeTaskFilesAlignment ); return super.assignTaskToNode( alignment ); } @@ -163,6 +170,11 @@ private void removeFromCopyingToNode(NodeLocation nodeLocation, HashMap< String, copyingToNode.get( nodeLocation ).keySet().removeAll( toRemove.keySet() ); } + /** + * + * @param alignment + * @return null if the task cannot be scheduled + */ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); @@ -177,13 +189,30 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { alignment.task.getConfig().getHash() ); - final HashMap> filesForCurrentNode = new HashMap<>(); final NodeLocation currentNode = alignment.node.getNodeLocation(); final HashMap> filesOnCurrentNode = copyingToNode.get(currentNode); + final TraceRecord traceRecord = alignment.task.getTraceRecord(); + + int filesOnNode = 0; + int filesOnNodeOtherTask = 0; + int filesNotOnNode = 0; + long filesOnNodeByte = 0; + long filesOnNodeOtherTaskByte = 0; + long filesNotOnNodeByte = 0; + for (Map.Entry> entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { - if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) continue; + if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) { + if (traceEnabled) { + filesOnNode = entry.getValue().size(); + filesOnNodeByte = entry.getValue() + .parallelStream() + .mapToLong(x -> x.file.getLocationWrapper(currentNode).getSizeInBytes()) + .sum(); + } + continue; + } final List collect = new LinkedList<>(); @@ -196,6 +225,10 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { else { //May be problematic if the task depending on fails/is stopped before all files are downloaded waitForTask.put( filePath.path, taskLocationTuple.getA() ); + if (traceEnabled) { + filesOnNodeOtherTask++; + filesOnNodeOtherTaskByte += filePath.locationWrapper.getSizeInBytes(); + } } } else { final LocationWrapper locationWrapper = filePath.file.getLocationWrapper(location); @@ -208,6 +241,10 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { ); collect.add(filePath.path ); filesForCurrentNode.put( filePath.path, new Tuple<>( alignment.task, location ) ); + if (traceEnabled) { + filesNotOnNode++; + filesNotOnNodeByte += filePath.locationWrapper.getSizeInBytes(); + } } } if( !collect.isEmpty() ) { @@ -217,6 +254,16 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { inputs.waitForTask( waitForTask ); inputs.symlinks.addAll(alignment.fileAlignment.symlinks); + if (traceEnabled) { + traceRecord.setSchedulerFilesNode(filesOnNode); + traceRecord.setSchedulerFilesNodeBytes(filesOnNodeByte); + traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); + traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); + traceRecord.setSchedulerFiles(filesOnNode + filesOnNodeOtherTask + filesNotOnNode); + traceRecord.setSchedulerFilesBytes(filesOnNodeByte + filesOnNodeOtherTaskByte + filesNotOnNodeByte); + traceRecord.setSchedulerDependingTask( (int) waitForTask.values().stream().distinct().count() ); + } + new ObjectMapper().writeValue( config, inputs ); return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode ); } catch (IOException e) { diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 3cbb9499..21edd4ea 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -209,3 +209,8 @@ def waitForFiles( syncFileTask, files, starttime ): if not waitForFiles( config[ "syncDir" ] + waitForTask, waitForFilesSet, starttime ): log.error( config[ "syncDir" ] + waitForTask + " was not successful" ) myExit( 200 ) + +traceFilePath = ".command.scheduler.trace" + +with open(traceFilePath, "a") as traceFile: + traceFile.write("scheduler_init_runtime=" + str(int((time.time()-starttime)*1000))) \ No newline at end of file From c5409e0068e79bed8118fbaea36c9f112f914be1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 29 Apr 2022 17:06:25 +0200 Subject: [PATCH 186/443] Fixed bug: wrong time in queue Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 50f1b1a9..2e668dc6 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -79,7 +79,7 @@ public void writeTrace(){ } public void setPod(PodWithAge pod) { - if( pod == null ) + if( this.pod == null ) timeAddedToQueue = System.currentTimeMillis(); this.pod = pod; } From 3335d9d751e563e09108dcb921f0995c14bd3ef0 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 11:32:24 +0200 Subject: [PATCH 187/443] Trace: data from how many nodes Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/tracing/TraceRecord.java | 4 ++++ .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 1 + 2 files changed, 5 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java index 113743ca..0e160651 100644 --- a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java @@ -52,6 +52,9 @@ public class TraceRecord { private int schedulerTriedToSchedule = 0; + @Getter + @Setter + private Integer schedulerNodesToCopyFrom = null; public void writeRecord( String tracePath ) throws IOException { @@ -68,6 +71,7 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_place_in_queue", schedulerPlaceInQueue, bw); writeValue("scheduler_location_count", schedulerLocationCount, bw); writeValue("scheduler_tried_to_schedule", schedulerTriedToSchedule, bw); + writeValue("scheduler_nodes_to_copy_from", schedulerNodesToCopyFrom, bw); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index f4040654..72b861b3 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -262,6 +262,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { traceRecord.setSchedulerFiles(filesOnNode + filesOnNodeOtherTask + filesNotOnNode); traceRecord.setSchedulerFilesBytes(filesOnNodeByte + filesOnNodeOtherTaskByte + filesNotOnNodeByte); traceRecord.setSchedulerDependingTask( (int) waitForTask.values().stream().distinct().count() ); + traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.nodeFileAlignment.size() - (filesOnNode > 0 ? 1 : 0) ); } new ObjectMapper().writeValue( config, inputs ); From 17bba761e6610ccd699309f8aeb7ce6e25e7f250 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 13:59:44 +0200 Subject: [PATCH 188/443] getMatchingFilesAndNodes in SchedulerWithDaemonSet Signed-off-by: Lehmann_Fabian --- .../scheduler/MatchingFilesAndNodes.java | 20 +++++++ .../scheduler/RandomLAScheduler.java | 59 ++++++------------- .../scheduler/SchedulerWithDaemonSet.java | 27 +++++++++ 3 files changed, 66 insertions(+), 40 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java diff --git a/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java b/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java new file mode 100644 index 00000000..31e56da8 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java @@ -0,0 +1,20 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.taskinputs.TaskInputs; +import lombok.Getter; + +import java.util.Set; + +@Getter +public class MatchingFilesAndNodes { + + private final Set nodes; + private final TaskInputs inputsOfTask; + + public MatchingFilesAndNodes( Set nodes, TaskInputs inputsOfTask ) { + this.nodes = nodes; + this.inputsOfTask = inputsOfTask; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index af272ea4..69ccc4b2 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -3,8 +3,6 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.NoAlignmentFoundException; -import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.NodeTaskAlignment; @@ -49,48 +47,29 @@ public ScheduleObject getTaskNodeAlignment( final PodWithAge pod = task.getPod(); log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); - final Set matchingNodes = getMatchingNodesForTask(availableByNode,task); - if( !matchingNodes.isEmpty() ) { + final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - final TaskInputs inputsOfTask; - try { - inputsOfTask = getInputsOfTask(task); - } catch (NoAlignmentFoundException e) { - continue; - } - - if( inputsOfTask == null ) { - log.info( "No node where the pod can start, pod: {}", pod.getName() ); - continue; - } - - filterNotMatchingNodesForTask( matchingNodes, inputsOfTask ); - - if( matchingNodes.isEmpty() ) { - log.info( "No node which fulfills all requirements {}", pod.getName() ); - continue; - } + if( matchingFilesAndNodes == null ){ + continue; + } - Optional node = selectNode( matchingNodes, task ); - if( node.isPresent() ) { - final FileAlignment fileAlignment = inputAlignment.getInputAlignment( task, inputsOfTask, node.get() ); - alignment.add(new NodeTaskFilesAlignment(node.get(), task, fileAlignment)); - availableByNode.get(node.get()).subFromThis(pod.getRequest()); - log.info("--> " + node.get().getName()); - if ( traceEnabled ){ - task.getTraceRecord().setSchedulerPlaceInQueue( index ); - task.getTraceRecord().setSchedulerLocationCount( - inputsOfTask.getFiles() - .parallelStream() - .mapToInt( x -> x.locations.size() ) - .sum() - ); - } + Optional node = selectNode( matchingFilesAndNodes.getNodes(), task ); + if( node.isPresent() ) { + final FileAlignment fileAlignment = inputAlignment.getInputAlignment( task, matchingFilesAndNodes.getInputsOfTask(), node.get() ); + alignment.add(new NodeTaskFilesAlignment(node.get(), task, fileAlignment)); + availableByNode.get(node.get()).subFromThis(pod.getRequest()); + log.info("--> " + node.get().getName()); + if ( traceEnabled ){ + task.getTraceRecord().setSchedulerPlaceInQueue( index ); + task.getTraceRecord().setSchedulerLocationCount( + matchingFilesAndNodes.getInputsOfTask().getFiles() + .parallelStream() + .mapToInt( x -> x.locations.size() ) + .sum() + ); } - - } else { - log.info( "No node with enough resources for {}", pod.getName() ); } + } final ScheduleObject scheduleObject = new ScheduleObject(alignment); scheduleObject.setCheckStillPossible( true ); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 72b861b3..3721f155 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -305,6 +305,33 @@ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileExce return new FileResponse( currentPath.toString(), node, getDaemonOnNode(node), node.equals(workflowEngineNode), symlinks, lastUpdate.getId() ); } + MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map availableByNode ){ + final Set matchingNodesForTask = getMatchingNodesForTask(availableByNode, task); + if( matchingNodesForTask.isEmpty() ) { + log.info( "No node with enough resources for {}", task.getConfig().getHash() ); + return null; + } + + final TaskInputs inputsOfTask; + try { + inputsOfTask = getInputsOfTask(task); + } catch (NoAlignmentFoundException e) { + return null; + } + if( inputsOfTask == null ) { + log.info( "No node where the pod can start, pod: {}", task.getConfig().getHash() ); + return null; + } + + filterNotMatchingNodesForTask( matchingNodesForTask, inputsOfTask ); + if( matchingNodesForTask.isEmpty() ) { + log.info( "No node which fulfills all requirements {}", task.getConfig().getHash() ); + return null; + } + + return new MatchingFilesAndNodes( matchingNodesForTask, inputsOfTask ); + } + public void addFile( String path, long size, long timestamp, long locationWrapperID, boolean overwrite, String node ){ final NodeLocation location = NodeLocation.getLocation( node == null ? workflowEngineNode : node ); From 61c0274f8aada85937cb64c8c4b15be30d8e1bb7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 15:39:48 +0200 Subject: [PATCH 189/443] added costs to FileAlignment Signed-off-by: Lehmann_Fabian --- .../filealignment/InputAlignment.java | 21 ++++++++++++++++++- .../filealignment/RandomAlignment.java | 4 ++-- .../fonda/scheduler/util/FileAlignment.java | 4 +++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java index 2da23a19..9a3d2177 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java @@ -8,6 +8,25 @@ public interface InputAlignment { - FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node); + /** + * Calculate a alignment for the input data. Stop if costs are higher than maxCost + * @param task + * @param inputsOfTask + * @param node + * @param maxCost + * @return null if no or no better than maxCost alignment is found + */ + FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, double maxCost ); + + /** + * Calculate a alignment for the input data + * @param task + * @param inputsOfTask + * @param node + * @return null if no alignment is found + */ + default FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node ){ + return getInputAlignment( task, inputsOfTask, node, Double.MAX_VALUE ); + } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index b6641d87..1c7b255a 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -19,7 +19,7 @@ public class RandomAlignment implements InputAlignment { private final Random random = new Random(); @Override - public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node) { + public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node, double maxCost) { final HashMap> map = new HashMap<>(); for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( @@ -30,7 +30,7 @@ public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTa final List pathsOfNode = map.get( nodeIdentifier ); pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file, locationWrapper ) ); } - return new FileAlignment( map, inputsOfTask.getSymlinks() ); + return new FileAlignment( map, inputsOfTask.getSymlinks(), 0); } } diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 36f6b4bf..b436ff04 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -15,10 +15,12 @@ public class FileAlignment { */ public final Map> nodeFileAlignment; public final List symlinks; + public final double cost; - public FileAlignment(Map> nodeFileAlignment, List symlinks) { + public FileAlignment(Map> nodeFileAlignment, List symlinks, double cost) { this.nodeFileAlignment = nodeFileAlignment; this.symlinks = symlinks; + this.cost = cost; } public List getAllLocationWrappers(){ From c3a25558ed5c9e39fe6f84b0a8ef6df90ceca528 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 15:42:58 +0200 Subject: [PATCH 190/443] Calculate data on nodes Signed-off-by: Lehmann_Fabian --- .../taskinputs/PathFileLocationTriple.java | 19 +++++++++++++++++++ .../model/taskinputs/TaskInputs.java | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index 98f1d0b4..a382851d 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -1,5 +1,6 @@ package fonda.scheduler.model.taskinputs; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealHierarchyFile; import lombok.EqualsAndHashCode; @@ -15,6 +16,7 @@ public class PathFileLocationTriple implements Input { public final Path path; public final RealHierarchyFile file; public final List locations; + private long size = -1; public PathFileLocationTriple(Path path, RealHierarchyFile file, List locations) { this.path = path; @@ -22,4 +24,21 @@ public PathFileLocationTriple(Path path, RealHierarchyFile file, List symlinks, List file this.excludedNodes = excludedNodes; } + public long calculateDataOnNode( Location loc ) { + long size = 0; + for ( PathFileLocationTriple fileLocation : files ) { + if (fileLocation.locatedOnLocation(loc)) { + size += fileLocation.getSizeInBytes(); + } + } + return size; + } + + public long calculateAvgSize() { + long size = 0; + for ( PathFileLocationTriple file : files ) { + size += file.getSizeInBytes(); + } + return size; + } + } From db73869967b6315b307fef9b3155b2dce6063501 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 16:32:19 +0200 Subject: [PATCH 191/443] TaskInput toString do not list all inputs Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/TaskInput.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/TaskInput.java b/src/main/java/fonda/scheduler/model/TaskInput.java index f4eaffea..817bf46f 100644 --- a/src/main/java/fonda/scheduler/model/TaskInput.java +++ b/src/main/java/fonda/scheduler/model/TaskInput.java @@ -4,7 +4,6 @@ import java.util.List; -@ToString public class TaskInput { public final List> booleanInputs; @@ -33,4 +32,13 @@ private TaskInput() { this.fileInputs = fileInputs; } + @Override + public String toString() { + return "TaskInput{" + + "booleanInputs=" + booleanInputs + + ", numberInputs=" + numberInputs + + ", stringInputs=" + stringInputs + + ", fileInputs=#" + (fileInputs != null ? fileInputs.size() : 0) + + '}'; + } } From 77e4a150400da97c195c0e874fd79633b5544763 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 16:32:57 +0200 Subject: [PATCH 192/443] Only inform scheduler if pod was removed the first time Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/client/KubernetesClient.java | 7 ++++--- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index cd0b2cc0..9af91ce5 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -165,10 +165,11 @@ public void eventReceived(Action action, Pod pod) { //Pod is finished case DELETED: case ERROR: - log.info("Pod has released its resources: {}", pod.getMetadata().getName()); //Delete Pod in any case - node.removePod( pod ); - kubernetesClient.informAllScheduler(); + if ( node.removePod( pod ) ){ + log.info("Pod has released its resources: {}", pod.getMetadata().getName()); + kubernetesClient.informAllScheduler(); + } break; default: log.warn("No implementation for {}", action); } diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 0e6e0f70..237cd91b 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -59,9 +59,9 @@ public void addPod( PodWithAge pod ){ } } - public void removePod( Pod pod ){ + public boolean removePod( Pod pod ){ synchronized (assignedPods) { - assignedPods.remove( pod.getMetadata().getUid() ); + return assignedPods.remove( pod.getMetadata().getUid() ) != null; } } From 3e2237215f3172a6e4584cd439f138adb72bbf7e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 16:33:21 +0200 Subject: [PATCH 193/443] Extended trace record Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/tracing/TraceRecord.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java index 0e160651..81008e59 100644 --- a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java @@ -56,6 +56,10 @@ public class TraceRecord { @Setter private Integer schedulerNodesToCopyFrom = null; + @Getter + @Setter + private Integer schedulerTimeToSchedule = null; + public void writeRecord( String tracePath ) throws IOException { try ( BufferedWriter bw = new BufferedWriter( new FileWriter( tracePath ) ) ) { @@ -72,6 +76,7 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_location_count", schedulerLocationCount, bw); writeValue("scheduler_tried_to_schedule", schedulerTriedToSchedule, bw); writeValue("scheduler_nodes_to_copy_from", schedulerNodesToCopyFrom, bw); + writeValue("scheduler_time_to_schedule", schedulerTimeToSchedule, bw); } } From 387c34b919ed01869766a26635657318d0c850e5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 16:34:55 +0200 Subject: [PATCH 194/443] First version of location aware scheduler Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java new file mode 100644 index 00000000..41a497a4 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -0,0 +1,137 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.*; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.data.NodeDataTuple; +import fonda.scheduler.scheduler.data.TaskData; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.Tuple; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@Slf4j +public class LocationAwareScheduler extends SchedulerWithDaemonSet { + + @Getter(AccessLevel.PACKAGE) + private final InputAlignment inputAlignment; + + public LocationAwareScheduler ( + String name, + KubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment) { + super( name, client, namespace, config ); + this.inputAlignment = inputAlignment; + } + + + + private NodeTaskFilesAlignment createNodeAlignment ( + final TaskData taskData, + final Map availableByNode, + AtomicInteger index + ) { + long startTime = System.nanoTime(); + log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); + final int currentIndex = index.incrementAndGet(); + final Tuple result = calculateBestNode(taskData, availableByNode); + if ( result == null ) return null; + final Task task = taskData.getTask(); + availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); + taskData.addNs( System.nanoTime()- startTime ); + if ( traceEnabled ){ + task.getTraceRecord().setSchedulerTimeToSchedule((int) (taskData.getTimeInNs() / 1_000_000)); + task.getTraceRecord().setSchedulerPlaceInQueue( currentIndex ); + task.getTraceRecord().setSchedulerLocationCount( + taskData.getMatchingFilesAndNodes().getInputsOfTask().getFiles() + .parallelStream() + .mapToInt( x -> x.locations.size() ) + .sum() + ); + } + return new NodeTaskFilesAlignment(result.getA(), task, result.getB()); + } + + @Override + public ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ){ + AtomicInteger index = new AtomicInteger( 0 ); + final List alignment = unscheduledTasks + .parallelStream() + .map( task -> calculateTaskData( task, availableByNode ) ) + .filter(Objects::nonNull) + .sorted(Comparator.reverseOrder()) + .sequential() + .map( taskData -> createNodeAlignment( taskData, availableByNode, index ) ) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + final ScheduleObject scheduleObject = new ScheduleObject(alignment); + scheduleObject.setCheckStillPossible( true ); + return scheduleObject; + } + + Tuple calculateBestNode( + final TaskData taskData, + final Map availableByNode + ){ + final Set matchingNodesForTask = getMatchingNodesForTask( availableByNode, taskData.getTask()); + FileAlignment bestAlignment = null; + NodeWithAlloc bestNode = null; + //Remove all nodes which do not fit anymore + final List nodeDataTuples = taskData.getNodeDataTuples(); + + for (NodeDataTuple nodeDataTuple : nodeDataTuples) { + final NodeWithAlloc currentNode = nodeDataTuple.getNode(); + if ( matchingNodesForTask.contains(currentNode) ) { + final FileAlignment fileAlignment = inputAlignment.getInputAlignment( + taskData.getTask(), + taskData.getMatchingFilesAndNodes().getInputsOfTask(), + currentNode, + bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost + ); + + if ( fileAlignment != null && (bestAlignment == null || bestAlignment.cost < fileAlignment.cost ) ){ + bestAlignment = fileAlignment; + bestNode = currentNode; + } + } + } + + return bestAlignment == null ? null : new Tuple<>( bestNode, bestAlignment ); + } + + + TaskData calculateTaskData( + final Task task, + final Map availableByNode + ) { + long startTime = System.nanoTime(); + final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); + if ( matchingFilesAndNodes.getNodes().isEmpty() ) return null; + final TaskInputs inputsOfTask = matchingFilesAndNodes.getInputsOfTask(); + long size = inputsOfTask.calculateAvgSize(); + final List nodeDataTuples = matchingFilesAndNodes + .getNodes() + .parallelStream() + .map(node -> new NodeDataTuple(node, inputsOfTask.calculateDataOnNode( node.getNodeLocation() ) ) ) + .sorted(Comparator.reverseOrder()) + .collect(Collectors.toList()); + final double fracOnNode = nodeDataTuples.get(0).getSizeInBytes() / (double) size; + final double antiStarvingFactor = 0; + final double value = fracOnNode + antiStarvingFactor; + return new TaskData( value, task, nodeDataTuples, matchingFilesAndNodes, System.nanoTime() - startTime ); + } + +} From c5fe7772f8127fee2920e78b9d7806667e972813 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 17:06:32 +0200 Subject: [PATCH 195/443] Refactoring Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 41a497a4..5ee2d185 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -44,7 +44,9 @@ private NodeTaskFilesAlignment createNodeAlignment ( long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); final int currentIndex = index.incrementAndGet(); - final Tuple result = calculateBestNode(taskData, availableByNode); + final Set matchingNodesForTask = getMatchingNodesForTask( availableByNode, taskData.getTask()); + if ( matchingNodesForTask.isEmpty() ) return null; + final Tuple result = calculateBestNode(taskData, matchingNodesForTask); if ( result == null ) return null; final Task task = taskData.getTask(); availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); @@ -70,7 +72,12 @@ public ScheduleObject getTaskNodeAlignment( AtomicInteger index = new AtomicInteger( 0 ); final List alignment = unscheduledTasks .parallelStream() - .map( task -> calculateTaskData( task, availableByNode ) ) + .map( task -> { + long startTime = System.nanoTime(); + final TaskData taskData = calculateTaskData(task, availableByNode); + taskData.addNs( System.nanoTime() - startTime ); + return taskData; + } ) .filter(Objects::nonNull) .sorted(Comparator.reverseOrder()) .sequential() @@ -84,9 +91,8 @@ public ScheduleObject getTaskNodeAlignment( Tuple calculateBestNode( final TaskData taskData, - final Map availableByNode + final Set matchingNodesForTask ){ - final Set matchingNodesForTask = getMatchingNodesForTask( availableByNode, taskData.getTask()); FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; //Remove all nodes which do not fit anymore @@ -113,11 +119,16 @@ Tuple calculateBestNode( } + /** + * Create a TaskData object for the input Task + * @param task + * @param availableByNode + * @return null if no nodes available + */ TaskData calculateTaskData( final Task task, final Map availableByNode ) { - long startTime = System.nanoTime(); final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); if ( matchingFilesAndNodes.getNodes().isEmpty() ) return null; final TaskInputs inputsOfTask = matchingFilesAndNodes.getInputsOfTask(); @@ -131,7 +142,7 @@ TaskData calculateTaskData( final double fracOnNode = nodeDataTuples.get(0).getSizeInBytes() / (double) size; final double antiStarvingFactor = 0; final double value = fracOnNode + antiStarvingFactor; - return new TaskData( value, task, nodeDataTuples, matchingFilesAndNodes, System.nanoTime() - startTime ); + return new TaskData( value, task, nodeDataTuples, matchingFilesAndNodes ); } } From 359b29a04999d1b587d90235391891778bed0209 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 17:07:04 +0200 Subject: [PATCH 196/443] Use LocationAware Scheduler for RandomLAScheduler Signed-off-by: Lehmann_Fabian --- .../scheduler/RandomLAScheduler.java | 79 ++++++++----------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 69ccc4b2..ca368da1 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -1,20 +1,24 @@ package fonda.scheduler.scheduler; import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.*; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.model.Task; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.scheduler.data.NodeDataTuple; +import fonda.scheduler.scheduler.data.TaskData; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.NodeTaskAlignment; -import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.Tuple; import lombok.extern.slf4j.Slf4j; import java.util.*; +import java.util.stream.Collectors; @Slf4j -public class RandomLAScheduler extends SchedulerWithDaemonSet { +public class RandomLAScheduler extends LocationAwareScheduler { - private final InputAlignment inputAlignment; private final Random random = new Random(); public RandomLAScheduler( @@ -24,8 +28,7 @@ public RandomLAScheduler( SchedulerConfig config, InputAlignment inputAlignment ) { - super(name, client, namespace, config); - this.inputAlignment = inputAlignment; + super(name, client, namespace, config, inputAlignment); } private Optional selectNode( Set matchingNodes, Task task ){ @@ -34,46 +37,32 @@ private Optional selectNode( Set matchingNodes, Ta : Optional.of( new LinkedList<>(matchingNodes).get(random.nextInt(matchingNodes.size()))); } - @Override - public ScheduleObject getTaskNodeAlignment( - final List unscheduledTasks, - final Map availableByNode + Tuple calculateBestNode( + final TaskData taskData, + final Set matchingNodesForTask ){ - List alignment = new LinkedList<>(); - - int index = 0; - for ( final Task task : unscheduledTasks ) { - index++; - final PodWithAge pod = task.getPod(); - log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); - - final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - - if( matchingFilesAndNodes == null ){ - continue; - } - - Optional node = selectNode( matchingFilesAndNodes.getNodes(), task ); - if( node.isPresent() ) { - final FileAlignment fileAlignment = inputAlignment.getInputAlignment( task, matchingFilesAndNodes.getInputsOfTask(), node.get() ); - alignment.add(new NodeTaskFilesAlignment(node.get(), task, fileAlignment)); - availableByNode.get(node.get()).subFromThis(pod.getRequest()); - log.info("--> " + node.get().getName()); - if ( traceEnabled ){ - task.getTraceRecord().setSchedulerPlaceInQueue( index ); - task.getTraceRecord().setSchedulerLocationCount( - matchingFilesAndNodes.getInputsOfTask().getFiles() - .parallelStream() - .mapToInt( x -> x.locations.size() ) - .sum() - ); - } - } + final NodeWithAlloc node = selectNode( matchingNodesForTask, taskData.getTask() ).get(); + final FileAlignment fileAlignment = getInputAlignment().getInputAlignment( + taskData.getTask(), + taskData.getMatchingFilesAndNodes().getInputsOfTask(), + node + ); + return new Tuple<>( node, fileAlignment ); + } - } - final ScheduleObject scheduleObject = new ScheduleObject(alignment); - scheduleObject.setCheckStillPossible( true ); - return scheduleObject; + TaskData calculateTaskData( + final Task task, + final Map availableByNode + ) { + final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); + if ( matchingFilesAndNodes.getNodes().isEmpty() ) return null; + final List nodeDataTuples = matchingFilesAndNodes + .getNodes() + .parallelStream() + .map(node -> new NodeDataTuple(node, 0 ) ) + .collect(Collectors.toList()); + return new TaskData( 0, task, nodeDataTuples, matchingFilesAndNodes ); } + } From 4f3462af6f06921cbc5c32baeb869376e572a883 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 2 May 2022 17:17:44 +0200 Subject: [PATCH 197/443] Added forgotten files Signed-off-by: Lehmann_Fabian --- .../scheduler/data/NodeDataTuple.java | 23 +++++++++++ .../scheduler/scheduler/data/TaskData.java | 38 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java create mode 100644 src/main/java/fonda/scheduler/scheduler/data/TaskData.java diff --git a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java new file mode 100644 index 00000000..313dc597 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java @@ -0,0 +1,23 @@ +package fonda.scheduler.scheduler.data; + +import fonda.scheduler.model.NodeWithAlloc; +import lombok.Getter; +import org.jetbrains.annotations.NotNull; + +@Getter +public +class NodeDataTuple implements Comparable { + + private final NodeWithAlloc node; + private final long sizeInBytes; + + public NodeDataTuple(NodeWithAlloc node, long sizeInBytes) { + this.node = node; + this.sizeInBytes = sizeInBytes; + } + + @Override + public int compareTo(@NotNull NodeDataTuple o) { + return Long.compare( sizeInBytes, o.sizeInBytes); + } +} \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java new file mode 100644 index 00000000..cfb6d948 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -0,0 +1,38 @@ +package fonda.scheduler.scheduler.data; + +import fonda.scheduler.model.Task; +import fonda.scheduler.scheduler.MatchingFilesAndNodes; +import lombok.Getter; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +@Getter +public class TaskData implements Comparable { + private final double value; + private final Task task; + private final List nodeDataTuples; + private final MatchingFilesAndNodes matchingFilesAndNodes; + private long timeInNs = 0; + + public TaskData( + double value, + Task task, + List nodeDataTuples, + MatchingFilesAndNodes matchingFilesAndNodes + ) { + this.value = value; + this.task = task; + this.nodeDataTuples = nodeDataTuples; + this.matchingFilesAndNodes = matchingFilesAndNodes; + } + + @Override + public int compareTo(@NotNull TaskData o) { + return Double.compare(value, o.value); + } + + public void addNs( long timeInNs ){ + this.timeInNs += timeInNs; + } + } \ No newline at end of file From 7c62c20413e646a91111d50e68df8e77bdb16931 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 May 2022 11:37:00 +0200 Subject: [PATCH 198/443] added lav1 schheduler Signed-off-by: Lehmann_Fabian --- .../rest/SchedulerRestController.java | 5 +++ .../scheduler/scheduler/LASchedulerV1.java | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index afa19076..ebba17b6 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -94,6 +94,11 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa ? new RandomLAScheduler( execution, client, namespace, config, new RandomAlignment() ) : new RandomScheduler( execution, client, namespace, config ); break; + case "lav1" : + if ( !config.locationAware ) + return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); + scheduler = new LASchedulerV1( execution, client, namespace, config, new RandomAlignment() ); + break; default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } diff --git a/src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java b/src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java new file mode 100644 index 00000000..39803d94 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java @@ -0,0 +1,32 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.*; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.NodeTaskAlignment; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.Tuple; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@Slf4j +public class LASchedulerV1 extends LocationAwareScheduler { + + public LASchedulerV1( + String name, + KubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment + ) { + super(name, client, namespace, config, inputAlignment); + } + +} From 80955a94c455512258bd03872ed20685f9354407 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 11:10:21 +0200 Subject: [PATCH 199/443] Added Null checks Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/LocationAwareScheduler.java | 4 ++-- .../java/fonda/scheduler/scheduler/RandomLAScheduler.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 5ee2d185..b2c5bca7 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -75,7 +75,7 @@ public ScheduleObject getTaskNodeAlignment( .map( task -> { long startTime = System.nanoTime(); final TaskData taskData = calculateTaskData(task, availableByNode); - taskData.addNs( System.nanoTime() - startTime ); + if ( taskData != null ) taskData.addNs( System.nanoTime() - startTime ); return taskData; } ) .filter(Objects::nonNull) @@ -130,7 +130,7 @@ TaskData calculateTaskData( final Map availableByNode ) { final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - if ( matchingFilesAndNodes.getNodes().isEmpty() ) return null; + if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) return null; final TaskInputs inputsOfTask = matchingFilesAndNodes.getInputsOfTask(); long size = inputsOfTask.calculateAvgSize(); final List nodeDataTuples = matchingFilesAndNodes diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index ca368da1..f954e7ae 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -55,7 +55,7 @@ TaskData calculateTaskData( final Map availableByNode ) { final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - if ( matchingFilesAndNodes.getNodes().isEmpty() ) return null; + if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) return null; final List nodeDataTuples = matchingFilesAndNodes .getNodes() .parallelStream() From e528071704f1f88495b3faf59406e1153961ad1d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 12:47:59 +0200 Subject: [PATCH 200/443] Do not always fetch metrics for problematic nodes Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/NodeWithAlloc.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 237cd91b..02cbca46 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -24,6 +24,8 @@ public class NodeWithAlloc extends Node implements Comparable { private final Map assignedPods; + private long lastMetricsFetchingProblem = 0; + @Getter private final NodeLocation nodeLocation; @@ -74,9 +76,16 @@ public Requirements getRequestedResources(){ requestedByPods = assignedPods.values().stream() .reduce(new Requirements(), Requirements::addToThis); } - final BigDecimal currentMemoryOfNode = kubernetesClient.getMemoryOfNode( this ); - if ( requestedByPods.getRam().compareTo(currentMemoryOfNode) == -1 ){ - requestedByPods.addRAMtoThis(currentMemoryOfNode.subtract(requestedByPods.getRam())); + if ( System.currentTimeMillis() > lastMetricsFetchingProblem + 10_000 ) { + try { + final BigDecimal currentMemoryOfNode = kubernetesClient.getMemoryOfNode(this); + if (requestedByPods.getRam().compareTo(currentMemoryOfNode) == -1) { + requestedByPods.addRAMtoThis(currentMemoryOfNode.subtract(requestedByPods.getRam())); + } + } catch (Exception e) { + log.warn( "Couldn't fetch metrics for {}", getName() ); + lastMetricsFetchingProblem = System.currentTimeMillis(); + } } return requestedByPods; } From 41184fead64c5b87c94238e0b5a0e1670cb4ad55 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 12:48:20 +0200 Subject: [PATCH 201/443] Fix problem: divide by 0 Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/LocationAwareScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index b2c5bca7..3cc5d63c 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -139,7 +139,7 @@ TaskData calculateTaskData( .map(node -> new NodeDataTuple(node, inputsOfTask.calculateDataOnNode( node.getNodeLocation() ) ) ) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); - final double fracOnNode = nodeDataTuples.get(0).getSizeInBytes() / (double) size; + final double fracOnNode = size == 0 ? 1 : nodeDataTuples.get(0).getSizeInBytes() / (double) size; final double antiStarvingFactor = 0; final double value = fracOnNode + antiStarvingFactor; return new TaskData( value, task, nodeDataTuples, matchingFilesAndNodes ); From dd48a2afeebe255815ff01ab4c1b4ebd5d8da3bd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 13:21:56 +0200 Subject: [PATCH 202/443] Check node is ready Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 5 +++++ src/main/java/fonda/scheduler/scheduler/Scheduler.java | 1 + 2 files changed, 6 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 02cbca46..bbfc2467 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -5,6 +5,7 @@ import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.client.internal.readiness.Readiness; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -131,4 +132,8 @@ public int hashCode() { result = 31 * result + (getNodeLocation() != null ? getNodeLocation().hashCode() : 0); return result; } + + public boolean isReady(){ + return Readiness.isNodeReady(this); + } } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index a0120cb8..07b51a0b 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -419,6 +419,7 @@ Map getAvailableByNode(){ List logInfo = new LinkedList<>(); logInfo.add("------------------------------------"); for (NodeWithAlloc item : getNodeList()) { + if ( !item.isReady() ) continue; final Requirements availableResources = item.getAvailableResources(); availableByNode.put(item, availableResources); logInfo.add("Node: " + item.getName() + " " + availableResources); From 28d6f30a56787ef14cea0f045af4a343d497c3e1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 13:22:19 +0200 Subject: [PATCH 203/443] Deleted unused method Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/PodWithAge.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index a6187ffe..1f5ce3e3 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -39,10 +39,6 @@ public Requirements getRequest(){ ).reduce( new Requirements(), Requirements::addToThis ); } - public boolean hasFinishedOrFailed(){ - return hasFinishedOrFailed( this ); - } - public String getName(){ return this.getMetadata().getName(); } From bf34f2de7f8ec8b2e261b343edea717b05550f14 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 15:10:54 +0200 Subject: [PATCH 204/443] Don't check metrics Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/NodeWithAlloc.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index bbfc2467..8122e984 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -25,8 +25,6 @@ public class NodeWithAlloc extends Node implements Comparable { private final Map assignedPods; - private long lastMetricsFetchingProblem = 0; - @Getter private final NodeLocation nodeLocation; @@ -77,17 +75,6 @@ public Requirements getRequestedResources(){ requestedByPods = assignedPods.values().stream() .reduce(new Requirements(), Requirements::addToThis); } - if ( System.currentTimeMillis() > lastMetricsFetchingProblem + 10_000 ) { - try { - final BigDecimal currentMemoryOfNode = kubernetesClient.getMemoryOfNode(this); - if (requestedByPods.getRam().compareTo(currentMemoryOfNode) == -1) { - requestedByPods.addRAMtoThis(currentMemoryOfNode.subtract(requestedByPods.getRam())); - } - } catch (Exception e) { - log.warn( "Couldn't fetch metrics for {}", getName() ); - lastMetricsFetchingProblem = System.currentTimeMillis(); - } - } return requestedByPods; } From 495f9bc519a99af3dc5506a74b4ff7856df6d8cd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 15:36:05 +0200 Subject: [PATCH 205/443] Updated versions Signed-off-by: Lehmann_Fabian --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 56eea8ec..d25f76ff 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ io.fabric8 kubernetes-client - 5.12.1 + 5.12.2 @@ -38,7 +38,7 @@ org.projectlombok lombok - 1.18.22 + 1.18.24 provided @@ -51,20 +51,20 @@ org.mockito mockito-inline - 4.3.1 + 4.5.1 test org.springframework.boot spring-boot-starter-web - 2.6.5 + 2.6.7 org.springframework.boot spring-boot-starter-test - 2.6.5 + 2.6.7 test @@ -92,7 +92,7 @@ commons-net commons-net - 3.6 + 3.8.0 From 5a413a3795d496001939a8bf9b666a3a6016669e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 16:32:49 +0200 Subject: [PATCH 206/443] FilePath getter + store triple Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 22 +++++++++--------- .../filealignment/RandomAlignment.java | 5 ++-- .../fonda/scheduler/util/FileAlignment.java | 2 +- .../java/fonda/scheduler/util/FilePath.java | 23 ++++++++++++++----- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 3721f155..ac172e43 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -208,7 +208,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { filesOnNode = entry.getValue().size(); filesOnNodeByte = entry.getValue() .parallelStream() - .mapToLong(x -> x.file.getLocationWrapper(currentNode).getSizeInBytes()) + .mapToLong(x -> x.getFile().getLocationWrapper(currentNode).getSizeInBytes()) .sum(); } continue; @@ -218,32 +218,32 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final NodeLocation location = NodeLocation.getLocation( entry.getKey() ); for (FilePath filePath : entry.getValue()) { - if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey( filePath.path ) ) { + if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey(filePath.getPath()) ) { //Node copies currently from somewhere else! - final Tuple taskLocationTuple = filesOnCurrentNode.get(filePath.path); + final Tuple taskLocationTuple = filesOnCurrentNode.get(filePath.getPath()); if ( taskLocationTuple.getB() != location ) return null; else { //May be problematic if the task depending on fails/is stopped before all files are downloaded - waitForTask.put( filePath.path, taskLocationTuple.getA() ); + waitForTask.put(filePath.getPath(), taskLocationTuple.getA() ); if (traceEnabled) { filesOnNodeOtherTask++; - filesOnNodeOtherTaskByte += filePath.locationWrapper.getSizeInBytes(); + filesOnNodeOtherTaskByte += filePath.getLocationWrapper().getSizeInBytes(); } } } else { - final LocationWrapper locationWrapper = filePath.file.getLocationWrapper(location); + final LocationWrapper locationWrapper = filePath.getFile().getLocationWrapper(location); inputFiles.add( new TaskInputFileLocationWrapper( - filePath.path, - filePath.file, + filePath.getPath(), + filePath.getFile(), locationWrapper.getCopyOf( currentNode ) ) ); - collect.add(filePath.path ); - filesForCurrentNode.put( filePath.path, new Tuple<>( alignment.task, location ) ); + collect.add(filePath.getPath()); + filesForCurrentNode.put(filePath.getPath(), new Tuple<>( alignment.task, location ) ); if (traceEnabled) { filesNotOnNode++; - filesNotOnNodeByte += filePath.locationWrapper.getSizeInBytes(); + filesNotOnNodeByte += filePath.getLocationWrapper().getSizeInBytes(); } } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index 1c7b255a..1fbaba30 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -26,9 +26,8 @@ public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTa random.nextInt( pathFileLocationTriple.locations.size() ) ); final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); - final List pathsOfNode = map.get( nodeIdentifier ); - pathsOfNode.add( new FilePath( pathFileLocationTriple.path.toString(), pathFileLocationTriple.file, locationWrapper ) ); + final List pathsOfNode = map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); + pathsOfNode.add( new FilePath( pathFileLocationTriple, locationWrapper ) ); } return new FileAlignment( map, inputsOfTask.getSymlinks(), 0); } diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index b436ff04..eb3b4f84 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -30,7 +30,7 @@ public List getAllLocationWrappers(){ .flatMap( l -> l .getValue() .parallelStream() - .map( p -> p.locationWrapper ) + .map( p -> p.getLocationWrapper() ) ) .collect(Collectors.toList()); } diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/fonda/scheduler/util/FilePath.java index 542498b1..e8ffe061 100644 --- a/src/main/java/fonda/scheduler/util/FilePath.java +++ b/src/main/java/fonda/scheduler/util/FilePath.java @@ -2,16 +2,27 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealHierarchyFile; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; public class FilePath { - public final String path; - public final RealHierarchyFile file; - public final LocationWrapper locationWrapper; + private final PathFileLocationTriple pathFileLocationTriple; + private final LocationWrapper locationWrapper; - public FilePath(String path, RealHierarchyFile file, LocationWrapper locationWrapper ) { - this.path = path; - this.file = file; + public FilePath( PathFileLocationTriple pathFileLocationTriple, LocationWrapper locationWrapper ) { + this.pathFileLocationTriple = pathFileLocationTriple; this.locationWrapper = locationWrapper; } + + public String getPath() { + return pathFileLocationTriple.path.toString(); + } + + public RealHierarchyFile getFile() { + return pathFileLocationTriple.file; + } + + public LocationWrapper getLocationWrapper() { + return locationWrapper; + } } From c8c4917d21b2572e9e7db936e864aaed056cdff8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 16:33:08 +0200 Subject: [PATCH 207/443] sortTaskInputs Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/taskinputs/TaskInputs.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java index 8cf81899..30d4b5dd 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -15,6 +15,8 @@ public class TaskInputs { private final List files; private final Set excludedNodes; + private boolean sorted = false; + public TaskInputs(List symlinks, List files, Set excludedNodes) { this.symlinks = symlinks; this.files = files; @@ -39,4 +41,13 @@ public long calculateAvgSize() { return size; } + public void sort(){ + synchronized ( files ) { + if (!sorted) { + files.sort((x, y) -> Long.compare(y.getSizeInBytes(), x.getSizeInBytes())); + sorted = true; + } + } + } + } From 1bfad9bf35f6356f92701a6ff7e459a238c1292c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 16:56:29 +0200 Subject: [PATCH 208/443] Use AlignmentWrapper Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 14 +++++------ .../filealignment/RandomAlignment.java | 9 ++++---- .../scheduler/util/AlignmentWrapper.java | 23 +++++++++++++++++++ .../fonda/scheduler/util/FileAlignment.java | 5 ++-- 4 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 src/main/java/fonda/scheduler/util/AlignmentWrapper.java diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index ac172e43..6fd18324 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -19,10 +19,7 @@ import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; -import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.NodeTaskAlignment; -import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.*; import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; @@ -202,11 +199,12 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { long filesOnNodeOtherTaskByte = 0; long filesNotOnNodeByte = 0; - for (Map.Entry> entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { + for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) { if (traceEnabled) { - filesOnNode = entry.getValue().size(); - filesOnNodeByte = entry.getValue() + final List alignment1 = entry.getValue().getAlignment(); + filesOnNode = alignment1.size(); + filesOnNodeByte = alignment1 .parallelStream() .mapToLong(x -> x.getFile().getLocationWrapper(currentNode).getSizeInBytes()) .sum(); @@ -217,7 +215,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final List collect = new LinkedList<>(); final NodeLocation location = NodeLocation.getLocation( entry.getKey() ); - for (FilePath filePath : entry.getValue()) { + for (FilePath filePath : entry.getValue().getAlignment()) { if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey(filePath.getPath()) ) { //Node copies currently from somewhere else! final Tuple taskLocationTuple = filesOnCurrentNode.get(filePath.getPath()); diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index 1fbaba30..5049b666 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -5,13 +5,12 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.util.AlignmentWrapper; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; import org.jetbrains.annotations.NotNull; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Random; public class RandomAlignment implements InputAlignment { @@ -20,14 +19,14 @@ public class RandomAlignment implements InputAlignment { @Override public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node, double maxCost) { - final HashMap> map = new HashMap<>(); + final HashMap map = new HashMap<>(); for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( random.nextInt( pathFileLocationTriple.locations.size() ) ); final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - final List pathsOfNode = map.computeIfAbsent(nodeIdentifier, k -> new LinkedList<>() ); - pathsOfNode.add( new FilePath( pathFileLocationTriple, locationWrapper ) ); + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(nodeIdentifier, k -> new AlignmentWrapper() ); + alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); } return new FileAlignment( map, inputsOfTask.getSymlinks(), 0); } diff --git a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java new file mode 100644 index 00000000..1548f73b --- /dev/null +++ b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java @@ -0,0 +1,23 @@ +package fonda.scheduler.util; + +import lombok.Getter; + +import java.util.LinkedList; +import java.util.List; + +@Getter +public class AlignmentWrapper { + + private final List alignment = new LinkedList(); + private double cost = 0; + + public void addAlignment( FilePath filePath, double cost ) { + alignment.add( filePath ); + this.cost = cost; + } + + public boolean empty() { + return alignment.isEmpty(); + } + +} diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index eb3b4f84..6514d660 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -13,11 +13,11 @@ public class FileAlignment { Key: node Value: Files from the node */ - public final Map> nodeFileAlignment; + public final Map nodeFileAlignment; public final List symlinks; public final double cost; - public FileAlignment(Map> nodeFileAlignment, List symlinks, double cost) { + public FileAlignment(Map nodeFileAlignment, List symlinks, double cost) { this.nodeFileAlignment = nodeFileAlignment; this.symlinks = symlinks; this.cost = cost; @@ -29,6 +29,7 @@ public List getAllLocationWrappers(){ .parallelStream() .flatMap( l -> l .getValue() + .getAlignment() .parallelStream() .map( p -> p.getLocationWrapper() ) ) From 8126223743e5407a6326127f353fd661101e9a99 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:03:26 +0200 Subject: [PATCH 209/443] Location and not String as key in alignmentmap Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 6 +++--- .../scheduler/scheduler/filealignment/RandomAlignment.java | 7 ++++--- src/main/java/fonda/scheduler/util/FileAlignment.java | 5 +++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 6fd18324..7a28dd0c 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -199,7 +199,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { long filesOnNodeOtherTaskByte = 0; long filesNotOnNodeByte = 0; - for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { + for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) { if (traceEnabled) { final List alignment1 = entry.getValue().getAlignment(); @@ -214,7 +214,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final List collect = new LinkedList<>(); - final NodeLocation location = NodeLocation.getLocation( entry.getKey() ); + final NodeLocation location = (NodeLocation) entry.getKey(); for (FilePath filePath : entry.getValue().getAlignment()) { if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey(filePath.getPath()) ) { //Node copies currently from somewhere else! @@ -246,7 +246,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { } } if( !collect.isEmpty() ) { - inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey()), entry.getKey(), collect)); + inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect)); } } inputs.waitForTask( waitForTask ); diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index 5049b666..2a0ead23 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -2,6 +2,7 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.TaskInputs; @@ -19,13 +20,13 @@ public class RandomAlignment implements InputAlignment { @Override public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node, double maxCost) { - final HashMap map = new HashMap<>(); + final HashMap map = new HashMap<>(); for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( random.nextInt( pathFileLocationTriple.locations.size() ) ); - final String nodeIdentifier = locationWrapper.getLocation().getIdentifier(); - final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(nodeIdentifier, k -> new AlignmentWrapper() ); + final Location location = locationWrapper.getLocation(); + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); } return new FileAlignment( map, inputsOfTask.getSymlinks(), 0); diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 6514d660..2587de12 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -1,5 +1,6 @@ package fonda.scheduler.util; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.SymlinkInput; @@ -13,11 +14,11 @@ public class FileAlignment { Key: node Value: Files from the node */ - public final Map nodeFileAlignment; + public final Map nodeFileAlignment; public final List symlinks; public final double cost; - public FileAlignment(Map nodeFileAlignment, List symlinks, double cost) { + public FileAlignment(Map nodeFileAlignment, List symlinks, double cost) { this.nodeFileAlignment = nodeFileAlignment; this.symlinks = symlinks; this.cost = cost; From d961263e5e4ba8ade1e181b6acdc9bdbb7587115 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:37:14 +0200 Subject: [PATCH 210/443] Fixed Bug: Format error in ftp.py Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 21edd4ea..99d9c81c 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -33,7 +33,7 @@ def getIP( node ): def clearLocatation( path ): if os.path.exists( path ): - log.debug( "Delete",path ) + log.debug( "Delete %s",path ) if os.path.islink( path ): os.unlink( path ) elif os.path.isdir( path ): From bd321d649766f138197a918ea2c417b21432fdf4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:38:51 +0200 Subject: [PATCH 211/443] Added cost functions Signed-off-by: Lehmann_Fabian --- .../costfunctions/CostFunction.java | 17 ++++++++++++ .../costfunctions/MinSizeCost.java | 27 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java new file mode 100644 index 00000000..a7737367 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java @@ -0,0 +1,17 @@ +package fonda.scheduler.scheduler.filealignment.costfunctions; + +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.util.AlignmentWrapper; + +public interface CostFunction { + + double getInitCost(); + double getCost( LocationWrapper fileToTest ); + + default double calculateCost(AlignmentWrapper alignment, LocationWrapper fileToTest ) { + double currentCost = alignment == null || alignment.empty() ? getInitCost() : alignment.getCost(); + return currentCost + getCost( fileToTest ); + } + + +} diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java new file mode 100644 index 00000000..4b2a3911 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java @@ -0,0 +1,27 @@ +package fonda.scheduler.scheduler.filealignment.costfunctions; + +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.util.AlignmentWrapper; +import fonda.scheduler.util.FilePath; + +import java.util.List; + +public class MinSizeCost implements CostFunction { + + private final double initCost; + + public MinSizeCost(double initCost) { + this.initCost = initCost; + } + + @Override + public double getInitCost() { + return initCost; + } + + @Override + public double getCost(LocationWrapper fileToTest) { + return fileToTest.getSizeInBytes(); + } + +} From 055e87ac7abd7824378b638646e74ef46808ccae Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:39:07 +0200 Subject: [PATCH 212/443] Added Greedy Alignment Signed-off-by: Lehmann_Fabian --- .../filealignment/GreedyAlignment.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java new file mode 100644 index 00000000..061c8599 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -0,0 +1,52 @@ +package fonda.scheduler.scheduler.filealignment; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import fonda.scheduler.util.AlignmentWrapper; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.FilePath; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; + +public class GreedyAlignment implements InputAlignment { + + private final CostFunction cf; + + public GreedyAlignment(CostFunction cf) { + this.cf = cf; + } + + @Override + public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, double maxCost) { + inputsOfTask.sort(); + final HashMap map = new HashMap<>(); + double cost = 0; + for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { + double minCost = Double.MAX_VALUE; + Location bestLoc = null; + LocationWrapper bestLocationWrapper = null; + for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { + final Location currentLoc = locationWrapper.getLocation(); + final AlignmentWrapper alignmentWrapper = map.get(currentLoc); + final double calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); + if ( calculatedCost < minCost ) { + bestLoc = currentLoc; + minCost = calculatedCost; + bestLocationWrapper = locationWrapper; + } + } + if ( minCost > maxCost ) return null; + if ( minCost > cost ) cost = minCost; + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); + alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost ); + } + return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); + } + +} From dd218ea95e7a7f250580fdec210ceaca6b207232 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:40:23 +0200 Subject: [PATCH 213/443] Use Greedy Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/rest/SchedulerRestController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index ebba17b6..f39a71ef 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -9,7 +9,9 @@ import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.*; +import fonda.scheduler.scheduler.filealignment.GreedyAlignment; import fonda.scheduler.scheduler.filealignment.RandomAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; @@ -76,7 +78,7 @@ private ResponseEntity noSchedulerFor( String execution ){ @PutMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { - log.trace("Register execution: {} strategy: {} config: {}", execution, strategy, config); + log.info("Register execution: {} strategy: {} config: {}", execution, strategy, config); Scheduler scheduler; @@ -97,7 +99,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa case "lav1" : if ( !config.locationAware ) return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); - scheduler = new LASchedulerV1( execution, client, namespace, config, new RandomAlignment() ); + scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(new MinSizeCost(0)) ); break; default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); From ce6677702c64a85a17f51cd4c10e8e79ca568e30 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:42:25 +0200 Subject: [PATCH 214/443] CanSchedule method for node Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 4 ++++ src/main/java/fonda/scheduler/scheduler/Scheduler.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 8122e984..6ab5d9f2 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -102,6 +102,10 @@ public int compareTo(NodeWithAlloc o) { } } + public boolean canScheduleNewPod(){ + return isReady() && ( getSpec().getUnschedulable() == null || !getSpec().getUnschedulable() ); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 07b51a0b..91ec6959 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -276,7 +276,9 @@ public TaskState getTaskState(String taskid) { * @return */ boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { - return availableByNode.higherOrEquals( pod.getRequest() ) && affinitiesMatch( pod, node ); + return node.canScheduleNewPod() + && availableByNode.higherOrEquals( pod.getRequest() ) + && affinitiesMatch( pod, node ); } boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ From bae3c06a2cfa935905c27ab650df08508f67ad16 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 17:43:29 +0200 Subject: [PATCH 215/443] Reuse getAvailable nodes for validSchedulerPlan Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/Scheduler.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 91ec6959..ec8d6702 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -119,14 +119,11 @@ void undoTaskScheduling( Task task ){} public boolean validSchedulePlan( List taskNodeAlignment ){ - List items = getNodeList(); - Map< NodeWithAlloc, Requirements> availableByNode = new HashMap<>(); - for ( NodeWithAlloc item : items ) { - final Requirements availableResources = item.getAvailableResources(); - availableByNode.put(item, availableResources); - } + Map< NodeWithAlloc, Requirements> availableByNode = getAvailableByNode(); for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { - availableByNode.get(nodeTaskAlignment.node).subFromThis(nodeTaskAlignment.task.getPod().getRequest()); + final Requirements requirements = availableByNode.get(nodeTaskAlignment.node); + if ( requirements == null ) return false; + requirements.subFromThis(nodeTaskAlignment.task.getPod().getRequest()); } for ( Map.Entry e : availableByNode.entrySet() ) { if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) return false; From 396f07e27e5de50149879b1f8efc3eaa5c5c40dc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 18:10:31 +0200 Subject: [PATCH 216/443] Prefer current node for files Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/filealignment/GreedyAlignment.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index 061c8599..2ef7a496 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -33,6 +33,12 @@ public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs i LocationWrapper bestLocationWrapper = null; for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { final Location currentLoc = locationWrapper.getLocation(); + if ( currentLoc != node.getNodeLocation() ) { + minCost = 0; + bestLoc = currentLoc; + bestLocationWrapper = locationWrapper; + break; + } final AlignmentWrapper alignmentWrapper = map.get(currentLoc); final double calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); if ( calculatedCost < minCost ) { From 421a011e6cb3cfa64d493f6f9acb113698dd3ef5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 18:11:09 +0200 Subject: [PATCH 217/443] Fixed problem current node was not recognized. Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 7a28dd0c..986f9e11 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -200,7 +200,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { long filesNotOnNodeByte = 0; for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { - if( entry.getKey().equals( alignment.node.getMetadata().getName() ) ) { + if( entry.getKey() == alignment.node.getNodeLocation() ) { if (traceEnabled) { final List alignment1 = entry.getValue().getAlignment(); filesOnNode = alignment1.size(); From c2fb7c466fed933ac6adf12b5296e4a66479e1a9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 4 May 2022 18:11:28 +0200 Subject: [PATCH 218/443] Refactored SchedulerWithDaemonSet Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 986f9e11..e877528b 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -83,11 +83,6 @@ private void freeLocations( List locationWrappers ){ locationWrappers.parallelStream().forEach( LocationWrapper::free ); } - private void calculateTrace( NodeTaskFilesAlignment nodeTaskFilesAlignment ){ - final Task task = nodeTaskFilesAlignment.task; - - } - @Override boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; @@ -100,10 +95,10 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); useLocations( allLocationWrappers ); - calculateTrace( nodeTaskFilesAlignment ); return super.assignTaskToNode( alignment ); } + @Override void undoTaskScheduling( Task task ){ if ( task.getInputFiles() != null ) { freeLocations( task.getInputFiles() ); @@ -347,7 +342,7 @@ private void handleProblematicInit( Task task ){ String file = this.localWorkDir + "/sync/" + task.getConfig().getHash(); try { Map wrapperByPath = new HashMap<>(); - task.getCopiedFiles().stream().forEach( x -> wrapperByPath.put( x.getPath(), x )); + task.getCopiedFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); log.info( "Get daemon on node {}; daemons: {}", task.getNode().getIdentifier(), daemonByNode ); final InputStream inputStream = getConnection(getDaemonOnNode(task.getNode().getIdentifier())).retrieveFileStream(file); if (inputStream == null) { @@ -355,7 +350,7 @@ private void handleProblematicInit( Task task ){ return; } Scanner scanner = new Scanner(inputStream); - Set openedFiles = new HashSet(); + Set openedFiles = new HashSet<>(); while( scanner.hasNext() ){ String line = scanner.nextLine(); if ( line.startsWith( "S-" ) ){ From 2350dfa486bc5e451b30cb19c21eddfcac6f0417 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 5 May 2022 11:09:59 +0200 Subject: [PATCH 219/443] Trace location aware scheduling Signed-off-by: Lehmann_Fabian --- .../scheduler/model/tracing/TraceRecord.java | 37 ++++++++++++++++--- .../scheduler/LocationAwareScheduler.java | 28 ++++++++++++-- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java index 81008e59..b84486db 100644 --- a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java @@ -6,6 +6,8 @@ import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; public class TraceRecord { @@ -40,15 +42,31 @@ public class TraceRecord { @Getter @Setter - private long schedulerTimeInQueue = -1; + private Long schedulerTimeInQueue = null; @Getter @Setter - private int schedulerPlaceInQueue = -1; + private Integer schedulerPlaceInQueue = null; @Getter @Setter - private int schedulerLocationCount = -1; + private Integer schedulerLocationCount = null; + + @Getter + @Setter + private Integer schedulerNodesTried = null; + + @Getter + @Setter + private List schedulerNodesCost = null; + + @Getter + @Setter + private Integer schedulerCouldStopFetching = null; + + @Getter + @Setter + private Double schedulerBestCost = null; private int schedulerTriedToSchedule = 0; @@ -74,6 +92,10 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_time_in_queue", schedulerTimeInQueue, bw); writeValue("scheduler_place_in_queue", schedulerPlaceInQueue, bw); writeValue("scheduler_location_count", schedulerLocationCount, bw); + writeValue("scheduler_nodes_tried", schedulerNodesTried, bw); + writeValue("scheduler_nodes_cost", schedulerNodesCost, bw); + writeValue("scheduler_could_stop_fetching", schedulerCouldStopFetching, bw); + writeValue("scheduler_best_cost", schedulerBestCost, bw); writeValue("scheduler_tried_to_schedule", schedulerTriedToSchedule, bw); writeValue("scheduler_nodes_to_copy_from", schedulerNodesToCopyFrom, bw); writeValue("scheduler_time_to_schedule", schedulerTimeToSchedule, bw); @@ -81,15 +103,18 @@ public void writeRecord( String tracePath ) throws IOException { } - private void writeValue( String name, Long value, BufferedWriter bw ) throws IOException { + private void writeValue( String name, T value, BufferedWriter bw ) throws IOException { if ( value != null ) { bw.write( name + '=' + value + '\n' ); } } - private void writeValue( String name, Integer value, BufferedWriter bw ) throws IOException { + private void writeValue( String name, List value, BufferedWriter bw ) throws IOException { if ( value != null ) { - bw.write( name + '=' + value + '\n' ); + final String collect = value.stream() + .map( x -> x==null ? "null" : x.toString() ) + .collect(Collectors.joining(";")); + bw.write( name + "=\"" + collect + "\"\n" ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 3cc5d63c..fc2c1ad8 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -3,6 +3,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.scheduler.data.NodeDataTuple; import fonda.scheduler.scheduler.data.TaskData; import fonda.scheduler.scheduler.filealignment.InputAlignment; @@ -97,25 +98,44 @@ Tuple calculateBestNode( NodeWithAlloc bestNode = null; //Remove all nodes which do not fit anymore final List nodeDataTuples = taskData.getNodeDataTuples(); - + int triedNodes = 0; + int couldStopFetching = 0; + final List costs = traceEnabled ? new LinkedList<>() : null; for (NodeDataTuple nodeDataTuple : nodeDataTuples) { final NodeWithAlloc currentNode = nodeDataTuple.getNode(); if ( matchingNodesForTask.contains(currentNode) ) { + triedNodes++; final FileAlignment fileAlignment = inputAlignment.getInputAlignment( taskData.getTask(), taskData.getMatchingFilesAndNodes().getInputsOfTask(), currentNode, bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost ); - - if ( fileAlignment != null && (bestAlignment == null || bestAlignment.cost < fileAlignment.cost ) ){ + if ( fileAlignment == null ){ + couldStopFetching++; + } else if ( bestAlignment == null || bestAlignment.cost < fileAlignment.cost ){ bestAlignment = fileAlignment; bestNode = currentNode; + log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); + } + if ( traceEnabled ) { + costs.add(fileAlignment == null ? null : fileAlignment.cost); } } } - return bestAlignment == null ? null : new Tuple<>( bestNode, bestAlignment ); + if ( bestAlignment != null ){ + if ( traceEnabled ){ + final TraceRecord record = taskData.getTask().getTraceRecord(); + record.setSchedulerNodesTried( triedNodes ); + record.setSchedulerNodesCost( costs ); + record.setSchedulerCouldStopFetching( couldStopFetching ); + record.setSchedulerBestCost( bestAlignment.cost ); + } + return new Tuple<>( bestNode, bestAlignment ); + } else { + return null; + } } From 9307b14314c14fcd6e32d41eaeaf730f2c1cb1de Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 5 May 2022 11:41:23 +0200 Subject: [PATCH 220/443] Cost function configurable Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/SchedulerConfig.java | 10 ++++++---- .../scheduler/rest/SchedulerRestController.java | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 14e620f3..2d4d42ea 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -12,18 +12,19 @@ public class SchedulerConfig { public final String workDir; public final String dns; public final String copyStrategy; - public final boolean locationAware; - public final boolean traceEnabled; + public final String costFunction; + public SchedulerConfig(List localClaims, List volumeClaims, String workDir, String dns, String copyStrategy, boolean locationAware, - boolean traceEnabled) { + boolean traceEnabled, + String costFunction) { this.localClaims = localClaims; this.volumeClaims = volumeClaims; this.workDir = workDir; @@ -31,10 +32,11 @@ public SchedulerConfig(List localClaims, this.copyStrategy = copyStrategy; this.locationAware = locationAware; this.traceEnabled = traceEnabled; + this.costFunction = costFunction; } private SchedulerConfig(){ - this(null,null,null,null,null,false,false); + this(null,null,null,null,null,false,false, null); } @ToString diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index f39a71ef..da1c54f7 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -11,6 +11,7 @@ import fonda.scheduler.scheduler.*; import fonda.scheduler.scheduler.filealignment.GreedyAlignment; import fonda.scheduler.scheduler.filealignment.RandomAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import lombok.extern.slf4j.Slf4j; import org.javatuples.Pair; @@ -78,7 +79,7 @@ private ResponseEntity noSchedulerFor( String execution ){ @PutMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { - log.info("Register execution: {} strategy: {} config: {}", execution, strategy, config); + log.info("Register execution: {} strategy: {} cf: {} config: {}", execution, strategy, config.costFunction, config); Scheduler scheduler; @@ -88,6 +89,14 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa return noSchedulerFor( execution ); } + CostFunction costFunction = null; + if ( config.costFunction != null ) { + switch (config.costFunction.toLowerCase()) { + case "minsize": costFunction = new MinSizeCost(0); break; + default: return new ResponseEntity<>( "No cost function: " + config.costFunction, HttpStatus.NOT_FOUND ); + } + } + switch ( strategy.toLowerCase() ){ case "fifo" : case "random" : @@ -99,7 +108,8 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa case "lav1" : if ( !config.locationAware ) return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); - scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(new MinSizeCost(0)) ); + if ( costFunction == null ) costFunction = new MinSizeCost( 0 ); + scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(costFunction) ); break; default: return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); From 9f78db55bf2b9909f1d78beeb220b61a69459e07 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 5 May 2022 11:53:51 +0200 Subject: [PATCH 221/443] Fixed potential bug Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/schedulingstrategy/Inputs.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index e0151875..f3293e5a 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -29,9 +29,9 @@ public Inputs( String dns, String syncDir, String hash ) { public void waitForTask( Map waitForTask ){ for (Map.Entry e : waitForTask.entrySet()) { final String hash = e.getValue().getConfig().getHash(); - if ( !waitForFilesOfTask.containsValue( hash ) ) waitForFilesOfTask.put( hash, new LinkedList<>() ); - final List listOfPathes = waitForFilesOfTask.get(hash); - listOfPathes.add( e.getKey() ); + if ( !waitForFilesOfTask.containsKey( hash ) ) waitForFilesOfTask.put( hash, new LinkedList<>() ); + final List listOfPaths = waitForFilesOfTask.get(hash); + listOfPaths.add( e.getKey() ); } } From 609b843493c0c4c99a1603690da4e84579d8f4b4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 5 May 2022 15:15:24 +0200 Subject: [PATCH 222/443] use computeIfAbsent Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/schedulingstrategy/Inputs.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index f3293e5a..4b5ca977 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -28,9 +28,8 @@ public Inputs( String dns, String syncDir, String hash ) { public void waitForTask( Map waitForTask ){ for (Map.Entry e : waitForTask.entrySet()) { - final String hash = e.getValue().getConfig().getHash(); - if ( !waitForFilesOfTask.containsKey( hash ) ) waitForFilesOfTask.put( hash, new LinkedList<>() ); - final List listOfPaths = waitForFilesOfTask.get(hash); + final String taskHash = e.getValue().getConfig().getHash(); + final List listOfPaths = waitForFilesOfTask.computeIfAbsent( taskHash, k -> new LinkedList<>() ); listOfPaths.add( e.getKey() ); } } From feaac48d886735403600ca64687d4bde7caac80f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 5 May 2022 15:54:03 +0200 Subject: [PATCH 223/443] Refactoring Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/NodeWithAlloc.java | 2 +- .../scheduler/model/SchedulerConfig.java | 27 +++----- src/main/java/fonda/scheduler/model/Task.java | 4 +- .../java/fonda/scheduler/model/TaskInput.java | 2 - .../scheduler/model/WriteConfigResult.java | 5 +- .../location/hierachy/RealHierarchyFile.java | 7 +- .../taskinputs/PathFileLocationTriple.java | 6 +- .../scheduler/scheduler/LASchedulerV1.java | 13 +--- .../scheduler/LocationAwareScheduler.java | 67 +++++++++++-------- .../scheduler/RandomLAScheduler.java | 6 +- .../fonda/scheduler/scheduler/Scheduler.java | 2 - .../scheduler/SchedulerWithDaemonSet.java | 10 +-- .../scheduler/copystrategy/CopyStrategy.java | 40 ++++++----- .../scheduler/data/NodeDataTuple.java | 5 +- .../scheduler/scheduler/data/TaskData.java | 2 + .../costfunctions/MinSizeCost.java | 4 -- .../scheduler/util/AlignmentWrapper.java | 2 +- .../fonda/scheduler/util/FileAlignment.java | 2 +- 18 files changed, 95 insertions(+), 111 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 6ab5d9f2..3c0b13a0 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -17,7 +17,7 @@ @Slf4j public class NodeWithAlloc extends Node implements Comparable { - private final KubernetesClient kubernetesClient; + private final transient KubernetesClient kubernetesClient; private static final long serialVersionUID = 1L; diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 2d4d42ea..615638fc 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -17,26 +17,15 @@ public class SchedulerConfig { public final String costFunction; - public SchedulerConfig(List localClaims, - List volumeClaims, - String workDir, - String dns, - String copyStrategy, - boolean locationAware, - boolean traceEnabled, - String costFunction) { - this.localClaims = localClaims; - this.volumeClaims = volumeClaims; - this.workDir = workDir; - this.dns = dns; - this.copyStrategy = copyStrategy; - this.locationAware = locationAware; - this.traceEnabled = traceEnabled; - this.costFunction = costFunction; - } - private SchedulerConfig(){ - this(null,null,null,null,null,false,false, null); + this.localClaims = null; + this.volumeClaims = null; + this.workDir = null; + this.dns = null; + this.copyStrategy = null; + this.locationAware = false; + this.traceEnabled = false; + this.costFunction = null; } @ToString diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 2e668dc6..a5f20a8b 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -12,8 +12,8 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import java.util.HashMap; import java.util.List; +import java.util.Map; @Slf4j public class Task { @@ -48,7 +48,7 @@ public class Task { @Getter @Setter - private HashMap< String, Tuple> copyingToNode; + private Map< String, Tuple> copyingToNode; @Getter private final TraceRecord traceRecord = new TraceRecord(); diff --git a/src/main/java/fonda/scheduler/model/TaskInput.java b/src/main/java/fonda/scheduler/model/TaskInput.java index 817bf46f..8e662482 100644 --- a/src/main/java/fonda/scheduler/model/TaskInput.java +++ b/src/main/java/fonda/scheduler/model/TaskInput.java @@ -1,7 +1,5 @@ package fonda.scheduler.model; -import lombok.ToString; - import java.util.List; public class TaskInput { diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java index ffbdac20..963adb0a 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -4,7 +4,6 @@ import fonda.scheduler.util.Tuple; import lombok.Getter; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -13,12 +12,12 @@ public class WriteConfigResult { private final List inputFiles; private final Map waitForTask; - private final HashMap< String, Tuple> copyingToNode; + private final Map< String, Tuple> copyingToNode; public WriteConfigResult( List inputFiles, Map waitForTask, - HashMap> copyingToNode + Map> copyingToNode ) { this.inputFiles = inputFiles; this.waitForTask = waitForTask; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index e4a94871..404a12bb 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -17,9 +17,10 @@ public class RealHierarchyFile extends AbstractHierarchyFile { */ @Getter private LocationWrapper[] locations; + static final String LOCATION_IS_NULL = "location is null"; public RealHierarchyFile(LocationWrapper location ) { - if ( location == null ) throw new IllegalArgumentException( "location is null" ); + if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); this.locations = new LocationWrapper[]{ location }; } @@ -34,7 +35,7 @@ public boolean isSymlink() { } public void removeLocation( LocationWrapper location ){ - if ( location == null ) throw new IllegalArgumentException( "location is null" ); + if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); synchronized ( this ){ for (int i = 0; i < locations.length; i++) { if ( location.getLocation().equals( locations[i].getLocation() ) ) { @@ -45,7 +46,7 @@ public void removeLocation( LocationWrapper location ){ } public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ - if ( location == null ) throw new IllegalArgumentException( "location is null" ); + if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); synchronized ( this ){ LocationWrapper locationWrapperToUpdate = null; for (int i = 0; i < locations.length; i++) { diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index a382851d..50ca783a 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -26,11 +26,11 @@ public PathFileLocationTriple(Path path, RealHierarchyFile file, List calculateBestNode( int couldStopFetching = 0; final List costs = traceEnabled ? new LinkedList<>() : null; for (NodeDataTuple nodeDataTuple : nodeDataTuples) { + final NodeWithAlloc currentNode = nodeDataTuple.getNode(); - if ( matchingNodesForTask.contains(currentNode) ) { + if ( !matchingNodesForTask.contains(currentNode) ) continue; + + final FileAlignment fileAlignment = inputAlignment.getInputAlignment( + taskData.getTask(), + taskData.getMatchingFilesAndNodes().getInputsOfTask(), + currentNode, + bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost + ); + if ( fileAlignment == null ){ + couldStopFetching++; + } else if ( bestAlignment == null || bestAlignment.cost < fileAlignment.cost ){ + bestAlignment = fileAlignment; + bestNode = currentNode; + log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); + } + if ( traceEnabled ) { triedNodes++; - final FileAlignment fileAlignment = inputAlignment.getInputAlignment( - taskData.getTask(), - taskData.getMatchingFilesAndNodes().getInputsOfTask(), - currentNode, - bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost - ); - if ( fileAlignment == null ){ - couldStopFetching++; - } else if ( bestAlignment == null || bestAlignment.cost < fileAlignment.cost ){ - bestAlignment = fileAlignment; - bestNode = currentNode; - log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); - } - if ( traceEnabled ) { - costs.add(fileAlignment == null ? null : fileAlignment.cost); - } + final Double thisRoundCost = fileAlignment == null + ? null + : fileAlignment.cost; + costs.add( thisRoundCost ); } } - if ( bestAlignment != null ){ - if ( traceEnabled ){ - final TraceRecord record = taskData.getTask().getTraceRecord(); - record.setSchedulerNodesTried( triedNodes ); - record.setSchedulerNodesCost( costs ); - record.setSchedulerCouldStopFetching( couldStopFetching ); - record.setSchedulerBestCost( bestAlignment.cost ); - } - return new Tuple<>( bestNode, bestAlignment ); - } else { - return null; - } + if ( bestAlignment == null ) return null; + storeTraceData( + taskData.getTask().getTraceRecord(), + triedNodes, + costs, + couldStopFetching, + bestAlignment.cost + ); + return new Tuple<>( bestNode, bestAlignment ); + } + + private void storeTraceData( final TraceRecord traceRecord, int triedNodes, List costs, int couldStopFetching, double bestCost ){ + if ( !traceEnabled ) return; + traceRecord.setSchedulerNodesTried( triedNodes ); + traceRecord.setSchedulerNodesCost( costs ); + traceRecord.setSchedulerCouldStopFetching( couldStopFetching ); + traceRecord.setSchedulerBestCost( bestCost ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index f954e7ae..e5e6c375 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -37,11 +37,14 @@ private Optional selectNode( Set matchingNodes, Ta : Optional.of( new LinkedList<>(matchingNodes).get(random.nextInt(matchingNodes.size()))); } + @Override Tuple calculateBestNode( final TaskData taskData, final Set matchingNodesForTask ){ - final NodeWithAlloc node = selectNode( matchingNodesForTask, taskData.getTask() ).get(); + final Optional nodeWithAlloc = selectNode(matchingNodesForTask, taskData.getTask()); + if (nodeWithAlloc.isEmpty()) return null; + final NodeWithAlloc node = nodeWithAlloc.get(); final FileAlignment fileAlignment = getInputAlignment().getInputAlignment( taskData.getTask(), taskData.getMatchingFilesAndNodes().getInputsOfTask(), @@ -50,6 +53,7 @@ Tuple calculateBestNode( return new Tuple<>( node, fileAlignment ); } + @Override TaskData calculateTaskData( final Task task, final Map availableByNode diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index ec8d6702..cb1bb228 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -315,8 +315,6 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ try(BufferedWriter printWriter = new BufferedWriter( new FileWriter( nodeFile ))){ printWriter.write( alignment.node.getName() ); printWriter.write( '\n' ); - } catch (FileNotFoundException e) { - e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index e877528b..084fbe0d 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -145,19 +145,19 @@ int terminateTasks(List finishedTasks) { return 0; } - private final Map< NodeLocation, HashMap< String, Tuple> > copyingToNode = new HashMap<>(); + private final Map< NodeLocation, Map< String, Tuple> > copyingToNode = new HashMap<>(); - private void addToCopyingToNode( NodeLocation nodeLocation, HashMap< String, Tuple > toAdd ){ + private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple > toAdd ){ if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); if ( copyingToNode.containsKey( nodeLocation ) ){ - final HashMap> stringTupleHashMap = copyingToNode.get( nodeLocation ); + final Map> stringTupleHashMap = copyingToNode.get( nodeLocation ); stringTupleHashMap.putAll( toAdd ); } else { copyingToNode.put( nodeLocation, toAdd ); } } - private void removeFromCopyingToNode(NodeLocation nodeLocation, HashMap< String, Tuple> toRemove ){ + private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tuple> toRemove ){ if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); copyingToNode.get( nodeLocation ).keySet().removeAll( toRemove.keySet() ); } @@ -183,7 +183,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final HashMap> filesForCurrentNode = new HashMap<>(); final NodeLocation currentNode = alignment.node.getNodeLocation(); - final HashMap> filesOnCurrentNode = copyingToNode.get(currentNode); + final Map> filesOnCurrentNode = copyingToNode.get(currentNode); final TraceRecord traceRecord = alignment.task.getTraceRecord(); diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java index a508d60f..23bec958 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -16,34 +16,32 @@ public void generateCopyScript( Task task ){ File file = new File(task.getWorkingDir() + '/' + ".command.init.run"); try (BufferedWriter pw = new BufferedWriter( new FileWriter( file) ) ) { + write( pw, file ); + } catch (IOException e) { + e.printStackTrace(); + } - ClassLoader classLoader = getClass().getClassLoader(); - - try (InputStream inputStream = classLoader.getResourceAsStream( getResource() )) { - assert inputStream != null; - try (InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(streamReader)) { - - String line; - while ((line = reader.readLine()) != null) { - pw.write( line ); - pw.write( '\n' ); - } + } - Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); - Files.setPosixFilePermissions( file.toPath(), executable ); + private void write( BufferedWriter pw, File file ){ + ClassLoader classLoader = getClass().getClassLoader(); + try (InputStream inputStream = classLoader.getResourceAsStream( getResource() )) { + assert inputStream != null; + try (InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(streamReader)) { + + String line; + while ((line = reader.readLine()) != null) { + pw.write( line ); + pw.write( '\n' ); } - } catch (IOException e) { - e.printStackTrace(); - } - - } catch (FileNotFoundException e) { - e.printStackTrace(); + Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); + Files.setPosixFilePermissions( file.toPath(), executable ); + } } catch (IOException e) { e.printStackTrace(); } - } abstract String getResource(); diff --git a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java index 313dc597..4462d9ed 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java +++ b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java @@ -1,12 +1,13 @@ package fonda.scheduler.scheduler.data; import fonda.scheduler.model.NodeWithAlloc; +import lombok.EqualsAndHashCode; import lombok.Getter; import org.jetbrains.annotations.NotNull; @Getter -public -class NodeDataTuple implements Comparable { +@EqualsAndHashCode +public class NodeDataTuple implements Comparable { private final NodeWithAlloc node; private final long sizeInBytes; diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index cfb6d948..87a19933 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -2,12 +2,14 @@ import fonda.scheduler.model.Task; import fonda.scheduler.scheduler.MatchingFilesAndNodes; +import lombok.EqualsAndHashCode; import lombok.Getter; import org.jetbrains.annotations.NotNull; import java.util.List; @Getter +@EqualsAndHashCode public class TaskData implements Comparable { private final double value; private final Task task; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java index 4b2a3911..13dd11e3 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java @@ -1,10 +1,6 @@ package fonda.scheduler.scheduler.filealignment.costfunctions; import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FilePath; - -import java.util.List; public class MinSizeCost implements CostFunction { diff --git a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java index 1548f73b..981815a6 100644 --- a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java +++ b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java @@ -8,7 +8,7 @@ @Getter public class AlignmentWrapper { - private final List alignment = new LinkedList(); + private final List alignment = new LinkedList<>(); private double cost = 0; public void addAlignment( FilePath filePath, double cost ) { diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 2587de12..68f8e7ca 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -32,7 +32,7 @@ public List getAllLocationWrappers(){ .getValue() .getAlignment() .parallelStream() - .map( p -> p.getLocationWrapper() ) + .map(FilePath::getLocationWrapper) ) .collect(Collectors.toList()); } From 8967d9208cf94185185bcbb324db4dc0ae08acac Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 5 May 2022 16:56:01 +0200 Subject: [PATCH 224/443] Logging instead of printStackTrace Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/TaskResultParser.java | 4 ++-- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 2 +- .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 8 ++++---- .../scheduler/scheduler/copystrategy/CopyStrategy.java | 6 ++++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index b397e43f..3f3ed968 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -32,7 +32,7 @@ private String getRootDir( File file ){ try ( Scanner sc = new Scanner( file ) ) { if( sc.hasNext() ) return sc.next().split(";")[0]; } catch (FileNotFoundException e) { - e.printStackTrace(); + log.error( "Cannot read " + file, e); } return null; } @@ -126,7 +126,7 @@ public Set getNewAndUpdatedFiles( return processOutput( out, inputdata, location, onlyUpdated, finishedTask, outputRootDir ); } catch (IOException e) { - e.printStackTrace(); + log.error( "Cannot read in/outfile in workdir: " + workdir, e); } return new HashSet<>(); diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index cb1bb228..3da63f7a 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -316,7 +316,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ printWriter.write( alignment.node.getName() ); printWriter.write( '\n' ); } catch (IOException e) { - e.printStackTrace(); + log.error( "Cannot read " + nodeFile, e); } alignment.task.setNode( alignment.node.getNodeLocation() ); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 084fbe0d..855ee29c 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -261,7 +261,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { new ObjectMapper().writeValue( config, inputs ); return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode ); } catch (IOException e) { - e.printStackTrace(); + log.error( "Cannot write " + config, e); } return null; } @@ -366,8 +366,7 @@ private void handleProblematicInit( Task task ){ log.info("task {}, file: {} deactivated on node {}", task.getConfig().getName(), openedFile, wrapperByPath.get( openedFile ).getWrapper().getLocation()); } } catch ( Exception e ){ - log.info( "Can't handle failed init from pod " + task.getPod().getName()); - e.printStackTrace(); + log.error( "Can't handle failed init from pod " + task.getPod().getName(), e); } } @@ -386,7 +385,8 @@ private FTPClient getConnection( String daemon ){ try { Thread.sleep((long) Math.pow(2, trial++)); } catch (InterruptedException ex) { - ex.printStackTrace(); + Thread.currentThread().interrupt(); + log.error( "Interrupted while waiting for retry to connect to FTP client", e); } } } diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java index 23bec958..5f43bbb0 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -1,6 +1,7 @@ package fonda.scheduler.scheduler.copystrategy; import fonda.scheduler.model.Task; +import lombok.extern.slf4j.Slf4j; import java.io.*; import java.nio.charset.StandardCharsets; @@ -9,6 +10,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; +@Slf4j public abstract class CopyStrategy { public void generateCopyScript( Task task ){ @@ -18,7 +20,7 @@ public void generateCopyScript( Task task ){ try (BufferedWriter pw = new BufferedWriter( new FileWriter( file) ) ) { write( pw, file ); } catch (IOException e) { - e.printStackTrace(); + log.error( "Cannot write " + file, e); } } @@ -40,7 +42,7 @@ private void write( BufferedWriter pw, File file ){ Files.setPosixFilePermissions( file.toPath(), executable ); } } catch (IOException e) { - e.printStackTrace(); + log.error( "Cannot write " + file, e); } } From f4ca27424ed45b2eb74d719a98194f17c3858623 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 May 2022 10:09:35 +0200 Subject: [PATCH 225/443] Search for best alignment Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/LocationAwareScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 09ae3aec..bb5496bb 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -114,7 +114,7 @@ Tuple calculateBestNode( ); if ( fileAlignment == null ){ couldStopFetching++; - } else if ( bestAlignment == null || bestAlignment.cost < fileAlignment.cost ){ + } else if ( bestAlignment == null || bestAlignment.cost > fileAlignment.cost ){ bestAlignment = fileAlignment; bestNode = currentNode; log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); From b9eff4b1dc97c201bc15200007892d0e1bc594d0 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 May 2022 10:24:37 +0200 Subject: [PATCH 226/443] Refactored code Signed-off-by: Lehmann_Fabian --- .../scheduler/filealignment/GreedyAlignment.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index 2ef7a496..4ae6799c 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -33,14 +33,13 @@ public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs i LocationWrapper bestLocationWrapper = null; for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { final Location currentLoc = locationWrapper.getLocation(); + final double calculatedCost; if ( currentLoc != node.getNodeLocation() ) { - minCost = 0; - bestLoc = currentLoc; - bestLocationWrapper = locationWrapper; - break; + final AlignmentWrapper alignmentWrapper = map.get(currentLoc); + calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); + } else { + calculatedCost = 0; } - final AlignmentWrapper alignmentWrapper = map.get(currentLoc); - final double calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); if ( calculatedCost < minCost ) { bestLoc = currentLoc; minCost = calculatedCost; From 865f263769e60097aad293ad5a7c71b4a3f454c8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 May 2022 13:51:32 +0200 Subject: [PATCH 227/443] Refactored ftp.py script v1 Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 66 ++++++++++++------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 99d9c81c..3e80d0e9 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -9,6 +9,7 @@ import signal import logging as log +exitIfFileWasNotFound = False CLOSE = False EXIT = 0 log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) @@ -41,41 +42,46 @@ def clearLocatation( path ): else: os.remove( path ) +def getFTP( node ): + connectionProblem = 0 + while( connectionProblem < 8 ): + try: + ip = getIP( node ) + log.info( "Try to connect to %s", ip ) + ftp = ftplib.FTP( ip ) + ftp.login("ftp", "pythonclient") + ftp.set_pasv(True) + ftp.encoding='utf-8' + log.info( "Connection established" ) + return ftp + except ConnectionRefusedError as err: + log.warning( "Connection refused! Try again..." ) + except BaseException as err: + log.exception( "Unexpected error" ) + connectionProblem += 1 + time.sleep(2**connectionProblem) + myExit( 8 ) + +def closeFTP( ftp ): + if ftp is None: + return + try: + ftp.quit() + ftp.close() + except BaseException as err: + log.exception( "Unexpected error" ) + def download( node, files, syncFile ): ftp = None size = len(files) - connectionProblem = 0 global CLOSE while not CLOSE and len(files) > 0: - if connectionProblem > 1: - ftp = None - if connectionProblem == 8: - myExit( 8 ) - log.warning( "Connection refused! Try again..." ) - time.sleep(2**connectionProblem) - if ftp is None: - try: - connectionProblem += 1 - ip = getIP( node ) - log.info( "Try to connect to %s", ip ) - ftp = ftplib.FTP( ip ) - ftp.login("ftp", "pythonclient") - ftp.set_pasv(True) - ftp.encoding='utf-8' - log.info( "Connection established" ) - connectionProblem = 0 - except ConnectionRefusedError as err: - continue - except BaseException as err: - log.exception( "Unexpected error" ) - continue - - + ftp = getFTP( node ) filename = files[0] index = size - len(files) + 1 log.info( "Download [%s/%s] - %s", str( index ).rjust( len( str( size ) ) ), str( size ), filename ) @@ -106,12 +112,8 @@ def download( node, files, syncFile ): files.pop(0) syncFile.write("F-" + filename + '\n') - if ftp is not None: - try: - ftp.quit() - ftp.close() - except BaseException as err: - log.exception( "Unexpected error" ) + closeFTP( ftp ) + def waitForFiles( syncFileTask, files, starttime ): #wait max. 10 seconds @@ -155,8 +157,6 @@ def waitForFiles( syncFileTask, files, starttime ): starttime = time.time() log.info( "Start to setup the environment" ) -exitIfFileWasNotFound = False - configFilePath = ".command.inputs.json" if not os.path.isfile( configFilePath ): From 4e107c4b9210ebcc0bd3b3fd143a77a8e476b62a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 May 2022 17:08:08 +0200 Subject: [PATCH 228/443] Refactored ftp.py script v2 Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 86 ++++++++++++------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 3e80d0e9..256f65a0 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -11,6 +11,7 @@ exitIfFileWasNotFound = False CLOSE = False +UNEXPECTED_ERROR = "Unexpected error" EXIT = 0 log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) @@ -21,12 +22,15 @@ def myExit( code ): CLOSE = True exit(EXIT) -def close( signalnum, syncfile ): +def close( signalnum, syncFile ): + log.info( "Killed: %s",str(signalnum)) + closeWithWarning( 50, syncFile ) + +def closeWithWarning( errorCode, syncFile ): syncFile.write('##FAILURE##\n') syncFile.flush() syncFile.close() - log.info( "Killed: %s",str(signalnum)) - myExit(50) + myExit( errorCode ) def getIP( node ): ip = urllib.request.urlopen(dns + node).read() @@ -42,7 +46,7 @@ def clearLocatation( path ): else: os.remove( path ) -def getFTP( node ): +def getFTP( node, syncFile ): connectionProblem = 0 while( connectionProblem < 8 ): try: @@ -54,13 +58,13 @@ def getFTP( node ): ftp.encoding='utf-8' log.info( "Connection established" ) return ftp - except ConnectionRefusedError as err: + except ConnectionRefusedError: log.warning( "Connection refused! Try again..." ) - except BaseException as err: - log.exception( "Unexpected error" ) + except BaseException: + log.exception( UNEXPECTED_ERROR ) connectionProblem += 1 time.sleep(2**connectionProblem) - myExit( 8 ) + closeWithWarning( 8, syncFile ) def closeFTP( ftp ): if ftp is None: @@ -68,52 +72,48 @@ def closeFTP( ftp ): try: ftp.quit() ftp.close() - except BaseException as err: - log.exception( "Unexpected error" ) + except BaseException: + log.exception( UNEXPECTED_ERROR ) -def download( node, files, syncFile ): +def downloadFile( ftp, filename, size, index, node ): + log.info( "Download [%s/%s] - %s", str( index ).rjust( len( str( size ) ) ), str( size ), filename ) + try: + syncFile.write("S-" + filename + '\n') + clearLocatation( filename ) + Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) + ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) + except ftplib.error_perm as err: + if( str(err) == "550 Failed to open file." ): + log.warning( "File not found node: %s file: %s", node, filename ) + if exitIfFileWasNotFound: + closeWithWarning( 40, syncFile ) + except FileNotFoundError: + log.warning( "File not found node: %s file: %s", node, filename ) + if exitIfFileWasNotFound: + closeWithWarning( 41, syncFile ) + except EOFError: + log.warning( "It seems the connection was lost! Try again..." ) + return False + except BaseException: + log.exception( UNEXPECTED_ERROR ) + return False + return True +def download( node, files, syncFile ): ftp = None size = len(files) - global CLOSE - while not CLOSE and len(files) > 0: - if ftp is None: - ftp = getFTP( node ) + ftp = getFTP( node, syncFile ) filename = files[0] index = size - len(files) + 1 - log.info( "Download [%s/%s] - %s", str( index ).rjust( len( str( size ) ) ), str( size ), filename ) - - try: - syncFile.write("S-" + filename + '\n') - clearLocatation( filename ) - Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) - ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) - except ftplib.error_perm as err: - if( str(err) == "550 Failed to open file." ): - log.warning( "File not found node: %s file: %s", node, filename ) - if exitIfFileWasNotFound: - myExit( 40 ) - except FileNotFoundError as err: - log.warning( "File not found node: %s file: %s", node, filename ) - if exitIfFileWasNotFound: - myExit( 41 ) - except EOFError as err: - log.warning( "It seems the connection was lost! Try again..." ) + if not downloadFile( ftp, filename, size, index, node ): ftp = None continue - except BaseException as err: - log.exception( "Unexpected error" ) - ftp = None - continue - files.pop(0) syncFile.write("F-" + filename + '\n') - - closeFTP( ftp ) - + closeFTP( ftp ) def waitForFiles( syncFileTask, files, starttime ): #wait max. 10 seconds @@ -160,7 +160,7 @@ def waitForFiles( syncFileTask, files, starttime ): configFilePath = ".command.inputs.json" if not os.path.isfile( configFilePath ): - log.error( "Config file not found:", configFilePath ) + log.error( "Config file not found: %s", configFilePath ) myExit( 102 ) with open(configFilePath,'r') as configFile: @@ -203,7 +203,7 @@ def waitForFiles( syncFileTask, files, starttime ): signal.signal( signal.SIGINT, lambda signalnum, handler: myExit( 1 ) ) signal.signal( signal.SIGTERM, lambda signalnum, handler: myExit( 1 ) ) -#No check for files of other tasks +#Now check for files of other tasks for waitForTask in config[ "waitForFilesOfTask" ]: waitForFilesSet = set( config[ "waitForFilesOfTask" ][ waitForTask ] ) if not waitForFiles( config[ "syncDir" ] + waitForTask, waitForFilesSet, starttime ): From be3cfc74abfbe3fc0f6869b5a28361871e6282f1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 9 May 2022 16:03:11 +0200 Subject: [PATCH 229/443] Refactored ftp.py script V3 Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 241 +++++++++++++---------- 1 file changed, 140 insertions(+), 101 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 256f65a0..badd84fc 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -15,118 +15,128 @@ EXIT = 0 log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) -def myExit( code ): + +def myExit(code): global EXIT EXIT = code global CLOSE CLOSE = True exit(EXIT) -def close( signalnum, syncFile ): - log.info( "Killed: %s",str(signalnum)) - closeWithWarning( 50, syncFile ) -def closeWithWarning( errorCode, syncFile ): +def close(signalnum, syncFile): + log.info("Killed: %s", str(signalnum)) + closeWithWarning(50, syncFile) + + +def closeWithWarning(errorCode, syncFile): syncFile.write('##FAILURE##\n') syncFile.flush() syncFile.close() - myExit( errorCode ) + myExit(errorCode) -def getIP( node ): + +def getIP(node, dns): ip = urllib.request.urlopen(dns + node).read() - return str(ip.decode("utf-8") ) - -def clearLocatation( path ): - if os.path.exists( path ): - log.debug( "Delete %s",path ) - if os.path.islink( path ): - os.unlink( path ) - elif os.path.isdir( path ): - shutil.rmtree( path ) + return str(ip.decode("utf-8")) + + +def clearLocatation(path): + if os.path.exists(path): + log.debug("Delete %s", path) + if os.path.islink(path): + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) else: - os.remove( path ) + os.remove(path) + -def getFTP( node, syncFile ): +def getFTP(node, dns, syncFile): connectionProblem = 0 - while( connectionProblem < 8 ): + while connectionProblem < 8: try: - ip = getIP( node ) - log.info( "Try to connect to %s", ip ) - ftp = ftplib.FTP( ip ) + ip = getIP(node, dns) + log.info("Try to connect to %s", ip) + ftp = ftplib.FTP(ip) ftp.login("ftp", "pythonclient") ftp.set_pasv(True) - ftp.encoding='utf-8' - log.info( "Connection established" ) + ftp.encoding = 'utf-8' + log.info("Connection established") return ftp except ConnectionRefusedError: - log.warning( "Connection refused! Try again..." ) + log.warning("Connection refused! Try again...") except BaseException: - log.exception( UNEXPECTED_ERROR ) + log.exception(UNEXPECTED_ERROR) connectionProblem += 1 - time.sleep(2**connectionProblem) - closeWithWarning( 8, syncFile ) + time.sleep(2 ** connectionProblem) + closeWithWarning(8, syncFile) + -def closeFTP( ftp ): +def closeFTP(ftp): if ftp is None: return try: ftp.quit() ftp.close() except BaseException: - log.exception( UNEXPECTED_ERROR ) + log.exception(UNEXPECTED_ERROR) -def downloadFile( ftp, filename, size, index, node ): - log.info( "Download [%s/%s] - %s", str( index ).rjust( len( str( size ) ) ), str( size ), filename ) + +def downloadFile(ftp, filename, size, index, node, syncFile): + log.info("Download [%s/%s] - %s", str(index).rjust(len(str(size))), str(size), filename) try: syncFile.write("S-" + filename + '\n') - clearLocatation( filename ) + clearLocatation(filename) Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) - ftp.retrbinary( 'RETR %s' % filename, open( filename, 'wb').write, 102400) + ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) except ftplib.error_perm as err: - if( str(err) == "550 Failed to open file." ): - log.warning( "File not found node: %s file: %s", node, filename ) + if str(err) == "550 Failed to open file.": + log.warning("File not found node: %s file: %s", node, filename) if exitIfFileWasNotFound: - closeWithWarning( 40, syncFile ) + closeWithWarning(40, syncFile) except FileNotFoundError: - log.warning( "File not found node: %s file: %s", node, filename ) + log.warning("File not found node: %s file: %s", node, filename) if exitIfFileWasNotFound: - closeWithWarning( 41, syncFile ) + closeWithWarning(41, syncFile) except EOFError: - log.warning( "It seems the connection was lost! Try again..." ) + log.warning("It seems the connection was lost! Try again...") return False except BaseException: - log.exception( UNEXPECTED_ERROR ) + log.exception(UNEXPECTED_ERROR) return False return True -def download( node, files, syncFile ): + +def download(node, files, dns, syncFile): ftp = None size = len(files) global CLOSE while not CLOSE and len(files) > 0: if ftp is None: - ftp = getFTP( node, syncFile ) + ftp = getFTP(node, dns, syncFile) filename = files[0] index = size - len(files) + 1 - if not downloadFile( ftp, filename, size, index, node ): + if not downloadFile(ftp, filename, size, index, node, syncFile): ftp = None continue files.pop(0) syncFile.write("F-" + filename + '\n') - closeFTP( ftp ) + closeFTP(ftp) -def waitForFiles( syncFileTask, files, starttime ): - #wait max. 10 seconds + +def waitForFiles(syncFilePath, files, starttime): + # wait max. 10 seconds while True: if starttime + 10 < time.time(): return False - if os.path.isfile( syncFileTask ): + if os.path.isfile(syncFilePath): break - log.debug( "Wait for file creation" ) + log.debug("Wait for file creation") time.sleep(0.1) - #Read file live - with open( syncFileTask, 'r' ) as syncFileTask: + # Read file live + with open(syncFilePath, 'r') as syncFileTask: current = [] while len(files) > 0: data = syncFileTask.read() @@ -139,78 +149,107 @@ def waitForFiles( syncFileTask, files, starttime ): else: text = ''.join(current) current = [] - if text.startswith( "S-" ): + if text.startswith("S-"): continue if text == "##FAILURE##": - log.debug( "Read FAILURE in %s", syncFileTask ) - myExit( 51 ) + log.debug("Read FAILURE in %s", syncFilePath) + myExit(51) if text == "##FINISHED##": - log.debug( "Read FINISHED in " + syncFileTask + " before all files were found" ) - myExit( 52 ) - log.debug( "Look for " + text[:2] + " with " + text[2:] + " in " + str(files) ) + log.debug("Read FINISHED in " + syncFilePath + " before all files were found") + myExit(52) + log.debug("Look for " + text[:2] + " with " + text[2:] + " in " + str(files)) if text[:2] == "F-" and text[2:] in files: - files.remove( text[2:] ) + files.remove(text[2:]) if len(files) == 0: return True return len(files) == 0 -starttime = time.time() -log.info( "Start to setup the environment" ) -configFilePath = ".command.inputs.json" +def loadConfig(configFilePath): + if not os.path.isfile(configFilePath): + log.error("Config file not found: %s", configFilePath) + myExit(102) -if not os.path.isfile( configFilePath ): - log.error( "Config file not found: %s", configFilePath ) - myExit( 102 ) + with open(configFilePath, 'r') as configFile: + config = json.load(configFile) -with open(configFilePath,'r') as configFile: - config = json.load( configFile ) + log.info(str(config)) -log.info( str(config) ) -dns = config["dns"] + os.makedirs(config["syncDir"], exist_ok=True) + return config -data = config[ "data" ] -symlinks = config[ "symlinks" ] -os.makedirs( config[ "syncDir" ], exist_ok=True) -with open( config[ "syncDir" ] + config[ "hash" ], 'w' ) as syncFile: +def registerSignal(syncFile): + signal.signal(signal.SIGINT, lambda signalnum, handler: close(signalnum, syncFile)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: close(signalnum, syncFile)) - signal.signal( signal.SIGINT, lambda signalnum, handler: close( signalnum, syncFile ) ) - signal.signal( signal.SIGTERM, lambda signalnum, handler: close( signalnum, syncFile ) ) - syncFile.write('##STARTED##\n') - syncFile.flush() +def registerSignal2(): + signal.signal(signal.SIGINT, lambda signalnum, handler: myExit(1)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: myExit(1)) + +def generateSymlinks(symlinks): for s in symlinks: src = s["src"] dst = s["dst"] - clearLocatation( src ) + clearLocatation(src) Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) - os.symlink( dst, src ) + os.symlink(dst, src) - syncFile.write('##SYMLINKS##\n') +def downloadAllData(data, dns, syncFile): for d in data: files = d["files"] - download( d["node"], files, syncFile ) - - if CLOSE: - log.debug( "Close with code %s", str(EXIT) ) - exit( EXIT ) - - syncFile.write('##FINISHED##\n') - - signal.signal( signal.SIGINT, lambda signalnum, handler: myExit( 1 ) ) - signal.signal( signal.SIGTERM, lambda signalnum, handler: myExit( 1 ) ) - -#Now check for files of other tasks -for waitForTask in config[ "waitForFilesOfTask" ]: - waitForFilesSet = set( config[ "waitForFilesOfTask" ][ waitForTask ] ) - if not waitForFiles( config[ "syncDir" ] + waitForTask, waitForFilesSet, starttime ): - log.error( config[ "syncDir" ] + waitForTask + " was not successful" ) - myExit( 200 ) - -traceFilePath = ".command.scheduler.trace" - -with open(traceFilePath, "a") as traceFile: - traceFile.write("scheduler_init_runtime=" + str(int((time.time()-starttime)*1000))) \ No newline at end of file + download(d["node"], files, dns, syncFile) + + +def waitForDependingTasks(waitForFilesOfTask, starttime, syncDir): + # Now check for files of other tasks + for waitForTask in waitForFilesOfTask: + waitForFilesSet = set(waitForFilesOfTask[waitForTask]) + if not waitForFiles(syncDir + waitForTask, waitForFilesSet, starttime): + log.error(syncDir + waitForTask + " was not successful") + myExit(200) + + +def writeTrace(traceFilePath, dataMap): + with open(traceFilePath, "a") as traceFile: + for d in dataMap: + traceFile.write(d + "=" + str(dataMap[d])) + + +def run(): + starttime = time.time() + log.info("Start to setup the environment") + config = loadConfig(".command.inputs.json") + + dns = config["dns"] + data = config["data"] + symlinks = config["symlinks"] + + with open(config["syncDir"] + config["hash"], 'w') as syncFile: + registerSignal(syncFile) + syncFile.write('##STARTED##\n') + syncFile.flush() + generateSymlinks(symlinks) + syncFile.write('##SYMLINKS##\n') + syncFile.flush() + downloadAllData(data, dns, syncFile) + if CLOSE: + log.debug("Closed with code %s", str(EXIT)) + exit(EXIT) + syncFile.write('##FINISHED##\n') + registerSignal2() + + waitForDependingTasks(config["waitForFilesOfTask"], starttime, config["syncDir"]) + + runtime = str(int((time.time() - starttime) * 1000)) + trace = { + "scheduler_init_runtime": runtime + } + writeTrace(".command.scheduler.trace", trace) + + +if __name__ == '__main__': + run() From 1eb0324670fc054dcf04ee2475cda04bfc6f1e48 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 08:59:48 +0200 Subject: [PATCH 230/443] Download in parallel Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index badd84fc..8051af5c 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -4,6 +4,7 @@ import os import json import urllib.request +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import shutil import signal @@ -84,7 +85,7 @@ def closeFTP(ftp): def downloadFile(ftp, filename, size, index, node, syncFile): - log.info("Download [%s/%s] - %s", str(index).rjust(len(str(size))), str(size), filename) + log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) try: syncFile.write("S-" + filename + '\n') clearLocatation(filename) @@ -199,9 +200,20 @@ def generateSymlinks(symlinks): def downloadAllData(data, dns, syncFile): - for d in data: - files = d["files"] - download(d["node"], files, dns, syncFile) + with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: + futures = [] + for d in data: + files = d["files"] + node = d["node"] + futures.append(executor.submit(download, node, files, dns, syncFile)) + trial = 0 + while len(futures) > 0: + if trial % 5 == 0: + log.info("Wait for %d threads to finish", len(futures)) + for f in futures[:]: + if f.done(): + futures.remove(f) + sleep(0.1) def waitForDependingTasks(waitForFilesOfTask, starttime, syncDir): From cc6fe73489fac1237f29192f1c792281c9bc7914 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 09:02:38 +0200 Subject: [PATCH 231/443] Reordered imports Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 8051af5c..61d5f9fa 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -1,14 +1,15 @@ #!/usr/bin/env python import ftplib -import time -import os import json +import logging as log +import os +import shutil +import signal +import time import urllib.request from concurrent.futures import ThreadPoolExecutor from pathlib import Path -import shutil -import signal -import logging as log +from time import sleep exitIfFileWasNotFound = False CLOSE = False From 777721e91b96fb6fdb7691c142efa6f8fab4a0c2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 09:03:12 +0200 Subject: [PATCH 232/443] Do not always request daemonset Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 61d5f9fa..18269eb1 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -54,11 +54,15 @@ def clearLocatation(path): os.remove(path) -def getFTP(node, dns, syncFile): +def getFTP(node, currentIP, dns, syncFile): connectionProblem = 0 while connectionProblem < 8: try: - ip = getIP(node, dns) + if currentIP is None: + log.info("Request ip for node: %s", node) + ip = getIP(node, dns) + else: + ip = currentIP log.info("Try to connect to %s", ip) ftp = ftplib.FTP(ip) ftp.login("ftp", "pythonclient") @@ -110,13 +114,14 @@ def downloadFile(ftp, filename, size, index, node, syncFile): return True -def download(node, files, dns, syncFile): +def download(node, currentIP, files, dns, syncFile): ftp = None size = len(files) global CLOSE while not CLOSE and len(files) > 0: if ftp is None: - ftp = getFTP(node, dns, syncFile) + ftp = getFTP(node, currentIP, dns, syncFile) + currentIP = None filename = files[0] index = size - len(files) + 1 if not downloadFile(ftp, filename, size, index, node, syncFile): @@ -206,7 +211,8 @@ def downloadAllData(data, dns, syncFile): for d in data: files = d["files"] node = d["node"] - futures.append(executor.submit(download, node, files, dns, syncFile)) + currentIP = d["currentIP"] + futures.append(executor.submit(download, node, currentIP, files, dns, syncFile)) trial = 0 while len(futures) > 0: if trial % 5 == 0: From d7094276e4cde795df1fc232c40bfb3563b5004b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 09:03:35 +0200 Subject: [PATCH 233/443] Log when step was finished Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 18269eb1..a65c3217 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -258,10 +258,12 @@ def run(): if CLOSE: log.debug("Closed with code %s", str(EXIT)) exit(EXIT) + log.info("Finished Download") syncFile.write('##FINISHED##\n') registerSignal2() waitForDependingTasks(config["waitForFilesOfTask"], starttime, config["syncDir"]) + log.info("Waited for all tasks") runtime = str(int((time.time() - starttime) * 1000)) trace = { From e7dcf18e896a1765edcdbaa86b9ca83030b9dd04 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 10:06:30 +0200 Subject: [PATCH 234/443] Trace init runtimes Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 49 +++++++++++++++++------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index a65c3217..57c6d2dd 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -16,6 +16,9 @@ UNEXPECTED_ERROR = "Unexpected error" EXIT = 0 log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) +trace = {} +traceFilePath = ".command.scheduler.trace" +errors = 0 def myExit(code): @@ -23,6 +26,7 @@ def myExit(code): EXIT = code global CLOSE CLOSE = True + writeTrace(trace) exit(EXIT) @@ -55,6 +59,7 @@ def clearLocatation(path): def getFTP(node, currentIP, dns, syncFile): + global errors connectionProblem = 0 while connectionProblem < 8: try: @@ -71,8 +76,10 @@ def getFTP(node, currentIP, dns, syncFile): log.info("Connection established") return ftp except ConnectionRefusedError: + errors += 1 log.warning("Connection refused! Try again...") except BaseException: + errors += 1 log.exception(UNEXPECTED_ERROR) connectionProblem += 1 time.sleep(2 ** connectionProblem) @@ -80,16 +87,19 @@ def getFTP(node, currentIP, dns, syncFile): def closeFTP(ftp): + global errors if ftp is None: return try: ftp.quit() ftp.close() except BaseException: + errors += 1 log.exception(UNEXPECTED_ERROR) def downloadFile(ftp, filename, size, index, node, syncFile): + global errors log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) try: syncFile.write("S-" + filename + '\n') @@ -97,18 +107,22 @@ def downloadFile(ftp, filename, size, index, node, syncFile): Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) except ftplib.error_perm as err: + errors += 1 if str(err) == "550 Failed to open file.": log.warning("File not found node: %s file: %s", node, filename) if exitIfFileWasNotFound: closeWithWarning(40, syncFile) except FileNotFoundError: + errors += 1 log.warning("File not found node: %s file: %s", node, filename) if exitIfFileWasNotFound: closeWithWarning(41, syncFile) except EOFError: + errors += 1 log.warning("It seems the connection was lost! Try again...") return False except BaseException: + errors += 1 log.exception(UNEXPECTED_ERROR) return False return True @@ -132,10 +146,10 @@ def download(node, currentIP, files, dns, syncFile): closeFTP(ftp) -def waitForFiles(syncFilePath, files, starttime): +def waitForFiles(syncFilePath, files, startTime): # wait max. 10 seconds while True: - if starttime + 10 < time.time(): + if startTime + 10 < time.time(): return False if os.path.isfile(syncFilePath): break @@ -223,23 +237,28 @@ def downloadAllData(data, dns, syncFile): sleep(0.1) -def waitForDependingTasks(waitForFilesOfTask, starttime, syncDir): +def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): # Now check for files of other tasks for waitForTask in waitForFilesOfTask: waitForFilesSet = set(waitForFilesOfTask[waitForTask]) - if not waitForFiles(syncDir + waitForTask, waitForFilesSet, starttime): + if not waitForFiles(syncDir + waitForTask, waitForFilesSet, startTime): log.error(syncDir + waitForTask + " was not successful") myExit(200) -def writeTrace(traceFilePath, dataMap): +def writeTrace(dataMap): + global errors + if len(dataMap) == 0 or errors > 0: + return with open(traceFilePath, "a") as traceFile: for d in dataMap: - traceFile.write(d + "=" + str(dataMap[d])) + traceFile.write(d + "=" + str(dataMap[d]) + "\n") + traceFile.write("scheduler_init_errors=" + str(errors) + "\n") def run(): - starttime = time.time() + global trace + startTime = time.time() log.info("Start to setup the environment") config = loadConfig(".command.inputs.json") @@ -251,10 +270,14 @@ def run(): registerSignal(syncFile) syncFile.write('##STARTED##\n') syncFile.flush() + startTimeSymlinks = time.time() generateSymlinks(symlinks) + trace["scheduler_init_symlinks_runtime"] = int((time.time() - startTimeSymlinks) * 1000) syncFile.write('##SYMLINKS##\n') syncFile.flush() + startTimeDownload = time.time() downloadAllData(data, dns, syncFile) + trace["scheduler_init_download_runtime"] = int((time.time() - startTimeDownload) * 1000) if CLOSE: log.debug("Closed with code %s", str(EXIT)) exit(EXIT) @@ -262,14 +285,14 @@ def run(): syncFile.write('##FINISHED##\n') registerSignal2() - waitForDependingTasks(config["waitForFilesOfTask"], starttime, config["syncDir"]) + startTimeDependingTasks = time.time() + waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) + trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) log.info("Waited for all tasks") - runtime = str(int((time.time() - starttime) * 1000)) - trace = { - "scheduler_init_runtime": runtime - } - writeTrace(".command.scheduler.trace", trace) + runtime = int((time.time() - startTime) * 1000) + trace["scheduler_init_runtime"] = runtime + writeTrace(trace) if __name__ == '__main__': From d33ec8f10d334d286d4865da0cd30cfbf2d06f7a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 10:34:25 +0200 Subject: [PATCH 235/443] Sort downloads by size desc Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 12 +++++++----- .../scheduler/schedulingstrategy/InputEntry.java | 13 +++++++++++-- .../scheduler/schedulingstrategy/Inputs.java | 5 +++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 855ee29c..a925f9a7 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -210,6 +210,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final List collect = new LinkedList<>(); final NodeLocation location = (NodeLocation) entry.getKey(); + long size = 0; for (FilePath filePath : entry.getValue().getAlignment()) { if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey(filePath.getPath()) ) { //Node copies currently from somewhere else! @@ -234,18 +235,19 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { ); collect.add(filePath.getPath()); filesForCurrentNode.put(filePath.getPath(), new Tuple<>( alignment.task, location ) ); - if (traceEnabled) { - filesNotOnNode++; - filesNotOnNodeByte += filePath.getLocationWrapper().getSizeInBytes(); - } + final long sizeInBytes = filePath.getLocationWrapper().getSizeInBytes(); + size += sizeInBytes; + filesNotOnNode++; } } + filesNotOnNodeByte += size; if( !collect.isEmpty() ) { - inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect)); + inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, size)); } } inputs.waitForTask( waitForTask ); inputs.symlinks.addAll(alignment.fileAlignment.symlinks); + inputs.sortData(); if (traceEnabled) { traceRecord.setSchedulerFilesNode(filesOnNode); diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java index 04dc903f..7eb2a07f 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java @@ -1,16 +1,25 @@ package fonda.scheduler.scheduler.schedulingstrategy; +import org.jetbrains.annotations.NotNull; + import java.util.List; -public class InputEntry { +public class InputEntry implements Comparable { public final String currentIP; public final String node; public final List files; + public final long size; - public InputEntry( String currentIP, String node, List files ) { + public InputEntry( String currentIP, String node, List files, long size ) { this.currentIP = currentIP; this.node = node; this.files = files; + this.size = size; + } + + @Override + public int compareTo(@NotNull InputEntry o) { + return Long.compare( size, o.size ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index 4b5ca977..18ff0638 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -3,6 +3,7 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.taskinputs.SymlinkInput; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -34,4 +35,8 @@ public void waitForTask( Map waitForTask ){ } } + public void sortData(){ + Collections.sort(data, Collections.reverseOrder()); + } + } From 664192b6a9bbb40767972ed7c1a3ab9f72cf12fc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 12:33:43 +0200 Subject: [PATCH 236/443] Trace throughput Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 57c6d2dd..f453c6cd 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -105,7 +105,13 @@ def downloadFile(ftp, filename, size, index, node, syncFile): syncFile.write("S-" + filename + '\n') clearLocatation(filename) Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) + start = time.time() ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) + end = time.time() + sizeInMB = os.path.getsize(filename) / 1_000_000 + delta = (end - start) + log.info("Speed: %.3f Mbit/s", sizeInMB / delta ) + return sizeInMB, delta except ftplib.error_perm as err: errors += 1 if str(err) == "550 Failed to open file.": @@ -120,30 +126,36 @@ def downloadFile(ftp, filename, size, index, node, syncFile): except EOFError: errors += 1 log.warning("It seems the connection was lost! Try again...") - return False + return None except BaseException: errors += 1 log.exception(UNEXPECTED_ERROR) - return False - return True + return None + return 0, 0 def download(node, currentIP, files, dns, syncFile): ftp = None size = len(files) global CLOSE + sizeInMB = 0 + downloadTime = 0 while not CLOSE and len(files) > 0: if ftp is None: ftp = getFTP(node, currentIP, dns, syncFile) currentIP = None filename = files[0] index = size - len(files) + 1 - if not downloadFile(ftp, filename, size, index, node, syncFile): + result = downloadFile(ftp, filename, size, index, node, syncFile) + if result is None: ftp = None continue + sizeInMB += result[0] + downloadTime += result[1] files.pop(0) syncFile.write("F-" + filename + '\n') closeFTP(ftp) + return node, sizeInMB / downloadTime def waitForFiles(syncFilePath, files, startTime): @@ -220,6 +232,8 @@ def generateSymlinks(symlinks): def downloadAllData(data, dns, syncFile): + global trace + throughput = [] with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: futures = [] for d in data: @@ -233,8 +247,10 @@ def downloadAllData(data, dns, syncFile): log.info("Wait for %d threads to finish", len(futures)) for f in futures[:]: if f.done(): + throughput.append(f.result()) futures.remove(f) sleep(0.1) + trace["scheduler_init_throughput"] = "\"" + ",".join( "{}:{:.3f}".format(*x) for x in throughput) + "\"" def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): From 1de8c87dbd2cf741982f5304476e98fb6ccf1c86 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 12:33:59 +0200 Subject: [PATCH 237/443] Don't log config Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index f453c6cd..b7e9f868 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -206,8 +206,6 @@ def loadConfig(configFilePath): with open(configFilePath, 'r') as configFile: config = json.load(configFile) - log.info(str(config)) - os.makedirs(config["syncDir"], exist_ok=True) return config From db729e36fae2108e80592fefebe2891c2dcaa38d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 May 2022 12:34:16 +0200 Subject: [PATCH 238/443] exitIfFileWasNotFound Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index b7e9f868..ee737d13 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -11,7 +11,7 @@ from pathlib import Path from time import sleep -exitIfFileWasNotFound = False +exitIfFileWasNotFound = True CLOSE = False UNEXPECTED_ERROR = "Unexpected error" EXIT = 0 From d7a66897a90702b4e8321df390f6b51edc9aeb1d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 11 May 2022 09:10:16 +0200 Subject: [PATCH 239/443] copying to Node handling more efficient Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index a925f9a7..f446121b 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -24,6 +24,7 @@ import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; +import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.net.ftp.FTPClient; @@ -49,6 +50,9 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); private final String localWorkDir; + @Getter(AccessLevel.PACKAGE) + private final Map< NodeLocation, Map< String, Tuple> > copyingToNode = new HashMap<>(); + SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); @@ -145,10 +149,9 @@ int terminateTasks(List finishedTasks) { return 0; } - private final Map< NodeLocation, Map< String, Tuple> > copyingToNode = new HashMap<>(); - private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple > toAdd ){ if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); + if ( toAdd.isEmpty() ) return; if ( copyingToNode.containsKey( nodeLocation ) ){ final Map> stringTupleHashMap = copyingToNode.get( nodeLocation ); stringTupleHashMap.putAll( toAdd ); @@ -159,7 +162,9 @@ private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple< private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tuple> toRemove ){ if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); - copyingToNode.get( nodeLocation ).keySet().removeAll( toRemove.keySet() ); + final Map> pathTasks = copyingToNode.get(nodeLocation); + if( pathTasks == null || toRemove == null || toRemove.isEmpty() ) return; + pathTasks.keySet().removeAll( toRemove.keySet() ); } /** From 57d5219935b5d0ed80b6919e5daee64e12736750 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 11 May 2022 09:40:32 +0200 Subject: [PATCH 240/443] Consider currently copied files in the scheduling decision Signed-off-by: Lehmann_Fabian --- .../scheduler/model/tracing/TraceRecord.java | 5 ++ .../scheduler/LocationAwareScheduler.java | 42 +++++++---- .../scheduler/RandomLAScheduler.java | 5 +- .../filealignment/GreedyAlignment.java | 74 +++++++++++++++---- .../filealignment/InputAlignment.java | 17 ++++- .../filealignment/RandomAlignment.java | 20 ++++- .../NoAligmentPossibleException.java | 8 ++ 7 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java index b84486db..24690702 100644 --- a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java @@ -78,6 +78,10 @@ public class TraceRecord { @Setter private Integer schedulerTimeToSchedule = null; + @Getter + @Setter + private Integer schedulerNoAlignmentFound = null; + public void writeRecord( String tracePath ) throws IOException { try ( BufferedWriter bw = new BufferedWriter( new FileWriter( tracePath ) ) ) { @@ -99,6 +103,7 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_tried_to_schedule", schedulerTriedToSchedule, bw); writeValue("scheduler_nodes_to_copy_from", schedulerNodesToCopyFrom, bw); writeValue("scheduler_time_to_schedule", schedulerTimeToSchedule, bw); + writeValue("scheduler_no_alignment_found", schedulerNoAlignmentFound, bw); } } diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index bb5496bb..72b5e3b0 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -2,11 +2,13 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.scheduler.data.NodeDataTuple; import fonda.scheduler.scheduler.data.TaskData; import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.NodeTaskAlignment; import fonda.scheduler.util.NodeTaskFilesAlignment; @@ -99,25 +101,33 @@ Tuple calculateBestNode( //Remove all nodes which do not fit anymore final List nodeDataTuples = taskData.getNodeDataTuples(); int triedNodes = 0; + int noAlignmentFound = 0; int couldStopFetching = 0; final List costs = traceEnabled ? new LinkedList<>() : null; for (NodeDataTuple nodeDataTuple : nodeDataTuples) { final NodeWithAlloc currentNode = nodeDataTuple.getNode(); if ( !matchingNodesForTask.contains(currentNode) ) continue; - - final FileAlignment fileAlignment = inputAlignment.getInputAlignment( - taskData.getTask(), - taskData.getMatchingFilesAndNodes().getInputsOfTask(), - currentNode, - bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost - ); - if ( fileAlignment == null ){ - couldStopFetching++; - } else if ( bestAlignment == null || bestAlignment.cost > fileAlignment.cost ){ - bestAlignment = fileAlignment; - bestNode = currentNode; - log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); + final Map> currentlyCopying = getCopyingToNode().get(currentNode.getNodeLocation()); + FileAlignment fileAlignment = null; + try { + fileAlignment = inputAlignment.getInputAlignment( + taskData.getTask(), + taskData.getMatchingFilesAndNodes().getInputsOfTask(), + currentNode, + currentlyCopying, + bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost + ); + if ( fileAlignment == null ){ + couldStopFetching++; + } else if ( bestAlignment == null || bestAlignment.cost > fileAlignment.cost ){ + bestAlignment = fileAlignment; + bestNode = currentNode; + log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); + } + } catch ( NoAligmentPossibleException e ){ + noAlignmentFound++; + log.info( "Task: {} - {}", taskData.getTask().getConfig().getName() , e.getMessage() ); } if ( traceEnabled ) { triedNodes++; @@ -134,17 +144,19 @@ Tuple calculateBestNode( triedNodes, costs, couldStopFetching, - bestAlignment.cost + bestAlignment.cost, + noAlignmentFound ); return new Tuple<>( bestNode, bestAlignment ); } - private void storeTraceData( final TraceRecord traceRecord, int triedNodes, List costs, int couldStopFetching, double bestCost ){ + private void storeTraceData(final TraceRecord traceRecord, int triedNodes, List costs, int couldStopFetching, double bestCost, int noAlignmentFound){ if ( !traceEnabled ) return; traceRecord.setSchedulerNodesTried( triedNodes ); traceRecord.setSchedulerNodesCost( costs ); traceRecord.setSchedulerCouldStopFetching( couldStopFetching ); traceRecord.setSchedulerBestCost( bestCost ); + traceRecord.setSchedulerNoAlignmentFound( noAlignmentFound ); } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index e5e6c375..84a4fdfb 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -5,6 +5,7 @@ import fonda.scheduler.model.Requirements; import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.scheduler.data.NodeDataTuple; import fonda.scheduler.scheduler.data.TaskData; @@ -45,10 +46,12 @@ Tuple calculateBestNode( final Optional nodeWithAlloc = selectNode(matchingNodesForTask, taskData.getTask()); if (nodeWithAlloc.isEmpty()) return null; final NodeWithAlloc node = nodeWithAlloc.get(); + final Map> currentlyCopying = getCopyingToNode().get(node.getNodeLocation()); final FileAlignment fileAlignment = getInputAlignment().getInputAlignment( taskData.getTask(), taskData.getMatchingFilesAndNodes().getInputsOfTask(), - node + node, + currentlyCopying ); return new Tuple<>( node, fileAlignment ); } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index 4ae6799c..ceed1725 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -7,13 +7,19 @@ import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import fonda.scheduler.util.AlignmentWrapper; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; +import fonda.scheduler.util.Tuple; +import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import java.util.HashMap; +import java.util.List; +import java.util.Map; +@Slf4j public class GreedyAlignment implements InputAlignment { private final CostFunction cf; @@ -22,8 +28,32 @@ public GreedyAlignment(CostFunction cf) { this.cf = cf; } + /** + * Check if another scheduled task is already copying a required file + * @param currentlyCopying + * @param path + * @param locations + * @return + */ + private LocationWrapper alreadyCopying(Map> currentlyCopying, String path, List locations){ + if ( currentlyCopying != null && currentlyCopying.containsKey( path ) ){ + final Location copyFrom = currentlyCopying.get(path).getB(); + for ( LocationWrapper locationWrapper : locations ) { + if ( locationWrapper.getLocation() == copyFrom ) { + return locationWrapper; + } + } + throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); + } + return null; + } + @Override - public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, double maxCost) { + public FileAlignment getInputAlignment(@NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + Map> currentlyCopying, + double maxCost) { inputsOfTask.sort(); final HashMap map = new HashMap<>(); double cost = 0; @@ -31,23 +61,35 @@ public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs i double minCost = Double.MAX_VALUE; Location bestLoc = null; LocationWrapper bestLocationWrapper = null; - for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { - final Location currentLoc = locationWrapper.getLocation(); - final double calculatedCost; - if ( currentLoc != node.getNodeLocation() ) { - final AlignmentWrapper alignmentWrapper = map.get(currentLoc); - calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); - } else { - calculatedCost = 0; - } - if ( calculatedCost < minCost ) { - bestLoc = currentLoc; - minCost = calculatedCost; - bestLocationWrapper = locationWrapper; + + final LocationWrapper copying = alreadyCopying( + currentlyCopying, + pathFileLocationTriple.path.toString(), + pathFileLocationTriple.locations + ); + if ( copying != null ) { + minCost = 0; + bestLoc = copying.getLocation(); + bestLocationWrapper = copying; + } else { + for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { + final Location currentLoc = locationWrapper.getLocation(); + final double calculatedCost; + if ( currentLoc != node.getNodeLocation() ) { + final AlignmentWrapper alignmentWrapper = map.get(currentLoc); + calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); + } else { + calculatedCost = 0; + } + if ( calculatedCost < minCost ) { + bestLoc = currentLoc; + minCost = calculatedCost; + bestLocationWrapper = locationWrapper; + } } + if ( minCost > maxCost ) return null; + if ( minCost > cost ) cost = minCost; } - if ( minCost > maxCost ) return null; - if ( minCost > cost ) cost = minCost; final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost ); } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java index 9a3d2177..4c62b92f 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java @@ -2,10 +2,14 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.Tuple; import org.jetbrains.annotations.NotNull; +import java.util.Map; + public interface InputAlignment { /** @@ -16,7 +20,11 @@ public interface InputAlignment { * @param maxCost * @return null if no or no better than maxCost alignment is found */ - FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, double maxCost ); + FileAlignment getInputAlignment( @NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + Map> currentlyCopying, + double maxCost ); /** * Calculate a alignment for the input data @@ -25,8 +33,11 @@ public interface InputAlignment { * @param node * @return null if no alignment is found */ - default FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node ){ - return getInputAlignment( task, inputsOfTask, node, Double.MAX_VALUE ); + default FileAlignment getInputAlignment( @NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + Map> currentlyCopying ){ + return getInputAlignment( task, inputsOfTask, node, currentlyCopying, Double.MAX_VALUE ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index 2a0ead23..549b7df6 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -9,9 +9,12 @@ import fonda.scheduler.util.AlignmentWrapper; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; +import fonda.scheduler.util.Tuple; import org.jetbrains.annotations.NotNull; import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import java.util.Random; public class RandomAlignment implements InputAlignment { @@ -19,12 +22,21 @@ public class RandomAlignment implements InputAlignment { private final Random random = new Random(); @Override - public FileAlignment getInputAlignment(Task task, @NotNull TaskInputs inputsOfTask, NodeWithAlloc node, double maxCost) { + public FileAlignment getInputAlignment(@NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + Map> currentlyCopying, + double maxCost) { final HashMap map = new HashMap<>(); for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { - final LocationWrapper locationWrapper = pathFileLocationTriple.locations.get( - random.nextInt( pathFileLocationTriple.locations.size() ) - ); + final Optional first = pathFileLocationTriple + .locations + .stream() + .filter(x -> x.getLocation() == node.getNodeLocation()) + .findFirst(); + final LocationWrapper locationWrapper = first.orElseGet(() -> pathFileLocationTriple.locations.get( + random.nextInt(pathFileLocationTriple.locations.size()) + )); final Location location = locationWrapper.getLocation(); final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java new file mode 100644 index 00000000..c8d0cb2d --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java @@ -0,0 +1,8 @@ +package fonda.scheduler.scheduler.filealignment.costfunctions; + +public class NoAligmentPossibleException extends RuntimeException { + + public NoAligmentPossibleException(String message) { + super(message); + } +} From 0c8ea7ac38637855151ae0a32f658f494bc85b70 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 11 May 2022 10:24:19 +0200 Subject: [PATCH 241/443] Code style Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 1 - .../location/hierachy/RealHierarchyFile.java | 16 ++++++++-------- .../scheduler/schedulingstrategy/Inputs.java | 2 +- .../scheduler/model/InputFileCollectorTest.java | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index a5f20a8b..8cab2e62 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -35,7 +35,6 @@ public class Task { private List< TaskInputFileLocationWrapper > copiedFiles; @Getter - @Setter private PodWithAge pod = null; @Getter diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index 404a12bb..ff1a381d 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -37,9 +37,9 @@ public boolean isSymlink() { public void removeLocation( LocationWrapper location ){ if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); synchronized ( this ){ - for (int i = 0; i < locations.length; i++) { - if ( location.getLocation().equals( locations[i].getLocation() ) ) { - locations[i].deactivate(); + for ( LocationWrapper locationWrapper : locations ) { + if ( location.getLocation().equals( locationWrapper.getLocation() ) ) { + locationWrapper.deactivate(); } } } @@ -49,15 +49,15 @@ public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper l if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); synchronized ( this ){ LocationWrapper locationWrapperToUpdate = null; - for (int i = 0; i < locations.length; i++) { - if ( location.getLocation().equals( locations[i].getLocation() ) ) { - locationWrapperToUpdate = locations[i]; - if ( overwrite || location.getTimestamp() > locations[i].getTimestamp() ) { + for (LocationWrapper locationWrapper : locations) { + if ( location.getLocation().equals( locationWrapper.getLocation() ) ) { + locationWrapperToUpdate = locationWrapper; + if ( overwrite || location.getTimestamp() > locationWrapper.getTimestamp() ) { locationWrapperToUpdate.update( location ); } if ( !overwrite ) return locationWrapperToUpdate; } else if ( overwrite ){ - locations[i].deactivate(); + locationWrapper.deactivate(); } } if ( overwrite && locationWrapperToUpdate != null ) return locationWrapperToUpdate; diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index 18ff0638..9421190e 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -36,7 +36,7 @@ public void waitForTask( Map waitForTask ){ } public void sortData(){ - Collections.sort(data, Collections.reverseOrder()); + data.sort(Collections.reverseOrder()); } } diff --git a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java index 5065e7bd..d75ecbd1 100644 --- a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java @@ -95,7 +95,7 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { fileInputs.add( b ); final Task task = new Task(taskConfig, dag); - final Set expected = new HashSet( List.of( file1, file2, file3 ) ); + final Set expected = new HashSet<>(List.of(file1, file2, file3)); TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); From 1fb67e962e9b18e063f72b393c62503604dabb68 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 11 May 2022 11:42:01 +0200 Subject: [PATCH 242/443] Create alignment for all unscheduled tasks in separate method Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 72b5e3b0..0055e200 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -18,7 +18,6 @@ import lombok.extern.slf4j.Slf4j; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @Slf4j @@ -38,15 +37,20 @@ public LocationAwareScheduler ( } - + /** + * Find the best alignment for one task + * @param taskData + * @param availableByNode + * @param index + * @return the alignment containing the node and the file alignments + */ private NodeTaskFilesAlignment createNodeAlignment ( final TaskData taskData, final Map availableByNode, - AtomicInteger index + int index ) { long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); - final int currentIndex = index.incrementAndGet(); final Set matchingNodesForTask = getMatchingNodesForTask( availableByNode, taskData.getTask()); if ( matchingNodesForTask.isEmpty() ) return null; final Tuple result = calculateBestNode(taskData, matchingNodesForTask); @@ -56,7 +60,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( taskData.addNs( System.nanoTime()- startTime ); if ( traceEnabled ){ task.getTraceRecord().setSchedulerTimeToSchedule((int) (taskData.getTimeInNs() / 1_000_000)); - task.getTraceRecord().setSchedulerPlaceInQueue( currentIndex ); + task.getTraceRecord().setSchedulerPlaceInQueue( index ); task.getTraceRecord().setSchedulerLocationCount( taskData.getMatchingFilesAndNodes().getInputsOfTask().getFiles() .parallelStream() @@ -67,26 +71,44 @@ private NodeTaskFilesAlignment createNodeAlignment ( return new NodeTaskFilesAlignment(result.getA(), task, result.getB()); } + /** + * Align all tasks to the best node + * @param unscheduledTasksSorted + * @param availableByNode + * @return a list with alignments + */ + List createAlignment( + final List unscheduledTasksSorted, + final Map availableByNode + ){ + int index = 0; + final List alignment = new LinkedList<>(); + for ( TaskData taskData : unscheduledTasksSorted ){ + final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, index); + if ( nodeAlignment != null ) alignment.add( nodeAlignment ); + } + return alignment; + } + @Override public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks, final Map availableByNode ){ - AtomicInteger index = new AtomicInteger( 0 ); - final List alignment = unscheduledTasks + final List unscheduledTasksSorted = unscheduledTasks .parallelStream() - .map( task -> { + .map(task -> { long startTime = System.nanoTime(); final TaskData taskData = calculateTaskData(task, availableByNode); - if ( taskData != null ) taskData.addNs( System.nanoTime() - startTime ); + if (taskData != null) taskData.addNs(System.nanoTime() - startTime); return taskData; - } ) + }) .filter(Objects::nonNull) .sorted(Comparator.reverseOrder()) - .sequential() - .map( taskData -> createNodeAlignment( taskData, availableByNode, index ) ) - .filter(Objects::nonNull) .collect(Collectors.toList()); + + + final List alignment = createAlignment(unscheduledTasksSorted, availableByNode); final ScheduleObject scheduleObject = new ScheduleObject(alignment); scheduleObject.setCheckStillPossible( true ); return scheduleObject; From dbb874160301627fa48f84c3443e36cdea458615 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 11 May 2022 16:19:32 +0200 Subject: [PATCH 243/443] Consider files fetched by tasks in the same batch Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/ScheduleObject.java | 1 + .../scheduler/LocationAwareScheduler.java | 43 ++++++-- .../scheduler/RandomLAScheduler.java | 8 +- .../fonda/scheduler/scheduler/Scheduler.java | 10 ++ .../scheduler/SchedulerWithDaemonSet.java | 34 +++--- .../filealignment/GreedyAlignment.java | 103 +++++++++++------- .../filealignment/InputAlignment.java | 7 +- .../filealignment/RandomAlignment.java | 3 +- .../scheduler/util/AlignmentWrapper.java | 29 ++++- .../fonda/scheduler/util/FileAlignment.java | 2 +- .../scheduler/util/FilePathWithTask.java | 16 +++ 11 files changed, 177 insertions(+), 79 deletions(-) create mode 100644 src/main/java/fonda/scheduler/util/FilePathWithTask.java diff --git a/src/main/java/fonda/scheduler/model/ScheduleObject.java b/src/main/java/fonda/scheduler/model/ScheduleObject.java index 602c6db3..e8c2ef56 100644 --- a/src/main/java/fonda/scheduler/model/ScheduleObject.java +++ b/src/main/java/fonda/scheduler/model/ScheduleObject.java @@ -12,6 +12,7 @@ public class ScheduleObject { private final List taskAlignments; private boolean checkStillPossible = false; + private boolean stopSubmitIfOneFails = false; public ScheduleObject( List taskAlignments ) { this.taskAlignments = taskAlignments; diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 0055e200..49abe0a2 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -9,10 +9,7 @@ import fonda.scheduler.scheduler.data.TaskData; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.NodeTaskAlignment; -import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.*; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -39,21 +36,24 @@ public LocationAwareScheduler ( /** * Find the best alignment for one task + * * @param taskData * @param availableByNode + * @param planedToCopy * @param index * @return the alignment containing the node and the file alignments */ private NodeTaskFilesAlignment createNodeAlignment ( final TaskData taskData, final Map availableByNode, + Map< Location, Map>> planedToCopy, int index ) { long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); final Set matchingNodesForTask = getMatchingNodesForTask( availableByNode, taskData.getTask()); if ( matchingNodesForTask.isEmpty() ) return null; - final Tuple result = calculateBestNode(taskData, matchingNodesForTask); + final Tuple result = calculateBestNode(taskData, matchingNodesForTask, planedToCopy); if ( result == null ) return null; final Task task = taskData.getTask(); availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); @@ -71,6 +71,22 @@ private NodeTaskFilesAlignment createNodeAlignment ( return new NodeTaskFilesAlignment(result.getA(), task, result.getB()); } + private void addAlignmentToPlanned ( + Map< Location, Map< String, Tuple> > planedToCopy, + final Map nodeFileAlignment, + Task task, + NodeWithAlloc node + ) { + for (Map.Entry entry : nodeFileAlignment.entrySet()) { + final Map> map = planedToCopy.computeIfAbsent(node.getNodeLocation(), k -> new HashMap<>()); + for (FilePath filePath : entry.getValue().getFilesToCopy()) { + if ( entry.getKey() != node.getNodeLocation() ) { + map.put(filePath.getPath(), new Tuple<>(task, entry.getKey())); + } + } + } + } + /** * Align all tasks to the best node * @param unscheduledTasksSorted @@ -83,9 +99,13 @@ List createAlignment( ){ int index = 0; final List alignment = new LinkedList<>(); + final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); for ( TaskData taskData : unscheduledTasksSorted ){ - final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, index); - if ( nodeAlignment != null ) alignment.add( nodeAlignment ); + final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, planedToCopy, index); + if ( nodeAlignment != null ) { + alignment.add(nodeAlignment); + addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.nodeFileAlignment, taskData.getTask(), nodeAlignment.node ); + } } return alignment; } @@ -106,18 +126,17 @@ public ScheduleObject getTaskNodeAlignment( .filter(Objects::nonNull) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); - - final List alignment = createAlignment(unscheduledTasksSorted, availableByNode); final ScheduleObject scheduleObject = new ScheduleObject(alignment); scheduleObject.setCheckStillPossible( true ); + scheduleObject.setStopSubmitIfOneFails( true ); return scheduleObject; } Tuple calculateBestNode( final TaskData taskData, - final Set matchingNodesForTask - ){ + final Set matchingNodesForTask, + Map< Location, Map>> planedToCopy){ FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; //Remove all nodes which do not fit anymore @@ -131,6 +150,7 @@ Tuple calculateBestNode( final NodeWithAlloc currentNode = nodeDataTuple.getNode(); if ( !matchingNodesForTask.contains(currentNode) ) continue; final Map> currentlyCopying = getCopyingToNode().get(currentNode.getNodeLocation()); + final Map> currentlyPlanedToCopy = planedToCopy.get(currentNode.getNodeLocation()); FileAlignment fileAlignment = null; try { fileAlignment = inputAlignment.getInputAlignment( @@ -138,6 +158,7 @@ Tuple calculateBestNode( taskData.getMatchingFilesAndNodes().getInputsOfTask(), currentNode, currentlyCopying, + currentlyPlanedToCopy, bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost ); if ( fileAlignment == null ){ diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 84a4fdfb..66339fff 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -41,17 +41,19 @@ private Optional selectNode( Set matchingNodes, Ta @Override Tuple calculateBestNode( final TaskData taskData, - final Set matchingNodesForTask - ){ + final Set matchingNodesForTask, + Map< Location, Map>> planedToCopy){ final Optional nodeWithAlloc = selectNode(matchingNodesForTask, taskData.getTask()); if (nodeWithAlloc.isEmpty()) return null; final NodeWithAlloc node = nodeWithAlloc.get(); final Map> currentlyCopying = getCopyingToNode().get(node.getNodeLocation()); + final Map> currentlyPlanetToCopy = planedToCopy.get(node.getNodeLocation()); final FileAlignment fileAlignment = getInputAlignment().getInputAlignment( taskData.getTask(), taskData.getMatchingFilesAndNodes().getInputsOfTask(), node, - currentlyCopying + currentlyCopying, + currentlyPlanetToCopy ); return new Tuple<>( node, fileAlignment ); } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 3da63f7a..2089545a 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -95,18 +95,28 @@ public int schedule( final List unscheduledTasks ) { } int failure = 0; + int scheduled = 0; for (NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment) { try { if (isClose()) return -1; if ( !assignTaskToNode( nodeTaskAlignment ) ){ + if ( scheduleObject.isStopSubmitIfOneFails() ) { + return taskNodeAlignment.size() - scheduled; + } failure++; continue; } } catch ( Exception e ){ + log.info( "Could not schedule task: {} undo all", nodeTaskAlignment.task.getConfig().getHash() ); + e.printStackTrace(); undoTaskScheduling( nodeTaskAlignment.task ); + if ( scheduleObject.isStopSubmitIfOneFails() ) { + return taskNodeAlignment.size() - scheduled; + } continue; } taskWasScheduled(nodeTaskAlignment.task); + scheduled++; } return unscheduledTasks.size() - taskNodeAlignment.size() + failure; } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index f446121b..44d8bf67 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -167,6 +167,17 @@ private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tup pathTasks.keySet().removeAll( toRemove.keySet() ); } + private void traceForCurrentNode( AlignmentWrapper alignmentWrapper, NodeLocation node, TraceRecord record ) { + final List all = alignmentWrapper.getAll(); + record.setSchedulerFilesNode(all.size()); + record.setSchedulerFilesNodeBytes( + all + .parallelStream() + .mapToLong(x -> x.getFile().getLocationWrapper(node).getSizeInBytes()) + .sum() + ); + } + /** * * @param alignment @@ -192,23 +203,14 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final TraceRecord traceRecord = alignment.task.getTraceRecord(); - int filesOnNode = 0; int filesOnNodeOtherTask = 0; int filesNotOnNode = 0; - long filesOnNodeByte = 0; long filesOnNodeOtherTaskByte = 0; long filesNotOnNodeByte = 0; for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { if( entry.getKey() == alignment.node.getNodeLocation() ) { - if (traceEnabled) { - final List alignment1 = entry.getValue().getAlignment(); - filesOnNode = alignment1.size(); - filesOnNodeByte = alignment1 - .parallelStream() - .mapToLong(x -> x.getFile().getLocationWrapper(currentNode).getSizeInBytes()) - .sum(); - } + if (traceEnabled) traceForCurrentNode( entry.getValue(), currentNode, traceRecord ); continue; } @@ -216,7 +218,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final NodeLocation location = (NodeLocation) entry.getKey(); long size = 0; - for (FilePath filePath : entry.getValue().getAlignment()) { + for (FilePath filePath : entry.getValue().getAll()) { if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey(filePath.getPath()) ) { //Node copies currently from somewhere else! final Tuple taskLocationTuple = filesOnCurrentNode.get(filePath.getPath()); @@ -255,14 +257,14 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { inputs.sortData(); if (traceEnabled) { - traceRecord.setSchedulerFilesNode(filesOnNode); - traceRecord.setSchedulerFilesNodeBytes(filesOnNodeByte); traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); - traceRecord.setSchedulerFiles(filesOnNode + filesOnNodeOtherTask + filesNotOnNode); - traceRecord.setSchedulerFilesBytes(filesOnNodeByte + filesOnNodeOtherTaskByte + filesNotOnNodeByte); + final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); + traceRecord.setSchedulerFiles(schedulerFilesNode + filesOnNodeOtherTask + filesNotOnNode); + final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); + traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); traceRecord.setSchedulerDependingTask( (int) waitForTask.values().stream().distinct().count() ); - traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.nodeFileAlignment.size() - (filesOnNode > 0 ? 1 : 0) ); + traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.nodeFileAlignment.size() - (schedulerFilesNode > 0 ? 1 : 0) ); } new ObjectMapper().writeValue( config, inputs ); diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index ceed1725..ad8e80a4 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -3,15 +3,13 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.*; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -30,17 +28,19 @@ public GreedyAlignment(CostFunction cf) { /** * Check if another scheduled task is already copying a required file + * * @param currentlyCopying * @param path * @param locations * @return */ - private LocationWrapper alreadyCopying(Map> currentlyCopying, String path, List locations){ + private Tuple alreadyCopying(Map> currentlyCopying, String path, List locations, String debug ){ if ( currentlyCopying != null && currentlyCopying.containsKey( path ) ){ - final Location copyFrom = currentlyCopying.get(path).getB(); + final Tuple taskLocationTuple = currentlyCopying.get(path); + final Location copyFrom = taskLocationTuple.getB(); for ( LocationWrapper locationWrapper : locations ) { if ( locationWrapper.getLocation() == copyFrom ) { - return locationWrapper; + return new Tuple(locationWrapper,taskLocationTuple.getA()); } } throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); @@ -48,50 +48,73 @@ private LocationWrapper alreadyCopying(Map> curren return null; } + private double findAlignmentForFile ( + PathFileLocationTriple pathFileLocationTriple, + NodeLocation nodeLocation, + HashMap map + ){ + double minCost = Double.MAX_VALUE; + Location bestLoc = null; + LocationWrapper bestLocationWrapper = null; + for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { + final Location currentLoc = locationWrapper.getLocation(); + final double calculatedCost; + if ( currentLoc != nodeLocation ) { + final AlignmentWrapper alignmentWrapper = map.get(currentLoc); + calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); + } else { + calculatedCost = 0; + } + if ( calculatedCost < minCost ) { + bestLoc = currentLoc; + minCost = calculatedCost; + bestLocationWrapper = locationWrapper; + } + } + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost ); + return minCost; + } + + private boolean canUseFileFromOtherTask ( + Map> currentlyCopying, + PathFileLocationTriple pathFileLocationTriple, + Map map, + String debug + ) { + final Tuple copying = alreadyCopying( + currentlyCopying, + pathFileLocationTriple.path.toString(), + pathFileLocationTriple.locations, + debug + ); + if ( copying != null ) { + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(copying.getA().getLocation(), k -> new AlignmentWrapper()); + alignmentWrapper.addAlignmentToWaitFor(new FilePathWithTask(pathFileLocationTriple, copying.getA(), copying.getB())); + return true; + } + return false; + } + @Override public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, Map> currentlyCopying, + Map> currentlyPlanedToCopy, double maxCost) { inputsOfTask.sort(); final HashMap map = new HashMap<>(); double cost = 0; for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { - double minCost = Double.MAX_VALUE; - Location bestLoc = null; - LocationWrapper bestLocationWrapper = null; - - final LocationWrapper copying = alreadyCopying( - currentlyCopying, - pathFileLocationTriple.path.toString(), - pathFileLocationTriple.locations - ); - if ( copying != null ) { - minCost = 0; - bestLoc = copying.getLocation(); - bestLocationWrapper = copying; - } else { - for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { - final Location currentLoc = locationWrapper.getLocation(); - final double calculatedCost; - if ( currentLoc != node.getNodeLocation() ) { - final AlignmentWrapper alignmentWrapper = map.get(currentLoc); - calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); - } else { - calculatedCost = 0; - } - if ( calculatedCost < minCost ) { - bestLoc = currentLoc; - minCost = calculatedCost; - bestLocationWrapper = locationWrapper; - } - } - if ( minCost > maxCost ) return null; - if ( minCost > cost ) cost = minCost; + if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) + && + ! canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) + ) { + final double newCost = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); + if ( newCost > maxCost ) return null; + if ( newCost > cost ) cost = newCost; } - final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); - alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost ); } return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java index 4c62b92f..ecaf7d57 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java @@ -24,6 +24,7 @@ FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, Map> currentlyCopying, + Map> currentlyPlanedToCopy, double maxCost ); /** @@ -36,8 +37,10 @@ FileAlignment getInputAlignment( @NotNull Task task, default FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, - Map> currentlyCopying ){ - return getInputAlignment( task, inputsOfTask, node, currentlyCopying, Double.MAX_VALUE ); + Map> currentlyCopying, + Map> currentlyPlanedToCopy + ){ + return getInputAlignment( task, inputsOfTask, node, currentlyCopying, currentlyPlanedToCopy, Double.MAX_VALUE ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index 549b7df6..cccb5c52 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -26,6 +26,7 @@ public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, Map> currentlyCopying, + Map> currentlyPlanedToCopy, double maxCost) { final HashMap map = new HashMap<>(); for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { @@ -39,7 +40,7 @@ public FileAlignment getInputAlignment(@NotNull Task task, )); final Location location = locationWrapper.getLocation(); final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); - alignmentWrapper.addAlignment( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); } return new FileAlignment( map, inputsOfTask.getSymlinks(), 0); } diff --git a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java index 981815a6..ca1681fd 100644 --- a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java +++ b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java @@ -5,19 +5,38 @@ import java.util.LinkedList; import java.util.List; -@Getter public class AlignmentWrapper { - private final List alignment = new LinkedList<>(); + @Getter + private final List filesToCopy = new LinkedList<>(); + + private final List waitFor = new LinkedList<>(); + + @Getter private double cost = 0; - public void addAlignment( FilePath filePath, double cost ) { - alignment.add( filePath ); + public void addAlignmentToCopy( FilePath filePath, double cost ) { + filesToCopy.add( filePath ); + this.cost = cost; + } + + public void addAlignmentToWaitFor( FilePathWithTask filePath, double cost ) { + waitFor.add( filePath ); this.cost = cost; } + public void addAlignmentToWaitFor( FilePathWithTask filePath ) { + waitFor.add( filePath ); + } + public boolean empty() { - return alignment.isEmpty(); + return filesToCopy.isEmpty() && waitFor.isEmpty(); + } + + public List getAll() { + List newList = new LinkedList<>(filesToCopy); + newList.addAll(waitFor); + return newList; } } diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 68f8e7ca..dfde2a6a 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -30,7 +30,7 @@ public List getAllLocationWrappers(){ .parallelStream() .flatMap( l -> l .getValue() - .getAlignment() + .getAll() .parallelStream() .map(FilePath::getLocationWrapper) ) diff --git a/src/main/java/fonda/scheduler/util/FilePathWithTask.java b/src/main/java/fonda/scheduler/util/FilePathWithTask.java new file mode 100644 index 00000000..4d6b6c7f --- /dev/null +++ b/src/main/java/fonda/scheduler/util/FilePathWithTask.java @@ -0,0 +1,16 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; + +public class FilePathWithTask extends FilePath { + + private final Task task; + + public FilePathWithTask(PathFileLocationTriple pathFileLocationTriple, LocationWrapper locationWrapper, Task task ) { + super(pathFileLocationTriple, locationWrapper); + this.task = task; + } + +} From 6c2fc766ea07086234c4ace74790d18d0dacb0e1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 20 May 2022 12:13:25 +0200 Subject: [PATCH 244/443] Nullcheck Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 44d8bf67..03fb61d8 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -257,6 +257,8 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { inputs.sortData(); if (traceEnabled) { + if (traceRecord.getSchedulerFilesNode() == null) traceRecord.setSchedulerFilesNode(0); + if (traceRecord.getSchedulerFilesNodeBytes() == null) traceRecord.setSchedulerFilesNodeBytes(0l); traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); From d9dc2ac51cde242f0e08ffcf97a8f446f037a5d7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 20 May 2022 15:22:56 +0200 Subject: [PATCH 245/443] Log place in queue correctly Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/LocationAwareScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 49abe0a2..f1a0e56b 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -101,7 +101,7 @@ List createAlignment( final List alignment = new LinkedList<>(); final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); for ( TaskData taskData : unscheduledTasksSorted ){ - final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, planedToCopy, index); + final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, planedToCopy, ++index); if ( nodeAlignment != null ) { alignment.add(nodeAlignment); addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.nodeFileAlignment, taskData.getTask(), nodeAlignment.node ); From b3e0d975c9e18022c296fd53fa01c0260922ebf3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 20 May 2022 17:22:08 +0200 Subject: [PATCH 246/443] transfer workDir via TaskConfig Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 3 +-- src/main/java/fonda/scheduler/model/TaskConfig.java | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 8cab2e62..29160f3b 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -60,8 +60,7 @@ public Task( TaskConfig config, DAG dag ) { } public String getWorkingDir(){ - if ( this.pod == null ) return null; - return pod.getSpec().getContainers().get(0).getWorkingDir(); + return config.getWorkDir(); } public boolean wasSuccessfullyExecuted(){ diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 0e08cef5..4e5cb4bc 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -19,6 +19,8 @@ public class TaskConfig { private final float cpus; private final long memoryInBytes; + private final String workDir; + private TaskConfig() { this( null ); } @@ -35,6 +37,7 @@ public TaskConfig(String task) { this.hash = null; this.cpus = 0; this.memoryInBytes = 0; + this.workDir = null; } @Getter From 1895fd15115d3d02d48081ac363b21325eee637c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 24 May 2022 13:47:43 +0200 Subject: [PATCH 247/443] unscheduledTasks as Queue, re-add tasks if best node is not available anymore Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 28 +++--- .../scheduler/RandomLAScheduler.java | 14 +-- .../scheduler/scheduler/data/TaskData.java | 87 ++++++++++++++----- 3 files changed, 86 insertions(+), 43 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index f1a0e56b..11a93cd4 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -51,9 +51,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( ) { long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); - final Set matchingNodesForTask = getMatchingNodesForTask( availableByNode, taskData.getTask()); - if ( matchingNodesForTask.isEmpty() ) return null; - final Tuple result = calculateBestNode(taskData, matchingNodesForTask, planedToCopy); + final Tuple result = calculateBestNode(taskData, planedToCopy); if ( result == null ) return null; final Task task = taskData.getTask(); availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); @@ -94,13 +92,21 @@ private void addAlignmentToPlanned ( * @return a list with alignments */ List createAlignment( - final List unscheduledTasksSorted, + final PriorityQueue unscheduledTasksSorted, final Map availableByNode ){ int index = 0; final List alignment = new LinkedList<>(); final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); - for ( TaskData taskData : unscheduledTasksSorted ){ + while( !unscheduledTasksSorted.isEmpty() ){ + TaskData taskData = unscheduledTasksSorted.poll(); + if ( taskData.calculate( availableByNode ) ){ + if ( !taskData.getNodeDataTuples().isEmpty() ) { + unscheduledTasksSorted.add(taskData); + } + continue; + } + final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, planedToCopy, ++index); if ( nodeAlignment != null ) { alignment.add(nodeAlignment); @@ -115,7 +121,7 @@ public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks, final Map availableByNode ){ - final List unscheduledTasksSorted = unscheduledTasks + final PriorityQueue unscheduledTasksSorted = unscheduledTasks .parallelStream() .map(task -> { long startTime = System.nanoTime(); @@ -124,8 +130,7 @@ public ScheduleObject getTaskNodeAlignment( return taskData; }) .filter(Objects::nonNull) - .sorted(Comparator.reverseOrder()) - .collect(Collectors.toList()); + .collect(Collectors.toCollection(() -> new PriorityQueue<>(Collections.reverseOrder()))); final List alignment = createAlignment(unscheduledTasksSorted, availableByNode); final ScheduleObject scheduleObject = new ScheduleObject(alignment); scheduleObject.setCheckStillPossible( true ); @@ -135,11 +140,9 @@ public ScheduleObject getTaskNodeAlignment( Tuple calculateBestNode( final TaskData taskData, - final Set matchingNodesForTask, Map< Location, Map>> planedToCopy){ FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; - //Remove all nodes which do not fit anymore final List nodeDataTuples = taskData.getNodeDataTuples(); int triedNodes = 0; int noAlignmentFound = 0; @@ -148,7 +151,6 @@ Tuple calculateBestNode( for (NodeDataTuple nodeDataTuple : nodeDataTuples) { final NodeWithAlloc currentNode = nodeDataTuple.getNode(); - if ( !matchingNodesForTask.contains(currentNode) ) continue; final Map> currentlyCopying = getCopyingToNode().get(currentNode.getNodeLocation()); final Map> currentlyPlanedToCopy = planedToCopy.get(currentNode.getNodeLocation()); FileAlignment fileAlignment = null; @@ -223,10 +225,8 @@ TaskData calculateTaskData( .map(node -> new NodeDataTuple(node, inputsOfTask.calculateDataOnNode( node.getNodeLocation() ) ) ) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); - final double fracOnNode = size == 0 ? 1 : nodeDataTuples.get(0).getSizeInBytes() / (double) size; final double antiStarvingFactor = 0; - final double value = fracOnNode + antiStarvingFactor; - return new TaskData( value, task, nodeDataTuples, matchingFilesAndNodes ); + return new TaskData( size, task, nodeDataTuples, matchingFilesAndNodes, antiStarvingFactor ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 66339fff..765cfe3d 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -32,18 +32,18 @@ public RandomLAScheduler( super(name, client, namespace, config, inputAlignment); } - private Optional selectNode( Set matchingNodes, Task task ){ - return matchingNodes.isEmpty() - ? Optional.empty() - : Optional.of( new LinkedList<>(matchingNodes).get(random.nextInt(matchingNodes.size()))); + private Optional selectNode( List matchingNodes, Task task ){ + if (matchingNodes.isEmpty()) return Optional.empty(); + else { + return Optional.of( new LinkedList<>(matchingNodes).get(random.nextInt(matchingNodes.size())).getNode()); + } } @Override Tuple calculateBestNode( final TaskData taskData, - final Set matchingNodesForTask, Map< Location, Map>> planedToCopy){ - final Optional nodeWithAlloc = selectNode(matchingNodesForTask, taskData.getTask()); + final Optional nodeWithAlloc = selectNode(taskData.getNodeDataTuples(), taskData.getTask()); if (nodeWithAlloc.isEmpty()) return null; final NodeWithAlloc node = nodeWithAlloc.get(); final Map> currentlyCopying = getCopyingToNode().get(node.getNodeLocation()); @@ -70,7 +70,7 @@ TaskData calculateTaskData( .parallelStream() .map(node -> new NodeDataTuple(node, 0 ) ) .collect(Collectors.toList()); - return new TaskData( 0, task, nodeDataTuples, matchingFilesAndNodes ); + return new TaskData( 0, task, nodeDataTuples, matchingFilesAndNodes, 0); } diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index 87a19933..950c8371 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -1,40 +1,83 @@ package fonda.scheduler.scheduler.data; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; import fonda.scheduler.model.Task; import fonda.scheduler.scheduler.MatchingFilesAndNodes; +import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; import org.jetbrains.annotations.NotNull; +import java.util.Iterator; import java.util.List; +import java.util.Map; @Getter @EqualsAndHashCode public class TaskData implements Comparable { - private final double value; - private final Task task; - private final List nodeDataTuples; - private final MatchingFilesAndNodes matchingFilesAndNodes; - private long timeInNs = 0; - - public TaskData( - double value, - Task task, - List nodeDataTuples, - MatchingFilesAndNodes matchingFilesAndNodes - ) { - this.value = value; - this.task = task; - this.nodeDataTuples = nodeDataTuples; - this.matchingFilesAndNodes = matchingFilesAndNodes; - } - @Override - public int compareTo(@NotNull TaskData o) { - return Double.compare(value, o.value); + private final double size; + + private final Task task; + + private final List nodeDataTuples; + + private final MatchingFilesAndNodes matchingFilesAndNodes; + + private long timeInNs = 0; + + private final double antiStarvingFactor; + + @Getter(AccessLevel.NONE) + private NodeDataTuple calulatedFor = null; + + private double value = 0; + + public TaskData( + double size, + Task task, + List nodeDataTuples, + MatchingFilesAndNodes matchingFilesAndNodes, + double antiStarvingFactor) { + this.size = size; + this.task = task; + this.nodeDataTuples = nodeDataTuples; + this.matchingFilesAndNodes = matchingFilesAndNodes; + this.antiStarvingFactor = antiStarvingFactor; + calc(); + } + + /** + * Recalculate the value. New value is smaller or equal old value + * @param availableByNode + * @return if the value has changed + */ + public boolean calculate( Map availableByNode ) { + final Iterator iterator = nodeDataTuples.iterator(); + while (iterator.hasNext()) { + if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) { + iterator.remove(); + calc(); + return true; + } } + return false; + } - public void addNs( long timeInNs ){ + private void calc() { + if ( nodeDataTuples.isEmpty() ) value = Double.MIN_VALUE; + if ( size == 0 ) value = 1; + else value = nodeDataTuples.get(0).getSizeInBytes() / (double) size; + } + + @Override + public int compareTo(@NotNull TaskData o) { + return Double.compare(getValue(), o.getValue()); + } + + public void addNs( long timeInNs ){ this.timeInNs += timeInNs; } - } \ No newline at end of file + +} \ No newline at end of file From ca00eb35d507f1f9fcd30acbdcffeb35263347e9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 24 May 2022 16:37:19 +0200 Subject: [PATCH 248/443] RandomAlignment filter files on Node Signed-off-by: Lehmann_Fabian --- .../filealignment/GreedyAlignment.java | 71 ++------------- .../filealignment/InputAlignmentClass.java | 89 +++++++++++++++++++ .../filealignment/RandomAlignment.java | 44 ++++----- 3 files changed, 113 insertions(+), 91 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index ad8e80a4..b57cb7d8 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -8,17 +8,18 @@ import fonda.scheduler.model.taskinputs.PathFileLocationTriple; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; -import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.util.*; +import fonda.scheduler.util.AlignmentWrapper; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.FilePath; +import fonda.scheduler.util.Tuple; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import java.util.HashMap; -import java.util.List; import java.util.Map; @Slf4j -public class GreedyAlignment implements InputAlignment { +public class GreedyAlignment extends InputAlignmentClass { private final CostFunction cf; @@ -26,31 +27,9 @@ public GreedyAlignment(CostFunction cf) { this.cf = cf; } - /** - * Check if another scheduled task is already copying a required file - * - * @param currentlyCopying - * @param path - * @param locations - * @return - */ - private Tuple alreadyCopying(Map> currentlyCopying, String path, List locations, String debug ){ - if ( currentlyCopying != null && currentlyCopying.containsKey( path ) ){ - final Tuple taskLocationTuple = currentlyCopying.get(path); - final Location copyFrom = taskLocationTuple.getB(); - for ( LocationWrapper locationWrapper : locations ) { - if ( locationWrapper.getLocation() == copyFrom ) { - return new Tuple(locationWrapper,taskLocationTuple.getA()); - } - } - throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); - } - return null; - } - - private double findAlignmentForFile ( + double findAlignmentForFile ( PathFileLocationTriple pathFileLocationTriple, - NodeLocation nodeLocation, + NodeLocation scheduledNode, HashMap map ){ double minCost = Double.MAX_VALUE; @@ -59,7 +38,7 @@ private double findAlignmentForFile ( for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { final Location currentLoc = locationWrapper.getLocation(); final double calculatedCost; - if ( currentLoc != nodeLocation ) { + if ( currentLoc != scheduledNode) { final AlignmentWrapper alignmentWrapper = map.get(currentLoc); calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); } else { @@ -76,26 +55,6 @@ private double findAlignmentForFile ( return minCost; } - private boolean canUseFileFromOtherTask ( - Map> currentlyCopying, - PathFileLocationTriple pathFileLocationTriple, - Map map, - String debug - ) { - final Tuple copying = alreadyCopying( - currentlyCopying, - pathFileLocationTriple.path.toString(), - pathFileLocationTriple.locations, - debug - ); - if ( copying != null ) { - final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(copying.getA().getLocation(), k -> new AlignmentWrapper()); - alignmentWrapper.addAlignmentToWaitFor(new FilePathWithTask(pathFileLocationTriple, copying.getA(), copying.getB())); - return true; - } - return false; - } - @Override public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @@ -104,19 +63,7 @@ public FileAlignment getInputAlignment(@NotNull Task task, Map> currentlyPlanedToCopy, double maxCost) { inputsOfTask.sort(); - final HashMap map = new HashMap<>(); - double cost = 0; - for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { - if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) - && - ! canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) - ) { - final double newCost = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); - if ( newCost > maxCost ) return null; - if ( newCost > cost ) cost = newCost; - } - } - return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); + return super.getInputAlignment( task, inputsOfTask, node, currentlyCopying, currentlyPlanedToCopy, maxCost ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java new file mode 100644 index 00000000..8a93c650 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -0,0 +1,89 @@ +package fonda.scheduler.scheduler.filealignment; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import fonda.scheduler.util.AlignmentWrapper; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.FilePathWithTask; +import fonda.scheduler.util.Tuple; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +abstract class InputAlignmentClass implements InputAlignment { + + /** + * Check if another scheduled task is already copying a required file + * + * @param currentlyCopying + * @param path + * @param locations + * @return + */ + private Tuple alreadyCopying(Map> currentlyCopying, String path, List locations, String debug ){ + if ( currentlyCopying != null && currentlyCopying.containsKey( path ) ){ + final Tuple taskLocationTuple = currentlyCopying.get(path); + final Location copyFrom = taskLocationTuple.getB(); + for ( LocationWrapper locationWrapper : locations ) { + if ( locationWrapper.getLocation() == copyFrom ) { + return new Tuple(locationWrapper,taskLocationTuple.getA()); + } + } + throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); + } + return null; + } + + private boolean canUseFileFromOtherTask ( + Map> currentlyCopying, + PathFileLocationTriple pathFileLocationTriple, + Map map, + String debug + ) { + final Tuple copying = alreadyCopying( + currentlyCopying, + pathFileLocationTriple.path.toString(), + pathFileLocationTriple.locations, + debug + ); + if ( copying != null ) { + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(copying.getA().getLocation(), k -> new AlignmentWrapper()); + alignmentWrapper.addAlignmentToWaitFor(new FilePathWithTask(pathFileLocationTriple, copying.getA(), copying.getB())); + return true; + } + return false; + } + + @Override + public FileAlignment getInputAlignment(@NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + Map> currentlyCopying, + Map> currentlyPlanedToCopy, + double maxCost) { + final HashMap map = new HashMap<>(); + double cost = 0; + for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { + if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) + && + ! canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) + ) { + final double newCost = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); + if ( newCost > maxCost ) return null; + if ( newCost > cost ) cost = newCost; + } + } + return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); + } + + abstract double findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map); + +} diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index cccb5c52..a1f379cc 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -1,48 +1,34 @@ package fonda.scheduler.scheduler.filealignment; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.Tuple; -import org.jetbrains.annotations.NotNull; import java.util.HashMap; -import java.util.Map; import java.util.Optional; import java.util.Random; -public class RandomAlignment implements InputAlignment { +public class RandomAlignment extends InputAlignmentClass { private final Random random = new Random(); @Override - public FileAlignment getInputAlignment(@NotNull Task task, - @NotNull TaskInputs inputsOfTask, - @NotNull NodeWithAlloc node, - Map> currentlyCopying, - Map> currentlyPlanedToCopy, - double maxCost) { - final HashMap map = new HashMap<>(); - for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { - final Optional first = pathFileLocationTriple - .locations - .stream() - .filter(x -> x.getLocation() == node.getNodeLocation()) - .findFirst(); - final LocationWrapper locationWrapper = first.orElseGet(() -> pathFileLocationTriple.locations.get( - random.nextInt(pathFileLocationTriple.locations.size()) - )); - final Location location = locationWrapper.getLocation(); - final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); - alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); - } - return new FileAlignment( map, inputsOfTask.getSymlinks(), 0); + double findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map) { + final Optional first = pathFileLocationTriple + .locations + .stream() + .filter(x -> x.getLocation() == scheduledNode) + .findFirst(); + final LocationWrapper locationWrapper = first.orElseGet(() -> pathFileLocationTriple.locations.get( + random.nextInt(pathFileLocationTriple.locations.size()) + )); + final Location location = locationWrapper.getLocation(); + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); + return 0; } } From 5abdc4b6292865c6819834d82894d4b8fe62a480 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 30 May 2022 13:06:10 +0200 Subject: [PATCH 249/443] Trace only if tracing is activated Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index ee737d13..591598d2 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -5,6 +5,7 @@ import os import shutil import signal +import sys import time import urllib.request from concurrent.futures import ThreadPoolExecutor @@ -261,13 +262,14 @@ def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): def writeTrace(dataMap): - global errors - if len(dataMap) == 0 or errors > 0: - return - with open(traceFilePath, "a") as traceFile: - for d in dataMap: - traceFile.write(d + "=" + str(dataMap[d]) + "\n") - traceFile.write("scheduler_init_errors=" + str(errors) + "\n") + if sys.argv[1] == 'true': + global errors + if len(dataMap) == 0 or errors > 0: + return + with open(traceFilePath, "a") as traceFile: + for d in dataMap: + traceFile.write(d + "=" + str(dataMap[d]) + "\n") + traceFile.write("scheduler_init_errors=" + str(errors) + "\n") def run(): From d313238313b450efe1ce7cb24789eb296ea9cf64 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 31 May 2022 10:58:44 +0200 Subject: [PATCH 250/443] Remove nodes without enough resources before scheduling Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 1 + .../scheduler/scheduler/data/TaskData.java | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 11a93cd4..7e2104b2 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -51,6 +51,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( ) { long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); + taskData.removeAllNodesWhichHaveNotEnoughResources( availableByNode ); final Tuple result = calculateBestNode(taskData, planedToCopy); if ( result == null ) return null; final Task task = taskData.getTask(); diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index 950c8371..31bbd195 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -55,14 +55,25 @@ public TaskData( */ public boolean calculate( Map availableByNode ) { final Iterator iterator = nodeDataTuples.iterator(); + boolean changed = false; while (iterator.hasNext()) { if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) { iterator.remove(); - calc(); - return true; + changed = true; + } else { + break; } } - return false; + calc(); + return changed; + } + + public void removeAllNodesWhichHaveNotEnoughResources( Map availableByNode ) { + final Iterator iterator = nodeDataTuples.iterator(); + while (iterator.hasNext()) { + if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) + iterator.remove(); + } } private void calc() { From 4d1adcd8dfbb65887dbb276653c9b0fd0c84149c Mon Sep 17 00:00:00 2001 From: Soeren Becker Date: Wed, 1 Jun 2022 09:27:31 +0200 Subject: [PATCH 251/443] Enable coverage reports in SonarQube (#5) --- Jenkinsfile | 5 ++--- jenkins-pod.yaml | 2 +- pom.xml | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d25f777a..2b77db2c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { junit 'target/surefire-reports/TEST-*.xml' jacoco classPattern: 'target/classes,target/test-classes', execPattern: 'target/coverage-reports/*.exec', inclusionPattern: '**/*.class', sourcePattern: 'src/main/java,src/test/java' archiveArtifacts 'target/surefire-reports/TEST-*.xml' - archiveArtifacts 'target/coverage-reports/*.exec' + archiveArtifacts 'target/*.exec' } } } @@ -58,8 +58,7 @@ pipeline { mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ - -Dsonar.junit.reportPaths=target/surefire-reports \ - -Dsonar.jacoco.reportPaths=$(find target/coverage-reports -name '*.exec' | paste -s -d , -) + -Dsonar.junit.reportPaths=target/surefire-reports ''' } } diff --git a/jenkins-pod.yaml b/jenkins-pod.yaml index 71628a4d..eb40c2b4 100644 --- a/jenkins-pod.yaml +++ b/jenkins-pod.yaml @@ -1,4 +1,4 @@ - +--- apiVersion: v1 kind: Pod spec: diff --git a/pom.xml b/pom.xml index d25f76ff..5de43cef 100644 --- a/pom.xml +++ b/pom.xml @@ -139,9 +139,6 @@ jacoco-maven-plugin 0.8.7 - ${project.build.directory}/coverage-reports/jacoco-unit.exec - ${project.build.directory}/coverage-reports/jacoco-unit.exec - ${project.reporting.outputDirectory}/jacoco-report surefireArgLine From 4cbea1f8bfc53af3376cd601fd5b10f632dd5896 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 7 Jun 2022 16:59:26 +0200 Subject: [PATCH 252/443] Rewrote writeInitConfig method Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 126 ++++++++---------- .../filealignment/GreedyAlignment.java | 2 +- .../filealignment/InputAlignmentClass.java | 2 +- .../filealignment/RandomAlignment.java | 2 +- .../scheduler/util/AlignmentWrapper.java | 17 +-- .../scheduler/util/FilePathWithTask.java | 2 + 6 files changed, 70 insertions(+), 81 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 03fb61d8..a0456c72 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -92,6 +92,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; final WriteConfigResult writeConfigResult = writeInitConfig(nodeTaskFilesAlignment); if ( writeConfigResult == null ) return false; + if ( traceEnabled ) traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); @@ -167,15 +168,35 @@ private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tup pathTasks.keySet().removeAll( toRemove.keySet() ); } - private void traceForCurrentNode( AlignmentWrapper alignmentWrapper, NodeLocation node, TraceRecord record ) { - final List all = alignmentWrapper.getAll(); - record.setSchedulerFilesNode(all.size()); - record.setSchedulerFilesNodeBytes( - all - .parallelStream() - .mapToLong(x -> x.getFile().getLocationWrapper(node).getSizeInBytes()) - .sum() - ); + private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult writeConfigResult ) { + final TraceRecord traceRecord = alignment.task.getTraceRecord(); + + int filesOnNodeOtherTask = 0; + int filesNotOnNode = 0; + long filesOnNodeOtherTaskByte = 0; + long filesNotOnNodeByte = 0; + + final NodeLocation currentNode = alignment.node.getNodeLocation(); + for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { + final AlignmentWrapper alignmentWrapper = entry.getValue(); + if( entry.getKey() == currentNode) { + traceRecord.setSchedulerFilesNode( alignmentWrapper.getFilesToCopy().size() + alignmentWrapper.getWaitFor().size() ); + traceRecord.setSchedulerFilesNodeBytes( alignmentWrapper.getToCopySize() + alignmentWrapper.getToWaitSize() ); + } else { + filesOnNodeOtherTask += alignmentWrapper.getWaitFor().size() + alignmentWrapper.getFilesToCopy().size(); + filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize() + alignmentWrapper.getToCopySize(); + } + } + if (traceRecord.getSchedulerFilesNode() == null) traceRecord.setSchedulerFilesNode(0); + if (traceRecord.getSchedulerFilesNodeBytes() == null) traceRecord.setSchedulerFilesNodeBytes(0l); + traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); + traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); + final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); + traceRecord.setSchedulerFiles(schedulerFilesNode + filesOnNodeOtherTask + filesNotOnNode); + final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); + traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); + traceRecord.setSchedulerDependingTask( (int) writeConfigResult.getWaitForTask().values().stream().distinct().count() ); + traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.nodeFileAlignment.size() - (schedulerFilesNode > 0 ? 1 : 0) ); } /** @@ -186,9 +207,10 @@ private void traceForCurrentNode( AlignmentWrapper alignmentWrapper, NodeLocatio WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); - LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); Map waitForTask = new HashMap<>(); + final HashMap> filesForCurrentNode = new HashMap<>(); + final NodeLocation currentNode = alignment.node.getNodeLocation(); try { final Inputs inputs = new Inputs( @@ -197,83 +219,47 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { alignment.task.getConfig().getHash() ); - final HashMap> filesForCurrentNode = new HashMap<>(); - final NodeLocation currentNode = alignment.node.getNodeLocation(); - final Map> filesOnCurrentNode = copyingToNode.get(currentNode); - - final TraceRecord traceRecord = alignment.task.getTraceRecord(); - - int filesOnNodeOtherTask = 0; - int filesNotOnNode = 0; - long filesOnNodeOtherTaskByte = 0; - long filesNotOnNodeByte = 0; - for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { - if( entry.getKey() == alignment.node.getNodeLocation() ) { - if (traceEnabled) traceForCurrentNode( entry.getValue(), currentNode, traceRecord ); + if( entry.getKey() == currentNode ) { continue; } - final List collect = new LinkedList<>(); - final NodeLocation location = (NodeLocation) entry.getKey(); - long size = 0; - for (FilePath filePath : entry.getValue().getAll()) { - if ( filesOnCurrentNode != null && filesOnCurrentNode.containsKey(filePath.getPath()) ) { - //Node copies currently from somewhere else! - final Tuple taskLocationTuple = filesOnCurrentNode.get(filePath.getPath()); - if ( taskLocationTuple.getB() != location ) return null; - else { - //May be problematic if the task depending on fails/is stopped before all files are downloaded - waitForTask.put(filePath.getPath(), taskLocationTuple.getA() ); - if (traceEnabled) { - filesOnNodeOtherTask++; - filesOnNodeOtherTaskByte += filePath.getLocationWrapper().getSizeInBytes(); - } - } - } else { - final LocationWrapper locationWrapper = filePath.getFile().getLocationWrapper(location); - inputFiles.add( - new TaskInputFileLocationWrapper( - filePath.getPath(), - filePath.getFile(), - locationWrapper.getCopyOf( currentNode ) - ) - ); - collect.add(filePath.getPath()); - filesForCurrentNode.put(filePath.getPath(), new Tuple<>( alignment.task, location ) ); - final long sizeInBytes = filePath.getLocationWrapper().getSizeInBytes(); - size += sizeInBytes; - filesNotOnNode++; - } + final AlignmentWrapper alignmentWrapper = entry.getValue(); + for (FilePathWithTask filePath : alignmentWrapper.getWaitFor()) { + //Node copies currently from somewhere else! + //May be problematic if the task depending on fails/is stopped before all files are downloaded + waitForTask.put( filePath.getPath(), filePath.getTask() ); + } + + final List collect = new LinkedList<>(); + for (FilePath filePath : alignmentWrapper.getFilesToCopy()) { + final LocationWrapper locationWrapper = filePath.getFile().getLocationWrapper(location); + inputFiles.add( + new TaskInputFileLocationWrapper( + filePath.getPath(), + filePath.getFile(), + locationWrapper.getCopyOf( currentNode ) + ) + ); + collect.add(filePath.getPath()); + filesForCurrentNode.put( filePath.getPath(), new Tuple<>( alignment.task, location ) ); } - filesNotOnNodeByte += size; if( !collect.isEmpty() ) { - inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, size)); + inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, alignmentWrapper.getToCopySize())); } } + inputs.waitForTask( waitForTask ); inputs.symlinks.addAll(alignment.fileAlignment.symlinks); inputs.sortData(); - - if (traceEnabled) { - if (traceRecord.getSchedulerFilesNode() == null) traceRecord.setSchedulerFilesNode(0); - if (traceRecord.getSchedulerFilesNodeBytes() == null) traceRecord.setSchedulerFilesNodeBytes(0l); - traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); - traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); - final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); - traceRecord.setSchedulerFiles(schedulerFilesNode + filesOnNodeOtherTask + filesNotOnNode); - final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); - traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); - traceRecord.setSchedulerDependingTask( (int) waitForTask.values().stream().distinct().count() ); - traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.nodeFileAlignment.size() - (schedulerFilesNode > 0 ? 1 : 0) ); - } - new ObjectMapper().writeValue( config, inputs ); return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode ); + } catch (IOException e) { log.error( "Cannot write " + config, e); } + return null; } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index b57cb7d8..6f5a02bc 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -51,7 +51,7 @@ public GreedyAlignment(CostFunction cf) { } } final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); - alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost, bestLocationWrapper.getSizeInBytes() ); return minCost; } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java index 8a93c650..71092ec6 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -56,7 +56,7 @@ private boolean canUseFileFromOtherTask ( ); if ( copying != null ) { final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(copying.getA().getLocation(), k -> new AlignmentWrapper()); - alignmentWrapper.addAlignmentToWaitFor(new FilePathWithTask(pathFileLocationTriple, copying.getA(), copying.getB())); + alignmentWrapper.addAlignmentToWaitFor(new FilePathWithTask(pathFileLocationTriple, copying.getA(), copying.getB()), copying.getA().getSizeInBytes()); return true; } return false; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index a1f379cc..6d10bb72 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -27,7 +27,7 @@ public class RandomAlignment extends InputAlignmentClass { )); final Location location = locationWrapper.getLocation(); final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); - alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0 ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0, locationWrapper.getSizeInBytes() ); return 0; } diff --git a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java index ca1681fd..22d8b16d 100644 --- a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java +++ b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java @@ -5,27 +5,28 @@ import java.util.LinkedList; import java.util.List; +@Getter public class AlignmentWrapper { - @Getter private final List filesToCopy = new LinkedList<>(); - + private long toCopySize = 0; private final List waitFor = new LinkedList<>(); - - @Getter + private long toWaitSize = 0; private double cost = 0; - public void addAlignmentToCopy( FilePath filePath, double cost ) { + public void addAlignmentToCopy( FilePath filePath, double cost, long size ) { filesToCopy.add( filePath ); + toCopySize += size; this.cost = cost; } - public void addAlignmentToWaitFor( FilePathWithTask filePath, double cost ) { - waitFor.add( filePath ); + public void addAlignmentToWaitFor( FilePathWithTask filePath, double cost, long size ) { + addAlignmentToWaitFor( filePath, size ); this.cost = cost; } - public void addAlignmentToWaitFor( FilePathWithTask filePath ) { + public void addAlignmentToWaitFor( FilePathWithTask filePath, long size ) { + toWaitSize += size; waitFor.add( filePath ); } diff --git a/src/main/java/fonda/scheduler/util/FilePathWithTask.java b/src/main/java/fonda/scheduler/util/FilePathWithTask.java index 4d6b6c7f..4b5cedee 100644 --- a/src/main/java/fonda/scheduler/util/FilePathWithTask.java +++ b/src/main/java/fonda/scheduler/util/FilePathWithTask.java @@ -3,9 +3,11 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import lombok.Getter; public class FilePathWithTask extends FilePath { + @Getter private final Task task; public FilePathWithTask(PathFileLocationTriple pathFileLocationTriple, LocationWrapper locationWrapper, Task task ) { From d6aba719f70412b66657fe4e63be299e01d8feee Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 14 Jun 2022 17:30:37 +0200 Subject: [PATCH 253/443] Receive OutLabel Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/OutLabel.java | 18 ++++++++++++++++++ .../java/fonda/scheduler/model/TaskConfig.java | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/main/java/fonda/scheduler/model/OutLabel.java diff --git a/src/main/java/fonda/scheduler/model/OutLabel.java b/src/main/java/fonda/scheduler/model/OutLabel.java new file mode 100644 index 00000000..1e071df5 --- /dev/null +++ b/src/main/java/fonda/scheduler/model/OutLabel.java @@ -0,0 +1,18 @@ +package fonda.scheduler.model; + +import lombok.Getter; +import lombok.ToString; + +@Getter +@ToString +public class OutLabel { + + private final String label; + private final double weight; + + public OutLabel() { + this.label = null; + this.weight = -1; + } + +} diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 4e5cb4bc..4e938fd5 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -18,7 +18,7 @@ public class TaskConfig { private final String hash; private final float cpus; private final long memoryInBytes; - + private final OutLabel outLabel; private final String workDir; private TaskConfig() { @@ -38,6 +38,7 @@ public TaskConfig(String task) { this.cpus = 0; this.memoryInBytes = 0; this.workDir = null; + this.outLabel = null; } @Getter From 3bb959334589fe6a7bb947f126ff6b7685976604 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 16 Jun 2022 12:54:14 +0200 Subject: [PATCH 254/443] Enable FTP Writes Signed-off-by: Lehmann_Fabian --- daemons/ftp/vsftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemons/ftp/vsftpd.conf b/daemons/ftp/vsftpd.conf index 8f851a17..9ea2fcc4 100644 --- a/daemons/ftp/vsftpd.conf +++ b/daemons/ftp/vsftpd.conf @@ -28,7 +28,7 @@ anonymous_enable=YES local_enable=NO # # Uncomment this to enable any form of FTP write command. -write_enable=NO +write_enable=YES # # Default umask for local users is 077. You may wish to change this to 022, # if your users expect that (022 is used by most other ftpd's) From eb448c23939628f40bd2f45225c1efe347933a5e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 20 Jun 2022 14:56:36 +0200 Subject: [PATCH 255/443] Consider OutLabels for scheduling Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/OutLabel.java | 6 +- src/main/java/fonda/scheduler/model/Task.java | 5 + .../scheduler/LocationAwareScheduler.java | 94 ++++++++++++++++--- .../scheduler/scheduler/OutLabelHolder.java | 55 +++++++++++ .../scheduler/OutLabelHolderMaxTasks.java | 25 +++++ .../scheduler/RandomLAScheduler.java | 7 +- .../scheduler/SchedulerWithDaemonSet.java | 9 +- .../scheduler/data/NodeDataTuple.java | 30 ++++-- .../scheduler/scheduler/data/TaskData.java | 49 +++++++++- .../fonda/scheduler/util/FileAlignment.java | 15 ++- .../scheduler/OutLabelHolderMaxTasksTest.java | 50 ++++++++++ 11 files changed, 312 insertions(+), 33 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java create mode 100644 src/main/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasks.java create mode 100644 src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java diff --git a/src/main/java/fonda/scheduler/model/OutLabel.java b/src/main/java/fonda/scheduler/model/OutLabel.java index 1e071df5..49c040bf 100644 --- a/src/main/java/fonda/scheduler/model/OutLabel.java +++ b/src/main/java/fonda/scheduler/model/OutLabel.java @@ -10,9 +10,13 @@ public class OutLabel { private final String label; private final double weight; - public OutLabel() { + private OutLabel() { this.label = null; this.weight = -1; } + public OutLabel(String label, double weight) { + this.label = label; + this.weight = weight; + } } diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 29160f3b..58bf8e42 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -86,6 +86,11 @@ public void submitted(){ traceRecord.setSchedulerTimeInQueue( System.currentTimeMillis() - timeAddedToQueue ); } + public String getOutLabel(){ + final OutLabel outLabel = config.getOutLabel(); + return outLabel == null ? null : outLabel.getLabel(); + } + @Override public String toString() { return "Task{" + diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 7e2104b2..1214690b 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -3,6 +3,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.scheduler.data.NodeDataTuple; @@ -52,7 +53,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); taskData.removeAllNodesWhichHaveNotEnoughResources( availableByNode ); - final Tuple result = calculateBestNode(taskData, planedToCopy); + final Tuple result = calculateBestNode(taskData, planedToCopy,availableByNode); if ( result == null ) return null; final Task task = taskData.getTask(); availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); @@ -101,7 +102,17 @@ List createAlignment( final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); while( !unscheduledTasksSorted.isEmpty() ){ TaskData taskData = unscheduledTasksSorted.poll(); - if ( taskData.calculate( availableByNode ) ){ + boolean changed = false; + log.info( "TaskData: {}", taskData.getTask().getPod().getName() ); + if ( !taskData.isWeightWasSet() ) { + log.info( "TaskData: {} weight was not set", taskData.getTask().getPod().getName() ); + final NodeLocation nodeForLabel = outLabelHolder.getNodeForLabel(taskData.getTask().getOutLabel()); + if ( nodeForLabel != null ) { + changed = true; + taskData.setNodeAndWeight( nodeForLabel, taskData.getTask().getConfig().getOutLabel().getWeight() ); + } + } + if ( taskData.calculate( availableByNode ) || changed ){ if ( !taskData.getNodeDataTuples().isEmpty() ) { unscheduledTasksSorted.add(taskData); } @@ -111,7 +122,8 @@ List createAlignment( final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, planedToCopy, ++index); if ( nodeAlignment != null ) { alignment.add(nodeAlignment); - addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.nodeFileAlignment, taskData.getTask(), nodeAlignment.node ); + outLabelHolder.scheduleTaskOnNode( taskData.getTask(), nodeAlignment.node.getNodeLocation() ); + addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.getNodeFileAlignment(), taskData.getTask(), nodeAlignment.node ); } } return alignment; @@ -139,9 +151,34 @@ public ScheduleObject getTaskNodeAlignment( return scheduleObject; } + /** + * Performs a stalemate between two possible alignments. + * @return True if the new alignment is better + */ + protected boolean stalemate( + NodeWithAlloc oldNode, + NodeWithAlloc newNode, + Map availableByNode) + { + final Requirements availableNodeA = availableByNode.get( oldNode ); + final Requirements availableNodeB = availableByNode.get( newNode ); + + //nodeA has less available CPU than nodeB + if( availableNodeA.getCpu().compareTo(availableNodeB.getCpu()) < 0 ) { + return true; + } else if ( availableNodeA.getCpu().compareTo(availableNodeB.getCpu()) == 0 ) { + if( availableNodeA.getRam().compareTo(availableNodeB.getRam() ) < 0 ) { + return true; + } + } + return false; + } + Tuple calculateBestNode( final TaskData taskData, - Map< Location, Map>> planedToCopy){ + Map< Location, Map>> planedToCopy, + Map availableByNode) + { FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; final List nodeDataTuples = taskData.getNodeDataTuples(); @@ -162,14 +199,32 @@ Tuple calculateBestNode( currentNode, currentlyCopying, currentlyPlanedToCopy, - bestAlignment == null ? Double.MAX_VALUE : bestAlignment.cost + bestAlignment == null ? Double.MAX_VALUE : bestAlignment.getCost() ); + if ( fileAlignment == null ){ couldStopFetching++; - } else if ( bestAlignment == null || bestAlignment.cost > fileAlignment.cost ){ - bestAlignment = fileAlignment; - bestNode = currentNode; - log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.cost ); + } else { + final boolean isOnOutLabelNode = nodeDataTuple.getNode().getNodeLocation() == taskData.getOutLabelNode(); + if ( isOnOutLabelNode ) { + fileAlignment.setWeight( taskData.getWeight() ); + } + log.info( "Task: {}, outLabelNode: {}, currentNode: {}, bestWeight: {}, currentWeight: {}", + taskData.getTask().getConfig().getName(), + taskData.getOutLabelNode(), + nodeDataTuple.getNode().getNodeLocation(), + bestAlignment == null ? null : bestAlignment.getWorth(), + fileAlignment.getWorth() + ); + if (bestAlignment == null || bestAlignment.getWorth() > fileAlignment.getWorth() || + //Alignment is comparable + ( bestAlignment.getWorth() + 1e8 > fileAlignment.getWorth() && + ( isOnOutLabelNode || stalemate( bestNode, currentNode, availableByNode ) ) ) + ) { + bestAlignment = fileAlignment; + bestNode = currentNode; + log.info("Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.getCost()); + } } } catch ( NoAligmentPossibleException e ){ noAlignmentFound++; @@ -179,7 +234,7 @@ Tuple calculateBestNode( triedNodes++; final Double thisRoundCost = fileAlignment == null ? null - : fileAlignment.cost; + : fileAlignment.getCost(); costs.add( thisRoundCost ); } } @@ -190,7 +245,7 @@ Tuple calculateBestNode( triedNodes, costs, couldStopFetching, - bestAlignment.cost, + bestAlignment.getCost(), noAlignmentFound ); return new Tuple<>( bestNode, bestAlignment ); @@ -219,15 +274,26 @@ TaskData calculateTaskData( final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) return null; final TaskInputs inputsOfTask = matchingFilesAndNodes.getInputsOfTask(); - long size = inputsOfTask.calculateAvgSize(); + final long size = inputsOfTask.calculateAvgSize(); + final OutLabel outLabel = task.getConfig().getOutLabel(); + final NodeLocation nodeForLabel = outLabel == null ? null : outLabelHolder.getNodeForLabel(outLabel.getLabel()); + final boolean weightWasSet = outLabel == null || nodeForLabel != null; + final boolean nodeForLabelNotNull = nodeForLabel != null; + final double outWeight = outLabel != null ? outLabel.getWeight() : 1.0; final List nodeDataTuples = matchingFilesAndNodes .getNodes() .parallelStream() - .map(node -> new NodeDataTuple(node, inputsOfTask.calculateDataOnNode( node.getNodeLocation() ) ) ) + .map(node -> { + double weight = (nodeForLabelNotNull && node.getNodeLocation() != nodeForLabel) ? outWeight : 1.0; + return new NodeDataTuple(node, inputsOfTask.calculateDataOnNode(node.getNodeLocation()), weight ); + } ) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); final double antiStarvingFactor = 0; - return new TaskData( size, task, nodeDataTuples, matchingFilesAndNodes, antiStarvingFactor ); + final TaskData taskData = new TaskData(size, task, nodeDataTuples, matchingFilesAndNodes, antiStarvingFactor, weightWasSet); + taskData.setOutLabelNode( nodeForLabel ); + taskData.setWeight( outWeight ); + return taskData; } } diff --git a/src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java b/src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java new file mode 100644 index 00000000..851f4e9b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java @@ -0,0 +1,55 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * The holder cannot be updated, if scheduling for a task fails. + */ +public abstract class OutLabelHolder { + + protected HashMap internalHolder = new HashMap<>(); + + /** + * Get Node with most Tasks running on it + * @param outLabel + * @return Null if no node was determined until now + */ + public NodeLocation getNodeForLabel(String outLabel) { + final InternalHolder holder = internalHolder.get(outLabel); + return ( holder == null ) ? null : holder.getBestNode(); + } + + public void scheduleTaskOnNode(Task task, NodeLocation node ){ + final String outLabel; + if ( (outLabel = task.getOutLabel()) == null ) return; + internalHolder.computeIfAbsent( outLabel, key -> new InternalHolder( this ) ).addTask( task, node ); + } + + abstract protected NodeLocation determineBestNode( Set>> set ); + + private class InternalHolder { + + private final OutLabelHolder outLabelHolder; + private final Map> tasksByNode = new HashMap<>(); + + public InternalHolder( OutLabelHolder outLabelHolder ) { + this.outLabelHolder = outLabelHolder; + } + + public NodeLocation getBestNode() { + return outLabelHolder.determineBestNode( tasksByNode.entrySet() ); + } + + public void addTask( Task task, NodeLocation node ) { + tasksByNode.computeIfAbsent( node, key -> new HashSet<>() ).add( task ); + } + + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasks.java b/src/main/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasks.java new file mode 100644 index 00000000..4cc71922 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasks.java @@ -0,0 +1,25 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; + +import java.util.Map; +import java.util.Set; + +public class OutLabelHolderMaxTasks extends OutLabelHolder { + + @Override + protected NodeLocation determineBestNode(Set>> set) { + int maxNumberOfTask = -1; + NodeLocation bestLocation = null; + for (Map.Entry> nodeLocationSetEntry : set) { + final int size = nodeLocationSetEntry.getValue().size(); + if ( size > maxNumberOfTask ) { + maxNumberOfTask = size; + bestLocation = nodeLocationSetEntry.getKey(); + } + } + return bestLocation; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 765cfe3d..4062787b 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -42,7 +42,9 @@ private Optional selectNode( List matchingNodes, T @Override Tuple calculateBestNode( final TaskData taskData, - Map< Location, Map>> planedToCopy){ + Map< Location, Map>> planedToCopy, + Map availableByNode + ){ final Optional nodeWithAlloc = selectNode(taskData.getNodeDataTuples(), taskData.getTask()); if (nodeWithAlloc.isEmpty()) return null; final NodeWithAlloc node = nodeWithAlloc.get(); @@ -70,7 +72,8 @@ TaskData calculateTaskData( .parallelStream() .map(node -> new NodeDataTuple(node, 0 ) ) .collect(Collectors.toList()); - return new TaskData( 0, task, nodeDataTuples, matchingFilesAndNodes, 0); + //Do not set any weights, as it is randomly scheduled + return new TaskData( 0, task, nodeDataTuples, matchingFilesAndNodes, 0, true ); } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index a0456c72..0ef0278f 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -49,6 +49,7 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final InputFileCollector inputFileCollector; private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); private final String localWorkDir; + protected final OutLabelHolder outLabelHolder = new OutLabelHolderMaxTasks() ; @Getter(AccessLevel.PACKAGE) private final Map< NodeLocation, Map< String, Tuple> > copyingToNode = new HashMap<>(); @@ -177,7 +178,7 @@ private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult long filesNotOnNodeByte = 0; final NodeLocation currentNode = alignment.node.getNodeLocation(); - for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { + for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { final AlignmentWrapper alignmentWrapper = entry.getValue(); if( entry.getKey() == currentNode) { traceRecord.setSchedulerFilesNode( alignmentWrapper.getFilesToCopy().size() + alignmentWrapper.getWaitFor().size() ); @@ -196,7 +197,7 @@ private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); traceRecord.setSchedulerDependingTask( (int) writeConfigResult.getWaitForTask().values().stream().distinct().count() ); - traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.nodeFileAlignment.size() - (schedulerFilesNode > 0 ? 1 : 0) ); + traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.getNodeFileAlignment().size() - (schedulerFilesNode > 0 ? 1 : 0) ); } /** @@ -219,7 +220,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { alignment.task.getConfig().getHash() ); - for (Map.Entry entry : alignment.fileAlignment.nodeFileAlignment.entrySet()) { + for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { if( entry.getKey() == currentNode ) { continue; } @@ -251,7 +252,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { } inputs.waitForTask( waitForTask ); - inputs.symlinks.addAll(alignment.fileAlignment.symlinks); + inputs.symlinks.addAll(alignment.fileAlignment.getSymlinks()); inputs.sortData(); new ObjectMapper().writeValue( config, inputs ); return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode ); diff --git a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java index 4462d9ed..8b185e8b 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java +++ b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java @@ -1,24 +1,38 @@ package fonda.scheduler.scheduler.data; import fonda.scheduler.model.NodeWithAlloc; +import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Setter; import org.jetbrains.annotations.NotNull; @Getter @EqualsAndHashCode public class NodeDataTuple implements Comparable { - private final NodeWithAlloc node; - private final long sizeInBytes; + private final NodeWithAlloc node; + private final long sizeInBytes; + @Getter(AccessLevel.NONE) + @Setter + private double weight; - public NodeDataTuple(NodeWithAlloc node, long sizeInBytes) { - this.node = node; - this.sizeInBytes = sizeInBytes; - } + public NodeDataTuple( NodeWithAlloc node, long sizeInBytes ) { + this( node, sizeInBytes, 1.0 ); + } + + public NodeDataTuple(NodeWithAlloc node, long sizeInBytes, double weight ) { + this.node = node; + this.sizeInBytes = sizeInBytes; + this.weight = weight; + } + + public double getWorth() { + return getSizeInBytes() / weight; + } - @Override + @Override public int compareTo(@NotNull NodeDataTuple o) { - return Long.compare( sizeInBytes, o.sizeInBytes); + return Double.compare( getWorth(), o.getWorth() ); } } \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index 31bbd195..816ef81a 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -3,14 +3,17 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Requirements; import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.scheduler.MatchingFilesAndNodes; import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Setter; import org.jetbrains.annotations.NotNull; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Map; @Getter @@ -34,20 +37,64 @@ public class TaskData implements Comparable { private double value = 0; + private boolean weightWasSet; + + @Setter + private NodeLocation outLabelNode = null; + + @Setter + private double weight = 1.0; + public TaskData( double size, Task task, List nodeDataTuples, MatchingFilesAndNodes matchingFilesAndNodes, - double antiStarvingFactor) { + double antiStarvingFactor, + boolean weightWasSet ) { this.size = size; this.task = task; this.nodeDataTuples = nodeDataTuples; this.matchingFilesAndNodes = matchingFilesAndNodes; this.antiStarvingFactor = antiStarvingFactor; + this.weightWasSet = weightWasSet; calc(); } + + /** + * After a location for a label was found, adjust the nodes. + * @param nodeForLabel + * @param weight + */ + public void setNodeAndWeight(NodeLocation nodeForLabel, double weight ) { + this.outLabelNode = nodeForLabel; + this.weight = weight; + final Iterator iterator = nodeDataTuples.iterator(); + NodeDataTuple onOutLabelNode = null; + while ( iterator.hasNext() ) { + final NodeDataTuple next = iterator.next(); + if (next.getNode().getNodeLocation() != nodeForLabel ) { + next.setWeight( weight ); + } else { + onOutLabelNode = next; + iterator.remove(); + } + } + //Insert the one element manuel. Sorting would be too expensive. + if ( onOutLabelNode != null ) { + final ListIterator listIterator = nodeDataTuples.listIterator(); + while ( listIterator.hasNext() ) { + if ( listIterator.next().getWorth() <= onOutLabelNode.getWorth() ) { + listIterator.previous(); + listIterator.add( onOutLabelNode ); + break; + } + } + } + weightWasSet = true; + } + /** * Recalculate the value. New value is smaller or equal old value * @param availableByNode diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index dfde2a6a..8cc68026 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -3,20 +3,25 @@ import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.SymlinkInput; +import lombok.Getter; +import lombok.Setter; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +@Getter public class FileAlignment { /* Key: node Value: Files from the node */ - public final Map nodeFileAlignment; - public final List symlinks; - public final double cost; + private final Map nodeFileAlignment; + private final List symlinks; + private final double cost; + @Setter + private double weight = 1.0; public FileAlignment(Map nodeFileAlignment, List symlinks, double cost) { this.nodeFileAlignment = nodeFileAlignment; @@ -24,6 +29,10 @@ public FileAlignment(Map nodeFileAlignment, List getAllLocationWrappers(){ return nodeFileAlignment .entrySet() diff --git a/src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java b/src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java new file mode 100644 index 00000000..de744cd9 --- /dev/null +++ b/src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java @@ -0,0 +1,50 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.Process; +import fonda.scheduler.dag.Vertex; +import fonda.scheduler.model.OutLabel; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.TaskConfig; +import fonda.scheduler.model.location.NodeLocation; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class OutLabelHolderMaxTasksTest { + + private class TaskConfigMock extends TaskConfig { + + public TaskConfigMock(String task) { + super(task); + } + + @Override + public OutLabel getOutLabel() { + return new OutLabel( "a", 2.0 ); + } + } + + @Test + void determineBestNode() { + final OutLabelHolderMaxTasks outLabelHolderMaxTasks = new OutLabelHolderMaxTasks(); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + dag.registerVertices(vertexList); + final TaskConfig taskConfig = new TaskConfigMock("processA"); + final NodeLocation nodeA = NodeLocation.getLocation("nodeA"); + final NodeLocation nodeB = NodeLocation.getLocation("nodeB"); + outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeA); + assertEquals( nodeA, outLabelHolderMaxTasks.getNodeForLabel( "a" ) ); + outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeB); + outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeB); + assertEquals( nodeB, outLabelHolderMaxTasks.getNodeForLabel( "a" ) ); + + } + +} \ No newline at end of file From 623867f083857938b72289cebc15279db06b7d98 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 20 Jun 2022 14:57:02 +0200 Subject: [PATCH 256/443] Remove unnecessary double casting Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/data/TaskData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index 816ef81a..79a342a0 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -126,7 +126,7 @@ public void removeAllNodesWhichHaveNotEnoughResources( Map Date: Tue, 21 Jun 2022 11:00:54 +0200 Subject: [PATCH 257/443] Moved OutLabel classes Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 ++ .../scheduler/scheduler/{ => outlabel}/OutLabelHolder.java | 1 - .../scheduler/{ => outlabel}/OutLabelHolderMaxTasks.java | 0 .../scheduler/{ => outlabel}/OutLabelHolderMaxTasksTest.java | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) rename src/main/java/fonda/scheduler/scheduler/{ => outlabel}/OutLabelHolder.java (99%) rename src/main/java/fonda/scheduler/scheduler/{ => outlabel}/OutLabelHolderMaxTasks.java (100%) rename src/test/java/fonda/scheduler/scheduler/{ => outlabel}/OutLabelHolderMaxTasksTest.java (97%) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 0ef0278f..40751a32 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -17,6 +17,8 @@ import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; +import fonda.scheduler.scheduler.outlabel.OutLabelHolder; +import fonda.scheduler.scheduler.outlabel.OutLabelHolderMaxTasks; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java similarity index 99% rename from src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java rename to src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java index 851f4e9b..bfc3dd97 100644 --- a/src/main/java/fonda/scheduler/scheduler/OutLabelHolder.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java @@ -34,7 +34,6 @@ public void scheduleTaskOnNode(Task task, NodeLocation node ){ abstract protected NodeLocation determineBestNode( Set>> set ); private class InternalHolder { - private final OutLabelHolder outLabelHolder; private final Map> tasksByNode = new HashMap<>(); diff --git a/src/main/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasks.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java similarity index 100% rename from src/main/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasks.java rename to src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java diff --git a/src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java b/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java similarity index 97% rename from src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java rename to src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java index de744cd9..3ab8aa56 100644 --- a/src/test/java/fonda/scheduler/scheduler/OutLabelHolderMaxTasksTest.java +++ b/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler; +package fonda.scheduler.scheduler.outlabel; import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Process; From 4ed0adb8f4950db903924a181b71b01f3d90dc01 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 21 Jun 2022 11:01:11 +0200 Subject: [PATCH 258/443] Changed package Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java | 2 +- .../scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java index bfc3dd97..a79b6271 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler; +package fonda.scheduler.scheduler.outlabel; import fonda.scheduler.model.Task; import fonda.scheduler.model.location.NodeLocation; diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java index 4cc71922..c6367f79 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler; +package fonda.scheduler.scheduler.outlabel; import fonda.scheduler.model.Task; import fonda.scheduler.model.location.NodeLocation; From 8c3531b301466c1261e2a52b69b771e7c366b507 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 21 Jun 2022 15:38:28 +0200 Subject: [PATCH 259/443] Fix Bug stalemate overwrote node with output label Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 1214690b..50a9b516 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -181,6 +181,7 @@ Tuple calculateBestNode( { FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; + boolean bestNodeHasOutLabel = false; final List nodeDataTuples = taskData.getNodeDataTuples(); int triedNodes = 0; int noAlignmentFound = 0; @@ -216,11 +217,24 @@ Tuple calculateBestNode( bestAlignment == null ? null : bestAlignment.getWorth(), fileAlignment.getWorth() ); - if (bestAlignment == null || bestAlignment.getWorth() > fileAlignment.getWorth() || + + if ( //Not set + bestAlignment == null + || + // better + bestAlignment.getWorth() > fileAlignment.getWorth() + || //Alignment is comparable - ( bestAlignment.getWorth() + 1e8 > fileAlignment.getWorth() && - ( isOnOutLabelNode || stalemate( bestNode, currentNode, availableByNode ) ) ) + ( bestAlignment.getWorth() + 1e8 > fileAlignment.getWorth() + && + //This is the outLabel node + ( isOnOutLabelNode + || + //Previous node was not the out label node and this alignment wins in stalemate + ( !bestNodeHasOutLabel && stalemate( bestNode, currentNode, availableByNode ) ) ) + ) ) { + bestNodeHasOutLabel = isOnOutLabelNode; bestAlignment = fileAlignment; bestNode = currentNode; log.info("Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.getCost()); From bfedd7190b947ea3a89d2c1f0529930d83820f7d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 21 Jun 2022 15:39:43 +0200 Subject: [PATCH 260/443] Refactor OutLabel holder. Cache results. Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 4 +- .../scheduler/outlabel/HolderMaxTasks.java | 22 ++++++++++ .../scheduler/outlabel/InternalHolder.java | 44 +++++++++++++++++++ .../scheduler/outlabel/OutLabelHolder.java | 32 +++----------- .../outlabel/OutLabelHolderMaxTasks.java | 25 ----------- .../outlabel/OutLabelHolderMaxTasksTest.java | 6 +-- 6 files changed, 77 insertions(+), 56 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java create mode 100644 src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java delete mode 100644 src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 40751a32..366cd6de 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -18,7 +18,7 @@ import fonda.scheduler.scheduler.copystrategy.CopyStrategy; import fonda.scheduler.scheduler.copystrategy.FTPstrategy; import fonda.scheduler.scheduler.outlabel.OutLabelHolder; -import fonda.scheduler.scheduler.outlabel.OutLabelHolderMaxTasks; +import fonda.scheduler.scheduler.outlabel.HolderMaxTasks; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.*; @@ -51,7 +51,7 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final InputFileCollector inputFileCollector; private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); private final String localWorkDir; - protected final OutLabelHolder outLabelHolder = new OutLabelHolderMaxTasks() ; + protected final OutLabelHolder outLabelHolder = new HolderMaxTasks() ; @Getter(AccessLevel.PACKAGE) private final Map< NodeLocation, Map< String, Tuple> > copyingToNode = new HashMap<>(); diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java b/src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java new file mode 100644 index 00000000..98073d10 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java @@ -0,0 +1,22 @@ +package fonda.scheduler.scheduler.outlabel; + +import fonda.scheduler.model.Task; + +import java.util.Set; + +public class HolderMaxTasks extends OutLabelHolder { + + @Override + protected InternalHolderMaxTasks create() { + return new InternalHolderMaxTasks(); + } + + private class InternalHolderMaxTasks extends InternalHolder { + + @Override + protected double calculateValue( Set input ) { + return input.size(); + } + + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java new file mode 100644 index 00000000..67a4f548 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java @@ -0,0 +1,44 @@ +package fonda.scheduler.scheduler.outlabel; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Slf4j +abstract class InternalHolder { + + private final Object syncHelperUpdateBestValue = new Object(); + private double bestValue = Double.MIN_VALUE; + private final Set currentlyBestNodes = new HashSet<>(); + private final Map> tasksByNode = new HashMap<>(); + + public void addTask(Task task, NodeLocation node) { + synchronized ( tasksByNode ) { + final Set tasks = tasksByNode.computeIfAbsent(node, key -> new HashSet<>()); + tasks.add(task); + final double calculatedValue = calculateValue(tasks); + if (calculatedValue > bestValue) { + bestValue = calculatedValue; + currentlyBestNodes.clear(); + } + if (calculatedValue >= bestValue) { + currentlyBestNodes.add(node); + } + } + } + + protected abstract double calculateValue( Set input ); + + public Set getBestNode() { + synchronized ( tasksByNode ){ + log.info("Best nodes: {}", currentlyBestNodes); + return currentlyBestNodes; + } + } + +} \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java index a79b6271..f34d1e49 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java @@ -3,10 +3,7 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.location.NodeLocation; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * The holder cannot be updated, if scheduling for a task fails. @@ -22,33 +19,18 @@ public abstract class OutLabelHolder { */ public NodeLocation getNodeForLabel(String outLabel) { final InternalHolder holder = internalHolder.get(outLabel); - return ( holder == null ) ? null : holder.getBestNode(); + return holder == null ? null : holder.getBestNode().stream().findFirst().orElse(null); } public void scheduleTaskOnNode(Task task, NodeLocation node ){ final String outLabel; if ( (outLabel = task.getOutLabel()) == null ) return; - internalHolder.computeIfAbsent( outLabel, key -> new InternalHolder( this ) ).addTask( task, node ); + internalHolder.computeIfAbsent( outLabel, key -> create() ).addTask( task, node ); } - abstract protected NodeLocation determineBestNode( Set>> set ); - - private class InternalHolder { - private final OutLabelHolder outLabelHolder; - private final Map> tasksByNode = new HashMap<>(); - - public InternalHolder( OutLabelHolder outLabelHolder ) { - this.outLabelHolder = outLabelHolder; - } - - public NodeLocation getBestNode() { - return outLabelHolder.determineBestNode( tasksByNode.entrySet() ); - } - - public void addTask( Task task, NodeLocation node ) { - tasksByNode.computeIfAbsent( node, key -> new HashSet<>() ).add( task ); - } - - } + /** + * @return new Instance of the required Holder + */ + protected abstract InternalHolder create(); } diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java deleted file mode 100644 index c6367f79..00000000 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasks.java +++ /dev/null @@ -1,25 +0,0 @@ -package fonda.scheduler.scheduler.outlabel; - -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; - -import java.util.Map; -import java.util.Set; - -public class OutLabelHolderMaxTasks extends OutLabelHolder { - - @Override - protected NodeLocation determineBestNode(Set>> set) { - int maxNumberOfTask = -1; - NodeLocation bestLocation = null; - for (Map.Entry> nodeLocationSetEntry : set) { - final int size = nodeLocationSetEntry.getValue().size(); - if ( size > maxNumberOfTask ) { - maxNumberOfTask = size; - bestLocation = nodeLocationSetEntry.getKey(); - } - } - return bestLocation; - } - -} diff --git a/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java b/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java index 3ab8aa56..de014661 100644 --- a/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java +++ b/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java @@ -9,11 +9,10 @@ import fonda.scheduler.model.location.NodeLocation; import org.junit.jupiter.api.Test; -import java.math.BigDecimal; import java.util.LinkedList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.Assert.assertEquals; class OutLabelHolderMaxTasksTest { @@ -31,7 +30,7 @@ public OutLabel getOutLabel() { @Test void determineBestNode() { - final OutLabelHolderMaxTasks outLabelHolderMaxTasks = new OutLabelHolderMaxTasks(); + final HolderMaxTasks outLabelHolderMaxTasks = new HolderMaxTasks(); DAG dag = new DAG(); List vertexList = new LinkedList<>(); vertexList.add(new Process("processA", 1)); @@ -44,7 +43,6 @@ void determineBestNode() { outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeB); outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeB); assertEquals( nodeB, outLabelHolderMaxTasks.getNodeForLabel( "a" ) ); - } } \ No newline at end of file From f4c172efaf89887e03e42c889630d3bcb31ac5cf Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 21 Jun 2022 15:42:10 +0200 Subject: [PATCH 261/443] Adjusted ftp config to allow uploads from Nextflow Signed-off-by: Lehmann_Fabian --- daemons/ftp/vsftpd.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/daemons/ftp/vsftpd.conf b/daemons/ftp/vsftpd.conf index 9ea2fcc4..5b695c36 100644 --- a/daemons/ftp/vsftpd.conf +++ b/daemons/ftp/vsftpd.conf @@ -37,11 +37,13 @@ write_enable=YES # Uncomment this to allow the anonymous FTP user to upload files. This only # has an effect if the above global write enable is activated. Also, you will # obviously need to create a directory writable by the FTP user. -anon_upload_enable=NO +anon_upload_enable=YES # # Uncomment this if you want the anonymous FTP user to be able to create # new directories. -anon_mkdir_write_enable=NO +anon_mkdir_write_enable=YES +# +anon_other_write_enable=YES # # Activate directory messages - messages given to remote users when they # go into a certain directory. From 4ebf68524727a7f7962a9885ee624dc5e230e77c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Jun 2022 10:43:22 +0200 Subject: [PATCH 262/443] Fix do not access empty list in TaskData.calc Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/data/TaskData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index 79a342a0..c09c17f0 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -125,7 +125,7 @@ public void removeAllNodesWhichHaveNotEnoughResources( Map Date: Wed, 22 Jun 2022 11:20:20 +0200 Subject: [PATCH 263/443] Code Style: brackets after if statement Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 8 +- src/main/java/fonda/scheduler/dag/DAG.java | 4 +- .../java/fonda/scheduler/dag/NotProcess.java | 8 +- .../java/fonda/scheduler/dag/Operator.java | 8 +- .../java/fonda/scheduler/dag/Process.java | 16 +++- .../scheduler/model/InputFileCollector.java | 8 +- .../fonda/scheduler/model/NodeWithAlloc.java | 12 ++- .../fonda/scheduler/model/PodWithAge.java | 12 ++- src/main/java/fonda/scheduler/model/Task.java | 3 +- .../scheduler/model/TaskResultParser.java | 23 +++-- .../scheduler/model/location/Location.java | 8 +- .../model/location/NodeLocation.java | 16 +++- .../model/location/hierachy/Folder.java | 6 +- .../location/hierachy/HierarchyWrapper.java | 50 ++++++++--- .../location/hierachy/LocationWrapper.java | 26 ++++-- .../location/hierachy/RealHierarchyFile.java | 89 ++++++++++++++----- .../scheduler/model/outfiles/OutputFile.java | 8 +- .../outfiles/PathLocationWrapperPair.java | 12 ++- .../model/outfiles/SymlinkOutput.java | 12 ++- .../taskinputs/PathFileLocationTriple.java | 8 +- .../rest/SchedulerRestController.java | 11 ++- .../scheduler/LocationAwareScheduler.java | 20 +++-- .../scheduler/RandomLAScheduler.java | 8 +- .../fonda/scheduler/scheduler/Scheduler.java | 44 ++++++--- .../scheduler/SchedulerWithDaemonSet.java | 44 ++++++--- .../scheduler/TaskprocessingThread.java | 4 +- .../scheduler/scheduler/data/TaskData.java | 13 ++- .../filealignment/InputAlignmentClass.java | 8 +- .../scheduler/outlabel/OutLabelHolder.java | 4 +- src/main/java/fonda/scheduler/util/Batch.java | 8 +- 30 files changed, 371 insertions(+), 130 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 9af91ce5..992c7ba4 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -108,7 +108,9 @@ public void eventReceived(Action action, Node node) { change = true; } } - if ( change ) kubernetesClient.informAllSchedulersNewNode( processedNode ); + if ( change ) { + kubernetesClient.informAllSchedulersNewNode( processedNode ); + } break; case DELETED: log.info("Node {} was deleted", node.getMetadata().getName()); @@ -118,7 +120,9 @@ public void eventReceived(Action action, Node node) { change = true; } } - if ( change ) kubernetesClient.informAllSchedulersRemovedNode( processedNode ); + if ( change ) { + kubernetesClient.informAllSchedulersRemovedNode( processedNode ); + } break; case ERROR: log.info("Node {} has an error", node.getMetadata().getName()); diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/fonda/scheduler/dag/DAG.java index 8644e97e..5ed9aa89 100644 --- a/src/main/java/fonda/scheduler/dag/DAG.java +++ b/src/main/java/fonda/scheduler/dag/DAG.java @@ -11,7 +11,9 @@ public class DAG { private Vertex getByUid( int uid ){ final Vertex vertex = vertices.get(uid); - if ( vertex == null ) throw new IllegalStateException( "Cannot find vertex with id " + uid ); + if ( vertex == null ) { + throw new IllegalStateException( "Cannot find vertex with id " + uid ); + } return vertex; } diff --git a/src/main/java/fonda/scheduler/dag/NotProcess.java b/src/main/java/fonda/scheduler/dag/NotProcess.java index e64d7ec1..cf35b2de 100644 --- a/src/main/java/fonda/scheduler/dag/NotProcess.java +++ b/src/main/java/fonda/scheduler/dag/NotProcess.java @@ -14,7 +14,9 @@ public Set getDescendants() { if ( !out.isEmpty() ) { for ( Edge edge : out ) { final Vertex to = edge.getTo(); - if ( to.getType() == Type.PROCESS ) results.add((Process) to); + if ( to.getType() == Type.PROCESS ) { + results.add((Process) to); + } results.addAll( to.getDescendants() ); } } @@ -25,7 +27,9 @@ public void addOutbound( Edge e ) { out.add( e ); final Vertex to = e.getTo(); final Set descendants = to.getDescendants(); - if ( to.getType() == Type.PROCESS ) descendants.add((Process) to); + if ( to.getType() == Type.PROCESS ) { + descendants.add((Process) to); + } final Set ancestors = this.getAncestors(); descendants.forEach( v -> v.ancestors.addAll( ancestors ) ); } diff --git a/src/main/java/fonda/scheduler/dag/Operator.java b/src/main/java/fonda/scheduler/dag/Operator.java index e502cb70..5e3b4fab 100644 --- a/src/main/java/fonda/scheduler/dag/Operator.java +++ b/src/main/java/fonda/scheduler/dag/Operator.java @@ -19,7 +19,9 @@ public Set getAncestors() { if ( !in.isEmpty() ){ for ( Edge edge : in ) { final Vertex from = edge.getFrom(); - if ( from.getType() == Type.PROCESS ) results.add((Process) from); + if ( from.getType() == Type.PROCESS ) { + results.add((Process) from); + } results.addAll( from.getAncestors() ); } } @@ -30,7 +32,9 @@ public void addInbound( Edge e ) { in.add( e ); final Vertex from = e.getFrom(); final Set ancestors = from.getAncestors(); - if ( from.getType() == Type.PROCESS ) ancestors.add((Process) from); + if ( from.getType() == Type.PROCESS ) { + ancestors.add((Process) from); + } final Set descendants = this.getDescendants(); ancestors.forEach( v -> v.descendants.addAll( descendants )); } diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/fonda/scheduler/dag/Process.java index bec8da9d..97bb1c83 100644 --- a/src/main/java/fonda/scheduler/dag/Process.java +++ b/src/main/java/fonda/scheduler/dag/Process.java @@ -38,10 +38,14 @@ public void addInbound( Edge e ) { final Vertex from = e.getFrom(); final Set fromAncestors = from.getAncestors(); - if ( from.getType() == Type.PROCESS ) fromAncestors.add((Process) from); + if ( from.getType() == Type.PROCESS ) { + fromAncestors.add((Process) from); + } this.ancestors.addAll(fromAncestors); - if ( from.getType() == Type.PROCESS ) ((Process) from).descendants.add( this ); + if ( from.getType() == Type.PROCESS ) { + ((Process) from).descendants.add( this ); + } final Set descendantsCopy = this.getDescendants(); fromAncestors.forEach( v -> { @@ -55,10 +59,14 @@ public void addOutbound( Edge e ) { final Vertex to = e.getTo(); final Set toDescendants = to.getDescendants(); - if ( to.getType() == Type.PROCESS ) toDescendants.add((Process) to); + if ( to.getType() == Type.PROCESS ) { + toDescendants.add((Process) to); + } this.descendants.addAll(toDescendants); - if ( to.getType() == Type.PROCESS ) ((Process)to).ancestors.add( this ); + if ( to.getType() == Type.PROCESS ) { + ((Process)to).ancestors.add( this ); + } final Set ancestorsCopy = this.getAncestors(); toDescendants.forEach( v -> { diff --git a/src/main/java/fonda/scheduler/model/InputFileCollector.java b/src/main/java/fonda/scheduler/model/InputFileCollector.java index 41bca3b8..0fc79d2c 100644 --- a/src/main/java/fonda/scheduler/model/InputFileCollector.java +++ b/src/main/java/fonda/scheduler/model/InputFileCollector.java @@ -29,7 +29,9 @@ private void processNext( ) throws NoAlignmentFoundException { final Tuple tuple = toProcess.removeLast(); final HierarchyFile file = tuple.getA(); - if( file == null ) return; + if( file == null ) { + return; + } final Path path = tuple.getB(); if ( file.isSymlink() ){ final Path linkTo = ((LinkHierarchyFile) file).getDst(); @@ -68,7 +70,9 @@ public TaskInputs getInputsOfTask( Task task, int numberNode ) throws NoAlignmen processNext( toProcess, symlinks, files, excludedLocations, task ); } - if( excludedLocations.size() == numberNode ) return null; + if( excludedLocations.size() == numberNode ) { + return null; + } return new TaskInputs( symlinks, files, excludedLocations ); diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 3c0b13a0..9d5a08e4 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -108,9 +108,15 @@ public boolean canScheduleNewPod(){ @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof NodeWithAlloc)) return false; - if (!super.equals(o)) return false; + if (this == o) { + return true; + } + if (!(o instanceof NodeWithAlloc)) { + return false; + } + if (!super.equals(o)) { + return false; + } NodeWithAlloc that = (NodeWithAlloc) o; return getMetadata().getName() != null ? getMetadata().getName().equals(that.getMetadata().getName()) : that.getMetadata().getName() == null; } diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/fonda/scheduler/model/PodWithAge.java index 1f5ce3e3..370cd967 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/fonda/scheduler/model/PodWithAge.java @@ -45,9 +45,15 @@ public String getName(){ @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof PodWithAge)) return false; - if (!super.equals(o)) return false; + if (this == o) { + return true; + } + if (!(o instanceof PodWithAge)) { + return false; + } + if (!super.equals(o)) { + return false; + } PodWithAge that = (PodWithAge) o; diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 58bf8e42..7d14155f 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -77,8 +77,9 @@ public void writeTrace(){ } public void setPod(PodWithAge pod) { - if( this.pod == null ) + if( this.pod == null ) { timeAddedToQueue = System.currentTimeMillis(); + } this.pod = pod; } diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 3f3ed968..05b950ea 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -30,7 +30,9 @@ public class TaskResultParser { private String getRootDir( File file ){ try ( Scanner sc = new Scanner( file ) ) { - if( sc.hasNext() ) return sc.next().split(";")[0]; + if( sc.hasNext() ) { + return sc.next().split(";")[0]; + } } catch (FileNotFoundException e) { log.error( "Cannot read " + file, e); } @@ -41,7 +43,9 @@ private void processInput( Stream in, final Map inputdat in.skip( 1 ) .forEach( line -> { String[] data = line.split(";"); - if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) { + return; + } String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ].substring( taskRootDir.length() + 1 ) : data[ REAL_PATH ]; String modificationDate = data[ MODIFICATION_DATE ]; inputdata.put( path , modificationDate ); @@ -60,11 +64,15 @@ private Set processOutput( out.skip( 1 ) .forEach( line -> { String[] data = line.split(";"); - if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) return; + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) { + return; + } boolean realFile = data[ REAL_PATH ].equals(""); String path = realFile ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; String modificationDate = data[ MODIFICATION_DATE ]; - if ( "directory".equals(data[ FILE_TYPE ]) ) return; + if ( "directory".equals(data[ FILE_TYPE ]) ) { + return; + } String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; if ( ( !inputdata.containsKey(lockupPath) && !onlyUpdated ) || @@ -105,13 +113,16 @@ public Set getNewAndUpdatedFiles( final String taskRootDir = getRootDir( infile.toFile() ); if( taskRootDir == null - && (finishedTask.getInputFiles() == null || finishedTask.getInputFiles().isEmpty()) ) + && (finishedTask.getInputFiles() == null || finishedTask.getInputFiles().isEmpty()) ) { throw new IllegalStateException("taskRootDir is null"); + } final String outputRootDir = getRootDir( outfile.toFile() ); //No outputs defined / found - if( outputRootDir == null ) return new HashSet<>(); + if( outputRootDir == null ) { + return new HashSet<>(); + } final Map inputdata = new HashMap<>(); diff --git a/src/main/java/fonda/scheduler/model/location/Location.java b/src/main/java/fonda/scheduler/model/location/Location.java index cf4f8b7b..3b057be6 100644 --- a/src/main/java/fonda/scheduler/model/location/Location.java +++ b/src/main/java/fonda/scheduler/model/location/Location.java @@ -11,8 +11,12 @@ public abstract class Location implements Serializable { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Location)) return false; + if (this == o) { + return true; + } + if (!(o instanceof Location)) { + return false; + } Location that = (Location) o; return ( getType() == that.getType() ) && ( getIdentifier().equals( that.getIdentifier() )); } diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java index 64eda2f1..5f862e16 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -24,7 +24,9 @@ public static NodeLocation getLocation( Node node ){ } public static NodeLocation getLocation( String node ){ - if ( node == null ) throw new IllegalArgumentException("Node cannot be null"); + if ( node == null ) { + throw new IllegalArgumentException("Node cannot be null"); + } final NodeLocation nodeLocation = locationHolder.get(node); if ( nodeLocation == null ){ locationHolder.putIfAbsent( node, new NodeLocation( node ) ); @@ -45,9 +47,15 @@ public String toString() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof NodeLocation)) return false; - if (!super.equals(o)) return false; + if (this == o) { + return true; + } + if (!(o instanceof NodeLocation)) { + return false; + } + if (!super.equals(o)) { + return false; + } NodeLocation that = (NodeLocation) o; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 8756df2e..7b088d06 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -62,8 +62,9 @@ private void getAllChildren(final Map result, Path public LocationWrapper addOrUpdateFile(final String name, boolean overwrite, final LocationWrapper location ) { final RealHierarchyFile file = (RealHierarchyFile) children.compute( name, (k, v) -> { - if (v == null || v.isDirectory() || v.isSymlink() ) + if (v == null || v.isDirectory() || v.isSymlink() ) { return new RealHierarchyFile( location ); + } return v; } ); return file.addOrUpdateLocation( overwrite, location ); @@ -71,8 +72,9 @@ public LocationWrapper addOrUpdateFile(final String name, boolean overwrite, fin public boolean addSymlink( final String name, final Path dst ){ children.compute( name, (k,v) -> { - if ( v == null || !v.isSymlink() || !((LinkHierarchyFile) v).getDst().equals(dst) ) + if ( v == null || !v.isSymlink() || !((LinkHierarchyFile) v).getDst().equals(dst) ) { return new LinkHierarchyFile( dst ); + } return v; } ); return true; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java index de3a0da9..555d38ea 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -17,7 +17,9 @@ public class HierarchyWrapper { private final ConcurrentMap workDirs = new ConcurrentHashMap<>(2); public HierarchyWrapper( String workdir ) { - if ( workdir == null ) throw new IllegalArgumentException( "Workdir is not defined" ); + if ( workdir == null ) { + throw new IllegalArgumentException( "Workdir is not defined" ); + } this.workdir = Paths.get( workdir ).normalize(); } @@ -26,9 +28,13 @@ private Path relativize( Path path ){ } private Folder getWorkdir( Iterator iterator, boolean create ){ - if(!iterator.hasNext()) return null; + if(!iterator.hasNext()) { + return null; + } final String hash1 = iterator.next().toString(); - if(!iterator.hasNext()) return null; + if(!iterator.hasNext()) { + return null; + } final String hash2 = iterator.next().toString(); final String key = hash1 + hash2; final Folder folder = workDirs.get( key ); @@ -48,7 +54,9 @@ public Map getAllFilesInDir(final Path path ){ final Path relativePath = relativize( path ); Iterator iterator = relativePath.iterator(); HierarchyFile current = getWorkdir( iterator, false ); - if( current == null ) return null; + if( current == null ) { + return null; + } while(iterator.hasNext()){ Path p = iterator.next(); if ( current != null && current.isDirectory() ){ @@ -57,10 +65,11 @@ public Map getAllFilesInDir(final Path path ){ return null; } } - if( current.isDirectory() ) + if( current.isDirectory() ) { return ((Folder) current).getAllChildren( path.normalize() ); - else + } else { return null; + } } /** @@ -77,8 +86,11 @@ public LocationWrapper addFile(final Path path, boolean overwrite, final Locatio final Folder folderToInsert = findFolderToInsert(path); - if( folderToInsert == null ) return null; - else return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, location ); + if( folderToInsert == null ) { + return null; + } else { + return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, location ); + } } @@ -86,8 +98,11 @@ public boolean addSymlink( final Path src, final Path dst ){ final Folder folderToInsert = findFolderToInsert( src ); - if( folderToInsert == null ) return false; - else return folderToInsert.addSymlink( src.getFileName().toString(), dst ); + if( folderToInsert == null ) { + return false; + } else { + return folderToInsert.addSymlink( src.getFileName().toString(), dst ); + } } @@ -98,7 +113,9 @@ private Folder findFolderToInsert( final Path path ){ } Iterator iterator = relativePath.iterator(); Folder current = getWorkdir( iterator, true ); - if( current == null ) return null; + if( current == null ) { + return null; + } while(iterator.hasNext()) { Path p = iterator.next(); if( iterator.hasNext() ){ @@ -125,8 +142,12 @@ public HierarchyFile getFile(Path path ){ } Iterator iterator = relativePath.iterator(); Folder current = getWorkdir( iterator, false ); - if( current == null ) return null; - if( !iterator.hasNext() ) return current; + if( current == null ) { + return null; + } + if( !iterator.hasNext() ) { + return current; + } while( iterator.hasNext() ) { Path p = iterator.next(); final HierarchyFile file = current.get( p.toString() ); @@ -135,8 +156,9 @@ public HierarchyFile getFile(Path path ){ current = (Folder) file; } else if ( !iterator.hasNext() ) { return file; - } else + } else { break; + } } return null; } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 887990c5..3fd5337f 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -39,8 +39,9 @@ private LocationWrapper(Location location, long timestamp, long sizeInBytes, Tas } public void update( LocationWrapper update ){ - if (location != update.location) + if (location != update.location) { throw new IllegalArgumentException( "Can only update LocationWrapper with the same location." ); + } synchronized ( this ) { this.timestamp = update.timestamp; this.sizeInBytes = update.sizeInBytes; @@ -79,17 +80,28 @@ public LocationWrapper getCopyOf( Location location ) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof LocationWrapper)) return false; + if (this == o) { + return true; + } + if (!(o instanceof LocationWrapper)) { + return false; + } LocationWrapper that = (LocationWrapper) o; synchronized ( this ) { - if (getTimestamp() != that.getTimestamp()) return false; - if (getSizeInBytes() != that.getSizeInBytes()) return false; - if (!getLocation().equals(that.getLocation())) return false; - if (getCreatedByTask() != null ? !getCreatedByTask().equals(that.getCreatedByTask()) : that.getCreatedByTask() != null) + if (getTimestamp() != that.getTimestamp()) { + return false; + } + if (getSizeInBytes() != that.getSizeInBytes()) { + return false; + } + if (!getLocation().equals(that.getLocation())) { + return false; + } + if (getCreatedByTask() != null ? !getCreatedByTask().equals(that.getCreatedByTask()) : that.getCreatedByTask() != null) { return false; + } return getCopyOf() != null ? getCopyOf().equals(that.getCopyOf()) : that.getCopyOf() == null; } } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index ff1a381d..caac0ef3 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -20,7 +20,9 @@ public class RealHierarchyFile extends AbstractHierarchyFile { static final String LOCATION_IS_NULL = "location is null"; public RealHierarchyFile(LocationWrapper location ) { - if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); + if ( location == null ) { + throw new IllegalArgumentException( LOCATION_IS_NULL ); + } this.locations = new LocationWrapper[]{ location }; } @@ -35,7 +37,9 @@ public boolean isSymlink() { } public void removeLocation( LocationWrapper location ){ - if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); + if ( location == null ) { + throw new IllegalArgumentException( LOCATION_IS_NULL ); + } synchronized ( this ){ for ( LocationWrapper locationWrapper : locations ) { if ( location.getLocation().equals( locationWrapper.getLocation() ) ) { @@ -46,7 +50,9 @@ public void removeLocation( LocationWrapper location ){ } public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ - if ( location == null ) throw new IllegalArgumentException( LOCATION_IS_NULL ); + if ( location == null ) { + throw new IllegalArgumentException( LOCATION_IS_NULL ); + } synchronized ( this ){ LocationWrapper locationWrapperToUpdate = null; for (LocationWrapper locationWrapper : locations) { @@ -55,12 +61,16 @@ public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper l if ( overwrite || location.getTimestamp() > locationWrapper.getTimestamp() ) { locationWrapperToUpdate.update( location ); } - if ( !overwrite ) return locationWrapperToUpdate; + if ( !overwrite ) { + return locationWrapperToUpdate; + } } else if ( overwrite ){ locationWrapper.deactivate(); } } - if ( overwrite && locationWrapperToUpdate != null ) return locationWrapperToUpdate; + if ( overwrite && locationWrapperToUpdate != null ) { + return locationWrapperToUpdate; + } final LocationWrapper[] newLocation = Arrays.copyOf(locations, locations.length + 1); newLocation[ locations.length ] = location; locations = newLocation; @@ -94,7 +104,9 @@ private List combineResultsWithInitial ( result = initial; } addAllLaterLocationsToResult( current, result, time ); - if( current == null ) addAllLaterLocationsToResult( ancestors, result, time ); + if( current == null ) { + addAllLaterLocationsToResult( ancestors, result, time ); + } addAllLaterLocationsToResult( unrelated, result, time ); addAllLaterLocationsToResult( descendants, result, time ); return result; @@ -107,18 +119,29 @@ private List combineResultsEmptyInitial ( LinkedList unrelated ) { LinkedList result = null; - if ( current != null ) result = current; + if ( current != null ) { + result = current; + } if ( ancestors != null ) { - if (current == null ) result = ancestors; - else result.addAll( ancestors ); + if (current == null ) { + result = ancestors; + } else { + result.addAll( ancestors ); + } } if ( unrelated != null ) { - if ( result == null ) result = unrelated; - else result.addAll( unrelated ); + if ( result == null ) { + result = unrelated; + } else { + result.addAll( unrelated ); + } } if ( descendants != null ) { - if ( result == null ) result = descendants; - else result.addAll( descendants ); + if ( result == null ) { + result = descendants; + } else { + result.addAll( descendants ); + } } return result; } @@ -133,7 +156,9 @@ private void addToAncestors(LinkedList ancestors, LocationWrapp if (locationProcess == currentProcess) { break; } else { - if( locationAncestors == null ) locationAncestors = locationProcess.getAncestors(); + if( locationAncestors == null ) { + locationAncestors = locationProcess.getAncestors(); + } if ( locationAncestors.contains(currentProcess) ) { iterator.remove(); } else if (locationProcess.getDescendants().contains(currentProcess)) { @@ -145,7 +170,9 @@ private void addToAncestors(LinkedList ancestors, LocationWrapp } private LinkedList addAndCreateList(LinkedList list, LocationWrapper toAdd ){ - if ( list == null ) list = new LinkedList<>(); + if ( list == null ) { + list = new LinkedList<>(); + } list.add( toAdd ); return list; } @@ -168,11 +195,15 @@ public MatchingLocationsPair getFilesForTask( Task task ) throws NoAlignmentFoun for ( LocationWrapper location : locationsRef ) { if( location.isInUse() ) { - if ( inUse == null ) inUse = new HashSet<>(); + if ( inUse == null ) { + inUse = new HashSet<>(); + } inUse.add(location.getLocation()); } - if ( !location.isActive() ) continue; + if ( !location.isActive() ) { + continue; + } //File was modified by an operator (no relation known) if ( location.getCreatedByTask() == null ) { @@ -183,8 +214,9 @@ public MatchingLocationsPair getFilesForTask( Task task ) throws NoAlignmentFoun final Process locationProcess = location.getCreatedByTask().getProcess(); if ( locationProcess == taskProcess) //Location was created by the same process == does definitely fit. + { current = addAndCreateList( current, location ); - else if ( taskAncestors.contains(locationProcess) ) { + } else if ( taskAncestors.contains(locationProcess) ) { // location is a direct ancestor if ( ancestors == null ) { ancestors = new LinkedList<>(); @@ -195,24 +227,31 @@ else if ( taskAncestors.contains(locationProcess) ) { } else if ( taskDescendants.contains(locationProcess) ) // location is a direct descendant + { descendants = addAndCreateList( descendants, location ); - else + } else // location was possibly generated in parallel + { unrelated = addAndCreateList( unrelated, location ); + } } final List matchingLocations = ( initial == null ) ? combineResultsEmptyInitial( current, ancestors, descendants, unrelated ) : combineResultsWithInitial( current, ancestors, descendants, unrelated, initial ); - if ( matchingLocations == null ) throw new NoAlignmentFoundException(); + if ( matchingLocations == null ) { + throw new NoAlignmentFoundException(); + } removeMatchingLocations( matchingLocations, inUse ); return new MatchingLocationsPair( matchingLocations, inUse ); } private void removeMatchingLocations( List matchingLocations, Set locations ){ - if ( locations == null || matchingLocations == null ) return; + if ( locations == null || matchingLocations == null ) { + return; + } for ( LocationWrapper matchingLocation : matchingLocations ) { if( matchingLocation.isInUse() ) { locations.remove(matchingLocation.getLocation()); @@ -236,7 +275,9 @@ private MatchingLocationsPair(List matchingLocations, Set list, List result, long time ){ if( list != null ) { for ( LocationWrapper l : list ) { - if( l.getCreateTime() >= time ) result.add( l ); + if( l.getCreateTime() >= time ) { + result.add( l ); + } } } } @@ -253,7 +294,9 @@ public LocationWrapper getLastUpdate( LocationType type ){ public LocationWrapper getLocationWrapper( Location location ){ for (LocationWrapper locationWrapper : locations) { - if ( locationWrapper.isActive() && locationWrapper.getLocation() == location ) return locationWrapper; + if ( locationWrapper.isActive() && locationWrapper.getLocation() == location ) { + return locationWrapper; + } } throw new RuntimeException( "Not found: " + location.getIdentifier() ); } diff --git a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java b/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java index 22e5ea1a..79f8dfea 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java +++ b/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java @@ -16,8 +16,12 @@ public OutputFile(Path path) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof OutputFile)) return false; + if (this == o) { + return true; + } + if (!(o instanceof OutputFile)) { + return false; + } OutputFile that = (OutputFile) o; return Objects.equals(getPath(), that.getPath()); } diff --git a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java index 42e2c1e8..289bdbac 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java +++ b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java @@ -27,9 +27,15 @@ public String toString() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof PathLocationWrapperPair)) return false; - if (!super.equals(o)) return false; + if (this == o) { + return true; + } + if (!(o instanceof PathLocationWrapperPair)) { + return false; + } + if (!super.equals(o)) { + return false; + } PathLocationWrapperPair that = (PathLocationWrapperPair) o; return Objects.equals(getLocationWrapper(), that.getLocationWrapper()); } diff --git a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java b/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java index dfbfd673..9fac8935 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java +++ b/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java @@ -18,9 +18,15 @@ public SymlinkOutput( String path, String dst ) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof SymlinkOutput)) return false; - if (!super.equals(o)) return false; + if (this == o) { + return true; + } + if (!(o instanceof SymlinkOutput)) { + return false; + } + if (!super.equals(o)) { + return false; + } SymlinkOutput that = (SymlinkOutput) o; return Objects.equals(dst, that.dst); } diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index 50ca783a..b0b9b1b9 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -25,7 +25,9 @@ public PathFileLocationTriple(Path path, RealHierarchyFile file, List 0); + if ( schedulerHolder.isEmpty() ) { + SpringApplication.exit(appContext, () -> 0); + } } } @@ -106,9 +108,12 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa : new RandomScheduler( execution, client, namespace, config ); break; case "lav1" : - if ( !config.locationAware ) + if ( !config.locationAware ) { return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); - if ( costFunction == null ) costFunction = new MinSizeCost( 0 ); + } + if ( costFunction == null ) { + costFunction = new MinSizeCost( 0 ); + } scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(costFunction) ); break; default: diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 50a9b516..47f1d983 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -54,7 +54,9 @@ private NodeTaskFilesAlignment createNodeAlignment ( log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); taskData.removeAllNodesWhichHaveNotEnoughResources( availableByNode ); final Tuple result = calculateBestNode(taskData, planedToCopy,availableByNode); - if ( result == null ) return null; + if ( result == null ) { + return null; + } final Task task = taskData.getTask(); availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); taskData.addNs( System.nanoTime()- startTime ); @@ -139,7 +141,9 @@ public ScheduleObject getTaskNodeAlignment( .map(task -> { long startTime = System.nanoTime(); final TaskData taskData = calculateTaskData(task, availableByNode); - if (taskData != null) taskData.addNs(System.nanoTime() - startTime); + if (taskData != null) { + taskData.addNs(System.nanoTime() - startTime); + } return taskData; }) .filter(Objects::nonNull) @@ -253,7 +257,9 @@ Tuple calculateBestNode( } } - if ( bestAlignment == null ) return null; + if ( bestAlignment == null ) { + return null; + } storeTraceData( taskData.getTask().getTraceRecord(), triedNodes, @@ -266,7 +272,9 @@ Tuple calculateBestNode( } private void storeTraceData(final TraceRecord traceRecord, int triedNodes, List costs, int couldStopFetching, double bestCost, int noAlignmentFound){ - if ( !traceEnabled ) return; + if ( !traceEnabled ) { + return; + } traceRecord.setSchedulerNodesTried( triedNodes ); traceRecord.setSchedulerNodesCost( costs ); traceRecord.setSchedulerCouldStopFetching( couldStopFetching ); @@ -286,7 +294,9 @@ TaskData calculateTaskData( final Map availableByNode ) { final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) return null; + if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) { + return null; + } final TaskInputs inputsOfTask = matchingFilesAndNodes.getInputsOfTask(); final long size = inputsOfTask.calculateAvgSize(); final OutLabel outLabel = task.getConfig().getOutLabel(); diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 4062787b..ee75d78d 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -46,7 +46,9 @@ Tuple calculateBestNode( Map availableByNode ){ final Optional nodeWithAlloc = selectNode(taskData.getNodeDataTuples(), taskData.getTask()); - if (nodeWithAlloc.isEmpty()) return null; + if (nodeWithAlloc.isEmpty()) { + return null; + } final NodeWithAlloc node = nodeWithAlloc.get(); final Map> currentlyCopying = getCopyingToNode().get(node.getNodeLocation()); final Map> currentlyPlanetToCopy = planedToCopy.get(node.getNodeLocation()); @@ -66,7 +68,9 @@ TaskData calculateTaskData( final Map availableByNode ) { final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) return null; + if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) { + return null; + } final List nodeDataTuples = matchingFilesAndNodes .getNodes() .parallelStream() diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 2089545a..784d80be 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -81,7 +81,9 @@ public abstract class Scheduler { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { - if( traceEnabled ) unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule() ); + if( traceEnabled ) { + unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule() ); + } final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode()); final List taskNodeAlignment = scheduleObject.getTaskAlignments(); @@ -98,7 +100,9 @@ public int schedule( final List unscheduledTasks ) { int scheduled = 0; for (NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment) { try { - if (isClose()) return -1; + if (isClose()) { + return -1; + } if ( !assignTaskToNode( nodeTaskAlignment ) ){ if ( scheduleObject.isStopSubmitIfOneFails() ) { return taskNodeAlignment.size() - scheduled; @@ -132,11 +136,15 @@ public boolean validSchedulePlan( List taskNodeAlignment ){ Map< NodeWithAlloc, Requirements> availableByNode = getAvailableByNode(); for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { final Requirements requirements = availableByNode.get(nodeTaskAlignment.node); - if ( requirements == null ) return false; + if ( requirements == null ) { + return false; + } requirements.subFromThis(nodeTaskAlignment.task.getPod().getRequest()); } for ( Map.Entry e : availableByNode.entrySet() ) { - if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) return false; + if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) { + return false; + } } return true; } @@ -161,7 +169,9 @@ void onPodTermination( PodWithAge pod ){ Task t = changeStateOfTask( pod, State.PROCESSING_FINISHED ); //If null, task was already changed - if( t == null ) return; + if( t == null ) { + return; + } t.setPod( pod ); synchronized (unfinishedTasks){ @@ -180,7 +190,9 @@ void taskWasFinished( Task task ){ public void schedulePod(PodWithAge pod ) { Task task = changeStateOfTask( pod, State.UNSCHEDULED ); //If null, task was already unscheduled - if ( task == null ) return; + if ( task == null ) { + return; + } task.setPod( pod ); if ( task.getBatch() == null ){ synchronized (unscheduledTasks){ @@ -291,9 +303,13 @@ boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeW boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ final Map podsNodeSelector = pod.getSpec().getNodeSelector(); final Map nodesLabels = node.getMetadata().getLabels(); - if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) return true; + if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) { + return true; + } //cannot be fulfilled if podsNodeSelector is not empty - if ( nodesLabels == null || nodesLabels.isEmpty() ) return false; + if ( nodesLabels == null || nodesLabels.isEmpty() ) { + return false; + } return nodesLabels.entrySet().containsAll( podsNodeSelector.entrySet() ); } @@ -357,7 +373,9 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); alignment.task.submitted(); - if( traceEnabled ) alignment.task.writeTrace(); + if( traceEnabled ) { + alignment.task.writeTrace(); + } return true; } @@ -426,7 +444,9 @@ Map getAvailableByNode(){ List logInfo = new LinkedList<>(); logInfo.add("------------------------------------"); for (NodeWithAlloc item : getNodeList()) { - if ( !item.isReady() ) continue; + if ( !item.isReady() ) { + continue; + } final Requirements availableResources = item.getAvailableResources(); availableByNode.put(item, availableResources); logInfo.add("Node: " + item.getName() + " " + availableResources); @@ -478,7 +498,9 @@ public void eventReceived(Action action, Pod pod) { scheduler.podEventReceived(action, pod); - if (!scheduler.name.equals(pod.getSpec().getSchedulerName())) return; + if (!scheduler.name.equals(pod.getSpec().getSchedulerName())) { + return; + } PodWithAge pwa = new PodWithAge(pod); if (pod.getMetadata().getLabels() != null) { diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 366cd6de..0217ccdc 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -60,7 +60,9 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { super(execution, client, namespace, config); this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); this.inputFileCollector = new InputFileCollector( hierarchyWrapper ); - if ( config.copyStrategy == null ) throw new IllegalArgumentException( "Copy strategy is null" ); + if ( config.copyStrategy == null ) { + throw new IllegalArgumentException( "Copy strategy is null" ); + } switch ( config.copyStrategy ){ case "ftp": case "copy": @@ -94,8 +96,12 @@ private void freeLocations( List locationWrappers ){ boolean assignTaskToNode( NodeTaskAlignment alignment ) { final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; final WriteConfigResult writeConfigResult = writeInitConfig(nodeTaskFilesAlignment); - if ( writeConfigResult == null ) return false; - if ( traceEnabled ) traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); + if ( writeConfigResult == null ) { + return false; + } + if ( traceEnabled ) { + traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); + } alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); @@ -154,8 +160,12 @@ int terminateTasks(List finishedTasks) { } private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple > toAdd ){ - if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); - if ( toAdd.isEmpty() ) return; + if ( nodeLocation == null ) { + throw new IllegalArgumentException( "NodeLocation cannot be null" ); + } + if ( toAdd.isEmpty() ) { + return; + } if ( copyingToNode.containsKey( nodeLocation ) ){ final Map> stringTupleHashMap = copyingToNode.get( nodeLocation ); stringTupleHashMap.putAll( toAdd ); @@ -165,9 +175,13 @@ private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple< } private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tuple> toRemove ){ - if ( nodeLocation == null ) throw new IllegalArgumentException( "NodeLocation cannot be null" ); + if ( nodeLocation == null ) { + throw new IllegalArgumentException( "NodeLocation cannot be null" ); + } final Map> pathTasks = copyingToNode.get(nodeLocation); - if( pathTasks == null || toRemove == null || toRemove.isEmpty() ) return; + if( pathTasks == null || toRemove == null || toRemove.isEmpty() ) { + return; + } pathTasks.keySet().removeAll( toRemove.keySet() ); } @@ -190,8 +204,12 @@ private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize() + alignmentWrapper.getToCopySize(); } } - if (traceRecord.getSchedulerFilesNode() == null) traceRecord.setSchedulerFilesNode(0); - if (traceRecord.getSchedulerFilesNodeBytes() == null) traceRecord.setSchedulerFilesNodeBytes(0l); + if (traceRecord.getSchedulerFilesNode() == null) { + traceRecord.setSchedulerFilesNode(0); + } + if (traceRecord.getSchedulerFilesNodeBytes() == null) { + traceRecord.setSchedulerFilesNodeBytes(0l); + } traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); @@ -292,7 +310,9 @@ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileExce } final RealHierarchyFile file = (RealHierarchyFile) currentFile; final LocationWrapper lastUpdate = file.getLastUpdate(LocationType.NODE); - if( lastUpdate == null ) return null; + if( lastUpdate == null ) { + return null; + } requestedLocations.put( lastUpdate.getId(), lastUpdate ); String node = lastUpdate.getLocation().getIdentifier(); return new FileResponse( currentPath.toString(), node, getDaemonOnNode(node), node.equals(workflowEngineNode), symlinks, lastUpdate.getId() ); @@ -380,7 +400,9 @@ private FTPClient getConnection( String daemon ){ f.enterLocalPassiveMode(); return f; } catch ( IOException e ) { - if ( trial > 5 ) throw new RuntimeException(e); + if ( trial > 5 ) { + throw new RuntimeException(e); + } log.error("Cannot create FTP client: {}", daemon); try { Thread.sleep((long) Math.pow(2, trial++)); diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java index 89429d31..96649596 100644 --- a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java @@ -29,7 +29,9 @@ public void run() { if (unscheduled == unprocessedTasks.size()) { unprocessedTasks.wait( 10000 ); } - if( Thread.interrupted() ) return; + if( Thread.interrupted() ) { + return; + } } while ( unprocessedTasks.isEmpty() ); tasks = new LinkedList<>(unprocessedTasks); } diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index c09c17f0..bf90b06d 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -118,15 +118,20 @@ public boolean calculate( Map availableByNode ) { public void removeAllNodesWhichHaveNotEnoughResources( Map availableByNode ) { final Iterator iterator = nodeDataTuples.iterator(); while (iterator.hasNext()) { - if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) + if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) { iterator.remove(); + } } } private void calc() { - if ( nodeDataTuples.isEmpty() ) value = Double.MIN_VALUE; - else if ( size == 0 ) value = 1; - else value = nodeDataTuples.get(0).getSizeInBytes() / size; + if ( nodeDataTuples.isEmpty() ) { + value = Double.MIN_VALUE; + } else if ( size == 0 ) { + value = 1; + } else { + value = nodeDataTuples.get(0).getSizeInBytes() / size; + } } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java index 71092ec6..0b0e5a9f 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -77,8 +77,12 @@ public FileAlignment getInputAlignment(@NotNull Task task, ! canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) ) { final double newCost = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); - if ( newCost > maxCost ) return null; - if ( newCost > cost ) cost = newCost; + if ( newCost > maxCost ) { + return null; + } + if ( newCost > cost ) { + cost = newCost; + } } } return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java index f34d1e49..96d7a213 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java @@ -24,7 +24,9 @@ public NodeLocation getNodeForLabel(String outLabel) { public void scheduleTaskOnNode(Task task, NodeLocation node ){ final String outLabel; - if ( (outLabel = task.getOutLabel()) == null ) return; + if ( (outLabel = task.getOutLabel()) == null ) { + return; + } internalHolder.computeIfAbsent( outLabel, key -> create() ).addTask( task, node ); } diff --git a/src/main/java/fonda/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/util/Batch.java index b8aafd5e..9fb5ba37 100644 --- a/src/main/java/fonda/scheduler/util/Batch.java +++ b/src/main/java/fonda/scheduler/util/Batch.java @@ -43,7 +43,9 @@ public void registerTask( Task task ){ public void informScheduable( Task task ){ synchronized ( unready ){ final boolean remove = unready.remove(task); - if ( remove ) ready.add( task ); + if ( remove ) { + ready.add( task ); + } } } @@ -52,7 +54,9 @@ public boolean canSchedule(){ } public List getTasksToScheduleAndDestroy(){ - if ( !closed ) throw new IllegalStateException("Batch was not yet closed!"); + if ( !closed ) { + throw new IllegalStateException("Batch was not yet closed!"); + } final List readyList = this.ready; this.ready = null; this.unready = null; From ef519f30d72cea735207a825ade9ab9eb19ee2b2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 29 Jun 2022 09:03:04 +0200 Subject: [PATCH 264/443] Use root account to avoid access problems Signed-off-by: Lehmann_Fabian --- daemons/ftp/Dockerfile | 3 +++ daemons/ftp/vsftpd.conf | 10 +++++----- src/main/resources/copystrategies/ftp.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/daemons/ftp/Dockerfile b/daemons/ftp/Dockerfile index c2db711a..07f872fc 100644 --- a/daemons/ftp/Dockerfile +++ b/daemons/ftp/Dockerfile @@ -7,4 +7,7 @@ RUN mkdir -p /var/run/vsftpd/empty COPY vsftpd.conf /etc/vsftpd.conf +USER root +RUN echo 'root:password' | chpasswd + ENTRYPOINT ["sh","-c","/usr/sbin/vsftpd /etc/vsftpd.conf"] \ No newline at end of file diff --git a/daemons/ftp/vsftpd.conf b/daemons/ftp/vsftpd.conf index 5b695c36..d14bf2df 100644 --- a/daemons/ftp/vsftpd.conf +++ b/daemons/ftp/vsftpd.conf @@ -25,7 +25,7 @@ listen_ipv6=NO anonymous_enable=YES # # Uncomment this to allow local users to log in. -local_enable=NO +local_enable=YES # # Uncomment this to enable any form of FTP write command. write_enable=YES @@ -37,13 +37,13 @@ write_enable=YES # Uncomment this to allow the anonymous FTP user to upload files. This only # has an effect if the above global write enable is activated. Also, you will # obviously need to create a directory writable by the FTP user. -anon_upload_enable=YES +anon_upload_enable=NO # # Uncomment this if you want the anonymous FTP user to be able to create # new directories. -anon_mkdir_write_enable=YES +anon_mkdir_write_enable=NO # -anon_other_write_enable=YES +anon_other_write_enable=NO # # Activate directory messages - messages given to remote users when they # go into a certain directory. @@ -144,7 +144,7 @@ connect_from_port_20=NO secure_chroot_dir=/var/run/vsftpd/empty # # This string is the name of the PAM service vsftpd will use. -pam_service_name=vsftpd +pam_service_name=ftp # # This option specifies the location of the RSA certificate to use for SSL # encrypted connections. diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 591598d2..f6f9bb20 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -71,7 +71,7 @@ def getFTP(node, currentIP, dns, syncFile): ip = currentIP log.info("Try to connect to %s", ip) ftp = ftplib.FTP(ip) - ftp.login("ftp", "pythonclient") + ftp.login("root", "password") ftp.set_pasv(True) ftp.encoding = 'utf-8' log.info("Connection established") From 02a819242e451ba513d82b66926d1cb379292133 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 1 Jul 2022 15:50:47 +0200 Subject: [PATCH 265/443] Work with new infiles format Signed-off-by: Lehmann_Fabian --- .../scheduler/model/TaskResultParser.java | 76 +++++++++++++++---- .../scheduler/model/TaskResultParserTest.java | 67 ++++++++++------ 2 files changed, 105 insertions(+), 38 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 05b950ea..42123f6c 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -28,10 +28,14 @@ public class TaskResultParser { static final int ACCESS_DATE = 6; static final int MODIFICATION_DATE = 7; - private String getRootDir( File file ){ + public String getIndex( File file, int index ){ try ( Scanner sc = new Scanner( file ) ) { - if( sc.hasNext() ) { - return sc.next().split(";")[0]; + int i = 0; + while( sc.hasNext() && i++ < index ) { + sc.nextLine(); + } + if ( sc.hasNext() ) { + return sc.nextLine(); } } catch (FileNotFoundException e) { log.error( "Cannot read " + file, e); @@ -39,26 +43,64 @@ private String getRootDir( File file ){ return null; } - private void processInput( Stream in, final Map inputdata, final String taskRootDir ){ - in.skip( 1 ) + private String getRootDir( File file, int index ){ + String data = getIndex( file, index ); + if ( data == null ) { + return null; + } + return data.split(";")[0]; + } + + private Long getDateDir(File file ){ + String data = getIndex( file, 0 ); + if ( data == null ) { + return null; + } + return DateParser.millisFromString( data ); + } + + public void processInput( Stream in, final Set inputdata, final String taskRootDir ){ + Map sourceTarget = new HashMap<>(); + in.skip( 2 ) .forEach( line -> { String[] data = line.split(";"); - if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) { + if( data[ FILE_EXISTS ].equals("0") ) { return; } + if ( data[ FILE_TYPE ].equals("directory") ) { + if ( !data[ VIRTUAL_PATH ].equals("") ) { + sourceTarget.put(Paths.get(data[VIRTUAL_PATH].substring(taskRootDir.length() + 1)), Paths.get(data[REAL_PATH])); + } + return; + } String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ].substring( taskRootDir.length() + 1 ) : data[ REAL_PATH ]; - String modificationDate = data[ MODIFICATION_DATE ]; - inputdata.put( path , modificationDate ); + if ( !path.startsWith("/") ){ + //Relative path + Path intialPath = Paths.get(path); + Path pathTmp = intialPath; + while ( (pathTmp = pathTmp.getParent()) != null ) { + if ( !sourceTarget.containsKey( pathTmp ) ){ + } else { + Path suffix = pathTmp.relativize( intialPath ); + path = sourceTarget.get( pathTmp ).resolve( suffix ).toString(); + //Fix to run tests on Windows + path = path.replace("\\","/"); + break; + } + } + } + inputdata.add( path ); }); } private Set processOutput( final Stream out, - final Map inputdata, + final Set inputdata, final Location location, final boolean onlyUpdated, final Task finishedTask, - final String outputRootDir + final String outputRootDir, + final long initailDate ){ final Set newOrUpdated = new HashSet<>(); out.skip( 1 ) @@ -74,9 +116,10 @@ private Set processOutput( return; } String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; - if ( ( !inputdata.containsKey(lockupPath) && !onlyUpdated ) + log.info( "lockupPath " + lockupPath + " contains: " + inputdata.contains( lockupPath ) + " Date: " + (DateParser.millisFromString(modificationDate) > initailDate) ); + if ( ( !inputdata.contains(lockupPath) && !onlyUpdated ) || - !modificationDate.equals( inputdata.get( lockupPath ) )) + DateParser.millisFromString(modificationDate) > initailDate ) { final LocationWrapper locationWrapper = new LocationWrapper( location, @@ -111,20 +154,20 @@ public Set getNewAndUpdatedFiles( final Path infile = workdir.resolve(".command.infiles"); final Path outfile = workdir.resolve(".command.outfiles"); - final String taskRootDir = getRootDir( infile.toFile() ); + final String taskRootDir = getRootDir( infile.toFile(), 1 ); if( taskRootDir == null && (finishedTask.getInputFiles() == null || finishedTask.getInputFiles().isEmpty()) ) { throw new IllegalStateException("taskRootDir is null"); } - final String outputRootDir = getRootDir( outfile.toFile() ); + final String outputRootDir = getRootDir( outfile.toFile(), 0 ); //No outputs defined / found if( outputRootDir == null ) { return new HashSet<>(); } - final Map inputdata = new HashMap<>(); + final Set inputdata = new HashSet<>(); try ( @@ -134,7 +177,8 @@ public Set getNewAndUpdatedFiles( processInput( in, inputdata, taskRootDir ); log.trace( "{}", inputdata ); - return processOutput( out, inputdata, location, onlyUpdated, finishedTask, outputRootDir ); + final Long initialDate = getDateDir( infile.toFile() ); + return processOutput( out, inputdata, location, onlyUpdated, finishedTask, outputRootDir, initialDate ); } catch (IOException e) { log.error( "Cannot read in/outfile in workdir: " + workdir, e); diff --git a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java index 4e2efda1..a5dc6ae3 100644 --- a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -13,13 +13,11 @@ import org.junit.Test; import java.io.BufferedWriter; +import java.io.File; import java.io.FileWriter; import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; import static org.junit.Assert.assertEquals; @@ -68,14 +66,15 @@ public void before(){ public void test1(){ String[] infiles = { - "/tmp/nxf.3iuGDWr6Id;0;;4096;directory;-;2021-11-10 12:58:11.210414589 +0000;2021-11-10 12:58:11.222414603 +0000", - "/tmp/nxf.3iuGDWr6Id/file.txt;1;/pvcdata/testfile.txt;6;regular file;-;2021-11-10 12:58:07.485035700 +0000;2021-11-10 12:58:07.219065500 +0000", - "/tmp/nxf.3iuGDWr6Id/.command.err;1;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000", - "/tmp/nxf.3iuGDWr6Id/.command.out;1;;0;regular empty file;-;2021-11-10 12:58:11.222414603 +0000;2021-11-10 12:58:11.222414603 +0000" + "2021-11-10 12:58:11.222414603 +0000", + "/tmp/nxf.3iuGDWr6Id;1;;4096;directory", + "/tmp/nxf.3iuGDWr6Id/file.txt;1;/pvcdata/testfile.txt;6;regular file", + "/tmp/nxf.3iuGDWr6Id/.command.err;1;;0;regular empty file", + "/tmp/nxf.3iuGDWr6Id/.command.out;1;;0;regular empty file" }; String[] outfiles = { - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650;0;;4096;directory;-;2021-11-10 12:58:11.278414667 +0000;2021-11-10 12:58:11.278414667 +0000", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650;1;;4096;directory;-;2021-11-10 12:58:11.278414667 +0000;2021-11-10 12:58:11.278414667 +0000", "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;1;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000", "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;1;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000" }; @@ -102,10 +101,11 @@ public void test1(){ public void test2(){ String[] infiles = { - "/tmp/nxf.IANFIlM3Kv;0;;4096;directory;-;2021-11-12 12:42:42.155614026 +0000;2021-11-12 12:42:42.171614019 +0000", - "/tmp/nxf.IANFIlM3Kv/file.txt;1;/pvcdata/testfile.txt;0;regular empty file;-;2021-11-12 12:42:29.000000000 +0000;2021-11-12 12:42:29.000000000 +0000", - "/tmp/nxf.IANFIlM3Kv/.command.err;1;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000", - "/tmp/nxf.IANFIlM3Kv/.command.out;1;;0;regular empty file;-;2021-11-12 12:42:42.171614019 +0000;2021-11-12 12:42:42.171614019 +0000" + "2021-11-12 12:42:42.171614019 +0000", + "/tmp/nxf.IANFIlM3Kv;1;;4096;directory", + "/tmp/nxf.IANFIlM3Kv/file.txt;1;/pvcdata/testfile.txt;0;regular empty file", + "/tmp/nxf.IANFIlM3Kv/.command.err;1;;0;regular empty file", + "/tmp/nxf.IANFIlM3Kv/.command.out;1;;0;regular empty file" }; String[] outfiles = { @@ -139,17 +139,24 @@ public void test2(){ @Test public void test3(){ + final TaskResultParser taskResultParser = new TaskResultParser(); + String[] infiles = { - "/tmp/nxf.9J6Y5mcXRD;0;;4096;directory;-;2021-11-12 14:42:20.941053482 +0000;2021-11-12 14:42:20.937053480 +0000", - "/tmp/nxf.9J6Y5mcXRD/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/tmp/nxf.9J6Y5mcXRD/.command.err;1;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000", - "/tmp/nxf.9J6Y5mcXRD/.command.out;1;;0;regular empty file;-;2021-11-12 14:42:20.937053480 +0000;2021-11-12 14:42:20.937053480 +0000" + "2021-11-12 14:42:20.941053482 +0000", + "/tmp/nxf.9J6Y5mcXRD;1;;4096;directory", + "/tmp/nxf.9J6Y5mcXRD/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory", + "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;1;;0;regular empty file", + "/tmp/nxf.9J6Y5mcXRD/t/b.txt;1;;2;regular file", + "/tmp/nxf.9J6Y5mcXRD/t/c.txt;1;;2;regular file", + "/tmp/nxf.9J6Y5mcXRD/t/a.txt;1;;2;regular file", + "/tmp/nxf.9J6Y5mcXRD/.command.err;1;;0;regular empty file", + "/tmp/nxf.9J6Y5mcXRD/.command.out;1;;0;regular empty file" }; + Set inputData = new HashSet(); + taskResultParser.processInput(Arrays.stream(infiles),inputData,"/tmp/nxf.9J6Y5mcXRD"); + log.info("INPUTS: {}", inputData); + String[] outfiles = { "/localdata/localwork/a2/f105825376b35dd6918824136adbf6;0;;4096;directory;-;2021-11-12 14:42:21.005053535 +0000;2021-11-12 14:42:21.009053539 +0000", "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", @@ -161,7 +168,6 @@ public void test3(){ final Path path = storeData(infiles, outfiles); - final TaskResultParser taskResultParser = new TaskResultParser(); final NodeLocation node1 = NodeLocation.getLocation("Node1"); final Task task = new Task( new TaskConfig("P1"), dag ); final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); @@ -179,4 +185,21 @@ public void test3(){ } + @Test + public void test4(){ + + String[] infiles = { + "---", + "---", + "/localdata/localwork/scratch/nxf.ZENnNNr9nq/wvdb;1;/input/FORCE2NXF-Rangeland/inputdata/wvdb;0;directory", + "/localdata/localwork/scratch/nxf.ZENnNNr9nq/wvdb/WVP_2020-12-16.txt;1;;420379;regular file", + }; + + final TaskResultParser taskResultParser = new TaskResultParser(); + Set inputData = new HashSet(); + taskResultParser.processInput(Arrays.stream(infiles),inputData,"/localdata/localwork/scratch/nxf.ZENnNNr9nq/"); + final HashSet expected = new HashSet<>(); + log.info("{}", inputData); + } + } \ No newline at end of file From 5d6617417acf565c1c87ca5307c5d2b279054e9e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 22 Jul 2022 11:10:38 +0200 Subject: [PATCH 266/443] Work with new data format Signed-off-by: Lehmann_Fabian --- .../scheduler/model/TaskResultParser.java | 47 ++-- .../scheduler/model/TaskResultParserTest.java | 200 ++++++++++++++---- 2 files changed, 173 insertions(+), 74 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 42123f6c..7a302dae 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -56,39 +56,20 @@ private Long getDateDir(File file ){ if ( data == null ) { return null; } - return DateParser.millisFromString( data ); + return Long.parseLong( data ); } - public void processInput( Stream in, final Set inputdata, final String taskRootDir ){ - Map sourceTarget = new HashMap<>(); + public void processInput( Stream in, final Set inputdata ){ in.skip( 2 ) .forEach( line -> { String[] data = line.split(";"); if( data[ FILE_EXISTS ].equals("0") ) { return; } - if ( data[ FILE_TYPE ].equals("directory") ) { - if ( !data[ VIRTUAL_PATH ].equals("") ) { - sourceTarget.put(Paths.get(data[VIRTUAL_PATH].substring(taskRootDir.length() + 1)), Paths.get(data[REAL_PATH])); - } + if ( data[ 3 ].equals("directory") ) { return; } - String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ].substring( taskRootDir.length() + 1 ) : data[ REAL_PATH ]; - if ( !path.startsWith("/") ){ - //Relative path - Path intialPath = Paths.get(path); - Path pathTmp = intialPath; - while ( (pathTmp = pathTmp.getParent()) != null ) { - if ( !sourceTarget.containsKey( pathTmp ) ){ - } else { - Path suffix = pathTmp.relativize( intialPath ); - path = sourceTarget.get( pathTmp ).resolve( suffix ).toString(); - //Fix to run tests on Windows - path = path.replace("\\","/"); - break; - } - } - } + String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; inputdata.add( path ); }); } @@ -109,28 +90,28 @@ private Set processOutput( if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) { return; } - boolean realFile = data[ REAL_PATH ].equals(""); - String path = realFile ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; + boolean isSymlink = !data[ REAL_PATH ].equals(""); + String path = isSymlink ? data[ REAL_PATH ] : data[ VIRTUAL_PATH ]; String modificationDate = data[ MODIFICATION_DATE ]; - if ( "directory".equals(data[ FILE_TYPE ]) ) { + if ( "directory".equals( data[ FILE_TYPE ] ) ) { return; } - String lockupPath = realFile ? path.substring( outputRootDir.length() + 1 ) : path; - log.info( "lockupPath " + lockupPath + " contains: " + inputdata.contains( lockupPath ) + " Date: " + (DateParser.millisFromString(modificationDate) > initailDate) ); + String lockupPath = isSymlink ? path : path.substring( outputRootDir.length() ); + long modificationDateNano = Long.parseLong( modificationDate ); if ( ( !inputdata.contains(lockupPath) && !onlyUpdated ) || - DateParser.millisFromString(modificationDate) > initailDate ) + modificationDateNano > initailDate ) { final LocationWrapper locationWrapper = new LocationWrapper( location, - DateParser.millisFromString(modificationDate), + modificationDateNano / (int) 1.0E6, Long.parseLong(data[ SIZE ]), finishedTask ); newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); } - if( !realFile ){ - newOrUpdated.add( new SymlinkOutput( data[ VIRTUAL_PATH ], data[ REAL_PATH ])); + if( isSymlink ){ + newOrUpdated.add( new SymlinkOutput( data[ VIRTUAL_PATH ], path )); } }); return newOrUpdated; @@ -175,7 +156,7 @@ public Set getNewAndUpdatedFiles( Stream out = Files.lines(outfile) ) { - processInput( in, inputdata, taskRootDir ); + processInput( in, inputdata ); log.trace( "{}", inputdata ); final Long initialDate = getDateDir( infile.toFile() ); return processOutput( out, inputdata, location, onlyUpdated, finishedTask, outputRootDir, initialDate ); diff --git a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java index a5dc6ae3..825620da 100644 --- a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -15,8 +15,9 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; -import java.nio.file.Files; -import java.nio.file.Path; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; import java.util.*; import static org.junit.Assert.assertEquals; @@ -66,17 +67,17 @@ public void before(){ public void test1(){ String[] infiles = { - "2021-11-10 12:58:11.222414603 +0000", + "1636549091222254911", "/tmp/nxf.3iuGDWr6Id;1;;4096;directory", - "/tmp/nxf.3iuGDWr6Id/file.txt;1;/pvcdata/testfile.txt;6;regular file", - "/tmp/nxf.3iuGDWr6Id/.command.err;1;;0;regular empty file", - "/tmp/nxf.3iuGDWr6Id/.command.out;1;;0;regular empty file" + "file.txt;1;/pvcdata/testfile.txt;6;regular file", + ".command.err;1;;0;regular empty file", + ".command.out;1;;0;regular empty file" }; String[] outfiles = { - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650;1;;4096;directory;-;2021-11-10 12:58:11.278414667 +0000;2021-11-10 12:58:11.278414667 +0000", - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;1;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000", - "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;1;/pvcdata/testfile.txt;13;regular file;-;2021-11-10 12:58:11.230039000 +0000;2021-11-10 12:58:11.230039000 +0000" + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;1;/pvcdata/testfile.txt;13;regular file;-;1636549091230400136;1636549091230935687", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;1;/pvcdata/testfile.txt;13;regular file;-;1636549091230676133;1636549091230588932", }; final Path path = storeData(infiles, outfiles); @@ -101,20 +102,20 @@ public void test1(){ public void test2(){ String[] infiles = { - "2021-11-12 12:42:42.171614019 +0000", + "1636720962171455407", "/tmp/nxf.IANFIlM3Kv;1;;4096;directory", - "/tmp/nxf.IANFIlM3Kv/file.txt;1;/pvcdata/testfile.txt;0;regular empty file", - "/tmp/nxf.IANFIlM3Kv/.command.err;1;;0;regular empty file", - "/tmp/nxf.IANFIlM3Kv/.command.out;1;;0;regular empty file" + "file.txt;1;/pvcdata/testfile.txt;0;regular empty file", + ".command.err;1;;0;regular empty file", + ".command.out;1;;0;regular empty file" }; String[] outfiles = { - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa;0;;4096;directory;-;2021-11-12 12:42:42.239613991 +0000;2021-11-12 12:42:42.243613989 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t;1;;4096;directory;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt;1;;0;regular empty file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt;1;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt;1;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000", - "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt;1;;2;regular file;-;2021-11-12 12:42:42.223613997 +0000;2021-11-12 12:42:42.223613997 +0000" + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t;1;;4096;directory;-;1636720962223377564;1636720962223927449", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt;1;;0;regular empty file;-;1636720962223648023;1636720962223304687", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt;1;;2;regular file;-;1636720962223821586;1636720962223246135", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt;1;;2;regular file;-;1636720962223035735;1636720962223994896", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt;1;;2;regular file;-;1636720962223163152;1636720962223785315" }; final Path path = storeData(infiles, outfiles); @@ -142,28 +143,24 @@ public void test3(){ final TaskResultParser taskResultParser = new TaskResultParser(); String[] infiles = { - "2021-11-12 14:42:20.941053482 +0000", - "/tmp/nxf.9J6Y5mcXRD;1;;4096;directory", - "/tmp/nxf.9J6Y5mcXRD/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory", - "/tmp/nxf.9J6Y5mcXRD/t/filenew.txt;1;;0;regular empty file", - "/tmp/nxf.9J6Y5mcXRD/t/b.txt;1;;2;regular file", - "/tmp/nxf.9J6Y5mcXRD/t/c.txt;1;;2;regular file", - "/tmp/nxf.9J6Y5mcXRD/t/a.txt;1;;2;regular file", - "/tmp/nxf.9J6Y5mcXRD/.command.err;1;;0;regular empty file", - "/tmp/nxf.9J6Y5mcXRD/.command.out;1;;0;regular empty file" + "1636728138941757518", + "/tmp/nxf.9J6Y5mcXRD;1;;directory", + "t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;directory", + "t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;regular empty file", + "t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;regular file", + "t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;regular file", + "t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;regular file", + ".command.err;1;;regular empty file", + ".command.out;1;;regular empty file" }; - Set inputData = new HashSet(); - taskResultParser.processInput(Arrays.stream(infiles),inputData,"/tmp/nxf.9J6Y5mcXRD"); - log.info("INPUTS: {}", inputData); - String[] outfiles = { - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6;0;;4096;directory;-;2021-11-12 14:42:21.005053535 +0000;2021-11-12 14:42:21.009053539 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;2021-11-12 14:42:17.009050211 +0000;2021-11-12 14:42:16.973050180 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;2021-11-12 14:42:16.969050176 +0000;2021-11-12 14:42:16.969050176 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:16.973050180 +0000", - "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;4;regular file;-;2021-11-12 14:42:16.973050180 +0000;2021-11-12 14:42:20.993053525 +0000" + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;1636728137009129688;1636728136973190140", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;1636728136969424493;1636728136969738095", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;1636728136973929300;1636728136973771278", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;1636728136973979318;1636728136973691162", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;4;regular file;-;1636728136973542646;1636728140993004364" }; final Path path = storeData(infiles, outfiles); @@ -191,15 +188,136 @@ public void test4(){ String[] infiles = { "---", "---", - "/localdata/localwork/scratch/nxf.ZENnNNr9nq/wvdb;1;/input/FORCE2NXF-Rangeland/inputdata/wvdb;0;directory", - "/localdata/localwork/scratch/nxf.ZENnNNr9nq/wvdb/WVP_2020-12-16.txt;1;;420379;regular file", + "wvdb;1;/input/FORCE2NXF-Rangeland/inputdata/wvdb;0;directory", + "wvdb/WVP_2020-12-16.txt;1;;420379;regular file", }; final TaskResultParser taskResultParser = new TaskResultParser(); Set inputData = new HashSet(); - taskResultParser.processInput(Arrays.stream(infiles),inputData,"/localdata/localwork/scratch/nxf.ZENnNNr9nq/"); + taskResultParser.processInput(Arrays.stream(infiles),inputData); final HashSet expected = new HashSet<>(); log.info("{}", inputData); } + @Test + public void test5(){ + + final TaskResultParser taskResultParser = new TaskResultParser(); + + String[] infiles = { + "1656925624971202680", + "/localdata/localwork/scratch/nxf.VX0s8jr100/;1;;directory", + ".command.err;1;;regular file", + ".command.out;1;;regular file" + }; + + String[] outfiles = { + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5;1;;4096;directory;1656925625099630891;1656925625099376329;1656925625219623172", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5;1;;4096;directory;1656925625099699272;1656925625099839483;1656925625219393367", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a;1;;4096;directory;1656925625151870586;1656925625151443598;1656925625207978673", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/file.txt;1;;0;regular empty file;1656925625019383511;1656925625019813106;1656925625019125236", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b;1;;4096;directory;1656925625163388439;1656925625163351190;1656925625195325883", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/file.txt;1;;0;regular empty file;1656925625023099305;1656925625023645725;1656925625023215656", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c;1;;4096;directory;1656925625171561519;1656925625171633246;1656925625175783837", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/file.txt;1;;0;regular empty file;1656925625023206366;1656925625023824131;1656925625023970434", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c/file.txt;1;;0;regular empty file;1656925625023995796;1656925625023981665;1656925625023053106" + }; + + final Path path = storeData(infiles, outfiles); + + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Task task = new Task( new TaskConfig("P1"), dag ); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); + + log.info("{}", newAndUpdatedFiles); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/file.txt"), new LocationWrapper( node1, 1656925625019L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); + assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + public void test6(){ + + final TaskResultParser taskResultParser = new TaskResultParser(); + + String[] infiles = { + "1658328555702440734", + "/localdata/localwork/scratch/nxf.QqsTpl7jcy/", + ".command.err;1;;0;regular file;1658328555691575355;1658328555691575355;1658328555691575355", + "JK2782_TGGCCGATCAACGA_L008_R2_001.fastq.gz.tengrand.fq.gz;1;/pvcdata/work/stage/da/08d4344c92af6ca44284ba0e17b484/JK2782_TGGCCGATCAACGA_L008_R2_001.fastq.gz.tengrand.fq.gz;111;symbolic link;1658328555659575265;1658328555659575265;1658328555659575265", + ".command.out;1;;0;regular file;1658328555691575355;1658328555691575355;1658328555691575355", + "JK2782_TGGCCGATCAACGA_L008_R1_001.fastq.gz.tengrand.fq.gz;1;/pvcdata/work/stage/db/a8a5b9f6d77f6fa9878977a278bc69/JK2782_TGGCCGATCAACGA_L008_R1_001.fastq.gz.tengrand.fq.gz;111;symbolic link;1658328555651575241;1658328555651575241;1658328555651575241", + + }; + + String[] outfiles = { + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9;1;;4096;directory;1658328570467617203;1658328570355616887;1658328570467617203", + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/a;1;;249228;regular file;1658328570427617091;1658328569675614962;1658328569683614985", + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/b;1;;260036;regular file;1658328570467617203;1658328559663586610;1658328569659614917", + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/c;1;;265581;regular file;1658328570391616989;1658328559231585386;1658328569707615053", + }; + + Set inputData = new HashSet(); + taskResultParser.processInput(Arrays.stream(infiles),inputData); + log.info("{}", inputData); + + final Path path = storeData(infiles, outfiles); + + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Task task = new Task( new TaskConfig("P1"), dag ); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); + + log.info("{}", newAndUpdatedFiles); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/a"), new LocationWrapper( node1, 1658328569683L, 249228, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/b"), new LocationWrapper( node1, 1658328569659L, 260036, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/c"), new LocationWrapper( node1, 1658328569707L, 265581, task )) ); + assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + public void test7(){ + long a = System.currentTimeMillis(); + long b = (long) (1658328570467617203l / 1.0E6); + log.info("{} {}", a - b, b); + } + + + @Test + public void fileWalker() throws IOException { + FileVisitor visitor = new FileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + log.info("preVisitDirectory: {}", dir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + log.info("visitFile: {}", file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + log.info("visitFileFailed: {}", file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + log.info("postVisitDirectory: {}", dir); + return FileVisitResult.CONTINUE; + } + }; + log.info("{}", Files.walkFileTree(Paths.get("C:\\Users\\Fabian Lehmann\\Documents\\VBox\\Paper\\test"), EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor)); + } + } \ No newline at end of file From aee9e7a19011c74cc6ffc62477a97350df633d3a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 22 Jul 2022 11:13:35 +0200 Subject: [PATCH 267/443] Trace more Signed-off-by: Lehmann_Fabian --- .../scheduler/model/tracing/TraceRecord.java | 59 ++++++++++++++++++- .../scheduler/LocationAwareScheduler.java | 3 +- .../fonda/scheduler/scheduler/Scheduler.java | 4 +- src/main/java/fonda/scheduler/util/Batch.java | 18 ++++++ 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java index 24690702..515c0c62 100644 --- a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java @@ -14,14 +14,17 @@ public class TraceRecord { @Getter @Setter + /*Filesize required for task*/ private Long schedulerFilesBytes = null; @Getter @Setter + /*Filesize required for task and already on node*/ private Long schedulerFilesNodeBytes = null; @Getter @Setter + /*Filesize required for task and already copied by other task*/ private Long schedulerFilesNodeOtherTaskBytes = null; @Getter @@ -82,6 +85,35 @@ public class TraceRecord { @Setter private Integer schedulerNoAlignmentFound = null; + private Integer schedulerDeltaScheduleSubmitted = null; + + private Integer schedulerDeltaScheduleAlignment = null; + + @Getter + @Setter + /*ID of the batch*/ + private Integer schedulerBatchId = null; + + @Getter + @Setter + /*Time delta between a batch was started and the scheduler received this task from the workflow engine*/ + private Integer schedulerDeltaBatchStartSubmitted = null; + + @Getter + @Setter + /*Time delta between a batch was started and the scheduler received the pod from the k8s API*/ + private Integer schedulerDeltaBatchStartReceived = null; + + @Getter + @Setter + /*Time delta between a batch was closed by the workflow engine and the scheduler received the pod from the k8s API*/ + private Integer schedulerDeltaBatchClosedBatchEnd = null; + + @Getter + @Setter + /*Time delta between a task was submitted and the batch became scheduable*/ + private Integer schedulerDeltaSubmittedBatchEnd = null; + public void writeRecord( String tracePath ) throws IOException { try ( BufferedWriter bw = new BufferedWriter( new FileWriter( tracePath ) ) ) { @@ -104,6 +136,14 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_nodes_to_copy_from", schedulerNodesToCopyFrom, bw); writeValue("scheduler_time_to_schedule", schedulerTimeToSchedule, bw); writeValue("scheduler_no_alignment_found", schedulerNoAlignmentFound, bw); + writeValue("scheduler_delta_schedule_submitted", schedulerDeltaScheduleSubmitted, bw); + writeValue("scheduler_delta_schedule_alignment", schedulerDeltaScheduleAlignment, bw); + writeValue("scheduler_batch_id", schedulerBatchId, bw); + writeValue("scheduler_delta_batch_start_submitted", schedulerDeltaBatchStartSubmitted, bw); + writeValue("scheduler_delta_batch_start_received", schedulerDeltaBatchStartReceived, bw); + writeValue("scheduler_delta_batch_closed_batch_end", schedulerDeltaBatchClosedBatchEnd, bw); + writeValue("scheduler_delta_submitted_batch_end", schedulerDeltaSubmittedBatchEnd, bw); + } } @@ -123,7 +163,24 @@ private void writeValue( String name, List value, BufferedWriter bw ) th } } - public void tryToSchedule(){ + private long startSchedule = 0; + + /** + * The task was submitted to a node + */ + public void submitted() { + schedulerDeltaScheduleSubmitted = (int) (System.currentTimeMillis() - startSchedule); + } + + /** + * Created an alignment for the task + */ + public void foundAlignment() { + schedulerDeltaScheduleAlignment = (int) (System.currentTimeMillis() - startSchedule); + } + + public void tryToSchedule( long startSchedule ){ + this.startSchedule = startSchedule; schedulerTriedToSchedule++; } diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 47f1d983..2cde3a3e 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -105,7 +105,7 @@ List createAlignment( while( !unscheduledTasksSorted.isEmpty() ){ TaskData taskData = unscheduledTasksSorted.poll(); boolean changed = false; - log.info( "TaskData: {}", taskData.getTask().getPod().getName() ); + log.trace( "TaskData: {}", taskData.getTask().getPod().getName() ); if ( !taskData.isWeightWasSet() ) { log.info( "TaskData: {} weight was not set", taskData.getTask().getPod().getName() ); final NodeLocation nodeForLabel = outLabelHolder.getNodeForLabel(taskData.getTask().getOutLabel()); @@ -280,6 +280,7 @@ private void storeTraceData(final TraceRecord traceRecord, int triedNodes, List< traceRecord.setSchedulerCouldStopFetching( couldStopFetching ); traceRecord.setSchedulerBestCost( bestCost ); traceRecord.setSchedulerNoAlignmentFound( noAlignmentFound ); + traceRecord.foundAlignment(); } diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 784d80be..f81c6243 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -81,8 +81,9 @@ public abstract class Scheduler { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { + long startSchedule = System.currentTimeMillis(); if( traceEnabled ) { - unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule() ); + unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule( startSchedule ) ); } final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode()); final List taskNodeAlignment = scheduleObject.getTaskAlignments(); @@ -374,6 +375,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ alignment.task.submitted(); if( traceEnabled ) { + alignment.task.getTraceRecord().submitted(); alignment.task.writeTrace(); } diff --git a/src/main/java/fonda/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/util/Batch.java index 9fb5ba37..0ccb9909 100644 --- a/src/main/java/fonda/scheduler/util/Batch.java +++ b/src/main/java/fonda/scheduler/util/Batch.java @@ -1,6 +1,7 @@ package fonda.scheduler.util; import fonda.scheduler.model.Task; +import fonda.scheduler.model.tracing.TraceRecord; import lombok.Getter; import java.util.HashSet; @@ -18,6 +19,10 @@ public class Batch { private Set unready = new HashSet<>(); private int tasksInBatch = -1; + private long createTime = System.currentTimeMillis(); + + private long closeTime; + public Batch(int id) { this.id = id; } @@ -25,6 +30,7 @@ public Batch(int id) { public void close( int tasksInBatch ){ this.closed = true; this.tasksInBatch = tasksInBatch; + closeTime = System.currentTimeMillis(); } public void registerTask( Task task ){ @@ -38,6 +44,10 @@ public void registerTask( Task task ){ unready.add( task ); } task.setBatch( this ); + final TraceRecord traceRecord = task.getTraceRecord(); + traceRecord.setSchedulerBatchId( id ); + traceRecord.setSchedulerDeltaBatchStartSubmitted((int) (System.currentTimeMillis() - createTime)); + } public void informScheduable( Task task ){ @@ -47,6 +57,7 @@ public void informScheduable( Task task ){ ready.add( task ); } } + task.getTraceRecord().setSchedulerDeltaBatchStartReceived((int) (System.currentTimeMillis() - createTime)); } public boolean canSchedule(){ @@ -58,6 +69,13 @@ public List getTasksToScheduleAndDestroy(){ throw new IllegalStateException("Batch was not yet closed!"); } final List readyList = this.ready; + long start = System.currentTimeMillis(); + int deltaCloseEnd = (int) (start - closeTime); + readyList.parallelStream().forEach( task -> { + final TraceRecord traceRecord = task.getTraceRecord(); + traceRecord.setSchedulerDeltaSubmittedBatchEnd((int) (start - createTime - traceRecord.getSchedulerDeltaBatchStartSubmitted())); + traceRecord.setSchedulerDeltaBatchClosedBatchEnd( deltaCloseEnd ); + }); this.ready = null; this.unready = null; return readyList; From 9e808db736ee6b67ef0e96e92be09cee1199bc48 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 22 Jul 2022 11:15:14 +0200 Subject: [PATCH 268/443] Sync addToCopyingToNode Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 0217ccdc..bc69014e 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -174,15 +174,23 @@ private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple< } } - private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tuple> toRemove ){ - if ( nodeLocation == null ) { - throw new IllegalArgumentException( "NodeLocation cannot be null" ); + private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tuple> toRemove ) { + if (nodeLocation == null) { + throw new IllegalArgumentException("NodeLocation cannot be null"); + } + if (toRemove == null || toRemove.isEmpty()) { + return; + } + final Map> pathTasks; + synchronized (copyingToNode) { + pathTasks = copyingToNode.get(nodeLocation); } - final Map> pathTasks = copyingToNode.get(nodeLocation); - if( pathTasks == null || toRemove == null || toRemove.isEmpty() ) { + if ( pathTasks == null ) { return; } - pathTasks.keySet().removeAll( toRemove.keySet() ); + synchronized ( pathTasks ) { + pathTasks.keySet().removeAll(toRemove.keySet()); + } } private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult writeConfigResult ) { From 4d45078d2a88dd0e281429a0c9aa1c89e9001d52 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 22 Jul 2022 11:15:47 +0200 Subject: [PATCH 269/443] Fix wrong byte number on node Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index bc69014e..dabb1575 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -208,8 +208,10 @@ private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult traceRecord.setSchedulerFilesNode( alignmentWrapper.getFilesToCopy().size() + alignmentWrapper.getWaitFor().size() ); traceRecord.setSchedulerFilesNodeBytes( alignmentWrapper.getToCopySize() + alignmentWrapper.getToWaitSize() ); } else { - filesOnNodeOtherTask += alignmentWrapper.getWaitFor().size() + alignmentWrapper.getFilesToCopy().size(); - filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize() + alignmentWrapper.getToCopySize(); + filesOnNodeOtherTask += alignmentWrapper.getWaitFor().size(); + filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize(); + filesNotOnNodeByte += alignmentWrapper.getToCopySize(); + filesNotOnNode += alignmentWrapper.getFilesToCopy().size(); } } if (traceRecord.getSchedulerFilesNode() == null) { From cac19e7dad412a7c665086d05273fa086454c165 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 22 Jul 2022 11:16:41 +0200 Subject: [PATCH 270/443] Changed log level Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/RandomScheduler.java | 2 +- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java index 04d9af71..80577e03 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java @@ -39,7 +39,7 @@ public ScheduleObject getTaskNodeAlignment( } } else { - log.info( "No node with enough resources for {}", pod.getName() ); + log.trace( "No node with enough resources for {}", pod.getName() ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index dabb1575..3d3539a6 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -331,7 +331,7 @@ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileExce MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map availableByNode ){ final Set matchingNodesForTask = getMatchingNodesForTask(availableByNode, task); if( matchingNodesForTask.isEmpty() ) { - log.info( "No node with enough resources for {}", task.getConfig().getHash() ); + log.trace( "No node with enough resources for {}", task.getConfig().getHash() ); return null; } From d27ae8e8da9024c64ebb81ea13b1917cae99c210 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 26 Aug 2022 15:06:24 +0200 Subject: [PATCH 271/443] Deal with init failures Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 3d3539a6..59b0b6b6 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -34,6 +34,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -132,23 +133,31 @@ int terminateTasks(List finishedTasks) { finishedTasks.parallelStream().forEach( finishedTask -> { try{ freeLocations( finishedTask.getInputFiles() ); - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( - Paths.get(finishedTask.getWorkingDir()), - finishedTask.getNode(), - !finishedTask.wasSuccessfullyExecuted(), - finishedTask - ); - for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { - if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { - hierarchyWrapper.addFile( - newAndUpdatedFile.getPath(), - ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() - ); - } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ - hierarchyWrapper.addSymlink( - newAndUpdatedFile.getPath(), - ((SymlinkOutput) newAndUpdatedFile).getDst() - ); + final Integer exitCode = finishedTask.getPod().getStatus().getContainerStatuses().get(0).getState().getTerminated().getExitCode(); + log.info( "Pod finished with exitCode: {}", exitCode ); + //Init failure + final Path workdir = Paths.get(finishedTask.getWorkingDir()); + if ( exitCode == 123 && Files.exists( workdir.resolve(".command.init.failure") ) ) { + log.info( "Task " + finishedTask.getConfig().getHash() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and outfiles" ); + } else { + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + workdir, + finishedTask.getNode(), + !finishedTask.wasSuccessfullyExecuted(), + finishedTask + ); + for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { + if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { + hierarchyWrapper.addFile( + newAndUpdatedFile.getPath(), + ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() + ); + } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ + hierarchyWrapper.addSymlink( + newAndUpdatedFile.getPath(), + ((SymlinkOutput) newAndUpdatedFile).getDst() + ); + } } } } catch ( Exception e ){ From dc7a416cb89a4d5053ba684af29b9ce254375318 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 26 Aug 2022 15:06:49 +0200 Subject: [PATCH 272/443] Make ftp script to use python3 default Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index f6f9bb20..18b4ac40 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import ftplib import json import logging as log @@ -109,7 +109,7 @@ def downloadFile(ftp, filename, size, index, node, syncFile): start = time.time() ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) end = time.time() - sizeInMB = os.path.getsize(filename) / 1_000_000 + sizeInMB = os.path.getsize(filename) / 1000000 delta = (end - start) log.info("Speed: %.3f Mbit/s", sizeInMB / delta ) return sizeInMB, delta @@ -160,9 +160,9 @@ def download(node, currentIP, files, dns, syncFile): def waitForFiles(syncFilePath, files, startTime): - # wait max. 10 seconds + # wait max. 60 seconds while True: - if startTime + 10 < time.time(): + if startTime + 60 < time.time(): return False if os.path.isfile(syncFilePath): break From f3c66ddc69f7499f697dc108328b652f6bdc1ed9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 26 Aug 2022 15:09:10 +0200 Subject: [PATCH 273/443] Create an empty script if nothing to copy Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/model/WriteConfigResult.java | 6 +++++- .../scheduler/scheduler/SchedulerWithDaemonSet.java | 9 ++++++--- .../scheduler/scheduler/copystrategy/CopyStrategy.java | 8 ++++---- src/main/resources/copystrategies/nothing.sh | 2 ++ 4 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 src/main/resources/copystrategies/nothing.sh diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java index 963adb0a..1ca4bc6e 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -14,13 +14,17 @@ public class WriteConfigResult { private final Map waitForTask; private final Map< String, Tuple> copyingToNode; + private final boolean wroteConfig; + public WriteConfigResult( List inputFiles, Map waitForTask, - Map> copyingToNode + Map> copyingToNode, + boolean wroteConfig ) { this.inputFiles = inputFiles; this.waitForTask = waitForTask; this.copyingToNode = copyingToNode; + this.wroteConfig = wroteConfig; } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 59b0b6b6..0c631df7 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -106,7 +106,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); - getCopyStrategy().generateCopyScript( alignment.task ); + getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); useLocations( allLocationWrappers ); @@ -293,8 +293,11 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { inputs.waitForTask( waitForTask ); inputs.symlinks.addAll(alignment.fileAlignment.getSymlinks()); inputs.sortData(); - new ObjectMapper().writeValue( config, inputs ); - return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode ); + final boolean allEmpty = inputs.data.isEmpty() && inputs.symlinks.isEmpty() && inputs.waitForFilesOfTask.isEmpty(); + if ( !allEmpty ) { + new ObjectMapper().writeValue( config, inputs ); + } + return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode, !allEmpty); } catch (IOException e) { log.error( "Cannot write " + config, e); diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java index 5f43bbb0..fe166d29 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -13,21 +13,21 @@ @Slf4j public abstract class CopyStrategy { - public void generateCopyScript( Task task ){ + public void generateCopyScript( Task task, boolean wroteConfig ){ File file = new File(task.getWorkingDir() + '/' + ".command.init.run"); try (BufferedWriter pw = new BufferedWriter( new FileWriter( file) ) ) { - write( pw, file ); + write( pw, file, wroteConfig ? getResource() : "copystrategies/nothing.sh" ); } catch (IOException e) { log.error( "Cannot write " + file, e); } } - private void write( BufferedWriter pw, File file ){ + private void write( BufferedWriter pw, File file, String resource ) { ClassLoader classLoader = getClass().getClassLoader(); - try (InputStream inputStream = classLoader.getResourceAsStream( getResource() )) { + try (InputStream inputStream = classLoader.getResourceAsStream( resource )) { assert inputStream != null; try (InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); BufferedReader reader = new BufferedReader(streamReader)) { diff --git a/src/main/resources/copystrategies/nothing.sh b/src/main/resources/copystrategies/nothing.sh new file mode 100644 index 00000000..03538c00 --- /dev/null +++ b/src/main/resources/copystrategies/nothing.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 \ No newline at end of file From 0ecc4cb18a92eeb9a429f4aa0f6308b1579c96e4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 12 Sep 2022 10:53:43 +0200 Subject: [PATCH 274/443] Don't fail when .command.outfiles does not exist Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/TaskResultParser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/fonda/scheduler/model/TaskResultParser.java index 7a302dae..032a1f14 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/fonda/scheduler/model/TaskResultParser.java @@ -134,6 +134,10 @@ public Set getNewAndUpdatedFiles( final Path infile = workdir.resolve(".command.infiles"); final Path outfile = workdir.resolve(".command.outfiles"); + if ( !outfile.toFile().exists() ) { + log.error( "Cannot find outfile " + infile ); + return new HashSet<>(); + } final String taskRootDir = getRootDir( infile.toFile(), 1 ); if( taskRootDir == null From 5e047dcd8bee13b64a527de6078da3fd88855a18 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 12 Sep 2022 10:56:47 +0200 Subject: [PATCH 275/443] Check if symlink is already correct and do not delete in this case. Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 18b4ac40..9c5b5a2e 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -48,15 +48,20 @@ def getIP(node, dns): return str(ip.decode("utf-8")) -def clearLocatation(path): +# True if the file was deleted or did not exist +def clearLocation(path, dst=None): if os.path.exists(path): log.debug("Delete %s", path) if os.path.islink(path): - os.unlink(path) + if dst is not None and os.readlink(path) == dst: + return False + else: + os.unlink(path) elif os.path.isdir(path): shutil.rmtree(path) else: os.remove(path) + return True def getFTP(node, currentIP, dns, syncFile): @@ -104,7 +109,7 @@ def downloadFile(ftp, filename, size, index, node, syncFile): log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) try: syncFile.write("S-" + filename + '\n') - clearLocatation(filename) + clearLocation(filename) Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) start = time.time() ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) @@ -225,9 +230,12 @@ def generateSymlinks(symlinks): for s in symlinks: src = s["src"] dst = s["dst"] - clearLocatation(src) - Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) - os.symlink(dst, src) + if clearLocation(src, dst): + Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) + try: + os.symlink(dst, src) + except FileExistsError: + log.warning("File exists: %s -> %s", src, dst) def downloadAllData(data, dns, syncFile): From 401e2d1c9cc64ce9e0310f36660b0c8de93048e0 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 12 Sep 2022 10:58:08 +0200 Subject: [PATCH 276/443] Do not log "Wait for %d threads to finish" continuously Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 9c5b5a2e..dcbf0501 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -248,10 +248,11 @@ def downloadAllData(data, dns, syncFile): node = d["node"] currentIP = d["currentIP"] futures.append(executor.submit(download, node, currentIP, files, dns, syncFile)) - trial = 0 + lastNum = -1 while len(futures) > 0: - if trial % 5 == 0: + if lastNum != len(futures): log.info("Wait for %d threads to finish", len(futures)) + lastNum = len(futures) for f in futures[:]: if f.done(): throughput.append(f.result()) From 60301cfb270ef1c89f2993b32c1194e1d31c6a09 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 12 Sep 2022 10:58:20 +0200 Subject: [PATCH 277/443] formatting Signed-off-by: Lehmann_Fabian --- src/main/resources/copystrategies/ftp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index dcbf0501..268b8f83 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -75,7 +75,7 @@ def getFTP(node, currentIP, dns, syncFile): else: ip = currentIP log.info("Try to connect to %s", ip) - ftp = ftplib.FTP(ip) + ftp = ftplib.FTP(ip, timeout=10) ftp.login("root", "password") ftp.set_pasv(True) ftp.encoding = 'utf-8' @@ -116,7 +116,7 @@ def downloadFile(ftp, filename, size, index, node, syncFile): end = time.time() sizeInMB = os.path.getsize(filename) / 1000000 delta = (end - start) - log.info("Speed: %.3f Mbit/s", sizeInMB / delta ) + log.info("Speed: %.3f Mbit/s", sizeInMB / delta) return sizeInMB, delta except ftplib.error_perm as err: errors += 1 @@ -258,7 +258,7 @@ def downloadAllData(data, dns, syncFile): throughput.append(f.result()) futures.remove(f) sleep(0.1) - trace["scheduler_init_throughput"] = "\"" + ",".join( "{}:{:.3f}".format(*x) for x in throughput) + "\"" + trace["scheduler_init_throughput"] = "\"" + ",".join("{}:{:.3f}".format(*x) for x in throughput) + "\"" def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): From 5099861af81e7c10eafa26161d07cdb738e9340d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 12 Sep 2022 11:01:10 +0200 Subject: [PATCH 278/443] New stalemate function Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/NodeWithAlloc.java | 15 ++++ .../scheduler/LocationAwareScheduler.java | 77 +++++++++++++++---- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 9d5a08e4..0736cfd6 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -3,6 +3,7 @@ import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.client.internal.readiness.Readiness; @@ -28,6 +29,15 @@ public class NodeWithAlloc extends Node implements Comparable { @Getter private final NodeLocation nodeLocation; + public NodeWithAlloc( String name ) { + this.kubernetesClient = null; + this.maxResources = null; + this.assignedPods = null; + this.nodeLocation = null; + this.setMetadata( new ObjectMeta() ); + this.getMetadata().setName( name ); + } + public NodeWithAlloc( Node node, KubernetesClient kubernetesClient ) { this.kubernetesClient = kubernetesClient; @@ -55,6 +65,7 @@ public NodeWithAlloc( Node node, KubernetesClient kubernetesClient ) { public void addPod( PodWithAge pod ){ Requirements request = pod.getRequest(); + log.info( "New Pod on node {} with request {}", this.getMetadata().getName(), request ); synchronized (assignedPods) { assignedPods.put( pod.getMetadata().getUid(), request ); } @@ -66,6 +77,10 @@ public boolean removePod( Pod pod ){ } } + public int getRunningPods(){ + return assignedPods.size(); + } + /** * @return max(Requested by all and currently used ) */ diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 2cde3a3e..d24514db 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -47,18 +47,20 @@ public LocationAwareScheduler ( private NodeTaskFilesAlignment createNodeAlignment ( final TaskData taskData, final Map availableByNode, + final Map assignedPodsByNode, Map< Location, Map>> planedToCopy, int index ) { long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); taskData.removeAllNodesWhichHaveNotEnoughResources( availableByNode ); - final Tuple result = calculateBestNode(taskData, planedToCopy,availableByNode); + final Tuple result = calculateBestNode(taskData, planedToCopy, availableByNode, assignedPodsByNode); if ( result == null ) { return null; } final Task task = taskData.getTask(); availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); + assignedPodsByNode.put(result.getA(), assignedPodsByNode.getOrDefault(result.getA(),0) + 1); taskData.addNs( System.nanoTime()- startTime ); if ( traceEnabled ){ task.getTraceRecord().setSchedulerTimeToSchedule((int) (taskData.getTimeInNs() / 1_000_000)); @@ -102,6 +104,7 @@ List createAlignment( int index = 0; final List alignment = new LinkedList<>(); final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); + final Map assignedPodsByNode = new HashMap<>(); while( !unscheduledTasksSorted.isEmpty() ){ TaskData taskData = unscheduledTasksSorted.poll(); boolean changed = false; @@ -121,7 +124,7 @@ List createAlignment( continue; } - final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, planedToCopy, ++index); + final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, assignedPodsByNode, planedToCopy, ++index); if ( nodeAlignment != null ) { alignment.add(nodeAlignment); outLabelHolder.scheduleTaskOnNode( taskData.getTask(), nodeAlignment.node.getNodeLocation() ); @@ -157,32 +160,80 @@ public ScheduleObject getTaskNodeAlignment( /** * Performs a stalemate between two possible alignments. + * First check for available cpu, then for the number of running tasks, than for the memory. + * Decide randomly if all three are equal. * @return True if the new alignment is better */ protected boolean stalemate( NodeWithAlloc oldNode, NodeWithAlloc newNode, - Map availableByNode) + Map availableByNode, + Map assignedPodsByNode, + Requirements taskRequest ) { - final Requirements availableNodeA = availableByNode.get( oldNode ); - final Requirements availableNodeB = availableByNode.get( newNode ); - //nodeA has less available CPU than nodeB - if( availableNodeA.getCpu().compareTo(availableNodeB.getCpu()) < 0 ) { + //Resources if this node executes the task + final Requirements availableNodeOld = availableByNode.get( oldNode ).sub( taskRequest ); + final Requirements availableNodeNew = availableByNode.get( newNode ).sub( taskRequest ); + + //Calculate percentage of resources that are available + final double availableCpuOld = availableNodeOld.getCpu().doubleValue() / oldNode.getMaxResources().getCpu().doubleValue(); + final double availableCpuNew = availableNodeNew.getCpu().doubleValue() / newNode.getMaxResources().getCpu().doubleValue(); + + double threshold = 0.05; + + if ( availableCpuOld + threshold < availableCpuNew ) { + log.trace( "Node {} has less available CPU than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); return true; - } else if ( availableNodeA.getCpu().compareTo(availableNodeB.getCpu()) == 0 ) { - if( availableNodeA.getRam().compareTo(availableNodeB.getRam() ) < 0 ) { + } else if ( availableCpuOld - threshold > availableCpuNew ) { + log.trace( "Node {} has more available CPU than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); + return false; + } else { + + //CPU load comparable, compare number of running tasks + + final int podsOnNewNode = newNode.getRunningPods() + assignedPodsByNode.getOrDefault( newNode, 0 ); + final int podsOnOldNode = oldNode.getRunningPods() + assignedPodsByNode.getOrDefault( oldNode, 0 ); + + if ( podsOnNewNode < podsOnOldNode ) { + log.trace( "Node {} has less running pods than node {}", newNode.getNodeLocation(), oldNode.getNodeLocation() ); return true; + } else if ( podsOnNewNode > podsOnOldNode ) { + log.trace( "Node {} has more running pods than node {}", newNode.getNodeLocation(), oldNode.getNodeLocation() ); + return false; + } else { + + //CPU and number of running tasks are comparable, compare memory + + final double availableRamOld = availableNodeOld.getRam().doubleValue() / oldNode.getMaxResources().getRam().doubleValue(); + final double availableRamNew = availableNodeNew.getRam().doubleValue() / newNode.getMaxResources().getRam().doubleValue(); + + if ( availableRamOld + threshold < availableRamNew ) { + log.trace( "Node {} has less available RAM than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); + return true; + } else if ( availableRamOld - threshold > availableRamNew ) { + log.trace( "Node {} has more available RAM than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); + return false; + } else { + + //Everything is comparable, decide randomly + log.trace( "Node {} and node {} are equally comparable -> decide randomly", oldNode.getNodeLocation(), newNode.getNodeLocation() ); + return Math.random() > 0.5; + + } + } + } - return false; + } Tuple calculateBestNode( final TaskData taskData, Map< Location, Map>> planedToCopy, - Map availableByNode) - { + Map availableByNode, + Map assignedPodsByNode + ) { FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; boolean bestNodeHasOutLabel = false; @@ -235,7 +286,7 @@ Tuple calculateBestNode( ( isOnOutLabelNode || //Previous node was not the out label node and this alignment wins in stalemate - ( !bestNodeHasOutLabel && stalemate( bestNode, currentNode, availableByNode ) ) ) + ( !bestNodeHasOutLabel && stalemate( bestNode, currentNode, availableByNode, assignedPodsByNode, taskData.getTask().getPod().getRequest() ) ) ) ) ) { bestNodeHasOutLabel = isOnOutLabelNode; From 0733170671e5ba0ce3c90347e5f8c36cbc56a411 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 22 Sep 2022 09:10:39 +0200 Subject: [PATCH 279/443] Method declaration not commited Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index ee75d78d..b8ce3135 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -43,7 +43,8 @@ private Optional selectNode( List matchingNodes, T Tuple calculateBestNode( final TaskData taskData, Map< Location, Map>> planedToCopy, - Map availableByNode + Map availableByNode, + Map assignedPodsByNode ){ final Optional nodeWithAlloc = selectNode(taskData.getNodeDataTuples(), taskData.getTask()); if (nodeWithAlloc.isEmpty()) { From 30d03fe4f261bab08ef005d3dc4289ec6d3f24d4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 22 Sep 2022 09:11:14 +0200 Subject: [PATCH 280/443] Added Swagger UI Signed-off-by: Lehmann_Fabian --- README.md | 4 ++++ pom.xml | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 674f9b61..51e21fd7 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ # Kubernetes Workflow Scheduler + +SWAGGER: http://localhost:8080/swagger-ui.html + +API-DOCS: http://localhost:8080/v3/api-docs/ \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5de43cef..6962f309 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,12 @@ 3.8.0 + + org.springdoc + springdoc-openapi-ui + 1.6.4 + + From 4d55def7d588c97f1b1b9d7efbce5dc3305b972b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 29 Sep 2022 15:27:38 +0200 Subject: [PATCH 281/443] Method to delete edges and vertices from DAG Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/DAG.java | 49 +- src/main/java/fonda/scheduler/dag/Edge.java | 10 +- .../java/fonda/scheduler/dag/InputEdge.java | 8 +- .../java/fonda/scheduler/dag/NotProcess.java | 20 +- .../java/fonda/scheduler/dag/Operator.java | 2 +- .../java/fonda/scheduler/dag/Process.java | 92 +++- src/main/java/fonda/scheduler/dag/Vertex.java | 80 +++- .../java/fonda/scheduler/dag/DAGTest.java | 451 ++++++++++++++++-- .../model/InputFileCollectorTest.java | 4 +- .../hierachy/RealHierarchyFileTest.java | 8 +- 10 files changed, 657 insertions(+), 67 deletions(-) diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/fonda/scheduler/dag/DAG.java index 5ed9aa89..f64dba95 100644 --- a/src/main/java/fonda/scheduler/dag/DAG.java +++ b/src/main/java/fonda/scheduler/dag/DAG.java @@ -7,9 +7,15 @@ public class DAG { private final Map vertices = new HashMap<>(); + private final Map edges = new HashMap<>(); private final Map processes = new HashMap<>(); - private Vertex getByUid( int uid ){ + /** + * not private for testing + * @param uid + * @return + */ + Vertex getByUid( int uid ){ final Vertex vertex = vertices.get(uid); if ( vertex == null ) { throw new IllegalStateException( "Cannot find vertex with id " + uid ); @@ -41,11 +47,50 @@ public void registerVertices( List vertices ){ public void registerEdges( List edges ){ synchronized ( this.vertices ) { for (InputEdge edge : edges) { - final Edge edgeNew = new Edge(edge.getLabel(), getByUid(edge.getFrom()), getByUid(edge.getTo())); + final Edge edgeNew = new Edge( edge.getUid(), edge.getLabel(), getByUid(edge.getFrom()), getByUid(edge.getTo())); + synchronized ( this.edges ) { + this.edges.put( edgeNew.getUid(), edgeNew ); + } edgeNew.getFrom().addOutbound(edgeNew); edgeNew.getTo().addInbound(edgeNew); } } } + /** + * This method removes vertices from the DAG plus the edges. + * @param verticesIds + */ + public void removeVertices( int... verticesIds ){ + synchronized ( this.vertices ) { + for ( int vertexId : verticesIds ) { + final Vertex remove = this.vertices.remove(vertexId); + if ( remove.getType() == Type.PROCESS ) { + processes.remove(remove.getLabel()); + } + remove.deleteItself(); + synchronized ( this.edges ) { + for (Edge edge : remove.getIn()) { + this.edges.remove(edge.getUid()); + } + for (Edge edge : remove.getOut()) { + this.edges.remove(edge.getUid()); + } + } + } + } + } + + public void removeEdges( int... edgesIds ){ + synchronized ( this.vertices ) { + for ( int edgeId : edgesIds ) { + final Edge remove; + synchronized ( this.edges ) { + remove = this.edges.remove( edgeId ); + } + remove.getTo().removeInbound(remove); + } + } + } + } diff --git a/src/main/java/fonda/scheduler/dag/Edge.java b/src/main/java/fonda/scheduler/dag/Edge.java index 8cb8fb3a..5a88381a 100644 --- a/src/main/java/fonda/scheduler/dag/Edge.java +++ b/src/main/java/fonda/scheduler/dag/Edge.java @@ -1,17 +1,25 @@ package fonda.scheduler.dag; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @Getter @ToString +@EqualsAndHashCode public class Edge { + private final int uid; private final String label; private final Vertex from; private final Vertex to; - Edge(String label, Vertex from, Vertex to) { + Edge(int uid, Vertex from, Vertex to) { + this(uid, null, from, to); + } + + Edge( int uid, String label, Vertex from, Vertex to) { + this.uid = uid; this.label = label; this.from = from; this.to = to; diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/fonda/scheduler/dag/InputEdge.java index 1ca4b8b1..89e29e41 100644 --- a/src/main/java/fonda/scheduler/dag/InputEdge.java +++ b/src/main/java/fonda/scheduler/dag/InputEdge.java @@ -7,15 +7,14 @@ @ToString public class InputEdge { + private final int uid; private final String label; private final int from; private final int to; @SuppressWarnings("unused") private InputEdge() { - this.label = null; - this.from = -1; - this.to = -1; + this(-1, -1, -1); } /** @@ -23,7 +22,8 @@ private InputEdge() { * @param from * @param to */ - public InputEdge(int from, int to) { + public InputEdge( int uid, int from, int to) { + this.uid = uid; label = null; this.from = from; this.to = to; diff --git a/src/main/java/fonda/scheduler/dag/NotProcess.java b/src/main/java/fonda/scheduler/dag/NotProcess.java index cf35b2de..6d7e9d1e 100644 --- a/src/main/java/fonda/scheduler/dag/NotProcess.java +++ b/src/main/java/fonda/scheduler/dag/NotProcess.java @@ -1,5 +1,6 @@ package fonda.scheduler.dag; +import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -15,7 +16,7 @@ public Set getDescendants() { for ( Edge edge : out ) { final Vertex to = edge.getTo(); if ( to.getType() == Type.PROCESS ) { - results.add((Process) to); + results.add( (Process) to ); } results.addAll( to.getDescendants() ); } @@ -28,10 +29,23 @@ public void addOutbound( Edge e ) { final Vertex to = e.getTo(); final Set descendants = to.getDescendants(); if ( to.getType() == Type.PROCESS ) { - descendants.add((Process) to); + descendants.add( (Process) to ); } final Set ancestors = this.getAncestors(); - descendants.forEach( v -> v.ancestors.addAll( ancestors ) ); + descendants.forEach( v -> v.addAncestor( ancestors ) ); + } + + void removeDescendant( Edge e, Collection p ) { + for ( Edge edge : in ) { + edge.getFrom().removeDescendant( edge, p ); + } + } + + + void removeAncestor( Edge e, Collection p ) { + for ( Edge edge : out ) { + edge.getTo().removeAncestor( edge, p ); + } } } diff --git a/src/main/java/fonda/scheduler/dag/Operator.java b/src/main/java/fonda/scheduler/dag/Operator.java index 5e3b4fab..7cf03c4a 100644 --- a/src/main/java/fonda/scheduler/dag/Operator.java +++ b/src/main/java/fonda/scheduler/dag/Operator.java @@ -36,7 +36,7 @@ public void addInbound( Edge e ) { ancestors.add((Process) from); } final Set descendants = this.getDescendants(); - ancestors.forEach( v -> v.descendants.addAll( descendants )); + ancestors.forEach( v -> v.addDescendant( descendants ) ); } } diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/fonda/scheduler/dag/Process.java index 97bb1c83..2dd16ee3 100644 --- a/src/main/java/fonda/scheduler/dag/Process.java +++ b/src/main/java/fonda/scheduler/dag/Process.java @@ -1,13 +1,85 @@ package fonda.scheduler.dag; +import java.util.Collection; import java.util.HashSet; +import java.util.LinkedList; import java.util.Set; public class Process extends Vertex { - final Set descendants; - final Set ancestors; + private final Set descendants; + private final Set ancestors; + + void addDescendant( Process p ) { + synchronized (descendants) { + descendants.add( p ); + } + } + + void addDescendant( Collection p ) { + synchronized (descendants) { + descendants.addAll( p ); + } + } + + void removeDescendant( Edge e, Collection pp ) { + synchronized (descendants) { + LinkedList toRemove = new LinkedList<>(); + for (Process p : pp) { + boolean found = false; + for (Edge edge : out) { + if ( edge != e && (edge.getTo().getDescendants().contains( p ) || edge.getTo() == p )) { + found = true; + break; + } + } + if ( !found ) { + descendants.remove( p ); + toRemove.add( p ); + } + } + if ( !toRemove.isEmpty() ) { + for (Edge edge : in) { + edge.getFrom().removeDescendant( edge, toRemove ); + } + } + } + } + + void addAncestor( Process p ) { + synchronized (ancestors) { + ancestors.add( p ); + } + } + + void addAncestor( Collection p ) { + synchronized (ancestors) { + ancestors.addAll( p ); + } + } + + void removeAncestor( Edge e, Collection pp ) { + synchronized (ancestors) { + LinkedList toRemove = new LinkedList<>(); + for (Process p : pp) { + boolean found = false; + for (Edge edge : in) { + if ( edge != e && (edge.getFrom().getAncestors().contains( p ) || edge.getFrom() == p) ) { + found = true; + break; + } + } + if ( !found ) { + ancestors.remove( p ); + toRemove.add( p ); + } + } + for (Edge edge : out) { + edge.getTo().removeAncestor( edge, toRemove ); + } + } + } /** * Only public for tests @@ -42,15 +114,15 @@ public void addInbound( Edge e ) { fromAncestors.add((Process) from); } - this.ancestors.addAll(fromAncestors); + this.addAncestor(fromAncestors); if ( from.getType() == Type.PROCESS ) { - ((Process) from).descendants.add( this ); + ((Process) from).addDescendant( this ); } final Set descendantsCopy = this.getDescendants(); fromAncestors.forEach( v -> { - v.descendants.addAll( descendantsCopy ); - v.descendants.add( this ); + v.addDescendant( descendantsCopy ); + v.addDescendant( this ); }); } @@ -63,15 +135,15 @@ public void addOutbound( Edge e ) { toDescendants.add((Process) to); } - this.descendants.addAll(toDescendants); + this.addDescendant(toDescendants); if ( to.getType() == Type.PROCESS ) { - ((Process)to).ancestors.add( this ); + ((Process)to).addAncestor( this ); } final Set ancestorsCopy = this.getAncestors(); toDescendants.forEach( v -> { - v.ancestors.addAll( ancestorsCopy ); - v.ancestors.add( this ); + v.addAncestor( ancestorsCopy ); + v.addAncestor( this ); }); } diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index 83dbfe9a..2f1fb5bd 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -3,8 +3,7 @@ import lombok.Getter; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -13,8 +12,8 @@ public abstract class Vertex { private final String label; private final int uid; - final List in = new LinkedList<>(); - final List out = new LinkedList<>(); + final Set in = new HashSet<>(); + final Set out = new HashSet<>(); Vertex( String label, int uid ) { this.label = label; @@ -25,12 +24,70 @@ public abstract class Vertex { public abstract void addInbound( Edge e ); + public void removeInbound( Edge e ) { + final boolean remove = in.remove(e); + if ( !remove ) { + throw new IllegalStateException("Edge " + e + " not found in " + this); + } + removeInboundIntern( e ); + } + + public void removeInboundIntern( Edge e ) { + final boolean remove = e.getFrom().out.remove(e); + if ( !remove ) { + throw new IllegalStateException("Edge " + e + " not found in " + this); + } + final Set ancestors = e.getFrom().getAncestors(); + if ( e.getFrom().getType() == Type.PROCESS ) { + ancestors.add((Process) e.getFrom()); + } + for (Edge edge : in) { + if ( edge != e ) { + ancestors.removeAll( edge.getFrom().getAncestors() ); + } + } + this.removeAncestor( e, ancestors ); + for (Edge edge : out) { + edge.getTo().removeAncestor( edge, ancestors ); + } + final Set descendants = getDescendants(); + if ( getType() == Type.PROCESS ) { + descendants.add((Process) this); + } + e.getFrom().removeDescendant( e, descendants ); + } + + public void removeOutbound( Edge e ) { + final boolean remove = out.remove(e); + if ( !remove ) { + throw new IllegalStateException("Edge " + e + " not found in " + this); + } + removeOutboundIntern( e ); + } + + public void removeOutboundIntern( Edge e ) { + e.getTo().removeInbound( e ); + } + + public void deleteItself(){ + for (Edge edge : in) { + removeInboundIntern( edge ); + } + for (Edge edge : out) { + removeOutboundIntern( edge ); + } + } + public abstract void addOutbound( Edge e ); public abstract Set getDescendants(); + abstract void removeDescendant( Edge edge, Collection p ); + public abstract Set getAncestors(); + abstract void removeAncestor( Edge edge, Collection p ); + private String collectionToString( Collection v ){ return v.stream().map( Process::getUid ).sorted().map( Object::toString ) .collect(Collectors.joining(",")); @@ -45,4 +102,19 @@ public String toString() { ", ancestors=" + collectionToString( getAncestors() ) + '}'; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Vertex)) return false; + + Vertex vertex = (Vertex) o; + + return getUid() == vertex.getUid(); + } + + @Override + public int hashCode() { + return getUid(); + } } diff --git a/src/test/java/fonda/scheduler/dag/DAGTest.java b/src/test/java/fonda/scheduler/dag/DAGTest.java index 766b287d..693145b2 100644 --- a/src/test/java/fonda/scheduler/dag/DAGTest.java +++ b/src/test/java/fonda/scheduler/dag/DAGTest.java @@ -4,13 +4,14 @@ import org.junit.Test; import java.util.*; +import java.util.stream.Collectors; import static org.junit.Assert.*; @Slf4j public class DAGTest { - private void compare(int uid, List vertices, int[] ancestorIds, int[] descedantsIds ){ + private void compare(int uid, List vertices, int[] ancestorIds, int[] descendantsIds ){ //noinspection OptionalGetWithoutIsPresent final Vertex vertex = vertices.stream().filter(v -> v.getUid() == uid).findFirst().get(); @@ -21,13 +22,69 @@ private void compare(int uid, List vertices, int[] ancestorIds, int[] de assertArrayEquals("Compare Ancestors for uid: " + uid, ancestorIds, a); } if( vertex.getDescendants() == null ){ - assertNull( descedantsIds ); + assertNull( descendantsIds ); } else { final int[] d = vertex.getDescendants().stream().mapToInt(Vertex::getUid).sorted().toArray(); - assertArrayEquals("Compare Descendants for uid: " + uid, descedantsIds, d); + assertArrayEquals("Compare Descendants for uid: " + uid, descendantsIds, d); } } + /** + * otherIds [edge id, destination id] + * @param dag + * @param idA + * @param otherIds + * @return + */ + private HashSet setFrom(DAG dag, int idA, int[]... otherIds){ + return Arrays.stream(otherIds).map(p -> { + Vertex a = dag.getByUid(idA); + Vertex b = dag.getByUid(p[1]); + int edgeId = p[0]; + return new Edge(edgeId, a, b); + }).collect(Collectors.toCollection(HashSet::new)); + } + + private HashSet setTo(DAG dag, int idA, int[]... otherIds){ + return Arrays.stream(otherIds).map(p -> { + Vertex a = dag.getByUid(idA); + Vertex b = dag.getByUid(p[1]); + int edgeId = p[0]; + return new Edge(edgeId, b, a); + }).collect(Collectors.toCollection(HashSet::new)); + } + + private void isSameEdge(HashSet expectedIn, HashSet expectedOut, Vertex vertex ){ + final String[] in = setToArrayEdges(expectedIn); + final String[] out = setToArrayEdges(expectedOut); + assertArrayEquals( "In of " + vertex.getLabel(), in, setToArrayEdges(vertex.getIn()) ); + assertArrayEquals( "Out of " + vertex.getLabel(), out, setToArrayEdges(vertex.getOut()) ); + } + + private void isSameProc(HashSet expectedAncestors, HashSet expectedDescendants, Vertex vertex ){ + final String[] descendants = setToArrayProcesses(expectedDescendants); + final String[] ancestors = setToArrayProcesses(expectedAncestors); + assertArrayEquals( "Descendants of " + vertex.getLabel(), descendants, setToArrayProcesses(vertex.getDescendants()) ); + assertArrayEquals( "Ancestors of " + vertex.getLabel(), ancestors, setToArrayProcesses(vertex.getAncestors()) ); + } + + private String[] setToArrayProcesses(Set set) { + return set.stream().map(Process::getLabel).sorted().collect(Collectors.toList()).toArray(new String[set.size()]); + } + + private String[] setToArrayEdges(Set set) { + return set + .stream() + .map( x -> String.format("(uid:%d,from:%s,to:%s)", x.getUid(), x.getFrom().getLabel(),x.getTo().getLabel())) + .sorted() + .collect(Collectors.toList()) + .toArray(new String[set.size()]); + } + + private HashSet set(DAG dag, String... elements){ + return Arrays.stream(elements).map(dag::getByProcess).collect(Collectors.toCollection(HashSet::new)); + } + public void expectedResult( List vertexList ) { compare( 1, vertexList, new int[]{}, new int[]{2,3,4,5,6,7,8,9,10,11,12} ); compare( 2, vertexList, new int[]{1} , new int[]{6,8,10,12} ); @@ -55,35 +112,28 @@ public List genVertexList(){ public List genEdgeList(){ List inputEdges = new LinkedList<>(); - inputEdges.add( new InputEdge(13,1) ); - inputEdges.add( new InputEdge(1,2) ); - inputEdges.add( new InputEdge(1,3) ); - inputEdges.add( new InputEdge(1,4) ); - inputEdges.add( new InputEdge(1,5) ); - inputEdges.add( new InputEdge(2,6) ); - inputEdges.add( new InputEdge(3,7) ); - inputEdges.add( new InputEdge(4,7) ); - inputEdges.add( new InputEdge(4,9) ); - inputEdges.add( new InputEdge(5,9) ); - inputEdges.add( new InputEdge(6,8) ); - inputEdges.add( new InputEdge(7,8) ); - inputEdges.add( new InputEdge(9,8) ); - inputEdges.add( new InputEdge(8,10) ); - inputEdges.add( new InputEdge(9,11) ); - inputEdges.add( new InputEdge(10,12) ); - inputEdges.add( new InputEdge(11,12) ); + inputEdges.add( new InputEdge(1,13,1) ); + inputEdges.add( new InputEdge(2,1,2) ); + inputEdges.add( new InputEdge(3,1,3) ); + inputEdges.add( new InputEdge(4,1,4) ); + inputEdges.add( new InputEdge(5,1,5) ); + inputEdges.add( new InputEdge(6,2,6) ); + inputEdges.add( new InputEdge(7,3,7) ); + inputEdges.add( new InputEdge(8,4,7) ); + inputEdges.add( new InputEdge(9,4,9) ); + inputEdges.add( new InputEdge(10,5,9) ); + inputEdges.add( new InputEdge(11,6,8) ); + inputEdges.add( new InputEdge(12,7,8) ); + inputEdges.add( new InputEdge(13,9,8) ); + inputEdges.add( new InputEdge(14,8,10) ); + inputEdges.add( new InputEdge(15,9,11) ); + inputEdges.add( new InputEdge(16,10,12) ); + inputEdges.add( new InputEdge(17,11,12) ); Collections.shuffle( inputEdges ); return inputEdges; } - public void debug( List vertices, int start, int end ){ - for (int i = start; i <= end; i++) { - final Vertex process = vertices.get( i ) ; - log.info( process.toString() ); - } - } - @Test public void testRelations() { for (int q = 0; q < 500 ; q++) { @@ -92,7 +142,6 @@ public void testRelations() { dag.registerVertices( vertexList ); List inputEdges = genEdgeList(); dag.registerEdges( inputEdges ); - debug( vertexList, 0, 11 ); expectedResult ( vertexList ); } } @@ -108,12 +157,11 @@ public void testRelations2() { final InputEdge remove = inputEdges.remove(index); final Vertex operator = new Operator("Operator", vertexList.size() + 1 ); vertexList.add( operator ); - inputEdges.add( new InputEdge( remove.getFrom(), operator.getUid() ) ); - inputEdges.add( new InputEdge( operator.getUid(), remove.getTo() ) ); + inputEdges.add( new InputEdge( i * 2, remove.getFrom(), operator.getUid() ) ); + inputEdges.add( new InputEdge( i * 2 + 1, operator.getUid(), remove.getTo() ) ); } dag.registerVertices( vertexList ); dag.registerEdges( inputEdges ); - debug( vertexList, 0, 11 ); expectedResult ( vertexList ); } } @@ -136,15 +184,13 @@ public void smallTest(){ vertexList.add(b); List inputEdges = new LinkedList<>(); - inputEdges.add( new InputEdge(1,2) ); - inputEdges.add( new InputEdge(2,3) ); - inputEdges.add( new InputEdge(3,4) ); + inputEdges.add( new InputEdge(1,1,2) ); + inputEdges.add( new InputEdge(2,2,3) ); + inputEdges.add( new InputEdge(3,3,4) ); dag.registerVertices( vertexList ); dag.registerEdges( inputEdges ); - debug( vertexList, 0, 3 ); - assertEquals( new HashSet<>(), a.getAncestors() ); final HashSet descA = new HashSet<>(); descA.add( b ); @@ -168,5 +214,338 @@ public void smallTest(){ } + /** + * + * o + * | + * a + * / \ + * f f2 <-- will be removed + * \ / + * b + * + */ + @Test + public void deleteTest(){ + + final DAG dag = new DAG(); + + + final Origin o = new Origin("o", 1); + final Process a = new Process("a", 2); + final Operator filter = new Operator("filter", 3); + final Operator filter2 = new Operator("filter", 5); + final Process b = new Process("b", 4); + + List vertexList = new LinkedList<>(); + vertexList.add(o); + vertexList.add(a); + vertexList.add(filter); + vertexList.add(filter2); + vertexList.add(b); + + List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(1,1,2) ); + inputEdges.add( new InputEdge(2,2,3) ); + inputEdges.add( new InputEdge(3,2,5) ); + inputEdges.add( new InputEdge(4,3,4) ); + inputEdges.add( new InputEdge(5,5,4) ); + + dag.registerVertices( vertexList ); + dag.registerEdges( inputEdges ); + + for (int i = 0; i < 2; i++) { + + if ( i == 1 ) { + dag.removeEdges( 3, 5 ); + dag.removeVertices( filter2.getUid() ); + } + + assertEquals( new HashSet<>(), a.getAncestors() ); + if ( i == 0 ) { + assertEquals( new HashSet<>( Arrays.asList( new Edge(2,a,filter), new Edge(3,a,filter2) ) ), a.getOut() ); + } else { + assertEquals( new HashSet<>(List.of(new Edge(2, a, filter))), a.getOut() ); + } + final HashSet descA = new HashSet<>(); + descA.add( b ); + assertEquals( descA, a.getDescendants() ); + + + final HashSet ancB = new HashSet<>(); + ancB.add( a ); + assertEquals( new HashSet<>(), b.getDescendants() ); + assertEquals( ancB, b.getAncestors() ); + + if ( i == 0 ) { + assertEquals( new HashSet<>( Arrays.asList( new Edge(4,filter,b), new Edge(5,filter2,b) ) ), b.getIn() ); + } else { + assertEquals( new HashSet<>(List.of(new Edge(4, filter, b))), b.getIn() ); + } + + assertEquals( ancB, filter.getAncestors() ); + assertEquals( descA, filter.getDescendants() ); + + assertEquals( new HashSet<>(),o.getAncestors() ); + final HashSet descO = new HashSet<>(); + + descO.add( a ); + descO.add( b ); + assertEquals( descO, o.getDescendants() ); + } + + } + + + + + @Test + public void deleteTest2() { + + final DAG dag = createDag(); + dag.removeEdges( 3 ); + isSameProc( new HashSet<>(), set( dag, "a","b","c","d","e","f","g","h","i" ), dag.getByUid( 1 ) ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByUid( 1 ) ); + isSameProc( new HashSet<>(), set( dag, "g","i" ), dag.getByProcess("a") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{4, 8} ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "c","d","e","f","h","i" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{6, 5},new int[]{5, 4} ), dag.getByProcess("b") ); + isSameProc( set( dag, "b" ), set( dag, "e","h","i" ), dag.getByProcess("c") ); + isSameEdge( setTo( dag, 4, new int[]{5, 3} ), setFrom( dag, 4, new int[]{7, 6} ), dag.getByProcess("c") ); + isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 5, new int[]{6, 3} ), setFrom( dag, 5, new int[]{8, 7} ), dag.getByProcess("d") ); + isSameProc( set( dag, "b","c" ), set( dag, "h","i" ), dag.getByProcess("e") ); + isSameEdge( setTo( dag, 6, new int[]{7, 4} ), setFrom( dag, 6, new int[]{9, 9} ), dag.getByProcess("e") ); + isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), dag.getByProcess("f") ); + isSameEdge( setTo( dag, 7, new int[]{8, 5} ), setFrom( dag, 7, new int[]{10, 9} ), dag.getByProcess("f") ); + isSameProc( set( dag, "a" ), set( dag, "i" ), dag.getByProcess("g") ); + isSameEdge( setTo( dag, 8, new int[]{4, 2} ), setFrom( dag, 8, new int[]{11, 10} ), dag.getByProcess("g") ); + isSameProc( set( dag, "b","c","d","e","f" ), set( dag, "i" ), dag.getByProcess("h") ); + isSameEdge( setTo( dag, 9, new int[]{10, 7},new int[]{9, 6} ), setFrom( dag, 9, new int[]{12, 10} ), dag.getByProcess("h") ); + isSameProc( set( dag, "a","b","c","d","e","f","g","h" ), new HashSet<>(), dag.getByProcess("i") ); + isSameEdge( setTo( dag, 10, new int[]{12, 9},new int[]{11, 8} ), new HashSet<>(), dag.getByProcess("i") ); + + } + + @Test + public void deleteTest3() { + + final DAG dag = createDag(); + dag.removeVertices( dag.getByProcess("c").getUid(), dag.getByProcess("e").getUid() ); + isSameProc( new HashSet<>(), set( dag, "a","b","d","f","g","h","i" ), dag.getByUid( 1 ) ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByUid( 1 ) ); + isSameProc( new HashSet<>(), set( dag, "g","i" ), dag.getByProcess("a") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{4, 8} ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "d","f","h","i" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{6, 5} ), dag.getByProcess("b") ); + isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 5, new int[]{6, 3} ), setFrom( dag, 5, new int[]{8, 7} ), dag.getByProcess("d") ); + isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), dag.getByProcess("f") ); + isSameEdge( setTo( dag, 7, new int[]{8, 5} ), setFrom( dag, 7, new int[]{10, 9} ), dag.getByProcess("f") ); + isSameProc( set( dag, "a" ), set( dag, "i" ), dag.getByProcess("g") ); + isSameEdge( setTo( dag, 8, new int[]{4, 2} ), setFrom( dag, 8, new int[]{11, 10} ), dag.getByProcess("g") ); + isSameProc( set( dag, "b","d","f" ), set( dag, "i" ), dag.getByProcess("h") ); + isSameEdge( setTo( dag, 9, new int[]{10, 7} ), setFrom( dag, 9, new int[]{12, 10} ), dag.getByProcess("h") ); + isSameProc( set( dag, "a","b","d","f","g","h" ), new HashSet<>(), dag.getByProcess("i") ); + isSameEdge( setTo( dag, 10, new int[]{12, 9},new int[]{11, 8} ), new HashSet<>(), dag.getByProcess("i") ); + assertThrows( IllegalStateException.class, () -> dag.getByProcess("c") ); + assertThrows( IllegalStateException.class, () -> dag.getByProcess("e") ); + + } + + /** + * a + * / \ + * b c + * \ /| + * d |<--- delete this edge + * |/ + * e + */ + @Test + public void deleteTest4() { + + final DAG dag = createDag(); + final Process a = new Process("a", 1); + final Process b = new Process("b", 2); + final Process c = new Process("c", 3); + final Process d = new Process("d", 4); + final Process e = new Process("e", 5); + List vertexList = Arrays.asList( a, b, c, d, e ); + List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(1, 1,2) ); + inputEdges.add( new InputEdge(2, 1,3) ); + inputEdges.add( new InputEdge(3, 2,4) ); + inputEdges.add( new InputEdge(4, 3,4) ); + inputEdges.add( new InputEdge(5, 3,5) ); + inputEdges.add( new InputEdge(6, 4,5) ); + dag.registerVertices( vertexList ); + dag.registerEdges(inputEdges); + + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4},new int[]{5, 5} ), dag.getByProcess("c") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5} ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameEdge( setTo( dag, 5, new int[]{5, 3},new int[]{6, 4} ), new HashSet<>(), dag.getByProcess("e") ); + + dag.removeEdges( 5 ); + + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5} ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameEdge( setTo( dag, 5, new int[]{6, 4} ), new HashSet<>(), dag.getByProcess("e") ); + + } + + /** + * a + * / \ + * b c + * \ / + * d + * | | <--- delete this edge + * e + */ + @Test + public void deleteTest5() { + + final DAG dag = new DAG(); + final Process a = new Process("a", 1); + final Process b = new Process("b", 2); + final Process c = new Process("c", 3); + final Process d = new Process("d", 4); + final Process e = new Process("e", 5); + List vertexList = Arrays.asList( a, b, c, d, e ); + List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(1, 1,2) ); + inputEdges.add( new InputEdge(2, 1,3) ); + inputEdges.add( new InputEdge(3, 2,4) ); + inputEdges.add( new InputEdge(4, 3,4) ); + inputEdges.add( new InputEdge(5, 4,5) ); + inputEdges.add( new InputEdge(6, 4,5) ); + dag.registerVertices( vertexList ); + dag.registerEdges(inputEdges); + + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5},new int[]{5, 5} ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameEdge( setTo( dag, 5, new int[]{6, 4},new int[]{5, 4} ), new HashSet<>(), dag.getByProcess("e") ); + + dag.removeEdges( 5 ); + + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5} ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameEdge( setTo( dag, 5, new int[]{6, 4} ), new HashSet<>(), dag.getByProcess("e") ); + + dag.removeEdges( 6 ); + + isSameProc( new HashSet<>(), set( dag, "b","c","d" ), dag.getByProcess("a") ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); + isSameProc( set( dag, "a" ), set( dag, "d" ), dag.getByProcess("b") ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d" ), dag.getByProcess("c") ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); + isSameProc( set( dag, "a","b","c" ), new HashSet<>(), dag.getByProcess("d") ); + isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), new HashSet<>(), dag.getByProcess("d") ); + isSameProc( new HashSet<>(), new HashSet<>(), dag.getByProcess("e") ); + isSameEdge( new HashSet<>(), new HashSet<>(), dag.getByProcess("e") ); + + } + + /** + * o + * | \ + * a b + * |\ |\ + * | -c d + * | | | + * | e f + * | |/ + * g h + * | / + * i + * + * @return the dag + */ + private DAG createDag() { + final DAG dag = new DAG(); + + final Origin o = new Origin("o", 1); + final Process a = new Process("a", 2); + final Process b = new Process("b", 3); + final Process c = new Process("c", 4); + final Process d = new Process("d", 5); + final Process e = new Process("e", 6); + final Process f = new Process("f", 7); + final Process g = new Process("g", 8); + final Process h = new Process("h", 9); + final Process i = new Process("i", 10); + + List vertexList = Arrays.asList(o, a, b, c, d, e, f, g, h, i); + + List inputEdges = Arrays.asList( + new InputEdge( 1, o.getUid(), a.getUid()), + new InputEdge( 2, o.getUid(), b.getUid()), + new InputEdge( 3, a.getUid(), c.getUid()), + new InputEdge( 4, a.getUid(), g.getUid()), + new InputEdge( 5, b.getUid(), c.getUid()), + new InputEdge( 6, b.getUid(), d.getUid()), + new InputEdge( 7, c.getUid(), e.getUid()), + new InputEdge( 8, d.getUid(), f.getUid()), + new InputEdge( 9, e.getUid(), h.getUid()), + new InputEdge( 10, f.getUid(), h.getUid()), + new InputEdge( 11, g.getUid(), i.getUid()), + new InputEdge( 12, h.getUid(), i.getUid()) + ); + + dag.registerVertices(vertexList); + dag.registerEdges(inputEdges); + + isSameProc( new HashSet<>(), set( dag, "a","b","c","d","e","f","g","h","i" ), o ); + isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), o ); + isSameProc( new HashSet<>(), set( dag, "c","e","g","h","i" ), a ); + isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4},new int[]{4, 8} ), a ); + isSameProc( new HashSet<>(), set( dag, "c","d","e","f","h","i" ), b ); + isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{6, 5},new int[]{5, 4} ), b ); + isSameProc( set( dag, "a","b" ), set( dag, "e","h","i" ), c ); + isSameEdge( setTo( dag, 4, new int[]{5, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{7, 6} ), c ); + isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), d ); + isSameEdge( setTo( dag, 5, new int[]{6, 3} ), setFrom( dag, 5, new int[]{8, 7} ), d ); + isSameProc( set( dag, "a","b","c" ), set( dag, "h","i" ), e ); + isSameEdge( setTo( dag, 6, new int[]{7, 4} ), setFrom( dag, 6, new int[]{9, 9} ), e ); + isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), f ); + isSameEdge( setTo( dag, 7, new int[]{8, 5} ), setFrom( dag, 7, new int[]{10, 9} ), f ); + isSameProc( set( dag, "a" ), set( dag, "i" ), g ); + isSameEdge( setTo( dag, 8, new int[]{4, 2} ), setFrom( dag, 8, new int[]{11, 10} ), g ); + isSameProc( set( dag, "a","b","c","d","e","f" ), set( dag, "i" ), h ); + isSameEdge( setTo( dag, 9, new int[]{10, 7},new int[]{9, 6} ), setFrom( dag, 9, new int[]{12, 10} ), h ); + isSameProc( set( dag, "a","b","c","d","e","f","g","h" ), new HashSet<>(), i ); + isSameEdge( setTo( dag, 10, new int[]{12, 9},new int[]{11, 8} ), new HashSet<>(), i ); + return dag; + } } \ No newline at end of file diff --git a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java index d75ecbd1..2c9164b7 100644 --- a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java @@ -144,7 +144,7 @@ public void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { vertexList.add(new Process("processB", 2)); vertexList.add(new Process("processC", 3)); dag.registerVertices(vertexList); - dag.registerEdges( List.of( new InputEdge(1,2),new InputEdge(2,3) ) ); + dag.registerEdges( List.of( new InputEdge(1,1,2),new InputEdge(2,2,3) ) ); final TaskConfig taskConfigA = new TaskConfig("processA"); final Task taskA = new Task(taskConfigA, dag); @@ -195,7 +195,7 @@ public void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundExceptio vertexList.add(new Process("processB", 2)); vertexList.add(new Process("processC", 3)); dag.registerVertices(vertexList); - dag.registerEdges( List.of( new InputEdge(1,2),new InputEdge(2,3) ) ); + dag.registerEdges( List.of( new InputEdge(1,1,2),new InputEdge(2,2,3) ) ); final TaskConfig taskConfigA = new TaskConfig("processA"); final Task taskA = new Task(taskConfigA, dag); diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java index 639e8d3b..47cd4fd7 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -38,10 +38,10 @@ public void before(){ vertexList.add(new Process("processE", 5)); dag.registerVertices(vertexList); List edgeList = new LinkedList<>(); - edgeList.add( new InputEdge(1,2) ); - edgeList.add( new InputEdge(2,4) ); - edgeList.add( new InputEdge(4,5) ); - edgeList.add( new InputEdge(1,3) ); + edgeList.add( new InputEdge(1,1,2) ); + edgeList.add( new InputEdge(2,2,4) ); + edgeList.add( new InputEdge(3,4,5) ); + edgeList.add( new InputEdge(4,1,3) ); dag.registerEdges(edgeList); processA = new Task( new TaskConfig("processA"), dag); processB = new Task( new TaskConfig("processB"), dag); From 1489cae6cbf4b8e3916fa9a193e14563dbc9fcb1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 29 Sep 2022 15:40:05 +0200 Subject: [PATCH 282/443] Adjust Scheduler to API Paper Signed-off-by: Lehmann_Fabian --- .../scheduler/model/SchedulerConfig.java | 7 +- .../fonda/scheduler/model/TaskConfig.java | 4 +- .../rest/SchedulerRestController.java | 201 +++++++++++------- .../scheduler/LocationAwareScheduler.java | 4 +- .../fonda/scheduler/scheduler/Scheduler.java | 62 ++++-- .../scheduler/SchedulerWithDaemonSet.java | 14 +- .../scheduler/schedulingstrategy/Inputs.java | 2 +- 7 files changed, 177 insertions(+), 117 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 615638fc..80d8359b 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -14,10 +14,11 @@ public class SchedulerConfig { public final String copyStrategy; public final boolean locationAware; public final boolean traceEnabled; - + public final String namespace; public final String costFunction; + public final String strategy; - private SchedulerConfig(){ + private SchedulerConfig() { this.localClaims = null; this.volumeClaims = null; this.workDir = null; @@ -26,6 +27,8 @@ private SchedulerConfig(){ this.locationAware = false; this.traceEnabled = false; this.costFunction = null; + this.namespace = null; + this.strategy = null; } @ToString diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/fonda/scheduler/model/TaskConfig.java index 4e938fd5..6798cd8d 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/fonda/scheduler/model/TaskConfig.java @@ -15,7 +15,7 @@ public class TaskConfig { private final String name; private final Map< String, List> schedulerParams; private final TaskInput inputs; - private final String hash; + private final String runName; private final float cpus; private final long memoryInBytes; private final OutLabel outLabel; @@ -34,7 +34,7 @@ public TaskConfig(String task) { this.name = null; this.schedulerParams = null; this.inputs = new TaskInput( null, null, null, new LinkedList<>() ); - this.hash = null; + this.runName = null; this.cpus = 0; this.memoryInBytes = 0; this.workDir = null; diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index dca43093..10ffdc12 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -14,7 +14,6 @@ import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import lombok.extern.slf4j.Slf4j; -import org.javatuples.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; @@ -44,7 +43,7 @@ public class SchedulerRestController { * Execution: String in lowercase * Scheduler: An instance of a scheduler with the requested type */ - private static final Map< Pair, Scheduler > schedulerHolder = new HashMap<>(); + private static final Map schedulerHolder = new HashMap<>(); public SchedulerRestController( @Autowired KubernetesClient client, @@ -71,23 +70,25 @@ private ResponseEntity noSchedulerFor( String execution ){ } /** - * Register a sheduler for a workflow execution - * @param namespace namespace where the workflow runs + * Register a scheduler for a workflow execution * @param execution unique name of the execution - * @param strategy which scheduling strategy should be used * @param config Additional parameters for the scheduler * @return */ - @PutMapping("/scheduler/registerScheduler/{namespace}/{execution}/{strategy}") - ResponseEntity registerScheduler(@PathVariable String namespace, @PathVariable String execution, @PathVariable String strategy, @RequestBody(required = false) SchedulerConfig config ) { + @PostMapping("/v1/scheduler/{execution}") + ResponseEntity registerScheduler( + @PathVariable String execution, + @RequestBody(required = false) SchedulerConfig config + ) { - log.info("Register execution: {} strategy: {} cf: {} config: {}", execution, strategy, config.costFunction, config); + final String namespace = config.namespace; + final String strategy = config.strategy; + log.info( "Register execution: {} strategy: {} cf: {} config: {}", execution, strategy, config.costFunction, config ); Scheduler scheduler; - final Pair key = getKey( namespace, execution ); - if( schedulerHolder.containsKey( key ) ) { + if ( schedulerHolder.containsKey( execution ) ) { return noSchedulerFor( execution ); } @@ -120,7 +121,7 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } - schedulerHolder.put( key, scheduler ); + schedulerHolder.put( execution, scheduler ); client.addScheduler( scheduler ); return new ResponseEntity<>( HttpStatus.OK ); @@ -129,35 +130,54 @@ ResponseEntity registerScheduler(@PathVariable String namespace, @PathVa /** * Register a task for the execution - * @param namespace namespace where the workflow runs + * * @param execution unique name of the execution * @param config The config contains the task name, input files, and optional task parameter the scheduler has to determine * @return Parameters the scheduler suggests for the task */ - @PutMapping("/scheduler/registerTask/{namespace}/{execution}") - ResponseEntity registerTask(@PathVariable String namespace, @PathVariable String execution, @RequestBody TaskConfig config ) { + @PostMapping("/v1/scheduler/{execution}/task/{id}") + ResponseEntity registerTask( @PathVariable String execution, @PathVariable int id, @RequestBody TaskConfig config ) { log.trace( execution + " " + config.getTask() + " got: " + config ); - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } - scheduler.addTask( config ); - Map schedulerParams = scheduler.getSchedulerParams(config.getTask(), config.getName()); + scheduler.addTask( id, config ); + Map schedulerParams = scheduler.getSchedulerParams( config.getTask(), config.getName() ); return new ResponseEntity<>( schedulerParams, HttpStatus.OK ); } - @PostMapping("/scheduler/startBatch/{namespace}/{execution}") - ResponseEntity startBatch(@PathVariable String namespace, @PathVariable String execution ) { + /** + * Delete a task, onl works if the batch of the task was closed and if no pod was yet submitted. + * If a pod was submitted, delete the pod instead. + * + * @param execution + * @param id + * @return + */ + @DeleteMapping("/v1/scheduler/{execution}/task/{id}") + ResponseEntity deleteTask( @PathVariable String execution, @PathVariable int id ) { - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { + return noSchedulerFor( execution ); + } + + final boolean found = scheduler.removeTask( id ); + return new ResponseEntity<>( found ? HttpStatus.OK : HttpStatus.NOT_FOUND ); + + } + + @PutMapping("/v1/scheduler/{execution}/startBatch") + ResponseEntity startBatch( @PathVariable String execution ) { + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } scheduler.startBatch(); @@ -165,12 +185,11 @@ ResponseEntity startBatch(@PathVariable String namespace, @PathVariable } - @PostMapping("/scheduler/endBatch/{namespace}/{execution}") - ResponseEntity endBatch(@PathVariable String namespace, @PathVariable String execution, @RequestBody int tasksInBatch ) { + @PutMapping("/v1/scheduler/{execution}/endBatch") + ResponseEntity endBatch( @PathVariable String execution, @RequestBody int tasksInBatch ) { - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } scheduler.endBatch( tasksInBatch ); @@ -180,55 +199,52 @@ ResponseEntity endBatch(@PathVariable String namespace, @PathVariable St /** * Check Task state - * @param namespace namespace where the workflow runs + * * @param execution unique name of the execution - * @param taskid unique name of task (nf-XYZ...) + * @param id unique id of task * @return boolean */ - @GetMapping("/scheduler/taskstate/{namespace}/{execution}/{taskid}") - ResponseEntity getTaskState(@PathVariable String namespace, @PathVariable String execution, @PathVariable String taskid ) { + @GetMapping("/v1/scheduler/{execution}/task/{id}") + ResponseEntity getTaskState( @PathVariable String execution, @PathVariable int id ) { - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } - return new ResponseEntity<>( scheduler.getTaskState( taskid ), HttpStatus.OK ); + return new ResponseEntity<>( scheduler.getTaskState( id ), HttpStatus.OK ); } /** * Call this after the execution has finished - * @param namespace namespace where the workflow runs + * * @param execution unique name of the execution * @return */ - @DeleteMapping ("/scheduler/{namespace}/{execution}") - ResponseEntity delete(@PathVariable String namespace, @PathVariable String execution) { + @DeleteMapping("/v1/scheduler/{execution}") + ResponseEntity delete( @PathVariable String execution ) { - log.info("Delete scheduler: " + execution); - final Pair key = getKey( namespace, execution ); + log.info( "Delete scheduler: " + execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } - schedulerHolder.remove( key ); + schedulerHolder.remove( execution ); client.removeScheduler( scheduler ); scheduler.close(); closedLastScheduler = System.currentTimeMillis(); return new ResponseEntity<>( HttpStatus.OK ); } - @GetMapping("/daemon/{namespace}/{execution}/{node}") - ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariable String execution, @PathVariable String node ) { + @GetMapping("/v1/daemon/{execution}/{node}") + ResponseEntity getDaemonName( @PathVariable String execution, @PathVariable String node ) { - log.info( "Asking for Daemon ns: {} exec: {} node: {}", namespace, execution, node ); + log.info( "Asking for Daemon exec: {} node: {}", execution, node ); - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if(!(scheduler instanceof SchedulerWithDaemonSet)){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { return noSchedulerFor( execution ); } @@ -242,14 +258,13 @@ ResponseEntity getDaemonName(@PathVariable String namespace, @PathVariab } - @GetMapping("/file/{namespace}/{execution}") - ResponseEntity getNodeForFile(@PathVariable String namespace, @PathVariable String execution, @RequestParam String path ) { + @GetMapping("/v1/file/{execution}") + ResponseEntity getNodeForFile( @PathVariable String execution, @RequestParam String path ) { - log.info( "Get file location request: {} {} {}", namespace, execution, path ); + log.info( "Get file location request: {} {}", execution, path ); - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if(!(scheduler instanceof SchedulerWithDaemonSet)){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { return noSchedulerFor( execution ); } @@ -269,20 +284,19 @@ ResponseEntity getNodeForFile(@PathVariable String namespace, } - @PostMapping("/file/location/{method}/{namespace}/{execution}") - ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @RequestBody PathAttributes pa ) { - return changeLocationForFile(method,namespace,execution,null,pa); + @PostMapping("/v1/file/{execution}/location/{method}") + ResponseEntity changeLocationForFile( @PathVariable String method, @PathVariable String execution, @RequestBody PathAttributes pa ) { + return changeLocationForFile( method, execution, null, pa ); } - @PostMapping("/file/location/{method}/{namespace}/{execution}/{node}") - ResponseEntity changeLocationForFile(@PathVariable String method, @PathVariable String namespace, @PathVariable String execution, @PathVariable String node, @RequestBody PathAttributes pa ) { + @PostMapping("/v1/file/{execution}/location/{method}/{node}") + ResponseEntity changeLocationForFile( @PathVariable String method, @PathVariable String execution, @PathVariable String node, @RequestBody PathAttributes pa ) { - log.info( "Change file location request: {} {} {} {}", method, namespace, execution, pa ); + log.info( "Change file location request: {} {} {}", method, execution, pa ); - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if(!(scheduler instanceof SchedulerWithDaemonSet)){ - log.info("No scheduler for: " + execution); + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { + log.info( "No scheduler for: " + execution ); return noSchedulerFor( execution ); } @@ -304,35 +318,45 @@ ResponseEntity checkHealth() { return new ResponseEntity<>( HttpStatus.OK ); } - private Pair getKey(String namespace, String execution ){ - return new Pair<>(namespace.toLowerCase(), execution.toLowerCase()); + @PostMapping("/v1/scheduler/{execution}/DAG/vertices") + ResponseEntity addVertices( @PathVariable String execution, @RequestBody List vertices ) { + + log.trace( "submit vertices: {}", vertices ); + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { + return noSchedulerFor( execution ); + } + + scheduler.getDag().registerVertices( vertices ); + + return new ResponseEntity<>( HttpStatus.OK ); + } - @PutMapping("/scheduler/DAG/addVertices/{namespace}/{execution}") - ResponseEntity addVertices(@PathVariable String namespace, @PathVariable String execution, @RequestBody List vertices ) { + @DeleteMapping("/v1/scheduler/{execution}/DAG/vertices") + ResponseEntity deleteVertices( @PathVariable String execution, @RequestBody int[] vertices ) { log.trace( "submit vertices: {}", vertices ); - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } - scheduler.getDag().registerVertices( vertices ); + scheduler.getDag().removeVertices( vertices ); return new ResponseEntity<>( HttpStatus.OK ); } - @PutMapping("/scheduler/DAG/addEdges/{namespace}/{execution}") - ResponseEntity addEdges(@PathVariable String namespace, @PathVariable String execution, @RequestBody List edges ) { + @PostMapping("/v1/scheduler/{execution}/DAG/edges") + ResponseEntity addEdges( @PathVariable String execution, @RequestBody List edges ) { log.trace( "submit edges: {}", edges ); - final Pair key = getKey( namespace, execution ); - final Scheduler scheduler = schedulerHolder.get( key ); - if( scheduler == null ){ + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { return noSchedulerFor( execution ); } @@ -343,4 +367,21 @@ ResponseEntity addEdges(@PathVariable String namespace, @PathVariable St } + @DeleteMapping("/v1/scheduler/{execution}/DAG/edges") + ResponseEntity deleteEdges( @PathVariable String execution, @RequestBody int[] edges ) { + + log.trace( "submit edges: {}", edges ); + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( scheduler == null ) { + return noSchedulerFor( execution ); + } + + final DAG dag = scheduler.getDag(); + dag.removeEdges( edges ); + + return new ResponseEntity<>( HttpStatus.OK ); + + } + } diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index d24514db..c7048ab9 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -52,7 +52,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( int index ) { long startTime = System.nanoTime(); - log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getHash(), taskData.getValue() ); + log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getRunName(), taskData.getValue() ); taskData.removeAllNodesWhichHaveNotEnoughResources( availableByNode ); final Tuple result = calculateBestNode(taskData, planedToCopy, availableByNode, assignedPodsByNode); if ( result == null ) { @@ -292,7 +292,7 @@ Tuple calculateBestNode( bestNodeHasOutLabel = isOnOutLabelNode; bestAlignment = fileAlignment; bestNode = currentNode; - log.info("Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getHash(), fileAlignment.getCost()); + log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getRunName(), fileAlignment.getCost() ); } } } catch ( NoAligmentPossibleException e ){ diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index f81c6243..2b5ac616 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -15,7 +15,10 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import java.io.*; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.util.*; @Slf4j @@ -41,9 +44,10 @@ public abstract class Scheduler { final KubernetesClient client; private final Set upcomingTasks = new HashSet<>(); - private final List unscheduledTasks = new ArrayList<>(100); - private final List unfinishedTasks = new ArrayList<>(100); - final Map tasksByHash = new HashMap<>(); + private final List unscheduledTasks = new ArrayList<>( 100 ); + private final List unfinishedTasks = new ArrayList<>( 100 ); + final Map tasksByPodName = new HashMap<>(); + final Map tasksById = new HashMap<>(); private final Watch watcher; private final TaskprocessingThread schedulingThread; private final TaskprocessingThread finishThread; @@ -112,7 +116,7 @@ public int schedule( final List unscheduledTasks ) { continue; } } catch ( Exception e ){ - log.info( "Could not schedule task: {} undo all", nodeTaskAlignment.task.getConfig().getHash() ); + log.info( "Could not schedule task: {} undo all", nodeTaskAlignment.task.getConfig().getRunName() ); e.printStackTrace(); undoTaskScheduling( nodeTaskAlignment.task ); if ( scheduleObject.isStopSubmitIfOneFails() ) { @@ -241,14 +245,19 @@ public void markPodAsDeleted( PodWithAge pod ) { /* External access to Tasks */ - public void addTask( TaskConfig conf ) { + public void addTask( int id, TaskConfig conf ) { final Task task = new Task( conf, dag ); - synchronized (tasksByHash) { - if( ! tasksByHash.containsKey( conf.getHash() ) ){ - tasksByHash.put( conf.getHash(), task ); + synchronized ( tasksByPodName ) { + if ( !tasksByPodName.containsKey( conf.getRunName() ) ) { + tasksByPodName.put( conf.getRunName(), task ); } } - synchronized ( upcomingTasks ){ + synchronized ( tasksById ) { + if ( !tasksById.containsKey( id ) ) { + tasksById.put( id, task ); + } + } + synchronized ( upcomingTasks ) { upcomingTasks.add( task ); } if( currentBatchInstance != null ){ @@ -256,13 +265,21 @@ public void addTask( TaskConfig conf ) { } } - TaskConfig getConfigFor( String hash ){ - synchronized (tasksByHash) { - if( tasksByHash.containsKey( hash ) ){ - return tasksByHash.get( hash ).getConfig(); + public boolean removeTask( int id ) { + final Task task; + synchronized ( tasksById ) { + task = tasksById.get( id ); } + if ( task == null ) { + return false; } - return null; + synchronized ( tasksByPodName ) { + tasksByPodName.remove( task.getConfig().getRunName() ); + } + synchronized ( upcomingTasks ) { + upcomingTasks.remove( task ); + } + return true; } /** @@ -275,10 +292,10 @@ public Map getSchedulerParams( String taskname, String name ){ return new HashMap<>(); } - public TaskState getTaskState(String taskid) { - synchronized (tasksByHash) { - if( tasksByHash.containsKey( taskid ) ){ - return tasksByHash.get( taskid ).getState(); + public TaskState getTaskState( int id ) { + synchronized ( tasksById ) { + if ( tasksById.containsKey( id ) ) { + return tasksById.get( id ).getState(); } } return null; @@ -400,7 +417,6 @@ public void endBatch( int tasksInBatch ){ } /** - * * @param pod * @param state * @return returns the task, if the state was changed @@ -428,9 +444,9 @@ public void informResourceChange() { Task getTaskByPod( Pod pod ) { Task t = null; - synchronized (tasksByHash) { - if( tasksByHash.containsKey( pod.getMetadata().getName() ) ){ - t = tasksByHash.get( pod.getMetadata().getName() ); + synchronized ( tasksByPodName ) { + if ( tasksByPodName.containsKey( pod.getMetadata().getName() ) ) { + t = tasksByPodName.get( pod.getMetadata().getName() ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 0c631df7..819d26fd 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -138,7 +138,7 @@ int terminateTasks(List finishedTasks) { //Init failure final Path workdir = Paths.get(finishedTask.getWorkingDir()); if ( exitCode == 123 && Files.exists( workdir.resolve(".command.init.failure") ) ) { - log.info( "Task " + finishedTask.getConfig().getHash() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and outfiles" ); + log.info( "Task " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and outfiles" ); } else { final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( workdir, @@ -161,7 +161,7 @@ int terminateTasks(List finishedTasks) { } } } catch ( Exception e ){ - log.info( "Problem while finishing task: " + finishedTask.getConfig().getHash() + " (" + finishedTask.getConfig().getName() + ")", e ); + log.info( "Problem while finishing task: " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ")", e ); } super.taskWasFinished( finishedTask ); }); @@ -256,7 +256,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final Inputs inputs = new Inputs( this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/", this.localWorkDir + "/sync/", - alignment.task.getConfig().getHash() + alignment.task.getConfig().getRunName() ); for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { @@ -343,7 +343,7 @@ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileExce MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map availableByNode ){ final Set matchingNodesForTask = getMatchingNodesForTask(availableByNode, task); if( matchingNodesForTask.isEmpty() ) { - log.trace( "No node with enough resources for {}", task.getConfig().getHash() ); + log.trace( "No node with enough resources for {}", task.getConfig().getRunName() ); return null; } @@ -354,13 +354,13 @@ MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map wrapperByPath = new HashMap<>(); task.getCopiedFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index 9421190e..a2d57ae7 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -29,7 +29,7 @@ public Inputs( String dns, String syncDir, String hash ) { public void waitForTask( Map waitForTask ){ for (Map.Entry e : waitForTask.entrySet()) { - final String taskHash = e.getValue().getConfig().getHash(); + final String taskHash = e.getValue().getConfig().getRunName(); final List listOfPaths = waitForFilesOfTask.computeIfAbsent( taskHash, k -> new LinkedList<>() ); listOfPaths.add( e.getKey() ); } From 042dc0dd245f405545a6912f827ede3a43191fc7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Oct 2022 10:21:32 +0200 Subject: [PATCH 283/443] Added comment and normalized indent Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/data/NodeDataTuple.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java index 8b185e8b..45593a48 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java +++ b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java @@ -27,12 +27,15 @@ public NodeDataTuple(NodeWithAlloc node, long sizeInBytes, double weight ) { this.weight = weight; } + /** + * @return reduce the worth, if this is not the outLabelNode + */ public double getWorth() { return getSizeInBytes() / weight; } @Override - public int compareTo(@NotNull NodeDataTuple o) { - return Double.compare( getWorth(), o.getWorth() ); - } + public int compareTo(@NotNull NodeDataTuple o) { + return Double.compare( getWorth(), o.getWorth() ); + } } \ No newline at end of file From 9d84b1ede76e4fae90cce7b7b4ac1b589bfb6575 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Oct 2022 10:30:15 +0200 Subject: [PATCH 284/443] Return for null label directly Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/outlabel/OutLabelHolder.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java index 96d7a213..c63f61c0 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java @@ -18,6 +18,9 @@ public abstract class OutLabelHolder { * @return Null if no node was determined until now */ public NodeLocation getNodeForLabel(String outLabel) { + if ( outLabel == null ) { + return null; + } final InternalHolder holder = internalHolder.get(outLabel); return holder == null ? null : holder.getBestNode().stream().findFirst().orElse(null); } From 7e628b5858eaed8646a1f4be07c62fb9cbd68ce7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Oct 2022 11:10:12 +0200 Subject: [PATCH 285/443] Sanitize wrong DNS Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 2b5ac616..8b3c7992 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -60,7 +60,7 @@ public abstract class Scheduler { this.namespace = namespace; log.trace( "Register scheduler for " + this.name ); this.client = client; - this.dns = config.dns; + this.dns = config.dns.endsWith( "/" ) ? config.dns : config.dns + "/"; this.dag = new DAG(); this.traceEnabled = config.traceEnabled; From beaa040c6490e521343e7536962d47c959129ddb Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Oct 2022 11:28:56 +0200 Subject: [PATCH 286/443] Sanitize wrong DNS 2 Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 819d26fd..19362096 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -254,7 +254,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { try { final Inputs inputs = new Inputs( - this.getDns() + "/daemon/" + getNamespace() + "/" + getExecution() + "/", + this.getDns() + "daemon/" + getNamespace() + "/" + getExecution() + "/", this.localWorkDir + "/sync/", alignment.task.getConfig().getRunName() ); From c78b395c52f4c33e424689763c4a83bdd2d568cc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 7 Oct 2022 09:59:35 +0200 Subject: [PATCH 287/443] Do not check for still valid nodes twice Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/LocationAwareScheduler.java | 2 +- .../java/fonda/scheduler/scheduler/data/TaskData.java | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index c7048ab9..5a63cd41 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -53,7 +53,6 @@ private NodeTaskFilesAlignment createNodeAlignment ( ) { long startTime = System.nanoTime(); log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getRunName(), taskData.getValue() ); - taskData.removeAllNodesWhichHaveNotEnoughResources( availableByNode ); final Tuple result = calculateBestNode(taskData, planedToCopy, availableByNode, assignedPodsByNode); if ( result == null ) { return null; @@ -117,6 +116,7 @@ List createAlignment( taskData.setNodeAndWeight( nodeForLabel, taskData.getTask().getConfig().getOutLabel().getWeight() ); } } + //Calculate the available nodes for this task + their weight. Re-add it to the queue, if it changed if ( taskData.calculate( availableByNode ) || changed ){ if ( !taskData.getNodeDataTuples().isEmpty() ) { unscheduledTasksSorted.add(taskData); diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index bf90b06d..2cad6d68 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -115,15 +115,6 @@ public boolean calculate( Map availableByNode ) { return changed; } - public void removeAllNodesWhichHaveNotEnoughResources( Map availableByNode ) { - final Iterator iterator = nodeDataTuples.iterator(); - while (iterator.hasNext()) { - if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) { - iterator.remove(); - } - } - } - private void calc() { if ( nodeDataTuples.isEmpty() ) { value = Double.MIN_VALUE; From 59cab437e441e289a1a21080f5e7e78e6906e2c8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 7 Oct 2022 10:00:28 +0200 Subject: [PATCH 288/443] Do not check for still valid nodes twice comment Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/data/TaskData.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index 2cad6d68..a11f3bf6 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -97,6 +97,7 @@ public void setNodeAndWeight(NodeLocation nodeForLabel, double weight ) { /** * Recalculate the value. New value is smaller or equal old value + * Delete all nodes that have not enough unassigned resources for the task. * @param availableByNode * @return if the value has changed */ From 7a97324222f4612b16b6052b1353e1c6501f3e25 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 7 Oct 2022 10:12:24 +0200 Subject: [PATCH 289/443] Only start X copy tasks per node Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 2 +- .../fonda/scheduler/model/NodeWithAlloc.java | 40 +++++++++++++++++-- src/main/java/fonda/scheduler/model/Task.java | 8 +++- .../scheduler/model/WriteConfigResult.java | 6 ++- .../rest/SchedulerRestController.java | 18 +++++++++ .../scheduler/LocationAwareScheduler.java | 36 ++++++++++++++++- .../fonda/scheduler/scheduler/Scheduler.java | 7 +++- .../scheduler/SchedulerWithDaemonSet.java | 25 ++++++++---- .../scheduler/schedulingstrategy/Inputs.java | 4 +- src/main/resources/copystrategies/ftp.py | 33 ++++++++++----- 10 files changed, 150 insertions(+), 29 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 992c7ba4..f9fdb7ed 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -158,7 +158,7 @@ public void eventReceived(Action action, Pod pod) { switch ( action ){ case ADDED: if ( !PodWithAge.hasFinishedOrFailed( pod ) ) { - node.addPod(new PodWithAge(pod)); + node.addPod(new PodWithAge(pod), false); } break; case MODIFIED: diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 0736cfd6..817105ab 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -11,8 +11,8 @@ import lombok.extern.slf4j.Slf4j; import java.math.BigDecimal; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; @Getter @Slf4j @@ -26,6 +26,8 @@ public class NodeWithAlloc extends Node implements Comparable { private final Map assignedPods; + private final List startingTaskCopyingData = new LinkedList<>(); + @Getter private final NodeLocation nodeLocation; @@ -63,24 +65,54 @@ public NodeWithAlloc( Node node, KubernetesClient kubernetesClient ) { log.info("Node {} has RAM: {} and CPU: {}", node.getMetadata().getName(), maxRam, maxCpu); } - public void addPod( PodWithAge pod ){ + public void addPod( PodWithAge pod, boolean withStartingTasks ) { Requirements request = pod.getRequest(); - log.info( "New Pod on node {} with request {}", this.getMetadata().getName(), request ); + if ( withStartingTasks ) { + synchronized ( startingTaskCopyingData ) { + if ( !startingTaskCopyingData.contains( pod ) ) { + startingTaskCopyingData.add( pod ); + } + } + } synchronized (assignedPods) { assignedPods.put( pod.getMetadata().getUid(), request ); } } + private void removeStartingTaskCopyingDataByUid( String uid ) { + synchronized ( startingTaskCopyingData ) { + final Iterator iterator = startingTaskCopyingData.iterator(); + while ( iterator.hasNext() ) { + final PodWithAge podWithAge = iterator.next(); + if ( podWithAge.getMetadata().getUid().equals( uid ) ) { + iterator.remove(); + break; + } + } + } + } + public boolean removePod( Pod pod ){ + removeStartingTaskCopyingDataByUid( pod.getMetadata().getUid() ); synchronized (assignedPods) { return assignedPods.remove( pod.getMetadata().getUid() ) != null; } } + + public void startingTaskCopyingDataFinished( Task task ) { + final String uid = task.getPod().getMetadata().getUid(); + removeStartingTaskCopyingDataByUid( uid ); + } + public int getRunningPods(){ return assignedPods.size(); } + public int getStartingPods(){ + return startingTaskCopyingData.size(); + } + /** * @return max(Requested by all and currently used ) */ diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 7d14155f..27e889ba 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -39,7 +39,7 @@ public class Task { @Getter @Setter - private NodeLocation node = null; + private NodeWithAlloc node = null; @Getter @Setter @@ -54,6 +54,10 @@ public class Task { private long timeAddedToQueue; + @Getter + @Setter + private boolean copiesDataToNode = false; + public Task( TaskConfig config, DAG dag ) { this.config = config; this.process = dag.getByProcess( config.getTask() ); @@ -97,7 +101,7 @@ public String toString() { return "Task{" + "state=" + state + ", pod=" + (pod == null ? "--" : pod.getMetadata().getName()) + - ", node='" + (node != null ? node.getIdentifier() : "--") + '\'' + + ", node='" + (node != null ? node.getNodeLocation().getIdentifier() : "--") + '\'' + ", workDir='" + getWorkingDir() + '\'' + '}'; } diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java index 1ca4bc6e..47e22d21 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -14,17 +14,21 @@ public class WriteConfigResult { private final Map waitForTask; private final Map< String, Tuple> copyingToNode; + private boolean copyDataToNode; + private final boolean wroteConfig; public WriteConfigResult( List inputFiles, Map waitForTask, Map> copyingToNode, - boolean wroteConfig + boolean wroteConfig, + boolean copyDataToNode ) { this.inputFiles = inputFiles; this.waitForTask = waitForTask; this.copyingToNode = copyingToNode; this.wroteConfig = wroteConfig; + this.copyDataToNode = copyDataToNode; } } diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 10ffdc12..ed3f67da 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -24,6 +24,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.bind.annotation.*; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -258,6 +259,23 @@ ResponseEntity getDaemonName( @PathVariable String execution, @PathVaria } + @PostMapping("/v1/downloadtask/{execution}") + ResponseEntity finishDownload( @PathVariable String execution, @RequestBody byte[] task ) { + + String name = new String( task, StandardCharsets.UTF_8 ); + if ( name.endsWith( "=" ) ) { + name = name.substring( 0, name.length() - 1 ); + } + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { + return noSchedulerFor( execution ); + } + ((SchedulerWithDaemonSet) scheduler).taskHasFinishedCopyTask( name ); + return new ResponseEntity<>( HttpStatus.OK ); + + } + @GetMapping("/v1/file/{execution}") ResponseEntity getNodeForFile( @PathVariable String execution, @RequestParam String path ) { diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 5a63cd41..20c437fe 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -104,6 +104,7 @@ List createAlignment( final List alignment = new LinkedList<>(); final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); final Map assignedPodsByNode = new HashMap<>(); + Map taskCountWhichCopyToNode = new HashMap<>(); while( !unscheduledTasksSorted.isEmpty() ){ TaskData taskData = unscheduledTasksSorted.poll(); boolean changed = false; @@ -125,7 +126,7 @@ List createAlignment( } final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, assignedPodsByNode, planedToCopy, ++index); - if ( nodeAlignment != null ) { + if ( nodeAlignment != null && onlyAllowXCopyToNodeTasks( nodeAlignment, taskCountWhichCopyToNode, 1 ) ) { alignment.add(nodeAlignment); outLabelHolder.scheduleTaskOnNode( taskData.getTask(), nodeAlignment.node.getNodeLocation() ); addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.getNodeFileAlignment(), taskData.getTask(), nodeAlignment.node ); @@ -134,6 +135,39 @@ List createAlignment( return alignment; } + + /** + * Check if the node is already used by x tasks which copy data to it + * @param nodeTaskAlignments + * @param taskCountWhichCopyToNode + * @param maxCopyToNodeTasks + * @return true if the task can be scheduled on that node and no if already maxCopyToNodeTasks tasks copy to that node + */ + private boolean onlyAllowXCopyToNodeTasks( NodeTaskFilesAlignment nodeTaskAlignments, Map taskCountWhichCopyToNode, int maxCopyToNodeTasks ){ + + final Map nodeFileAlignment = nodeTaskAlignments.fileAlignment.getNodeFileAlignment(); + + //Does this task needs to copy data to the node? + boolean copyDataToNode = false; + for ( Map.Entry locationAlignmentWrapperEntry : nodeFileAlignment.entrySet() ) { + if ( locationAlignmentWrapperEntry.getValue().getFilesToCopy().size() > 0 ) { + copyDataToNode = true; + break; + } + } + + //Only keep if there are no more than maxCopyToNodeTasks + if ( copyDataToNode ) { + final int alreadyAssignedToNode = taskCountWhichCopyToNode.computeIfAbsent( nodeTaskAlignments.node, NodeWithAlloc::getStartingPods ); + if ( alreadyAssignedToNode >= maxCopyToNodeTasks ) { + return false; + } else { + taskCountWhichCopyToNode.put( nodeTaskAlignments.node, alreadyAssignedToNode + 1 ); + } + } + return true; + } + @Override public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks, diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 8b3c7992..dab4655b 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -363,11 +363,11 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ log.error( "Cannot read " + nodeFile, e); } - alignment.task.setNode( alignment.node.getNodeLocation() ); + alignment.task.setNode( alignment.node ); final PodWithAge pod = alignment.task.getPod(); - alignment.node.addPod( pod ); + alignment.node.addPod( pod, alignment.task.isCopiesDataToNode() ); log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + alignment.node.getMetadata().getName() ); @@ -436,6 +436,9 @@ Task changeStateOfTask(Pod pod, State state){ return null; } + /** + * starts the scheduling routine + */ public void informResourceChange() { synchronized (unscheduledTasks){ unscheduledTasks.notifyAll(); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 19362096..28cb6896 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -107,6 +107,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); + alignment.task.setCopiesDataToNode( writeConfigResult.isCopyDataToNode() ); final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); useLocations( allLocationWrappers ); @@ -120,7 +121,7 @@ void undoTaskScheduling( Task task ){ task.setInputFiles( null ); } if ( task.getCopyingToNode() != null ) { - removeFromCopyingToNode( task.getNode(), task.getCopyingToNode()); + removeFromCopyingToNode( task.getNode().getNodeLocation(), task.getCopyingToNode()); task.setCopyingToNode( null ); } task.setCopiedFiles( null ); @@ -142,7 +143,7 @@ int terminateTasks(List finishedTasks) { } else { final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( workdir, - finishedTask.getNode(), + finishedTask.getNode().getNodeLocation(), !finishedTask.wasSuccessfullyExecuted(), finishedTask ); @@ -254,7 +255,8 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { try { final Inputs inputs = new Inputs( - this.getDns() + "daemon/" + getNamespace() + "/" + getExecution() + "/", + this.getDns(), + getExecution(), this.localWorkDir + "/sync/", alignment.task.getConfig().getRunName() ); @@ -290,14 +292,15 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { } } + boolean copyDataToNode = !inputs.data.isEmpty(); inputs.waitForTask( waitForTask ); inputs.symlinks.addAll(alignment.fileAlignment.getSymlinks()); inputs.sortData(); - final boolean allEmpty = inputs.data.isEmpty() && inputs.symlinks.isEmpty() && inputs.waitForFilesOfTask.isEmpty(); + final boolean allEmpty = !copyDataToNode && inputs.symlinks.isEmpty() && inputs.waitForFilesOfTask.isEmpty(); if ( !allEmpty ) { new ObjectMapper().writeValue( config, inputs ); } - return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode, !allEmpty); + return new WriteConfigResult( inputFiles, waitForTask, filesForCurrentNode, !allEmpty, copyDataToNode); } catch (IOException e) { log.error( "Cannot write " + config, e); @@ -385,8 +388,8 @@ private void handleProblematicInit( Task task ){ try { Map wrapperByPath = new HashMap<>(); task.getCopiedFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); - log.info( "Get daemon on node {}; daemons: {}", task.getNode().getIdentifier(), daemonByNode ); - final InputStream inputStream = getConnection(getDaemonOnNode(task.getNode().getIdentifier())).retrieveFileStream(file); + log.info( "Get daemon on node {}; daemons: {}", task.getNode().getNodeLocation().getIdentifier(), daemonByNode ); + final InputStream inputStream = getConnection(getDaemonOnNode(task.getNode().getNodeLocation().getIdentifier())).retrieveFileStream(file); if (inputStream == null) { //Init has not even started return; @@ -441,7 +444,7 @@ private void podWasInitialized( Pod pod ){ final Task task = changeStateOfTask(pod, exitCode == 0 ? State.PREPARED : State.INIT_WITH_ERRORS); task.setPod( new PodWithAge( pod ) ); log.info( "Pod {}, Init Code: {}", pod.getMetadata().getName(), exitCode); - removeFromCopyingToNode( task.getNode(), task.getCopyingToNode() ); + removeFromCopyingToNode( task.getNode().getNodeLocation(), task.getCopyingToNode() ); if( exitCode == 0 ){ task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); } else { @@ -473,6 +476,12 @@ void filterNotMatchingNodesForTask(Set matchingNodes, TaskInputs } } + public void taskHasFinishedCopyTask( String name ){ + final Task task = tasksByPodName.get( name ); + task.getNode().startingTaskCopyingDataFinished( task ); + informResourceChange(); + } + /** * Since task was not yet initialized: set scheduled * @param task task that was scheduled diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index a2d57ae7..4ee5174f 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -12,14 +12,16 @@ public class Inputs { public final String dns; + public final String execution; public final String hash; public final List data; public final List symlinks; public final String syncDir; public final Map> waitForFilesOfTask; - public Inputs( String dns, String syncDir, String hash ) { + public Inputs( String dns, String execution, String syncDir, String hash ) { this.dns = dns; + this.execution = execution; this.syncDir = syncDir; this.hash = hash; this.data = new LinkedList<>(); diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 268b8f83..0c711d8e 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -8,6 +8,7 @@ import sys import time import urllib.request +import urllib.parse from concurrent.futures import ThreadPoolExecutor from pathlib import Path from time import sleep @@ -43,8 +44,8 @@ def closeWithWarning(errorCode, syncFile): myExit(errorCode) -def getIP(node, dns): - ip = urllib.request.urlopen(dns + node).read() +def getIP(node, dns, execution): + ip = urllib.request.urlopen(dns + "daemon/" + execution + "/" + node).read() return str(ip.decode("utf-8")) @@ -64,14 +65,14 @@ def clearLocation(path, dst=None): return True -def getFTP(node, currentIP, dns, syncFile): +def getFTP(node, currentIP, dns, execution, syncFile): global errors connectionProblem = 0 while connectionProblem < 8: try: if currentIP is None: log.info("Request ip for node: %s", node) - ip = getIP(node, dns) + ip = getIP(node, dns, execution) else: ip = currentIP log.info("Try to connect to %s", ip) @@ -140,7 +141,7 @@ def downloadFile(ftp, filename, size, index, node, syncFile): return 0, 0 -def download(node, currentIP, files, dns, syncFile): +def download(node, currentIP, files, dns, execution, syncFile): ftp = None size = len(files) global CLOSE @@ -148,7 +149,7 @@ def download(node, currentIP, files, dns, syncFile): downloadTime = 0 while not CLOSE and len(files) > 0: if ftp is None: - ftp = getFTP(node, currentIP, dns, syncFile) + ftp = getFTP(node, currentIP, dns, execution, syncFile) currentIP = None filename = files[0] index = size - len(files) + 1 @@ -238,7 +239,7 @@ def generateSymlinks(symlinks): log.warning("File exists: %s -> %s", src, dst) -def downloadAllData(data, dns, syncFile): +def downloadAllData(data, dns, execution, syncFile): global trace throughput = [] with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: @@ -247,7 +248,7 @@ def downloadAllData(data, dns, syncFile): files = d["files"] node = d["node"] currentIP = d["currentIP"] - futures.append(executor.submit(download, node, currentIP, files, dns, syncFile)) + futures.append(executor.submit(download, node, currentIP, files, dns, execution, syncFile)) lastNum = -1 while len(futures) > 0: if lastNum != len(futures): @@ -281,6 +282,16 @@ def writeTrace(dataMap): traceFile.write("scheduler_init_errors=" + str(errors) + "\n") +def finishedDownload(dns, execution, taskname): + try: + dns = dns + "downloadtask/" + execution + log.info("Request: %s", dns) + urllib.request.urlopen(dns, taskname.encode( "utf-8" )) + except BaseException as err: + log.exception(err) + myExit(100) + + def run(): global trace startTime = time.time() @@ -288,8 +299,10 @@ def run(): config = loadConfig(".command.inputs.json") dns = config["dns"] + execution = config["execution"] data = config["data"] symlinks = config["symlinks"] + taskname = config["hash"] with open(config["syncDir"] + config["hash"], 'w') as syncFile: registerSignal(syncFile) @@ -301,7 +314,7 @@ def run(): syncFile.write('##SYMLINKS##\n') syncFile.flush() startTimeDownload = time.time() - downloadAllData(data, dns, syncFile) + downloadAllData(data, dns, execution, syncFile) trace["scheduler_init_download_runtime"] = int((time.time() - startTimeDownload) * 1000) if CLOSE: log.debug("Closed with code %s", str(EXIT)) @@ -310,6 +323,8 @@ def run(): syncFile.write('##FINISHED##\n') registerSignal2() + finishedDownload(dns, execution, taskname) + startTimeDependingTasks = time.time() waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) From ab0ac3209efbc22450327206ea68db03455f06d7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 7 Oct 2022 11:13:48 +0200 Subject: [PATCH 290/443] Submit maxCopyTasksPerNode to the scheduler Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/SchedulerConfig.java | 6 ++++++ .../fonda/scheduler/scheduler/LocationAwareScheduler.java | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 80d8359b..28eb7b81 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -18,6 +18,10 @@ public class SchedulerConfig { public final String costFunction; public final String strategy; + public final Integer maxCopyTasksPerNode; + + public final Integer maxWaitingCopyTasksPerNode; + private SchedulerConfig() { this.localClaims = null; this.volumeClaims = null; @@ -29,6 +33,8 @@ private SchedulerConfig() { this.costFunction = null; this.namespace = null; this.strategy = null; + this.maxCopyTasksPerNode = null; + this.maxWaitingCopyTasksPerNode = null; } @ToString diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 20c437fe..8c8a83ff 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -23,6 +23,10 @@ public class LocationAwareScheduler extends SchedulerWithDaemonSet { @Getter(AccessLevel.PACKAGE) private final InputAlignment inputAlignment; + @Getter(AccessLevel.PACKAGE) + private final int maxCopyTasksPerNode; + @Getter(AccessLevel.PACKAGE) + private final int maxWaitingCopyTasksPerNode; public LocationAwareScheduler ( String name, @@ -32,6 +36,8 @@ public LocationAwareScheduler ( InputAlignment inputAlignment) { super( name, client, namespace, config ); this.inputAlignment = inputAlignment; + this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; + this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; } @@ -126,7 +132,7 @@ List createAlignment( } final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, assignedPodsByNode, planedToCopy, ++index); - if ( nodeAlignment != null && onlyAllowXCopyToNodeTasks( nodeAlignment, taskCountWhichCopyToNode, 1 ) ) { + if ( nodeAlignment != null && onlyAllowXCopyToNodeTasks( nodeAlignment, taskCountWhichCopyToNode, getMaxCopyTasksPerNode() ) ) { alignment.add(nodeAlignment); outLabelHolder.scheduleTaskOnNode( taskData.getTask(), nodeAlignment.node.getNodeLocation() ); addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.getNodeFileAlignment(), taskData.getTask(), nodeAlignment.node ); From 1002fce851a176aa35f7bfbe07e675815c9bd933 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Oct 2022 10:19:40 +0200 Subject: [PATCH 291/443] Calculate Task Rank Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/dag/NotProcess.java | 4 + .../java/fonda/scheduler/dag/Process.java | 8 ++ src/main/java/fonda/scheduler/dag/Vertex.java | 42 +++++++ .../java/fonda/scheduler/dag/DAGTest.java | 111 +++++++++--------- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/src/main/java/fonda/scheduler/dag/NotProcess.java b/src/main/java/fonda/scheduler/dag/NotProcess.java index 6d7e9d1e..31150c22 100644 --- a/src/main/java/fonda/scheduler/dag/NotProcess.java +++ b/src/main/java/fonda/scheduler/dag/NotProcess.java @@ -48,4 +48,8 @@ void removeAncestor( Edge e, Collection p ) { } } + @Override + int incRank( int rank ) { + return rank; + } } diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/fonda/scheduler/dag/Process.java index 2dd16ee3..ec916c57 100644 --- a/src/main/java/fonda/scheduler/dag/Process.java +++ b/src/main/java/fonda/scheduler/dag/Process.java @@ -145,6 +145,14 @@ public void addOutbound( Edge e ) { v.addAncestor( ancestorsCopy ); v.addAncestor( this ); }); + + final int rank = e.getTo().getRank(); + informNewDescendent( rank + 1 ); + } + @Override + int incRank( int rank ) { + return rank + 1; + } } diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index 2f1fb5bd..9611aedd 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -1,6 +1,8 @@ package fonda.scheduler.dag; +import lombok.AccessLevel; import lombok.Getter; +import lombok.Setter; import java.util.Collection; import java.util.HashSet; @@ -15,6 +17,9 @@ public abstract class Vertex { final Set in = new HashSet<>(); final Set out = new HashSet<>(); + @Setter(AccessLevel.PACKAGE) + private int rank = 0; + Vertex( String label, int uid ) { this.label = label; this.uid = uid; @@ -55,6 +60,7 @@ public void removeInboundIntern( Edge e ) { descendants.add((Process) this); } e.getFrom().removeDescendant( e, descendants ); + e.getFrom().informDeletedDescendent(); } public void removeOutbound( Edge e ) { @@ -117,4 +123,40 @@ public boolean equals(Object o) { public int hashCode() { return getUid(); } + + /** + * For processes add one, otherwise return the input + * @param rank + * @return + */ + abstract int incRank( int rank ); + + void informNewDescendent( int rank ) { + if ( rank > getRank() ) { + setRank( rank ); + for ( Edge edge : in ) { + edge.getFrom().informNewDescendent( incRank( rank ) ); + } + } + } + + void informDeletedDescendent() { + int rank = 0; + if ( !out.isEmpty() ) { + for ( Edge edge : out ) { + final Vertex to = edge.getTo(); + final int toRank = to.incRank( to.getRank() ); + if ( toRank > rank ) { + rank = toRank; + } + } + } + if ( rank != getRank() ) { + setRank( rank ); + for ( Edge edge : in ) { + edge.getFrom().informDeletedDescendent(); + } + } + } + } diff --git a/src/test/java/fonda/scheduler/dag/DAGTest.java b/src/test/java/fonda/scheduler/dag/DAGTest.java index 693145b2..4a12c5c7 100644 --- a/src/test/java/fonda/scheduler/dag/DAGTest.java +++ b/src/test/java/fonda/scheduler/dag/DAGTest.java @@ -54,18 +54,19 @@ private HashSet setTo(DAG dag, int idA, int[]... otherIds){ }).collect(Collectors.toCollection(HashSet::new)); } - private void isSameEdge(HashSet expectedIn, HashSet expectedOut, Vertex vertex ){ + private void isSameEdge( HashSet expectedIn, HashSet expectedOut, Vertex vertex ){ final String[] in = setToArrayEdges(expectedIn); final String[] out = setToArrayEdges(expectedOut); assertArrayEquals( "In of " + vertex.getLabel(), in, setToArrayEdges(vertex.getIn()) ); assertArrayEquals( "Out of " + vertex.getLabel(), out, setToArrayEdges(vertex.getOut()) ); } - private void isSameProc(HashSet expectedAncestors, HashSet expectedDescendants, Vertex vertex ){ + private void isSameProc( HashSet expectedAncestors, HashSet expectedDescendants, Vertex vertex, int rank ){ final String[] descendants = setToArrayProcesses(expectedDescendants); final String[] ancestors = setToArrayProcesses(expectedAncestors); assertArrayEquals( "Descendants of " + vertex.getLabel(), descendants, setToArrayProcesses(vertex.getDescendants()) ); assertArrayEquals( "Ancestors of " + vertex.getLabel(), ancestors, setToArrayProcesses(vertex.getAncestors()) ); + assertEquals( rank, vertex.getRank() ); } private String[] setToArrayProcesses(Set set) { @@ -304,25 +305,25 @@ public void deleteTest2() { final DAG dag = createDag(); dag.removeEdges( 3 ); - isSameProc( new HashSet<>(), set( dag, "a","b","c","d","e","f","g","h","i" ), dag.getByUid( 1 ) ); + isSameProc( new HashSet<>(), set( dag, "a","b","c","d","e","f","g","h","i" ), dag.getByUid( 1 ), 5 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByUid( 1 ) ); - isSameProc( new HashSet<>(), set( dag, "g","i" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "g","i" ), dag.getByProcess("a"), 2 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{4, 8} ), dag.getByProcess("a") ); - isSameProc( new HashSet<>(), set( dag, "c","d","e","f","h","i" ), dag.getByProcess("b") ); + isSameProc( new HashSet<>(), set( dag, "c","d","e","f","h","i" ), dag.getByProcess("b"), 4 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{6, 5},new int[]{5, 4} ), dag.getByProcess("b") ); - isSameProc( set( dag, "b" ), set( dag, "e","h","i" ), dag.getByProcess("c") ); + isSameProc( set( dag, "b" ), set( dag, "e","h","i" ), dag.getByProcess("c"), 3 ); isSameEdge( setTo( dag, 4, new int[]{5, 3} ), setFrom( dag, 4, new int[]{7, 6} ), dag.getByProcess("c") ); - isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), dag.getByProcess("d") ); + isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), dag.getByProcess("d"), 3 ); isSameEdge( setTo( dag, 5, new int[]{6, 3} ), setFrom( dag, 5, new int[]{8, 7} ), dag.getByProcess("d") ); - isSameProc( set( dag, "b","c" ), set( dag, "h","i" ), dag.getByProcess("e") ); + isSameProc( set( dag, "b","c" ), set( dag, "h","i" ), dag.getByProcess("e"), 2 ); isSameEdge( setTo( dag, 6, new int[]{7, 4} ), setFrom( dag, 6, new int[]{9, 9} ), dag.getByProcess("e") ); - isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), dag.getByProcess("f") ); + isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), dag.getByProcess("f"), 2 ); isSameEdge( setTo( dag, 7, new int[]{8, 5} ), setFrom( dag, 7, new int[]{10, 9} ), dag.getByProcess("f") ); - isSameProc( set( dag, "a" ), set( dag, "i" ), dag.getByProcess("g") ); + isSameProc( set( dag, "a" ), set( dag, "i" ), dag.getByProcess("g"), 1 ); isSameEdge( setTo( dag, 8, new int[]{4, 2} ), setFrom( dag, 8, new int[]{11, 10} ), dag.getByProcess("g") ); - isSameProc( set( dag, "b","c","d","e","f" ), set( dag, "i" ), dag.getByProcess("h") ); + isSameProc( set( dag, "b","c","d","e","f" ), set( dag, "i" ), dag.getByProcess("h"), 1 ); isSameEdge( setTo( dag, 9, new int[]{10, 7},new int[]{9, 6} ), setFrom( dag, 9, new int[]{12, 10} ), dag.getByProcess("h") ); - isSameProc( set( dag, "a","b","c","d","e","f","g","h" ), new HashSet<>(), dag.getByProcess("i") ); + isSameProc( set( dag, "a","b","c","d","e","f","g","h" ), new HashSet<>(), dag.getByProcess("i"), 0 ); isSameEdge( setTo( dag, 10, new int[]{12, 9},new int[]{11, 8} ), new HashSet<>(), dag.getByProcess("i") ); } @@ -332,21 +333,21 @@ public void deleteTest3() { final DAG dag = createDag(); dag.removeVertices( dag.getByProcess("c").getUid(), dag.getByProcess("e").getUid() ); - isSameProc( new HashSet<>(), set( dag, "a","b","d","f","g","h","i" ), dag.getByUid( 1 ) ); + isSameProc( new HashSet<>(), set( dag, "a","b","d","f","g","h","i" ), dag.getByUid( 1 ), 5 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByUid( 1 ) ); - isSameProc( new HashSet<>(), set( dag, "g","i" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "g","i" ), dag.getByProcess("a"), 2 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{4, 8} ), dag.getByProcess("a") ); - isSameProc( new HashSet<>(), set( dag, "d","f","h","i" ), dag.getByProcess("b") ); + isSameProc( new HashSet<>(), set( dag, "d","f","h","i" ), dag.getByProcess("b"), 4 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{6, 5} ), dag.getByProcess("b") ); - isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), dag.getByProcess("d") ); + isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), dag.getByProcess("d"), 3 ); isSameEdge( setTo( dag, 5, new int[]{6, 3} ), setFrom( dag, 5, new int[]{8, 7} ), dag.getByProcess("d") ); - isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), dag.getByProcess("f") ); + isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), dag.getByProcess("f"), 2 ); isSameEdge( setTo( dag, 7, new int[]{8, 5} ), setFrom( dag, 7, new int[]{10, 9} ), dag.getByProcess("f") ); - isSameProc( set( dag, "a" ), set( dag, "i" ), dag.getByProcess("g") ); + isSameProc( set( dag, "a" ), set( dag, "i" ), dag.getByProcess("g"), 1 ); isSameEdge( setTo( dag, 8, new int[]{4, 2} ), setFrom( dag, 8, new int[]{11, 10} ), dag.getByProcess("g") ); - isSameProc( set( dag, "b","d","f" ), set( dag, "i" ), dag.getByProcess("h") ); + isSameProc( set( dag, "b","d","f" ), set( dag, "i" ), dag.getByProcess("h"), 1 ); isSameEdge( setTo( dag, 9, new int[]{10, 7} ), setFrom( dag, 9, new int[]{12, 10} ), dag.getByProcess("h") ); - isSameProc( set( dag, "a","b","d","f","g","h" ), new HashSet<>(), dag.getByProcess("i") ); + isSameProc( set( dag, "a","b","d","f","g","h" ), new HashSet<>(), dag.getByProcess("i"), 0 ); isSameEdge( setTo( dag, 10, new int[]{12, 9},new int[]{11, 8} ), new HashSet<>(), dag.getByProcess("i") ); assertThrows( IllegalStateException.class, () -> dag.getByProcess("c") ); assertThrows( IllegalStateException.class, () -> dag.getByProcess("e") ); @@ -382,28 +383,28 @@ public void deleteTest4() { dag.registerVertices( vertexList ); dag.registerEdges(inputEdges); - isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a"), 3 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b"), 2 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c"), 2 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4},new int[]{5, 5} ), dag.getByProcess("c") ); - isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d"), 1 ); isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5} ), dag.getByProcess("d") ); - isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e"), 0 ); isSameEdge( setTo( dag, 5, new int[]{5, 3},new int[]{6, 4} ), new HashSet<>(), dag.getByProcess("e") ); dag.removeEdges( 5 ); - isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a"), 3 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b"), 2 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c"), 2 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); - isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d"), 1 ); isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5} ), dag.getByProcess("d") ); - isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e"), 0 ); isSameEdge( setTo( dag, 5, new int[]{6, 4} ), new HashSet<>(), dag.getByProcess("e") ); } @@ -437,41 +438,41 @@ public void deleteTest5() { dag.registerVertices( vertexList ); dag.registerEdges(inputEdges); - isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a"), 3 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b"), 2 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c"), 2 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); - isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d"), 1 ); isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5},new int[]{5, 5} ), dag.getByProcess("d") ); - isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e"), 0 ); isSameEdge( setTo( dag, 5, new int[]{6, 4},new int[]{5, 4} ), new HashSet<>(), dag.getByProcess("e") ); dag.removeEdges( 5 ); - isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "b","c","d","e" ), dag.getByProcess("a"), 3 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("b"), 2 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); - isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c") ); + isSameProc( set( dag, "a" ), set( dag, "d","e" ), dag.getByProcess("c"), 2 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); - isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c" ), set( dag, "e" ), dag.getByProcess("d"), 1 ); isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{6, 5} ), dag.getByProcess("d") ); - isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e") ); + isSameProc( set( dag, "a","b","c","d" ), new HashSet<>(), dag.getByProcess("e"), 0 ); isSameEdge( setTo( dag, 5, new int[]{6, 4} ), new HashSet<>(), dag.getByProcess("e") ); dag.removeEdges( 6 ); - isSameProc( new HashSet<>(), set( dag, "b","c","d" ), dag.getByProcess("a") ); + isSameProc( new HashSet<>(), set( dag, "b","c","d" ), dag.getByProcess("a"), 2 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), dag.getByProcess("a") ); - isSameProc( set( dag, "a" ), set( dag, "d" ), dag.getByProcess("b") ); + isSameProc( set( dag, "a" ), set( dag, "d" ), dag.getByProcess("b"), 1 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4} ), dag.getByProcess("b") ); - isSameProc( set( dag, "a" ), set( dag, "d" ), dag.getByProcess("c") ); + isSameProc( set( dag, "a" ), set( dag, "d" ), dag.getByProcess("c"), 1 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{4, 4} ), dag.getByProcess("c") ); - isSameProc( set( dag, "a","b","c" ), new HashSet<>(), dag.getByProcess("d") ); + isSameProc( set( dag, "a","b","c" ), new HashSet<>(), dag.getByProcess("d"), 0 ); isSameEdge( setTo( dag, 4, new int[]{4, 3},new int[]{3, 2} ), new HashSet<>(), dag.getByProcess("d") ); - isSameProc( new HashSet<>(), new HashSet<>(), dag.getByProcess("e") ); + isSameProc( new HashSet<>(), new HashSet<>(), dag.getByProcess("e"), 0 ); isSameEdge( new HashSet<>(), new HashSet<>(), dag.getByProcess("e") ); } @@ -525,25 +526,25 @@ private DAG createDag() { dag.registerVertices(vertexList); dag.registerEdges(inputEdges); - isSameProc( new HashSet<>(), set( dag, "a","b","c","d","e","f","g","h","i" ), o ); + isSameProc( new HashSet<>(), set( dag, "a","b","c","d","e","f","g","h","i" ), o, 5 ); isSameEdge( new HashSet<>(), setFrom( dag, 1, new int[]{2, 3},new int[]{1, 2} ), o ); - isSameProc( new HashSet<>(), set( dag, "c","e","g","h","i" ), a ); + isSameProc( new HashSet<>(), set( dag, "c","e","g","h","i" ), a, 4 ); isSameEdge( setTo( dag, 2, new int[]{1, 1} ), setFrom( dag, 2, new int[]{3, 4},new int[]{4, 8} ), a ); - isSameProc( new HashSet<>(), set( dag, "c","d","e","f","h","i" ), b ); + isSameProc( new HashSet<>(), set( dag, "c","d","e","f","h","i" ), b, 4 ); isSameEdge( setTo( dag, 3, new int[]{2, 1} ), setFrom( dag, 3, new int[]{6, 5},new int[]{5, 4} ), b ); - isSameProc( set( dag, "a","b" ), set( dag, "e","h","i" ), c ); + isSameProc( set( dag, "a","b" ), set( dag, "e","h","i" ), c, 3 ); isSameEdge( setTo( dag, 4, new int[]{5, 3},new int[]{3, 2} ), setFrom( dag, 4, new int[]{7, 6} ), c ); - isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), d ); + isSameProc( set( dag, "b" ), set( dag, "f","h","i" ), d, 3 ); isSameEdge( setTo( dag, 5, new int[]{6, 3} ), setFrom( dag, 5, new int[]{8, 7} ), d ); - isSameProc( set( dag, "a","b","c" ), set( dag, "h","i" ), e ); + isSameProc( set( dag, "a","b","c" ), set( dag, "h","i" ), e, 2 ); isSameEdge( setTo( dag, 6, new int[]{7, 4} ), setFrom( dag, 6, new int[]{9, 9} ), e ); - isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), f ); + isSameProc( set( dag, "b","d" ), set( dag, "h","i" ), f, 2 ); isSameEdge( setTo( dag, 7, new int[]{8, 5} ), setFrom( dag, 7, new int[]{10, 9} ), f ); - isSameProc( set( dag, "a" ), set( dag, "i" ), g ); + isSameProc( set( dag, "a" ), set( dag, "i" ), g, 1 ); isSameEdge( setTo( dag, 8, new int[]{4, 2} ), setFrom( dag, 8, new int[]{11, 10} ), g ); - isSameProc( set( dag, "a","b","c","d","e","f" ), set( dag, "i" ), h ); + isSameProc( set( dag, "a","b","c","d","e","f" ), set( dag, "i" ), h, 1 ); isSameEdge( setTo( dag, 9, new int[]{10, 7},new int[]{9, 6} ), setFrom( dag, 9, new int[]{12, 10} ), h ); - isSameProc( set( dag, "a","b","c","d","e","f","g","h" ), new HashSet<>(), i ); + isSameProc( set( dag, "a","b","c","d","e","f","g","h" ), new HashSet<>(), i, 0 ); isSameEdge( setTo( dag, 10, new int[]{12, 9},new int[]{11, 8} ), new HashSet<>(), i ); return dag; } From e95779a28d4ba82ac626ff0e197478e4367eda7a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Nov 2022 14:02:34 +0100 Subject: [PATCH 292/443] Added different prioritization and assignment strategies Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/client/Informable.java | 12 +++ .../scheduler/client/KubernetesClient.java | 46 ++++++----- src/main/java/fonda/scheduler/model/Task.java | 23 +++++- .../rest/SchedulerRestController.java | 52 +++++++++---- .../scheduler/PrioritizeAssignScheduler.java | 56 ++++++++++++++ .../scheduler/scheduler/RandomScheduler.java | 51 ------------- .../fonda/scheduler/scheduler/Scheduler.java | 7 +- .../scheduler/nodeassign/FairAssign.java | 53 +++++++++++++ .../scheduler/nodeassign/NodeAssign.java | 23 ++++++ .../nodeassign/RandomNodeAssign.java | 42 ++++++++++ .../nodeassign/RoundRobinAssign.java | 76 +++++++++++++++++++ .../scheduler/prioritize/FifoPrioritize.java | 15 ++++ .../prioritize/MaxInputPrioritize.java | 15 ++++ .../prioritize/MinInputPrioritize.java | 15 ++++ .../scheduler/prioritize/Prioritize.java | 11 +++ .../prioritize/RandomPrioritize.java | 15 ++++ .../scheduler/prioritize/RankPrioritize.java | 20 +++++ .../prioritize/RankPrioritizeTest.java | 56 ++++++++++++++ 18 files changed, 495 insertions(+), 93 deletions(-) create mode 100644 src/main/java/fonda/scheduler/client/Informable.java create mode 100644 src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java delete mode 100644 src/main/java/fonda/scheduler/scheduler/RandomScheduler.java create mode 100644 src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java create mode 100644 src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java create mode 100644 src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java create mode 100644 src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java create mode 100644 src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java diff --git a/src/main/java/fonda/scheduler/client/Informable.java b/src/main/java/fonda/scheduler/client/Informable.java new file mode 100644 index 00000000..a5c2ec56 --- /dev/null +++ b/src/main/java/fonda/scheduler/client/Informable.java @@ -0,0 +1,12 @@ +package fonda.scheduler.client; + +import fonda.scheduler.model.NodeWithAlloc; + +public interface Informable { + + void informResourceChange(); + void newNode( NodeWithAlloc node ); + void removedNode( NodeWithAlloc node ); + + +} diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index f9fdb7ed..e34b85dc 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -19,7 +19,7 @@ public class KubernetesClient extends DefaultKubernetesClient { private final Map nodeHolder= new HashMap<>(); - private final List schedulerList = new LinkedList<>(); + private final List informables = new LinkedList<>(); public KubernetesClient(){ @@ -30,40 +30,38 @@ public KubernetesClient(){ this.nodes().watch( new NodeWatcher( this ) ); } - public void addScheduler( Scheduler scheduler ){ - log.info("Added scheduler {}", scheduler.getName()); - synchronized ( schedulerList ){ - schedulerList.add( scheduler ); + public void addInformable( Informable informable ){ + synchronized ( informables ){ + informables.add( informable ); } } - public void removeScheduler( Scheduler scheduler ){ - log.info("Removed scheduler {}", scheduler.getName()); - synchronized ( schedulerList ){ - schedulerList.remove( scheduler ); + public void removeInformable( Informable informable ){ + synchronized ( informables ){ + informables.remove( informable ); } } - private void informAllScheduler(){ - synchronized ( schedulerList ){ - for (Scheduler scheduler : schedulerList) { - scheduler.informResourceChange(); + private void informAllInformable(){ + synchronized ( informables ){ + for (Informable informable : informables ) { + informable.informResourceChange(); } } } - private void informAllSchedulersNewNode( NodeWithAlloc node ){ - synchronized ( schedulerList ){ - for (Scheduler scheduler : schedulerList) { - scheduler.newNode( node ); + private void informAllNewNode( NodeWithAlloc node ){ + synchronized ( informables ){ + for (Informable informable : informables ) { + informable.newNode( node ); } } } - private void informAllSchedulersRemovedNode( NodeWithAlloc node ){ - synchronized ( schedulerList ){ - for (Scheduler scheduler : schedulerList) { - scheduler.removedNode( node ); + private void informAllRemovedNode( NodeWithAlloc node ){ + synchronized ( informables ){ + for (Informable informable : informables ) { + informable.removedNode( node ); } } } @@ -109,7 +107,7 @@ public void eventReceived(Action action, Node node) { } } if ( change ) { - kubernetesClient.informAllSchedulersNewNode( processedNode ); + kubernetesClient.informAllNewNode( processedNode ); } break; case DELETED: @@ -121,7 +119,7 @@ public void eventReceived(Action action, Node node) { } } if ( change ) { - kubernetesClient.informAllSchedulersRemovedNode( processedNode ); + kubernetesClient.informAllRemovedNode( processedNode ); } break; case ERROR: @@ -172,7 +170,7 @@ public void eventReceived(Action action, Pod pod) { //Delete Pod in any case if ( node.removePod( pod ) ){ log.info("Pod has released its resources: {}", pod.getMetadata().getName()); - kubernetesClient.informAllScheduler(); + kubernetesClient.informAllInformable(); } break; default: log.warn("No implementation for {}", action); diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 27e889ba..653636da 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -3,7 +3,6 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Process; import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.util.Batch; @@ -12,12 +11,16 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import java.io.File; import java.util.List; import java.util.Map; @Slf4j public class Task { + @Getter + private final long submissionTime = System.currentTimeMillis(); + @Getter private final TaskConfig config; @Getter @@ -96,6 +99,24 @@ public String getOutLabel(){ return outLabel == null ? null : outLabel.getLabel(); } + private long inputSize = -1; + + public long getInputSize(){ + synchronized ( this ) { + if ( inputSize == -1 ) { + //calculate + inputSize = getConfig() + .getInputs() + .fileInputs + .parallelStream() + .mapToLong( input -> new File(input.value.sourceObj).length() ) + .sum(); + } + } + //return cached value + return inputSize; + } + @Override public String toString() { return "Task{" + diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index ed3f67da..e7a0da25 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -4,15 +4,18 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.InputEdge; import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.model.TaskConfig; +import fonda.scheduler.model.*; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.*; import fonda.scheduler.scheduler.filealignment.GreedyAlignment; -import fonda.scheduler.scheduler.filealignment.RandomAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; +import fonda.scheduler.scheduler.nodeassign.FairAssign; +import fonda.scheduler.scheduler.nodeassign.NodeAssign; +import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; +import fonda.scheduler.scheduler.nodeassign.RoundRobinAssign; +import fonda.scheduler.scheduler.prioritize.*; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -102,13 +105,6 @@ ResponseEntity registerScheduler( } switch ( strategy.toLowerCase() ){ - case "fifo" : - case "random" : - case "fifo-random" : - scheduler = config.locationAware - ? new RandomLAScheduler( execution, client, namespace, config, new RandomAlignment() ) - : new RandomScheduler( execution, client, namespace, config ); - break; case "lav1" : if ( !config.locationAware ) { return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); @@ -118,12 +114,40 @@ ResponseEntity registerScheduler( } scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(costFunction) ); break; - default: - return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); + default: { + final String[] split = strategy.split( "-" ); + Prioritize prioritize = null; + NodeAssign assign = null; + if ( split.length <= 2 ) { + switch ( split[0].toLowerCase() ) { + case "fifo": prioritize = new FifoPrioritize(); break; + case "rank": prioritize = new RankPrioritize(); break; + case "random": case "r": prioritize = new RandomPrioritize(); break; + case "max": prioritize = new MaxInputPrioritize(); break; + case "min": prioritize = new MinInputPrioritize(); break; + default: + return new ResponseEntity<>( "No Prioritize for: " + split[0], HttpStatus.NOT_FOUND ); + } + if ( split.length == 2 ) { + switch ( split[1].toLowerCase() ) { + case "random": case "r": assign = new RandomNodeAssign(); break; + case "roundrobin": case "rr": assign = new RoundRobinAssign(); break; + case "fair": case "f": assign = new FairAssign(); break; + default: + return new ResponseEntity<>( "No Assign for: " + split[1], HttpStatus.NOT_FOUND ); + } + } else { + assign = new RoundRobinAssign(); + } + scheduler = new PrioritizeAssignScheduler( execution, client, namespace, config, prioritize, assign ); + } else { + return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); + } + } } schedulerHolder.put( execution, scheduler ); - client.addScheduler( scheduler ); + client.addInformable( scheduler ); return new ResponseEntity<>( HttpStatus.OK ); @@ -233,7 +257,7 @@ ResponseEntity delete( @PathVariable String execution ) { return noSchedulerFor( execution ); } schedulerHolder.remove( execution ); - client.removeScheduler( scheduler ); + client.removeInformable( scheduler ); scheduler.close(); closedLastScheduler = System.currentTimeMillis(); return new ResponseEntity<>( HttpStatus.OK ); diff --git a/src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java new file mode 100644 index 00000000..d466e685 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -0,0 +1,56 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.client.Informable; +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.*; +import fonda.scheduler.scheduler.nodeassign.NodeAssign; +import fonda.scheduler.scheduler.prioritize.Prioritize; +import fonda.scheduler.util.NodeTaskAlignment; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class PrioritizeAssignScheduler extends Scheduler { + + private final Prioritize prioritize; + private final NodeAssign nodeAssigner; + + public PrioritizeAssignScheduler( String execution, + KubernetesClient client, + String namespace, + SchedulerConfig config, + Prioritize prioritize, + NodeAssign nodeAssigner ) { + super(execution, client, namespace, config); + this.prioritize = prioritize; + this.nodeAssigner = nodeAssigner; + nodeAssigner.registerScheduler( this ); + if ( nodeAssigner instanceof Informable ){ + client.addInformable( (Informable) nodeAssigner ); + } + } + + @Override + public void close() { + super.close(); + if ( nodeAssigner instanceof Informable ){ + client.removeInformable( (Informable) nodeAssigner ); + } + } + + @Override + public ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ){ + + prioritize.sortTasks( unscheduledTasks ); + List alignment = nodeAssigner.getTaskNodeAlignment(unscheduledTasks, availableByNode); + + final ScheduleObject scheduleObject = new ScheduleObject(alignment); + scheduleObject.setCheckStillPossible( false ); + return scheduleObject; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java deleted file mode 100644 index 80577e03..00000000 --- a/src/main/java/fonda/scheduler/scheduler/RandomScheduler.java +++ /dev/null @@ -1,51 +0,0 @@ -package fonda.scheduler.scheduler; - -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.*; -import fonda.scheduler.util.NodeTaskAlignment; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; - -@Slf4j -public class RandomScheduler extends Scheduler { - - public RandomScheduler( String execution, - KubernetesClient client, - String namespace, - SchedulerConfig config ) { - super(execution, client, namespace, config); - } - - @Override - public ScheduleObject getTaskNodeAlignment( - final List unscheduledTasks, - final Map availableByNode - ){ - List alignment = new LinkedList<>(); - - for ( final Task task : unscheduledTasks ) { - final PodWithAge pod = task.getPod(); - log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest()); - - final Set matchingNodes = getMatchingNodesForTask(availableByNode,task); - if( !matchingNodes.isEmpty() ) { - - Optional node = matchingNodes.stream().findAny(); - if( node.isPresent() ) { - alignment.add(new NodeTaskAlignment(node.get(), task)); - availableByNode.get(node.get()).subFromThis(pod.getRequest()); - log.info("--> " + node.get().getName()); - } - - } else { - log.trace( "No node with enough resources for {}", pod.getName() ); - } - - } - final ScheduleObject scheduleObject = new ScheduleObject(alignment); - scheduleObject.setCheckStillPossible( true ); - return scheduleObject; - } - -} diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index dab4655b..bc4f737e 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -1,5 +1,6 @@ package fonda.scheduler.scheduler; +import fonda.scheduler.client.Informable; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.dag.DAG; import fonda.scheduler.model.*; @@ -22,7 +23,7 @@ import java.util.*; @Slf4j -public abstract class Scheduler { +public abstract class Scheduler implements Informable { //Visible variables @Getter @@ -312,7 +313,7 @@ public TaskState getTaskState( int id ) { * @param node * @return */ - boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + public boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { return node.canScheduleNewPod() && availableByNode.higherOrEquals( pod.getRequest() ) && affinitiesMatch( pod, node ); @@ -483,7 +484,7 @@ Map getAvailableByNode(){ * @param task * @return */ - Set getMatchingNodesForTask(Map availableByNode, Task task ){ + public Set getMatchingNodesForTask( Map availableByNode, Task task ){ Set result = new HashSet<>(); for (Map.Entry entry : availableByNode.entrySet()) { if ( this.canSchedulePodOnNode( entry.getValue(), task.getPod(), entry.getKey() ) ){ diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java new file mode 100644 index 00000000..17c4b64d --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java @@ -0,0 +1,53 @@ +package fonda.scheduler.scheduler.nodeassign; + +import fonda.scheduler.client.Informable; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.PodWithAge; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.Task; +import fonda.scheduler.util.NodeTaskAlignment; +import lombok.extern.slf4j.Slf4j; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +@Slf4j +public class FairAssign extends NodeAssign { + + @Override + public List getTaskNodeAlignment( List unscheduledTasks, Map availableByNode ) { + + LinkedList alignment = new LinkedList<>(); + for ( final Task task : unscheduledTasks ) { + final PodWithAge pod = task.getPod(); + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest() ); + NodeWithAlloc bestNode = null; + Double bestScore = null; + final BigDecimal podRequest = pod.getRequest().getCpu(); + for ( Map.Entry e : availableByNode.entrySet() ) { + if ( scheduler.canSchedulePodOnNode( availableByNode.get( e.getKey() ), pod, e.getKey() ) ) { + final BigDecimal maxValue = e.getKey().getMaxResources().getCpu(); + //how much is available if we assign this pod + final BigDecimal newValue = e.getValue().getCpu().subtract( podRequest ); + //larger values are better => more resources available + final double score = newValue.doubleValue() / maxValue.doubleValue(); + if ( bestScore == null || score > bestScore ) { + bestScore = score; + bestNode = e.getKey(); + } + } + } + if ( bestNode != null ) { + alignment.add( new NodeTaskAlignment( bestNode, task ) ); + availableByNode.get( bestNode ).subFromThis( pod.getRequest() ); + log.info( "--> " + bestNode.getName() ); + } + } + return alignment; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java new file mode 100644 index 00000000..616eeb83 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java @@ -0,0 +1,23 @@ +package fonda.scheduler.scheduler.nodeassign; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.Task; +import fonda.scheduler.scheduler.Scheduler; +import fonda.scheduler.util.NodeTaskAlignment; + +import java.util.List; +import java.util.Map; + +public abstract class NodeAssign { + + Scheduler scheduler; + + public abstract List getTaskNodeAlignment( List unscheduledTasks, Map availableByNode ); + + public void registerScheduler(Scheduler scheduler) { + this.scheduler = scheduler; + } + + +} diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java new file mode 100644 index 00000000..7bfd28ab --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java @@ -0,0 +1,42 @@ +package fonda.scheduler.scheduler.nodeassign; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.PodWithAge; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.Task; +import fonda.scheduler.util.NodeTaskAlignment; +import lombok.extern.slf4j.Slf4j; + +import java.math.BigDecimal; +import java.util.*; + +@Slf4j +public class RandomNodeAssign extends NodeAssign { + + @Override + public List getTaskNodeAlignment( List unscheduledTasks, Map availableByNode ) { + LinkedList alignment = new LinkedList<>(); + final ArrayList> entries = new ArrayList<>( availableByNode.entrySet() ); + for ( final Task task : unscheduledTasks ) { + final PodWithAge pod = task.getPod(); + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest() ); + Collections.shuffle( entries ); + boolean assigned = false; + for ( Map.Entry e : entries ) { + final NodeWithAlloc node = e.getKey(); + if ( scheduler.canSchedulePodOnNode( availableByNode.get( node ), pod, node ) ) { + alignment.add(new NodeTaskAlignment( node, task)); + availableByNode.get( node ).subFromThis(pod.getRequest()); + log.info("--> " + node.getName()); + assigned = true; + break; + } + } + if ( !assigned ) { + log.trace( "No node with enough resources for {}", pod.getName() ); + } + } + return alignment; + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java new file mode 100644 index 00000000..52992985 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java @@ -0,0 +1,76 @@ +package fonda.scheduler.scheduler.nodeassign; + +import fonda.scheduler.client.Informable; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.PodWithAge; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.Task; +import fonda.scheduler.util.NodeTaskAlignment; +import io.swagger.v3.oas.models.links.Link; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class RoundRobinAssign extends NodeAssign implements Informable { + + private ArrayList nodes = null; + private int nextNode = 0; + + @Override + public List getTaskNodeAlignment( List unscheduledTasks, Map availableByNode ) { + synchronized ( this ) { + if ( nodes == null ) { + nodes = new ArrayList<>( availableByNode.keySet() ); + } + } + + LinkedList alignment = new LinkedList<>(); + for ( final Task task : unscheduledTasks ) { + final PodWithAge pod = task.getPod(); + log.info("Pod: " + pod.getName() + " Requested Resources: " + pod.getRequest() ); + synchronized ( this ) { + int firstTrial = nextNode; + do { + final NodeWithAlloc node = nodes.get( nextNode ); + log.info( "Next node: " + node.getName() + "--( " + nextNode + " )" ); + nextNode = ( nextNode + 1 ) % nodes.size(); + if ( scheduler.canSchedulePodOnNode( availableByNode.get( node ), pod, node ) ) { + alignment.add( new NodeTaskAlignment( node, task ) ); + availableByNode.get( node ).subFromThis( pod.getRequest() ); + log.info( "--> " + node.getName() ); + break; + } + } while ( nextNode != firstTrial ); + } + } + return alignment; + } + + @Override + public void informResourceChange() {} + + @Override + public void newNode( NodeWithAlloc node ) { + synchronized ( this ) { + if ( nodes != null & !nodes.contains( node ) ) { + nodes.add( node ); + } + } + } + + @Override + public void removedNode( NodeWithAlloc node ) { + synchronized ( this ) { + if ( nodes != null ) { + final int index = nodes.indexOf( node ); + if ( index != -1 ) { + nodes.remove( index ); + if ( index < nextNode ) { + nextNode--; + } + } + } + } + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java new file mode 100644 index 00000000..233b5b0b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java @@ -0,0 +1,15 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.Comparator; +import java.util.List; + +public class FifoPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + tasks.sort( Comparator.comparing( Task::getSubmissionTime ) ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java new file mode 100644 index 00000000..188f088f --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java @@ -0,0 +1,15 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.Comparator; +import java.util.List; + +public class MaxInputPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + tasks.sort( Comparator.comparing( Task::getInputSize ).reversed() ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java new file mode 100644 index 00000000..c6e34a9f --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java @@ -0,0 +1,15 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.Comparator; +import java.util.List; + +public class MinInputPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + tasks.sort( Comparator.comparing( Task::getInputSize ) ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java new file mode 100644 index 00000000..0c454ee6 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java @@ -0,0 +1,11 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.List; + +public interface Prioritize { + + void sortTasks( List tasks); + +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java new file mode 100644 index 00000000..9417a2b1 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java @@ -0,0 +1,15 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.Collections; +import java.util.List; + +public class RandomPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + Collections.shuffle( tasks ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java new file mode 100644 index 00000000..fea37c70 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java @@ -0,0 +1,20 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.List; + +public class RankPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + tasks.sort( ( o1, o2 ) -> { + if ( o1.getProcess().getRank() == o2.getProcess().getRank() ) { + return (int) (o1.getSubmissionTime() - o2.getSubmissionTime()); + } + //Prefer larger ranks + return o2.getProcess().getRank() - o1.getProcess().getRank(); + } ); + } + +} diff --git a/src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java b/src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java new file mode 100644 index 00000000..c6a8aca5 --- /dev/null +++ b/src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java @@ -0,0 +1,56 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.dag.DAG; +import fonda.scheduler.dag.InputEdge; +import fonda.scheduler.dag.Process; +import fonda.scheduler.dag.Vertex; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.TaskConfig; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class RankPrioritizeTest { + + @Test + void sortTasks() throws InterruptedException { + + final DAG dag = new DAG(); + final Process a = new Process("a", 1); + final Process b = new Process("b", 2); + final Process c = new Process("c", 3); + List vertexList = Arrays.asList( a, b, c ); + List inputEdges = new LinkedList<>(); + inputEdges.add( new InputEdge(1, 1,2) ); + inputEdges.add( new InputEdge(2, 2,3) ); + dag.registerVertices( vertexList ); + dag.registerEdges(inputEdges); + + final Task c1 = new Task( new TaskConfig( "c" ), dag ); + Thread.sleep( 10 ); + final Task b1 = new Task( new TaskConfig( "b" ), dag ); + Thread.sleep( 10 ); + final Task a1 = new Task( new TaskConfig( "a" ), dag ); + Thread.sleep( 10 ); + final Task a2 = new Task( new TaskConfig( "a" ), dag ); + Thread.sleep( 10 ); + final Task b2 = new Task( new TaskConfig( "b" ), dag ); + Thread.sleep( 10 ); + final Task c2 = new Task( new TaskConfig( "c" ), dag ); + Thread.sleep( 10 ); + + final List tasks = Arrays.asList( c1, b1, a1, a2, b2, c2 ); + new RankPrioritize().sortTasks( tasks ); + assertEquals( Arrays.asList( a1, a2, b1, b2, c1, c2 ), tasks ); + + + final List tasks2 = Arrays.asList( c2, b2, a2, a1, b1, c1 ); + new RankPrioritize().sortTasks( tasks2 ); + assertEquals( Arrays.asList( a1, a2, b1, b2, c1, c2 ), tasks2 ); + + } +} \ No newline at end of file From d1e73510a53b334847f73ff134251476c8fd8591 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 1 Nov 2022 14:03:01 +0100 Subject: [PATCH 293/443] changed visibility Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 28cb6896..81824618 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -456,7 +456,7 @@ private void podWasInitialized( Pod pod ){ } @Override - boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + public boolean canSchedulePodOnNode( Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { return this.getDaemonOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); } From d6bac2973a8571aa3a5949237c030ce94cb54656 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Nov 2022 14:32:53 +0100 Subject: [PATCH 294/443] Use Id to number tasks Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 5 ++++- .../fonda/scheduler/scheduler/prioritize/FifoPrioritize.java | 2 +- .../fonda/scheduler/scheduler/prioritize/RankPrioritize.java | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 653636da..d13238ae 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -14,12 +14,15 @@ import java.io.File; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; @Slf4j public class Task { + private static AtomicInteger idCounter = new AtomicInteger(0); + @Getter - private final long submissionTime = System.currentTimeMillis(); + private final int id = idCounter.getAndIncrement(); @Getter private final TaskConfig config; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java index 233b5b0b..be0682c7 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java @@ -9,7 +9,7 @@ public class FifoPrioritize implements Prioritize { @Override public void sortTasks( List tasks ) { - tasks.sort( Comparator.comparing( Task::getSubmissionTime ) ); + tasks.sort( Comparator.comparing( Task::getId ) ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java index fea37c70..0b3376d7 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java @@ -10,7 +10,7 @@ public class RankPrioritize implements Prioritize { public void sortTasks( List tasks ) { tasks.sort( ( o1, o2 ) -> { if ( o1.getProcess().getRank() == o2.getProcess().getRank() ) { - return (int) (o1.getSubmissionTime() - o2.getSubmissionTime()); + return o1.getId() - o2.getId(); } //Prefer larger ranks return o2.getProcess().getRank() - o1.getProcess().getRank(); From ecd36987fd9080bd76cf661f7b1b532fbdb22bcc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Nov 2022 14:35:56 +0100 Subject: [PATCH 295/443] New Rank strategies Signed-off-by: Lehmann_Fabian --- .../prioritize/RankMaxPrioritize.java | 20 +++++++++++++++++++ .../prioritize/RankMinPrioritize.java | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java create mode 100644 src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java new file mode 100644 index 00000000..5c9e1d44 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java @@ -0,0 +1,20 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.List; + +public class RankMaxPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + tasks.sort( ( o1, o2 ) -> { + if ( o1.getProcess().getRank() == o2.getProcess().getRank() ) { + return (int) (o2.getInputSize() - o1.getInputSize()); + } + //Prefer larger ranks + return o2.getProcess().getRank() - o1.getProcess().getRank(); + } ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java b/src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java new file mode 100644 index 00000000..351423cf --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java @@ -0,0 +1,20 @@ +package fonda.scheduler.scheduler.prioritize; + +import fonda.scheduler.model.Task; + +import java.util.List; + +public class RankMinPrioritize implements Prioritize { + + @Override + public void sortTasks( List tasks ) { + tasks.sort( ( o1, o2 ) -> { + if ( o1.getProcess().getRank() == o2.getProcess().getRank() ) { + return (int) (o1.getInputSize() - o2.getInputSize()); + } + //Prefer larger ranks + return o2.getProcess().getRank() - o1.getProcess().getRank(); + } ); + } + +} From e4248508daf0d2e0bbe69c74a93254778471102e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Nov 2022 14:36:32 +0100 Subject: [PATCH 296/443] Sychronize tryToScheduleBatch Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/Scheduler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index bc4f737e..23f677fc 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -211,10 +211,16 @@ public void schedulePod(PodWithAge pod ) { } else { Batch batch = task.getBatch(); batch.informScheduable( task ); - tryToScheduleBatch( batch ); + synchronized (batchHelper) { + tryToScheduleBatch( batch ); + } } } + /** + * Synchronize calls via batchHelper + * @param batch + */ private void tryToScheduleBatch( Batch batch ){ if ( batch.canSchedule() ){ synchronized (unscheduledTasks){ @@ -314,6 +320,9 @@ public TaskState getTaskState( int id ) { * @return */ public boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + if ( availableByNode == null ) { + return false; + } return node.canScheduleNewPod() && availableByNode.higherOrEquals( pod.getRequest() ) && affinitiesMatch( pod, node ); From da7c12d8305e0323b3df70156474af557d9ca871 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 28 Nov 2022 14:37:03 +0100 Subject: [PATCH 297/443] Inefficient map iteration Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/nodeassign/FairAssign.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java index 17c4b64d..2a38afb1 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java @@ -29,7 +29,7 @@ public List getTaskNodeAlignment( List unscheduledTasks Double bestScore = null; final BigDecimal podRequest = pod.getRequest().getCpu(); for ( Map.Entry e : availableByNode.entrySet() ) { - if ( scheduler.canSchedulePodOnNode( availableByNode.get( e.getKey() ), pod, e.getKey() ) ) { + if ( scheduler.canSchedulePodOnNode( e.getValue(), pod, e.getKey() ) ) { final BigDecimal maxValue = e.getKey().getMaxResources().getCpu(); //how much is available if we assign this pod final BigDecimal newValue = e.getValue().getCpu().subtract( podRequest ); From a56a77836613b98c3745c917ae55e5affff61517 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 16 Dec 2022 15:07:44 +0100 Subject: [PATCH 298/443] Rank by size callable about REST Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/rest/SchedulerRestController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index e7a0da25..26a42484 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -122,6 +122,8 @@ ResponseEntity registerScheduler( switch ( split[0].toLowerCase() ) { case "fifo": prioritize = new FifoPrioritize(); break; case "rank": prioritize = new RankPrioritize(); break; + case "rank_min": prioritize = new RankMinPrioritize(); break; + case "rank_max": prioritize = new RankMaxPrioritize(); break; case "random": case "r": prioritize = new RandomPrioritize(); break; case "max": prioritize = new MaxInputPrioritize(); break; case "min": prioritize = new MinInputPrioritize(); break; From dedc0cdf5c28ec5dad35242dca8659b6a85dd0b1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 16 Dec 2022 15:13:08 +0100 Subject: [PATCH 299/443] added REST documentation Signed-off-by: Lehmann_Fabian --- .../rest/SchedulerRestController.java | 80 ++++++++++++++++++- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 26a42484..a5c9a02a 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -8,14 +8,15 @@ import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.*; -import fonda.scheduler.scheduler.filealignment.GreedyAlignment; -import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; -import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import fonda.scheduler.scheduler.nodeassign.FairAssign; import fonda.scheduler.scheduler.nodeassign.NodeAssign; import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; import fonda.scheduler.scheduler.nodeassign.RoundRobinAssign; import fonda.scheduler.scheduler.prioritize.*; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -73,6 +74,14 @@ private ResponseEntity noSchedulerFor( String execution ){ return new ResponseEntity<>( "There is no scheduler for " + execution, HttpStatus.BAD_REQUEST ); } + @Operation(summary = "Register a new execution") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Execution successfully registered", + content = @Content), + @ApiResponse(responseCode = "404", description = "Scheduling algorithm or cost function not found", + content = @Content), + @ApiResponse(responseCode = "400", description = "Scheduling algorithm does not work with current config", + content = @Content) }) /** * Register a scheduler for a workflow execution * @param execution unique name of the execution @@ -155,6 +164,12 @@ ResponseEntity registerScheduler( } + @Operation(summary = "Register a task for execution") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Task successfully registered", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) /** * Register a task for the execution * @@ -179,8 +194,17 @@ ResponseEntity registerTask( @PathVariable String execution, @ } + + @Operation(summary = "Delete a task of execution") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Task successfully deleted", + content = @Content), + @ApiResponse(responseCode = "404", description = "Task not found", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) /** - * Delete a task, onl works if the batch of the task was closed and if no pod was yet submitted. + * Delete a task, only works if the batch of the task was closed and if no pod was yet submitted. * If a pod was submitted, delete the pod instead. * * @param execution @@ -200,6 +224,12 @@ ResponseEntity deleteTask( @PathVariable String execution, @Pa } + @Operation(summary = "") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Successfully started batch", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) @PutMapping("/v1/scheduler/{execution}/startBatch") ResponseEntity startBatch( @PathVariable String execution ) { @@ -212,6 +242,12 @@ ResponseEntity startBatch( @PathVariable String execution ) { } + @Operation(summary = "End a batch") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Successfully ended batch", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) @PutMapping("/v1/scheduler/{execution}/endBatch") ResponseEntity endBatch( @PathVariable String execution, @RequestBody int tasksInBatch ) { @@ -224,6 +260,12 @@ ResponseEntity endBatch( @PathVariable String execution, @RequestBody in } + @Operation(summary = "Check the state of a task") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Task found", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) /** * Check Task state * @@ -243,6 +285,12 @@ ResponseEntity getTaskState( @PathVariable String execution, @ } + @Operation(summary = "Delete an execution after it has finished or crashed") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Execution successfully deleted", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) /** * Call this after the execution has finished * @@ -362,6 +410,12 @@ ResponseEntity checkHealth() { return new ResponseEntity<>( HttpStatus.OK ); } + @Operation(summary = "Register DAG vertices") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Vertices successfully registered", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) @PostMapping("/v1/scheduler/{execution}/DAG/vertices") ResponseEntity addVertices( @PathVariable String execution, @RequestBody List vertices ) { @@ -378,6 +432,12 @@ ResponseEntity addVertices( @PathVariable String execution, @RequestBody } + @Operation(summary = "Delete DAG vertices") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Vertices successfully removed", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) @DeleteMapping("/v1/scheduler/{execution}/DAG/vertices") ResponseEntity deleteVertices( @PathVariable String execution, @RequestBody int[] vertices ) { @@ -394,6 +454,12 @@ ResponseEntity deleteVertices( @PathVariable String execution, @RequestB } + @Operation(summary = "Register DAG edges") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Edges successfully registered", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) @PostMapping("/v1/scheduler/{execution}/DAG/edges") ResponseEntity addEdges( @PathVariable String execution, @RequestBody List edges ) { @@ -411,6 +477,12 @@ ResponseEntity addEdges( @PathVariable String execution, @RequestBody Li } + @Operation(summary = "Delete DAG edges") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Edges successfully removed", + content = @Content), + @ApiResponse(responseCode = "400", description = "No scheduler found for this execution", + content = @Content) }) @DeleteMapping("/v1/scheduler/{execution}/DAG/edges") ResponseEntity deleteEdges( @PathVariable String execution, @RequestBody int[] edges ) { From 22b2d3caa7930e39a7ab1b4cb36ab2f1649e1bed Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 16 Dec 2022 15:19:28 +0100 Subject: [PATCH 300/443] Readded imports Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/rest/SchedulerRestController.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index a5c9a02a..6986bf80 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -8,6 +8,9 @@ import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.*; +import fonda.scheduler.scheduler.filealignment.GreedyAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import fonda.scheduler.scheduler.nodeassign.FairAssign; import fonda.scheduler.scheduler.nodeassign.NodeAssign; import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; @@ -125,8 +128,8 @@ ResponseEntity registerScheduler( break; default: { final String[] split = strategy.split( "-" ); - Prioritize prioritize = null; - NodeAssign assign = null; + Prioritize prioritize; + NodeAssign assign; if ( split.length <= 2 ) { switch ( split[0].toLowerCase() ) { case "fifo": prioritize = new FifoPrioritize(); break; From 40c050935b2df9c7946233b6372d3565929cc6bc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 19 Dec 2022 14:02:01 +0100 Subject: [PATCH 301/443] Update Fabric8 version Signed-off-by: Lehmann_Fabian --- pom.xml | 2 +- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 6962f309..4e561fb3 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ io.fabric8 kubernetes-client - 5.12.2 + 6.3.1 diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 817105ab..e832c7e0 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -6,13 +6,12 @@ import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Quantity; -import io.fabric8.kubernetes.client.internal.readiness.Readiness; +import io.fabric8.kubernetes.client.readiness.Readiness; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.math.BigDecimal; import java.util.*; -import java.util.stream.Collectors; @Getter @Slf4j From b6d560ca3bfb42d6e198c6cde8ddf6a1065ac586 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 19 Dec 2022 14:45:44 +0100 Subject: [PATCH 302/443] Removed unused initContainer Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 50 +++++++++++++++++-- .../fonda/scheduler/scheduler/Scheduler.java | 20 ++------ .../scheduler/SchedulerWithDaemonSet.java | 23 ++++++--- .../util/NodeTaskFilesAlignment.java | 6 +++ 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index e34b85dc..500c2f75 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -3,10 +3,7 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; import fonda.scheduler.scheduler.Scheduler; -import io.fabric8.kubernetes.api.model.ContainerStatus; -import io.fabric8.kubernetes.api.model.Node; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; @@ -84,6 +81,51 @@ public BigDecimal getMemoryOfNode(NodeWithAlloc node ){ return Quantity.getAmountInBytes(memory); } + private void forceDeletePod( Pod pod ) { + this.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .withName(pod.getMetadata().getName()) + .withGracePeriod(0) + .withPropagationPolicy( DeletionPropagation.BACKGROUND ) + .delete(); + } + + private void createPod( Pod pod ) { + this.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .resource( pod ) + .create(); + } + + public void assignPodToNode( PodWithAge pod, String node ) { + Binding b1 = new Binding(); + + ObjectMeta om = new ObjectMeta(); + om.setName( pod.getMetadata().getName()); + om.setNamespace( pod.getMetadata().getNamespace()); + b1.setMetadata(om); + + ObjectReference objectReference = new ObjectReference(); + objectReference.setApiVersion("v1"); + objectReference.setKind("Node"); + objectReference.setName( node ); + + b1.setTarget(objectReference); + + bindings().create(b1); + } + + public void assignPodToNodeAndRemoveInit( PodWithAge pod, String node ) { + pod.getSpec().getInitContainers().remove( 0 ); + pod.getMetadata().setResourceVersion( null ); + pod.getMetadata().setManagedFields( null ); + pod.getSpec().setNodeName( node ); + + forceDeletePod( pod ); + createPod( pod ); + } + + static class NodeWatcher implements Watcher{ private final KubernetesClient kubernetesClient; diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 23f677fc..974e27c3 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -362,6 +362,10 @@ boolean canPodBeScheduled( PodWithAge pod, NodeWithAlloc node ){ return node.canSchedule( pod ); } + void assignPodToNode( PodWithAge pod, NodeTaskAlignment alignment ){ + client.assignPodToNode( pod, alignment.node.getMetadata().getName() ); + } + boolean assignTaskToNode( NodeTaskAlignment alignment ){ final File nodeFile = new File(alignment.task.getWorkingDir() + '/' + ".command.node"); @@ -381,21 +385,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + alignment.node.getMetadata().getName() ); - Binding b1 = new Binding(); - - ObjectMeta om = new ObjectMeta(); - om.setName(pod.getMetadata().getName()); - om.setNamespace(pod.getMetadata().getNamespace()); - b1.setMetadata(om); - - ObjectReference objectReference = new ObjectReference(); - objectReference.setApiVersion("v1"); - objectReference.setKind("Node"); - objectReference.setName(alignment.node.getMetadata().getName()); - - b1.setTarget(objectReference); - - client.bindings().create(b1); + assignPodToNode( pod, alignment ); pod.getSpec().setNodeName( alignment.node.getMetadata().getName() ); log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 81824618..ba2da6eb 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -22,18 +22,14 @@ import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.*; -import io.fabric8.kubernetes.api.model.ContainerStatus; -import io.fabric8.kubernetes.api.model.Node; -import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.Watcher; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.net.ftp.FTPClient; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -103,10 +99,13 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { if ( traceEnabled ) { traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); } + nodeTaskFilesAlignment.setRemoveInit( !writeConfigResult.isWroteConfig() ); alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); - getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); + if ( writeConfigResult.isWroteConfig() ) { + getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); + } alignment.task.setCopiesDataToNode( writeConfigResult.isCopyDataToNode() ); final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); alignment.task.setInputFiles( allLocationWrappers ); @@ -114,6 +113,16 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { return super.assignTaskToNode( alignment ); } + @Override + void assignPodToNode( PodWithAge pod, NodeTaskAlignment alignment ) { + if ( ((NodeTaskFilesAlignment) alignment).isRemoveInit() ) { + log.info( "Removing init container from pod {}", pod.getMetadata().getName() ); + client.assignPodToNodeAndRemoveInit( pod, alignment.node.getName() ); + } else { + super.assignPodToNode( pod, alignment ); + } + } + @Override void undoTaskScheduling( Task task ){ if ( task.getInputFiles() != null ) { diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java index a171a218..0d2dd958 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java @@ -2,12 +2,18 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; +import lombok.Getter; +import lombok.Setter; public class NodeTaskFilesAlignment extends NodeTaskAlignment { public final FileAlignment fileAlignment; + @Getter + @Setter + private boolean removeInit = false; + public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, FileAlignment fileAlignment ) { super(node, task); this.fileAlignment = fileAlignment; From e825df84cd2461c7b51c8388fc815cddf7bf7b56 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 19 Dec 2022 17:26:42 +0100 Subject: [PATCH 303/443] Fix problem: modified pod is two times added Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/NodeWithAlloc.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index e832c7e0..702b8af0 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -68,16 +68,26 @@ public void addPod( PodWithAge pod, boolean withStartingTasks ) { Requirements request = pod.getRequest(); if ( withStartingTasks ) { synchronized ( startingTaskCopyingData ) { - if ( !startingTaskCopyingData.contains( pod ) ) { - startingTaskCopyingData.add( pod ); + final Iterator iterator = startingTaskCopyingData.iterator(); + while ( iterator.hasNext() ) { + final PodWithAge copyPod = iterator.next(); + if ( getPodInternalId( pod ).equals( getPodInternalId( copyPod ) )) { + iterator.remove(); + break; + } } + startingTaskCopyingData.add( pod ); } } synchronized (assignedPods) { - assignedPods.put( pod.getMetadata().getUid(), request ); + assignedPods.put( getPodInternalId( pod ) , request ); } } + private String getPodInternalId( Pod pod ) { + return pod.getMetadata().getNamespace() + "||" + pod.getMetadata().getName(); + } + private void removeStartingTaskCopyingDataByUid( String uid ) { synchronized ( startingTaskCopyingData ) { final Iterator iterator = startingTaskCopyingData.iterator(); @@ -94,7 +104,7 @@ private void removeStartingTaskCopyingDataByUid( String uid ) { public boolean removePod( Pod pod ){ removeStartingTaskCopyingDataByUid( pod.getMetadata().getUid() ); synchronized (assignedPods) { - return assignedPods.remove( pod.getMetadata().getUid() ) != null; + return assignedPods.remove( getPodInternalId( pod ) ) != null; } } From 16c19e80aa214009c4c518304f0e946e34d0eedd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 19 Dec 2022 17:27:23 +0100 Subject: [PATCH 304/443] Fix problem: OutOfBound if pod has no init container Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/client/KubernetesClient.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 500c2f75..b1b47c71 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -116,7 +116,9 @@ public void assignPodToNode( PodWithAge pod, String node ) { } public void assignPodToNodeAndRemoveInit( PodWithAge pod, String node ) { - pod.getSpec().getInitContainers().remove( 0 ); + if ( pod.getSpec().getInitContainers().size() > 0 ) { + pod.getSpec().getInitContainers().remove( 0 ); + } pod.getMetadata().setResourceVersion( null ); pod.getMetadata().setManagedFields( null ); pod.getSpec().setNodeName( node ); From 77ed1725deba9d9dcd38b28f36cc879a8526be36 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 23 Dec 2022 12:33:57 +0100 Subject: [PATCH 305/443] First version of separate copy tasks (LAv2) Signed-off-by: Lehmann_Fabian --- daemons/ftp/Dockerfile | 4 + daemons/ftp/ftp.py | 346 ++++++++++++++++++ .../location/hierachy/LocationWrapper.java | 10 + .../location/hierachy/RealHierarchyFile.java | 6 + .../taskinputs/PathFileLocationTriple.java | 9 + .../rest/SchedulerRestController.java | 12 +- .../scheduler/data/TaskInputsNodes.java | 20 + .../scheduler/util/CopyToNodeManager.java | 56 +++ .../fonda/scheduler/util/DaemonHolder.java | 55 +++ .../java/fonda/scheduler/util/DataOnNode.java | 50 +++ .../util/NodeTaskLocalFilesAlignment.java | 24 ++ 11 files changed, 591 insertions(+), 1 deletion(-) create mode 100644 daemons/ftp/ftp.py create mode 100644 src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java create mode 100644 src/main/java/fonda/scheduler/util/CopyToNodeManager.java create mode 100644 src/main/java/fonda/scheduler/util/DaemonHolder.java create mode 100644 src/main/java/fonda/scheduler/util/DataOnNode.java create mode 100644 src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java diff --git a/daemons/ftp/Dockerfile b/daemons/ftp/Dockerfile index 07f872fc..57099808 100644 --- a/daemons/ftp/Dockerfile +++ b/daemons/ftp/Dockerfile @@ -10,4 +10,8 @@ COPY vsftpd.conf /etc/vsftpd.conf USER root RUN echo 'root:password' | chpasswd +COPY ftp.py /code/ftp.py + +WORKDIR /code + ENTRYPOINT ["sh","-c","/usr/sbin/vsftpd /etc/vsftpd.conf"] \ No newline at end of file diff --git a/daemons/ftp/ftp.py b/daemons/ftp/ftp.py new file mode 100644 index 00000000..f1f5691e --- /dev/null +++ b/daemons/ftp/ftp.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +import ftplib +import json +import logging as log +import os +import shutil +import signal +import sys +import time +import urllib.request +import urllib.parse +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from time import sleep + +######################################################################################## +# Call this class with three arguments: trace enabled, name to store logs, config json # +######################################################################################## + +exitIfFileWasNotFound = True +CLOSE = False +UNEXPECTED_ERROR = "Unexpected error" +EXIT = 0 +log.basicConfig( + format='%(levelname)s: %(message)s', + level=log.DEBUG, + handlers=[ + log.FileHandler(".command.init." + sys.argv[2] + ".log"), + log.StreamHandler() + ] +) +trace = {} +traceFilePath = ".command.scheduler.trace" +errors = 0 + + +def myExit(code): + global EXIT + EXIT = code + global CLOSE + CLOSE = True + writeTrace(trace) + exit(EXIT) + + +def close(signalnum, syncFile): + log.info("Killed: %s", str(signalnum)) + closeWithWarning(50, syncFile) + + +def closeWithWarning(errorCode, syncFile): + syncFile.write('##FAILURE##\n') + syncFile.flush() + syncFile.close() + myExit(errorCode) + + +def getIP(node, dns, execution): + ip = urllib.request.urlopen(dns + "daemon/" + execution + "/" + node).read() + return str(ip.decode("utf-8")) + + +# True if the file was deleted or did not exist +def clearLocation(path, dst=None): + if os.path.exists(path): + log.debug("Delete %s", path) + if os.path.islink(path): + if dst is not None and os.readlink(path) == dst: + return False + else: + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + return True + + +def getFTP(node, currentIP, dns, execution, syncFile): + global errors + connectionProblem = 0 + while connectionProblem < 8: + try: + if currentIP is None: + log.info("Request ip for node: %s", node) + ip = getIP(node, dns, execution) + else: + ip = currentIP + log.info("Try to connect to %s", ip) + ftp = ftplib.FTP(ip, timeout=10) + ftp.login("root", "password") + ftp.set_pasv(True) + ftp.encoding = 'utf-8' + log.info("Connection established") + return ftp + except ConnectionRefusedError: + errors += 1 + log.warning("Connection refused! Try again...") + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + connectionProblem += 1 + time.sleep(2 ** connectionProblem) + closeWithWarning(8, syncFile) + + +def closeFTP(ftp): + global errors + if ftp is None: + return + try: + ftp.quit() + ftp.close() + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + + +def downloadFile(ftp, filename, size, index, node, syncFile): + global errors + log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) + try: + syncFile.write("S-" + filename + '\n') + clearLocation(filename) + Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) + start = time.time() + ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) + end = time.time() + sizeInMB = os.path.getsize(filename) / 1000000 + delta = (end - start) + log.info("Speed: %.3f Mbit/s", sizeInMB / delta) + return sizeInMB, delta + except ftplib.error_perm as err: + errors += 1 + if str(err) == "550 Failed to open file.": + log.warning("File not found node: %s file: %s", node, filename) + if exitIfFileWasNotFound: + closeWithWarning(40, syncFile) + except FileNotFoundError: + errors += 1 + log.warning("File not found node: %s file: %s", node, filename) + if exitIfFileWasNotFound: + closeWithWarning(41, syncFile) + except EOFError: + errors += 1 + log.warning("It seems the connection was lost! Try again...") + return None + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + return None + return 0, 0 + + +def download(node, currentIP, files, dns, execution, syncFile): + ftp = None + size = len(files) + global CLOSE + sizeInMB = 0 + downloadTime = 0 + while not CLOSE and len(files) > 0: + if ftp is None: + ftp = getFTP(node, currentIP, dns, execution, syncFile) + currentIP = None + filename = files[0] + index = size - len(files) + 1 + result = downloadFile(ftp, filename, size, index, node, syncFile) + if result is None: + ftp = None + continue + sizeInMB += result[0] + downloadTime += result[1] + files.pop(0) + syncFile.write("F-" + filename + '\n') + closeFTP(ftp) + return node, sizeInMB / downloadTime + + +def waitForFiles(syncFilePath, files, startTime): + # wait max. 60 seconds + while True: + if startTime + 60 < time.time(): + return False + if os.path.isfile(syncFilePath): + break + log.debug("Wait for file creation") + time.sleep(0.1) + + # Read file live + with open(syncFilePath, 'r') as syncFileTask: + current = [] + while len(files) > 0: + data = syncFileTask.read() + if not data: + time.sleep(0.3) + else: + for d in data: + if d != "\n": + current.append(d) + else: + text = ''.join(current) + current = [] + if text.startswith("S-"): + continue + if text == "##FAILURE##": + log.debug("Read FAILURE in %s", syncFilePath) + myExit(51) + if text == "##FINISHED##": + log.debug("Read FINISHED in " + syncFilePath + " before all files were found") + myExit(52) + log.debug("Look for " + text[:2] + " with " + text[2:] + " in " + str(files)) + if text[:2] == "F-" and text[2:] in files: + files.remove(text[2:]) + if len(files) == 0: + return True + return len(files) == 0 + + +def loadConfig(): + log.info("Load config") + log.info( "Parse: --" + sys.argv[3] + "--" ) + config = json.loads(sys.argv[3]) + os.makedirs(config["syncDir"], exist_ok=True) + return config + + +def registerSignal(syncFile): + signal.signal(signal.SIGINT, lambda signalnum, handler: close(signalnum, syncFile)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: close(signalnum, syncFile)) + + +def registerSignal2(): + signal.signal(signal.SIGINT, lambda signalnum, handler: myExit(1)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: myExit(1)) + + +def generateSymlinks(symlinks): + for s in symlinks: + src = s["src"] + dst = s["dst"] + if clearLocation(src, dst): + Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) + try: + os.symlink(dst, src) + except FileExistsError: + log.warning("File exists: %s -> %s", src, dst) + + +def downloadAllData(data, dns, execution, syncFile): + global trace + throughput = [] + with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: + futures = [] + for d in data: + files = d["files"] + node = d["node"] + currentIP = d["currentIP"] + futures.append(executor.submit(download, node, currentIP, files, dns, execution, syncFile)) + lastNum = -1 + while len(futures) > 0: + if lastNum != len(futures): + log.info("Wait for %d threads to finish", len(futures)) + lastNum = len(futures) + for f in futures[:]: + if f.done(): + throughput.append(f.result()) + futures.remove(f) + sleep(0.1) + trace["scheduler_init_throughput"] = "\"" + ",".join("{}:{:.3f}".format(*x) for x in throughput) + "\"" + + +def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): + # Now check for files of other tasks + for waitForTask in waitForFilesOfTask: + waitForFilesSet = set(waitForFilesOfTask[waitForTask]) + if not waitForFiles(syncDir + waitForTask, waitForFilesSet, startTime): + log.error(syncDir + waitForTask + " was not successful") + myExit(200) + + +def writeTrace(dataMap): + if sys.argv[1] == 'true': + global errors + if len(dataMap) == 0 or errors > 0: + return + with open(traceFilePath, "a") as traceFile: + for d in dataMap: + traceFile.write(d + "=" + str(dataMap[d]) + "\n") + traceFile.write("scheduler_init_errors=" + str(errors) + "\n") + + +def finishedDownload(dns, execution, taskname): + try: + dns = dns + "downloadtask/" + execution + log.info("Request: %s", dns) + urllib.request.urlopen(dns, taskname.encode( "utf-8" )) + except BaseException as err: + log.exception(err) + myExit(100) + + +def run(): + global trace + startTime = time.time() + log.info("Start to setup the environment") + config = loadConfig() + + dns = config["dns"] + execution = config["execution"] + data = config["data"] + symlinks = config["symlinks"] + taskname = config["hash"] + + with open(config["syncDir"] + config["hash"], 'w') as syncFile: + registerSignal(syncFile) + syncFile.write('##STARTED##\n') + syncFile.flush() + startTimeSymlinks = time.time() + generateSymlinks(symlinks) + trace["scheduler_init_symlinks_runtime"] = int((time.time() - startTimeSymlinks) * 1000) + syncFile.write('##SYMLINKS##\n') + syncFile.flush() + startTimeDownload = time.time() + downloadAllData(data, dns, execution, syncFile) + trace["scheduler_init_download_runtime"] = int((time.time() - startTimeDownload) * 1000) + if CLOSE: + log.debug("Closed with code %s", str(EXIT)) + exit(EXIT) + log.info("Finished Download") + syncFile.write('##FINISHED##\n') + registerSignal2() + + #finishedDownload(dns, execution, taskname) + + #startTimeDependingTasks = time.time() + #waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) + #trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) + #log.info("Waited for all tasks") + + #runtime = int((time.time() - startTime) * 1000) + #trace["scheduler_init_runtime"] = runtime + #writeTrace(trace) + + +if __name__ == '__main__': + run() diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java index 3fd5337f..c737775f 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java @@ -56,18 +56,28 @@ public void deactivate(){ this.active = false; } + /** + * use the file, if you copy it to a node, or a task uses it as input + */ public void use(){ synchronized ( this ) { inUse++; } } + /** + * free the file, if you finished copy it to a node, or a task the task finished that used it as an input + */ public void free(){ synchronized ( this ) { inUse--; } } + /** + * Any task currently reading or writing to this file + * @return + */ public boolean isInUse(){ return inUse > 0; } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java index caac0ef3..cbd2bbd9 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -177,6 +177,12 @@ private LinkedList addAndCreateList(LinkedList return list; } + /** + * This method is used to find all possible LocationWrappers of a file for a specific task. + * @param task + * @return a list of all LocationWrapper of this file that could be used and a list of all Locations that are in use and are not in a version that this task could use. + * @throws NoAlignmentFoundException + */ public MatchingLocationsPair getFilesForTask( Task task ) throws NoAlignmentFoundException { LocationWrapper[] locationsRef = this.locations; diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index b0b9b1b9..ac48d116 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -36,6 +36,15 @@ public long getSizeInBytes() { return this.size; } + public LocationWrapper locationWrapperOnLocation(Location loc){ + for (LocationWrapper location : locations) { + if ( location.getLocation().equals(loc) ) { + return location; + } + } + throw new IllegalStateException("LocationWrapper not found for location " + loc); + } + public boolean locatedOnLocation(Location loc){ for (LocationWrapper location : locations) { if ( location.getLocation() == loc ) { diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 6986bf80..af9b5f05 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -16,6 +16,7 @@ import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; import fonda.scheduler.scheduler.nodeassign.RoundRobinAssign; import fonda.scheduler.scheduler.prioritize.*; +import fonda.scheduler.scheduler.LocationAwareSchedulerV2Simple; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -126,6 +127,15 @@ ResponseEntity registerScheduler( } scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(costFunction) ); break; + case "lav2" : + if ( !config.locationAware ) { + return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); + } + if ( costFunction == null ) { + costFunction = new MinSizeCost( 0 ); + } + scheduler = new LocationAwareSchedulerV2Simple( execution, client, namespace, config, new GreedyAlignment(costFunction) ); + break; default: { final String[] split = strategy.split( "-" ); Prioritize prioritize; @@ -326,7 +336,7 @@ ResponseEntity getDaemonName( @PathVariable String execution, @PathVaria return noSchedulerFor( execution ); } - String daemon = ((SchedulerWithDaemonSet) scheduler).getDaemonOnNode( node ); + String daemon = ((SchedulerWithDaemonSet) scheduler).getDaemonIpOnNode( node ); if ( daemon == null ){ return new ResponseEntity<>( "No daemon for node found: " + node , HttpStatus.NOT_FOUND ); diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java b/src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java new file mode 100644 index 00000000..0880bd64 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java @@ -0,0 +1,20 @@ +package fonda.scheduler.scheduler.data; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.taskinputs.TaskInputs; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +/** + * This class is used to store the data of a task, the task, and the nodes that contain all data for this task. + */ +@Getter +@RequiredArgsConstructor +public class TaskInputsNodes { + private final Task task; + private final List nodesWithAllData; + private final TaskInputs inputsOfTask; +} \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/util/CopyToNodeManager.java b/src/main/java/fonda/scheduler/util/CopyToNodeManager.java new file mode 100644 index 00000000..69541a48 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/CopyToNodeManager.java @@ -0,0 +1,56 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class CopyToNodeManager { + + private final Map> taskOnNodes = new HashMap<>(); + private final Map> nodesForTask = new HashMap<>(); + + public void copyToNode( Task task, NodeWithAlloc node, double part ){ + synchronized ( this ) { + List tasks = taskOnNodes.get( node ); + if ( tasks == null ) { + tasks = new LinkedList<>(); + taskOnNodes.put( node, tasks ); + } + tasks.add( task ); + + List nodes = nodesForTask.get( task ); + if ( nodes == null ) { + nodes = new LinkedList<>(); + nodesForTask.put( task, nodes ); + } + nodes.add( node ); + } + } + + public void finishedCopyToNode( Task task, NodeWithAlloc node ){ + synchronized ( this ) { + List tasks = taskOnNodes.get( node ); + if ( tasks != null ) { + tasks.remove( task ); + } + + List nodes = nodesForTask.get( task ); + if ( nodes != null ) { + nodes.remove( node ); + } + } + } + + public Map getCurrentlyCopyingTasksOnNode() { + Map result = new HashMap<>(); + for ( Map.Entry> entry : taskOnNodes.entrySet() ) { + result.put( entry.getKey(), entry.getValue().size() ); + } + return result; + } + +} diff --git a/src/main/java/fonda/scheduler/util/DaemonHolder.java b/src/main/java/fonda/scheduler/util/DaemonHolder.java new file mode 100644 index 00000000..83a262d2 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/DaemonHolder.java @@ -0,0 +1,55 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.location.NodeLocation; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class DaemonHolder { + + final Map daemonByNode = new HashMap<>(); + + public void removeDaemon(String nodeName) { + daemonByNode.remove(nodeName); + } + + public void addDaemon(String node, String daemonName, String daemonIp ) { + daemonByNode.put(node, new DaemonData( daemonName, daemonIp ) ); + } + + public String getDaemonIp(String node) { + final DaemonData daemonData = daemonByNode.get( node ); + return daemonData == null ? null : daemonData.getIp(); + } + + public String getDaemonName(String node) { + final DaemonData daemonData = daemonByNode.get( node ); + return daemonData == null ? null : daemonData.getName(); + } + + public String getDaemonIp( NodeLocation node) { + return getDaemonIp(node.getIdentifier()); + } + + public String getDaemonName(NodeLocation node) { + return getDaemonName(node.getIdentifier()); + } + + @Getter + @ToString + @AllArgsConstructor + private class DaemonData { + private String name; + private String ip; + } + + @Override + public String toString() { + return daemonByNode.toString(); + } +} diff --git a/src/main/java/fonda/scheduler/util/DataOnNode.java b/src/main/java/fonda/scheduler/util/DataOnNode.java new file mode 100644 index 00000000..0793adc0 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/DataOnNode.java @@ -0,0 +1,50 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.taskinputs.TaskInputs; +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class DataOnNode { + + + private final Map dataOnNode = new HashMap<>(); + + @Getter + private final long overAllData; + @Getter + private final Task task; + @Getter + private int nodesWithAllData; + @Getter + private final TaskInputs inputsOfTask; + + public DataOnNode( Task task, TaskInputs inputsOfTask ) { + this.task = task; + this.inputsOfTask = inputsOfTask; + this.overAllData = inputsOfTask.calculateAvgSize(); + } + + public void allData( NodeWithAlloc node ){ + nodesWithAllData++; + addData( node, 1 ); + } + + public void addData( NodeWithAlloc node, double part ){ + dataOnNode.put( node, part ); + } + + public double getPart( NodeWithAlloc node ){ + return dataOnNode.get( node ); + } + + public Set getNodes() { + return dataOnNode.keySet(); + } + +} diff --git a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java new file mode 100644 index 00000000..a154ce8d --- /dev/null +++ b/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java @@ -0,0 +1,24 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.taskinputs.SymlinkInput; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +public class NodeTaskLocalFilesAlignment extends NodeTaskAlignment { + + + public final List symlinks; + public final List locationWrappers; + + public NodeTaskLocalFilesAlignment( NodeWithAlloc node, Task task, List symlinks, List locationWrappers ) { + super(node, task); + this.symlinks = symlinks; + this.locationWrappers = locationWrappers; + } + +} From 8ae21a8f86e473e2ed6fa924171afae47388f9ba Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 23 Dec 2022 12:38:46 +0100 Subject: [PATCH 306/443] First version of separate copy tasks (LAv2) Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 21 +- src/main/java/fonda/scheduler/model/Task.java | 6 +- .../scheduler/model/WriteConfigResult.java | 5 +- .../scheduler/LocationAwareScheduler.java | 96 +++- .../scheduler/LocationAwareSchedulerV2.java | 467 ++++++++++++++++++ .../LocationAwareSchedulerV2Simple.java | 61 +++ .../scheduler/RandomLAScheduler.java | 8 +- .../fonda/scheduler/scheduler/Scheduler.java | 31 +- .../scheduler/SchedulerWithDaemonSet.java | 192 +++---- .../scheduler/TaskprocessingThread.java | 12 +- .../filealignment/GreedyAlignment.java | 5 +- .../filealignment/InputAlignment.java | 9 +- .../filealignment/InputAlignmentClass.java | 24 +- .../scheduler/util/AlignmentWrapper.java | 2 + .../java/fonda/scheduler/util/CopyTask.java | 40 ++ .../fonda/scheduler/util/FileAlignment.java | 15 + .../java/fonda/scheduler/util/FilePath.java | 5 + .../fonda/scheduler/util/LogCopyTask.java | 50 ++ .../fonda/scheduler/util/MyExecListner.java | 15 + src/main/java/fonda/scheduler/util/Tuple.java | 2 + .../scheduler/util/copying/CopySource.java | 15 + .../util/copying/CurrentlyCopying.java | 48 ++ .../util/copying/CurrentlyCopyingOnNode.java | 65 +++ 23 files changed, 1024 insertions(+), 170 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java create mode 100644 src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java create mode 100644 src/main/java/fonda/scheduler/util/CopyTask.java create mode 100644 src/main/java/fonda/scheduler/util/LogCopyTask.java create mode 100644 src/main/java/fonda/scheduler/util/MyExecListner.java create mode 100644 src/main/java/fonda/scheduler/util/copying/CopySource.java create mode 100644 src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java create mode 100644 src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index b1b47c71..22b00dc3 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -2,13 +2,17 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.scheduler.Scheduler; +import fonda.scheduler.scheduler.LocationAwareSchedulerV2; +import fonda.scheduler.util.MyExecListner; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.dsl.ExecListener; +import io.fabric8.kubernetes.client.dsl.ExecWatch; import lombok.extern.slf4j.Slf4j; +import java.io.ByteArrayOutputStream; import java.math.BigDecimal; import java.util.*; @@ -127,6 +131,21 @@ public void assignPodToNodeAndRemoveInit( PodWithAge pod, String node ) { createPod( pod ); } + public void execCommand( String podName, String namespace, String[] command, MyExecListner listener ){ + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream error = new ByteArrayOutputStream(); + final ExecWatch exec = this.pods() + .inNamespace( namespace ) + .withName( podName ) + .writingOutput( out ) + .writingError( error ) + .usingListener( listener ) + .exec( command ); + listener.setExec( exec ); + listener.setError( error ); + listener.setOut( out ); + } + static class NodeWatcher implements Watcher{ diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index d13238ae..b6ec6c75 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -2,18 +2,16 @@ import fonda.scheduler.dag.DAG; import fonda.scheduler.dag.Process; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.util.Batch; -import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @Slf4j @@ -53,7 +51,7 @@ public class Task { @Getter @Setter - private Map< String, Tuple> copyingToNode; + private CurrentlyCopyingOnNode copyingToNode; @Getter private final TraceRecord traceRecord = new TraceRecord(); diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java index 47e22d21..ee69cb5f 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -2,6 +2,7 @@ import fonda.scheduler.model.location.Location; import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; import java.util.List; @@ -12,7 +13,7 @@ public class WriteConfigResult { private final List inputFiles; private final Map waitForTask; - private final Map< String, Tuple> copyingToNode; + private final CurrentlyCopyingOnNode copyingToNode; private boolean copyDataToNode; @@ -21,7 +22,7 @@ public class WriteConfigResult { public WriteConfigResult( List inputFiles, Map waitForTask, - Map> copyingToNode, + CurrentlyCopyingOnNode copyingToNode, boolean wroteConfig, boolean copyDataToNode ) { diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 8c8a83ff..b230c94f 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -4,6 +4,7 @@ import fonda.scheduler.model.*; import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.scheduler.data.NodeDataTuple; @@ -11,6 +12,8 @@ import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import fonda.scheduler.util.*; +import fonda.scheduler.util.copying.CurrentlyCopying; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -54,7 +57,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( final TaskData taskData, final Map availableByNode, final Map assignedPodsByNode, - Map< Location, Map>> planedToCopy, + CurrentlyCopying planedToCopy, int index ) { long startTime = System.nanoTime(); @@ -80,22 +83,6 @@ private NodeTaskFilesAlignment createNodeAlignment ( return new NodeTaskFilesAlignment(result.getA(), task, result.getB()); } - private void addAlignmentToPlanned ( - Map< Location, Map< String, Tuple> > planedToCopy, - final Map nodeFileAlignment, - Task task, - NodeWithAlloc node - ) { - for (Map.Entry entry : nodeFileAlignment.entrySet()) { - final Map> map = planedToCopy.computeIfAbsent(node.getNodeLocation(), k -> new HashMap<>()); - for (FilePath filePath : entry.getValue().getFilesToCopy()) { - if ( entry.getKey() != node.getNodeLocation() ) { - map.put(filePath.getPath(), new Tuple<>(task, entry.getKey())); - } - } - } - } - /** * Align all tasks to the best node * @param unscheduledTasksSorted @@ -108,7 +95,7 @@ List createAlignment( ){ int index = 0; final List alignment = new LinkedList<>(); - final Map< Location, Map< String, Tuple> > planedToCopy = new HashMap<>(); + final CurrentlyCopying planedToCopy = new CurrentlyCopying(); final Map assignedPodsByNode = new HashMap<>(); Map taskCountWhichCopyToNode = new HashMap<>(); while( !unscheduledTasksSorted.isEmpty() ){ @@ -270,7 +257,7 @@ protected boolean stalemate( Tuple calculateBestNode( final TaskData taskData, - Map< Location, Map>> planedToCopy, + CurrentlyCopying planedToCopy, Map availableByNode, Map assignedPodsByNode ) { @@ -285,8 +272,8 @@ Tuple calculateBestNode( for (NodeDataTuple nodeDataTuple : nodeDataTuples) { final NodeWithAlloc currentNode = nodeDataTuple.getNode(); - final Map> currentlyCopying = getCopyingToNode().get(currentNode.getNodeLocation()); - final Map> currentlyPlanedToCopy = planedToCopy.get(currentNode.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyCopying = getCurrentlyCopying().get(currentNode.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(currentNode.getNodeLocation()); FileAlignment fileAlignment = null; try { fileAlignment = inputAlignment.getInputAlignment( @@ -412,4 +399,71 @@ TaskData calculateTaskData( return taskData; } + @Override + boolean assignTaskToNode( NodeTaskAlignment alignment ) { + final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; + final WriteConfigResult writeConfigResult = writeInitConfig(nodeTaskFilesAlignment); + if ( writeConfigResult == null ) { + return false; + } + if ( traceEnabled ) { + traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); + } + nodeTaskFilesAlignment.setRemoveInit( !writeConfigResult.isWroteConfig() ); + alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); + addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); + alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); + if ( writeConfigResult.isWroteConfig() ) { + getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); + } + alignment.task.setCopiesDataToNode( writeConfigResult.isCopyDataToNode() ); + final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); + alignment.task.setInputFiles( allLocationWrappers ); + useLocations( allLocationWrappers ); + return super.assignTaskToNode( alignment ); + } + + private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult writeConfigResult ) { + final TraceRecord traceRecord = alignment.task.getTraceRecord(); + + int filesOnNodeOtherTask = 0; + int filesNotOnNode = 0; + long filesOnNodeOtherTaskByte = 0; + long filesNotOnNodeByte = 0; + + final NodeLocation currentNode = alignment.node.getNodeLocation(); + for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { + final AlignmentWrapper alignmentWrapper = entry.getValue(); + if( entry.getKey() == currentNode) { + traceRecord.setSchedulerFilesNode( alignmentWrapper.getFilesToCopy().size() + alignmentWrapper.getWaitFor().size() ); + traceRecord.setSchedulerFilesNodeBytes( alignmentWrapper.getToCopySize() + alignmentWrapper.getToWaitSize() ); + } else { + filesOnNodeOtherTask += alignmentWrapper.getWaitFor().size(); + filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize(); + filesNotOnNodeByte += alignmentWrapper.getToCopySize(); + filesNotOnNode += alignmentWrapper.getFilesToCopy().size(); + } + } + if (traceRecord.getSchedulerFilesNode() == null) { + traceRecord.setSchedulerFilesNode(0); + } + if (traceRecord.getSchedulerFilesNodeBytes() == null) { + traceRecord.setSchedulerFilesNodeBytes(0l); + } + traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); + traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); + final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); + traceRecord.setSchedulerFiles(schedulerFilesNode + filesOnNodeOtherTask + filesNotOnNode); + final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); + traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); + traceRecord.setSchedulerDependingTask( (int) writeConfigResult.getWaitForTask().values().stream().distinct().count() ); + traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.getNodeFileAlignment().size() - (schedulerFilesNode > 0 ? 1 : 0) ); + } + + @Override + public boolean canSchedulePodOnNode( Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { + return this.getDaemonIpOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); + } + + } diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java new file mode 100644 index 00000000..27bf5d88 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -0,0 +1,467 @@ +package fonda.scheduler.scheduler; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.*; +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.model.location.hierachy.NoAlignmentFoundException; +import fonda.scheduler.model.taskinputs.SymlinkInput; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.data.TaskInputsNodes; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; +import fonda.scheduler.scheduler.schedulingstrategy.Inputs; +import fonda.scheduler.util.*; +import fonda.scheduler.util.copying.CurrentlyCopying; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import io.fabric8.kubernetes.api.model.Status; +import io.fabric8.kubernetes.client.dsl.ExecListener; +import io.fabric8.kubernetes.client.dsl.ExecWatch; +import lombok.*; +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public abstract class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { + + @Getter(AccessLevel.PACKAGE) + private final InputAlignment inputAlignment; + @Getter(AccessLevel.PACKAGE) + private final int maxCopyTasksPerNode; + @Getter(AccessLevel.PACKAGE) + private final int maxWaitingCopyTasksPerNode; + + @Getter(AccessLevel.PACKAGE) + private final CopyToNodeManager copyToNodeManager = new CopyToNodeManager(); + + private final LogCopyTask logCopyTask = new LogCopyTask(); + + /** + * This lock is used to syncronize the creation of the copy tasks and the finishing. + * Otherwise, it could happen that: + * 1. Copy tasks checks for data already on the node + * 2. Copy tasks finished, data is removed from currently copying and added to onNode + * 3. Task checks if data is currently copied -> no -> create a new task + */ + private final Object copyLock = new Object(); + + public LocationAwareSchedulerV2( + String name, + KubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment) { + super( name, client, namespace, config ); + this.inputAlignment = inputAlignment; + this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; + this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; + } + + protected abstract List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ); + + @Override + public ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ){ + final List tasksAndData = unscheduledTasks + .parallelStream() + .map( task -> { + final TaskInputs inputsOfTask = extractInputsOfData( task ); + if ( inputsOfTask == null ) return null; + //all nodes that contain all files + final List nodesWithAllData = availableByNode + .keySet() + .stream() + .filter( node -> { + final CurrentlyCopyingOnNode copyingFilesToNode = getCurrentlyCopying().get( node.getNodeLocation() ); + //File version does not match and is in use + return !inputsOfTask.getExcludedNodes().contains( node.getNodeLocation() ) + //Affinities are correct and the node can run new pods + && canSchedulePodOnNode( task.getPod(), node ) + //All files are on the node and no copy task is overwriting them + && inputsOfTask.allFilesAreOnLocationAndNotOverwritten( node.getNodeLocation(), copyingFilesToNode.getAllFilesCurrentlyCopying() ); + } ) + .collect( Collectors.toList() ); + return new TaskInputsNodes( task, nodesWithAllData, inputsOfTask ); + } ) + .filter( Objects::nonNull ) + .collect( Collectors.toList() ); + + final List taskWithAllData = tasksAndData + .stream() + .filter( td -> !td.getNodesWithAllData().isEmpty() ) + .collect( Collectors.toList() ); + final List alignment = createAlignmentForTasksWithAllDataOnNode(taskWithAllData, availableByNode); + final ScheduleObject scheduleObject = new ScheduleObject( (List) alignment ); + scheduleObject.setCheckStillPossible( true ); + scheduleObject.setStopSubmitIfOneFails( true ); + return scheduleObject; + } + + @Override + void postScheduling( List unscheduledTasks ) { + final List allNodes = client.getAllNodes(); + final List nodeTaskFilesAlignments; + synchronized ( copyLock ) { + final List tasksAndData = unscheduledTasks + .parallelStream() + .map( task -> { + final TaskInputs inputsOfTask = extractInputsOfData( task ); + if ( inputsOfTask == null ) return null; + final DataOnNode dataOnNode = getDataOnNode( task, inputsOfTask, allNodes ); + return dataOnNode; + } ) + .filter( Objects::nonNull ) + .collect( Collectors.toList() ); + nodeTaskFilesAlignments = createCopyTasks( tasksAndData ); + } + nodeTaskFilesAlignments.parallelStream().forEach( this::startCopyTasks ); + } + + private void startCopyTasks( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + final CopyTask copyTask = initializeCopyTask( nodeTaskFilesAlignment ); + String[] command = new String[3]; + command[0] = "/bin/bash"; + command[1] = "-c"; + command[2] = "cd " + nodeTaskFilesAlignment.task.getWorkingDir() + " && "; + final String json; + try { + json = new ObjectMapper() + .writeValueAsString( copyTask.getInputs() ) + .replace( "\"", "\\\"" ); + } catch ( JsonProcessingException e ) { + throw new RuntimeException( e ); + } + command[2] += "/code/ftp.py false " + nodeTaskFilesAlignment.node.getName() + " \"" + json + "\""; + String name = nodeTaskFilesAlignment.task.getConfig().getName() + "-copy-" + nodeTaskFilesAlignment.node.getName(); + log.info( "Starting {} to node {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); + String files = ""; + for (TaskInputFileLocationWrapper t : copyTask.getInputFiles()) { + files += t.getPath() + ", "; + } + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start (" + files + ")" ); + client.execCommand( getDaemonNameOnNode( copyTask.getNodeLocation().getIdentifier() ), getNamespace(), command, new LaListener( copyTask, name, nodeTaskFilesAlignment ) ); + } + + @RequiredArgsConstructor + private class LaListener implements MyExecListner { + + @Setter + private ExecWatch exec; + private final CopyTask copyTask; + private final String name; + @Setter + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + @Setter + private ByteArrayOutputStream error = new ByteArrayOutputStream(); + private boolean finished = false; + + private final NodeTaskFilesAlignment nodeTaskFilesAlignment; + + @Override + public void onClose( int exitCode, String reason ) { + if ( !finished ) { + log.error( "Copy task was not finished, but closed. ExitCode: " + exitCode + " Reason: " + reason ); + copyTaskFinished( copyTask, exitCode == 0 ); + } + informResourceChange(); + } + + @Override + public void onFailure( Throwable t, Response failureResponse ) { + log.info( name + " failed, output: ", t ); + log.info( name + " Exec Output: {} ", out ); + log.info( name + " Exec Error Output: {} ", error ); + exec.close(); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "failed" ); + } + + @Override + public void onExit( int exitCode, Status reason ) { + finished = true; + log.info( name + " was finished exitCode = {}, reason = {}", exitCode, reason ); + log.debug( name + " Exec Output: {} ", out ); + log.debug( name + " Exec Error Output: {} ", error ); + copyTaskFinished( copyTask, exitCode == 0 ); + exec.close(); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "finished(" + exitCode + ")" ); + } + } + + /** + * Creates config and reserves files + * + * @param nodeTaskFilesAlignment + * @return + */ + CopyTask initializeCopyTask( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + final CopyTask copyTask = createCopyTask(nodeTaskFilesAlignment); + copyTask.setNodeLocation( nodeTaskFilesAlignment.node.getNodeLocation() ); + final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); + copyTask.setAllLocationWrapper( allLocationWrappers ); + + log.info( "addToCopyingToNode task: {}, node: {}, data: {}, currentlyCopying: {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getFilesForCurrentNode(), getCurrentlyCopying().get( nodeTaskFilesAlignment.node.getNodeLocation() ) ); + //Files that will be copied + addToCopyingToNode( nodeTaskFilesAlignment.node.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + useLocations( allLocationWrappers ); + return copyTask; + } + + private void copyTaskFinished( CopyTask copyTask, boolean success ) { + if( success ){ + synchronized ( copyLock ) { + copyTask.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); + removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + } + freeLocations( copyTask.getAllLocationWrapper() ); + } else { + synchronized ( copyLock ) { + removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + handleProblematicInit( copyTask ); + } + } + } + + private void handleProblematicInit( CopyTask copyTask ){ + String file = this.localWorkDir + "/sync/" + copyTask.getInputs().execution; + try { + Map wrapperByPath = new HashMap<>(); + copyTask.getInputFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); + log.info( "Get daemon on node {}; daemons: {}", copyTask.getNodeLocation().getIdentifier(), daemonHolder ); + final InputStream inputStream = getConnection( getDaemonIpOnNode(copyTask.getNodeLocation().getIdentifier())).retrieveFileStream(file); + if (inputStream == null) { + //Init has not even started + return; + } + Scanner scanner = new Scanner(inputStream); + Set openedFiles = new HashSet<>(); + while( scanner.hasNext() ){ + String line = scanner.nextLine(); + if ( line.startsWith( "S-" ) ){ + openedFiles.add( line.substring( 2 ) ); + } else if ( line.startsWith( "F-" ) ){ + openedFiles.remove( line.substring( 2 ) ); + wrapperByPath.get( line.substring( 2 ) ).success(); + log.info("task {}, file: {} success", copyTask.getInputs().execution, line); + } + } + for ( String openedFile : openedFiles ) { + wrapperByPath.get( openedFile ).failure(); + log.info("task {}, file: {} deactivated on node {}", copyTask.getInputs().execution, openedFile, wrapperByPath.get( openedFile ).getWrapper().getLocation()); + } + } catch ( Exception e ){ + log.error( "Can't handle failed init from pod " + copyTask.getInputs().execution, e); + } + } + + + /** + * @param alignment + * @return null if the task cannot be scheduled + */ + CopyTask createCopyTask( NodeTaskFilesAlignment alignment ) { + LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); + final CurrentlyCopyingOnNode filesForCurrentNode = new CurrentlyCopyingOnNode(); + final NodeLocation currentNode = alignment.node.getNodeLocation(); + final Inputs inputs = new Inputs( + this.getDns(), + getExecution(), + this.localWorkDir + "/sync/", + alignment.task.getConfig().getRunName() + ); + + for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { + if( entry.getKey() == currentNode ) { + continue; + } + + final NodeLocation location = (NodeLocation) entry.getKey(); + final AlignmentWrapper alignmentWrapper = entry.getValue(); + + final List collect = new LinkedList<>(); + for (FilePath filePath : alignmentWrapper.getFilesToCopy()) { + final LocationWrapper locationWrapper = filePath.getFile().getLocationWrapper(location); + inputFiles.add( + new TaskInputFileLocationWrapper( + filePath.getPath(), + filePath.getFile(), + locationWrapper.getCopyOf( currentNode ) + ) + ); + collect.add(filePath.getPath()); + filesForCurrentNode.add( filePath.getPath(), alignment.task, location ); + } + if( !collect.isEmpty() ) { + inputs.data.add(new InputEntry( getDaemonIpOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, alignmentWrapper.getToCopySize())); + } + } + + inputs.sortData(); + return new CopyTask( inputs, inputFiles, filesForCurrentNode); + } + + List createCopyTasks( List tasksAndData ){ + final CurrentlyCopying planedToCopy = new CurrentlyCopying(); + tasksAndData.sort( Comparator.comparing( DataOnNode::getNodesWithAllData ).reversed() ); + final Map assignedPodsByNode = copyToNodeManager.getCurrentlyCopyingTasksOnNode(); + List nodeTaskAlignments = new LinkedList<>(); + for ( DataOnNode dataOnNode : tasksAndData ) { + final Tuple result = calculateBestNode( dataOnNode, planedToCopy ); + if ( result != null && assignedPodsByNode.getOrDefault( result.getA(), 0 ) < getMaxCopyTasksPerNode() ) { + addAlignmentToPlanned( planedToCopy, result.getB().getNodeFileAlignment(), dataOnNode.getTask(), result.getA() ); + nodeTaskAlignments.add( new NodeTaskFilesAlignment( result.getA(), dataOnNode.getTask(), result.getB() ) ); + assignedPodsByNode.put( result.getA(), assignedPodsByNode.getOrDefault( result.getA(), 0 ) + 1 ); + } + } + return nodeTaskAlignments; + } + + Tuple calculateBestNode( final DataOnNode taskData, CurrentlyCopying planedToCopy ) { + FileAlignment bestAlignment = null; + NodeWithAlloc bestNode = null; + final Set matchingNodes = taskData.getNodes(); + int triedNodes = 0; + int noAlignmentFound = 0; + int couldStopFetching = 0; + final List costs = traceEnabled ? new LinkedList<>() : null; + for ( final NodeWithAlloc node : matchingNodes) { + + final CurrentlyCopyingOnNode currentlyCopying = getCurrentlyCopying().get(node.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(node.getNodeLocation()); + FileAlignment fileAlignment = null; + try { + fileAlignment = inputAlignment.getInputAlignment( + taskData.getTask(), + taskData.getInputsOfTask(), + node, + currentlyCopying, + currentlyPlanedToCopy, + bestAlignment == null ? Double.MAX_VALUE : bestAlignment.getCost() + ); + + if ( fileAlignment == null ){ + couldStopFetching++; + } else { + log.info( "Task: {}, node: {}, bestWeight: {}, currentWeight: {}", + taskData.getTask().getConfig().getName(), + node.getNodeLocation(), + bestAlignment == null ? null : bestAlignment.getWorth(), + fileAlignment.getWorth() + ); + + if ( + //Anything to copy? + fileAlignment.copyFromSomewhere( node.getNodeLocation() ) && + //Not set or better than current best + ( bestAlignment == null || bestAlignment.getWorth() > fileAlignment.getWorth() ) ) { + bestAlignment = fileAlignment; + bestNode = node; + log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getRunName(), fileAlignment.getCost() ); + } + } + } catch ( NoAligmentPossibleException e ){ + noAlignmentFound++; + log.info( "Task: {} - {}", taskData.getTask().getConfig().getName() , e.getMessage() ); + } + if ( traceEnabled ) { + triedNodes++; + final Double thisRoundCost = fileAlignment == null + ? null + : fileAlignment.getCost(); + costs.add( thisRoundCost ); + } + } + + if ( bestAlignment == null ) { + return null; + } + + logCopyTask.log( taskData.getTask().getConfig().getName() + " " + bestAlignment ); + + return new Tuple<>( bestNode, bestAlignment ); + } + + + private TaskInputs extractInputsOfData( Task task ) { + if ( task == null ) return null; + final TaskInputs inputsOfTask; + try { + inputsOfTask = getInputsOfTask( task ); + } catch ( NoAlignmentFoundException e ) { + return null; + } + if ( inputsOfTask == null ) { + log.info( "No node where the pod can start, pod: {}", task.getConfig().getRunName() ); + return null; + } + return inputsOfTask; + } + + + private DataOnNode getDataOnNode( Task task, TaskInputs inputsOfTask, List allNodes ) { + final DataOnNode dataOnNode = new DataOnNode( task, inputsOfTask ); + for ( NodeWithAlloc node : allNodes ) { + final long avgSize = inputsOfTask.calculateAvgSize(); + if ( !inputsOfTask.getExcludedNodes().contains( node ) && affinitiesMatch( task.getPod(), node ) ) { + final Tuple booleanLongTuple = inputsOfTask.calculateDataOnNodeAdditionalInfo( node.getNodeLocation() ); + if ( booleanLongTuple.getA() ) { + dataOnNode.allData( node ); + } else if ( booleanLongTuple.getB() >= avgSize ) { + //As we only calculate the average size of all files, it can happen that more than 100% are on the node + dataOnNode.addData( node, 0.999 ); + } else { + dataOnNode.addData( node, booleanLongTuple.getB() / avgSize ); + } + } + } + return dataOnNode; + } + + @Override + boolean assignTaskToNode( NodeTaskAlignment alignment ) { + final NodeTaskLocalFilesAlignment nodeTaskFilesAlignment = (NodeTaskLocalFilesAlignment) alignment; + + if ( !nodeTaskFilesAlignment.symlinks.isEmpty() ) { + try ( BufferedWriter writer = new BufferedWriter( new FileWriter( alignment.task.getWorkingDir() + '/' + ".command.symlinks" ) ) ) { + writer.write( "create_symlink() {" ); + writer.newLine(); + writer.write( " rm -rf \"$2\"" ); + writer.newLine(); + writer.write( " mkdir -p \"$(dirname \"$2\")\"" ); + writer.newLine(); + writer.write( " ln -s \"$1\" \"$2\"" ); + writer.newLine(); + writer.write( "}" ); + writer.newLine(); + for ( SymlinkInput symlink : nodeTaskFilesAlignment.symlinks ) { + writer.write( "create_symlink \"" + symlink.getDst().replace( "\"", "\\\"" ) + "\" \"" + symlink.getSrc().replace( "\"", "\\\"" ) + "\"" ); + writer.newLine(); + } + } catch ( IOException ex ) { + ex.printStackTrace(); + } + } + + alignment.task.setInputFiles( nodeTaskFilesAlignment.locationWrappers ); + useLocations( nodeTaskFilesAlignment.locationWrappers ); + + return super.assignTaskToNode( alignment ); + } + + @Override + public void close() { + logCopyTask.close(); + super.close(); + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java new file mode 100644 index 00000000..0e405ed6 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java @@ -0,0 +1,61 @@ +package fonda.scheduler.scheduler; + +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.SchedulerConfig; +import fonda.scheduler.scheduler.LocationAwareSchedulerV2; +import fonda.scheduler.scheduler.data.TaskInputsNodes; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.NodeTaskLocalFilesAlignment; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class LocationAwareSchedulerV2Simple extends LocationAwareSchedulerV2 { + + public LocationAwareSchedulerV2Simple( String name, KubernetesClient client, String namespace, SchedulerConfig config, InputAlignment inputAlignment ) { + super( name, client, namespace, config, inputAlignment ); + } + + /** + * Select the first node that fits the requirements. + * @param taskWithAllData + * @param availableByNode + * @return + */ + @Override + protected List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ) { + final List alignment = new LinkedList<>(); + final Iterator iterator = taskWithAllData.iterator(); + while( iterator.hasNext() ) { + final TaskInputsNodes taskInputsNodes = iterator.next(); + final Requirements taskRequest = taskInputsNodes.getTask().getPod().getRequest(); + for ( NodeWithAlloc nodeWithAll : taskInputsNodes.getNodesWithAllData() ) { + final Requirements availableOnNode = availableByNode.get( nodeWithAll ); + if ( availableOnNode != null && + availableOnNode.higherOrEquals( taskRequest ) ) { + alignment.add( + new NodeTaskLocalFilesAlignment( + nodeWithAll, + taskInputsNodes.getTask(), + taskInputsNodes.getInputsOfTask().getSymlinks(), + taskInputsNodes.getInputsOfTask().allLocationWrapperOnLocation( nodeWithAll.getNodeLocation() ) + ) + ); + availableOnNode.subFromThis( taskRequest ); + iterator.remove(); + break; + } + } + } + return alignment; + } +} diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index b8ce3135..2d9ed7c3 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -12,6 +12,8 @@ import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CurrentlyCopying; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.extern.slf4j.Slf4j; import java.util.*; @@ -42,7 +44,7 @@ private Optional selectNode( List matchingNodes, T @Override Tuple calculateBestNode( final TaskData taskData, - Map< Location, Map>> planedToCopy, + CurrentlyCopying planedToCopy, Map availableByNode, Map assignedPodsByNode ){ @@ -51,8 +53,8 @@ Tuple calculateBestNode( return null; } final NodeWithAlloc node = nodeWithAlloc.get(); - final Map> currentlyCopying = getCopyingToNode().get(node.getNodeLocation()); - final Map> currentlyPlanetToCopy = planedToCopy.get(node.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyCopying = getCurrentlyCopying().get(node.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyPlanetToCopy = planedToCopy.get(node.getNodeLocation()); final FileAlignment fileAlignment = getInputAlignment().getInputAlignment( taskData.getTask(), taskData.getMatchingFilesAndNodes().getInputsOfTask(), diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 974e27c3..cdc88ced 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -86,6 +86,7 @@ public abstract class Scheduler implements Informable { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { + final LinkedList unscheduledTasksCopy = new LinkedList<>( unscheduledTasks ); long startSchedule = System.currentTimeMillis(); if( traceEnabled ) { unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule( startSchedule ) ); @@ -98,6 +99,7 @@ public int schedule( final List unscheduledTasks ) { boolean possible = validSchedulePlan ( taskNodeAlignment ); if (!possible) { log.info("The whole scheduling plan is not possible anymore."); + informOtherResourceChange(); return taskNodeAlignment.size(); } } @@ -126,11 +128,20 @@ public int schedule( final List unscheduledTasks ) { continue; } taskWasScheduled(nodeTaskAlignment.task); + unscheduledTasksCopy.remove( nodeTaskAlignment.task ); scheduled++; } + //Use instance object that does not contain yet scheduled tasks + postScheduling( unscheduledTasksCopy ); return unscheduledTasks.size() - taskNodeAlignment.size() + failure; } + /** + * This method is called when a SchedulePlan was successfully executed. + * @param unscheduledTasks + */ + void postScheduling( final List unscheduledTasks ) {} + /** * Call this method in case of any scheduling problems * @param task @@ -143,12 +154,14 @@ public boolean validSchedulePlan( List taskNodeAlignment ){ for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { final Requirements requirements = availableByNode.get(nodeTaskAlignment.node); if ( requirements == null ) { + log.info( "Node {} is not available anymore", nodeTaskAlignment.node.getMetadata().getName() ); return false; } requirements.subFromThis(nodeTaskAlignment.task.getPod().getRequest()); } for ( Map.Entry e : availableByNode.entrySet() ) { if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) { + log.info( "Node {} has not enough resources. Available: {}, After assignment it would be: {}", e.getKey().getMetadata().getName(), e.getKey().getAvailableResources(), e.getValue() ); return false; } } @@ -323,9 +336,11 @@ public boolean canSchedulePodOnNode(Requirements availableByNode, PodWithAge pod if ( availableByNode == null ) { return false; } - return node.canScheduleNewPod() - && availableByNode.higherOrEquals( pod.getRequest() ) - && affinitiesMatch( pod, node ); + return canSchedulePodOnNode( pod, node ) && availableByNode.higherOrEquals( pod.getRequest() ); + } + + public boolean canSchedulePodOnNode(PodWithAge pod, NodeWithAlloc node ) { + return node.canScheduleNewPod() && affinitiesMatch( pod, node ); } boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ @@ -374,7 +389,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ printWriter.write( alignment.node.getName() ); printWriter.write( '\n' ); } catch (IOException e) { - log.error( "Cannot read " + nodeFile, e); + log.error( "Cannot write " + nodeFile, e); } alignment.task.setNode( alignment.node ); @@ -440,11 +455,19 @@ Task changeStateOfTask(Pod pod, State state){ * starts the scheduling routine */ public void informResourceChange() { + informOtherResourceChange(); synchronized (unscheduledTasks){ unscheduledTasks.notifyAll(); } } + /** + * Call if something was changed withing the scheduling loop. In all other cases, call informResourceChange() + */ + private void informOtherResourceChange() { + schedulingThread.otherResourceChange(); + } + Task getTaskByPod( Pod pod ) { Task t = null; synchronized ( tasksByPodName ) { diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index ba2da6eb..ad050838 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -12,7 +12,6 @@ import fonda.scheduler.model.outfiles.SymlinkOutput; import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.model.tracing.TraceRecord; import fonda.scheduler.rest.exceptions.NotARealFileException; import fonda.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.copystrategy.CopyStrategy; @@ -22,6 +21,8 @@ import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.*; +import fonda.scheduler.util.copying.CurrentlyCopying; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.Watcher; import lombok.AccessLevel; @@ -39,7 +40,8 @@ @Slf4j public abstract class SchedulerWithDaemonSet extends Scheduler { - private final Map daemonByNode = new HashMap<>(); + @Getter(AccessLevel.PROTECTED) + final DaemonHolder daemonHolder = new DaemonHolder(); @Getter private String workflowEngineNode = null; @Getter @@ -47,11 +49,14 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { final HierarchyWrapper hierarchyWrapper; private final InputFileCollector inputFileCollector; private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); - private final String localWorkDir; + final String localWorkDir; protected final OutLabelHolder outLabelHolder = new HolderMaxTasks() ; + /** + * Which node is currently copying files from which node + */ @Getter(AccessLevel.PACKAGE) - private final Map< NodeLocation, Map< String, Tuple> > copyingToNode = new HashMap<>(); + private final CurrentlyCopying currentlyCopying = new CurrentlyCopying(); SchedulerWithDaemonSet(String execution, KubernetesClient client, String namespace, SchedulerConfig config) { super(execution, client, namespace, config); @@ -71,51 +76,37 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { this.localWorkDir = config.workDir; } - public String getDaemonOnNode( String node ){ - synchronized ( daemonByNode ) { - return daemonByNode.get( node ); - } + public String getDaemonIpOnNode( String node ){ + return daemonHolder.getDaemonIp( node ); } - String getDaemonOnNode( Node node ){ - return getDaemonOnNode( node.getMetadata().getName() ); + public String getDaemonNameOnNode( String node ){ + return daemonHolder.getDaemonName( node ); } - private void useLocations( List locationWrappers ){ - locationWrappers.parallelStream().forEach( LocationWrapper::use ); + String getDaemonIpOnNode( Node node ){ + return getDaemonIpOnNode( node.getMetadata().getName() ); } - private void freeLocations( List locationWrappers ){ - locationWrappers.parallelStream().forEach( LocationWrapper::free ); + /** + * Mark all locationWrappers as used + * @param locationWrappers + */ + void useLocations( List locationWrappers ){ + locationWrappers.parallelStream().forEach( LocationWrapper::use ); } - @Override - boolean assignTaskToNode( NodeTaskAlignment alignment ) { - final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; - final WriteConfigResult writeConfigResult = writeInitConfig(nodeTaskFilesAlignment); - if ( writeConfigResult == null ) { - return false; - } - if ( traceEnabled ) { - traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); - } - nodeTaskFilesAlignment.setRemoveInit( !writeConfigResult.isWroteConfig() ); - alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); - addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); - alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); - if ( writeConfigResult.isWroteConfig() ) { - getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); - } - alignment.task.setCopiesDataToNode( writeConfigResult.isCopyDataToNode() ); - final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); - alignment.task.setInputFiles( allLocationWrappers ); - useLocations( allLocationWrappers ); - return super.assignTaskToNode( alignment ); + /** + * Mark all locationWrappers as unused + * @param locationWrappers + */ + void freeLocations( List locationWrappers ){ + locationWrappers.parallelStream().forEach( LocationWrapper::free ); } @Override void assignPodToNode( PodWithAge pod, NodeTaskAlignment alignment ) { - if ( ((NodeTaskFilesAlignment) alignment).isRemoveInit() ) { + if ( !pod.getSpec().getInitContainers().isEmpty() && ((NodeTaskFilesAlignment) alignment).isRemoveInit() ) { log.info( "Removing init container from pod {}", pod.getMetadata().getName() ); client.assignPodToNodeAndRemoveInit( pod, alignment.node.getName() ); } else { @@ -178,75 +169,28 @@ int terminateTasks(List finishedTasks) { return 0; } - private void addToCopyingToNode( NodeLocation nodeLocation, Map< String, Tuple > toAdd ){ + /** + * Register that file is copied to node + * @param nodeLocation + * @param toAdd + */ + void addToCopyingToNode( NodeLocation nodeLocation, CurrentlyCopyingOnNode toAdd ){ if ( nodeLocation == null ) { throw new IllegalArgumentException( "NodeLocation cannot be null" ); } - if ( toAdd.isEmpty() ) { - return; - } - if ( copyingToNode.containsKey( nodeLocation ) ){ - final Map> stringTupleHashMap = copyingToNode.get( nodeLocation ); - stringTupleHashMap.putAll( toAdd ); - } else { - copyingToNode.put( nodeLocation, toAdd ); - } + currentlyCopying.add( nodeLocation, toAdd ); } - private void removeFromCopyingToNode(NodeLocation nodeLocation, Map< String, Tuple> toRemove ) { + /** + * Remove that file is copied to node + * @param nodeLocation + * @param toRemove + */ + void removeFromCopyingToNode( NodeLocation nodeLocation, CurrentlyCopyingOnNode toRemove ) { if (nodeLocation == null) { throw new IllegalArgumentException("NodeLocation cannot be null"); } - if (toRemove == null || toRemove.isEmpty()) { - return; - } - final Map> pathTasks; - synchronized (copyingToNode) { - pathTasks = copyingToNode.get(nodeLocation); - } - if ( pathTasks == null ) { - return; - } - synchronized ( pathTasks ) { - pathTasks.keySet().removeAll(toRemove.keySet()); - } - } - - private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult writeConfigResult ) { - final TraceRecord traceRecord = alignment.task.getTraceRecord(); - - int filesOnNodeOtherTask = 0; - int filesNotOnNode = 0; - long filesOnNodeOtherTaskByte = 0; - long filesNotOnNodeByte = 0; - - final NodeLocation currentNode = alignment.node.getNodeLocation(); - for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { - final AlignmentWrapper alignmentWrapper = entry.getValue(); - if( entry.getKey() == currentNode) { - traceRecord.setSchedulerFilesNode( alignmentWrapper.getFilesToCopy().size() + alignmentWrapper.getWaitFor().size() ); - traceRecord.setSchedulerFilesNodeBytes( alignmentWrapper.getToCopySize() + alignmentWrapper.getToWaitSize() ); - } else { - filesOnNodeOtherTask += alignmentWrapper.getWaitFor().size(); - filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize(); - filesNotOnNodeByte += alignmentWrapper.getToCopySize(); - filesNotOnNode += alignmentWrapper.getFilesToCopy().size(); - } - } - if (traceRecord.getSchedulerFilesNode() == null) { - traceRecord.setSchedulerFilesNode(0); - } - if (traceRecord.getSchedulerFilesNodeBytes() == null) { - traceRecord.setSchedulerFilesNodeBytes(0l); - } - traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); - traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); - final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); - traceRecord.setSchedulerFiles(schedulerFilesNode + filesOnNodeOtherTask + filesNotOnNode); - final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); - traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); - traceRecord.setSchedulerDependingTask( (int) writeConfigResult.getWaitForTask().values().stream().distinct().count() ); - traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.getNodeFileAlignment().size() - (schedulerFilesNode > 0 ? 1 : 0) ); + currentlyCopying.remove( nodeLocation, toRemove ); } /** @@ -259,7 +203,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); Map waitForTask = new HashMap<>(); - final HashMap> filesForCurrentNode = new HashMap<>(); + final CurrentlyCopyingOnNode filesForCurrentNode = new CurrentlyCopyingOnNode(); final NodeLocation currentNode = alignment.node.getNodeLocation(); try { @@ -294,10 +238,10 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { ) ); collect.add(filePath.getPath()); - filesForCurrentNode.put( filePath.getPath(), new Tuple<>( alignment.task, location ) ); + filesForCurrentNode.add( filePath.getPath(), alignment.task, location ); } if( !collect.isEmpty() ) { - inputs.data.add(new InputEntry(getDaemonOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, alignmentWrapper.getToCopySize())); + inputs.data.add(new InputEntry( getDaemonIpOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, alignmentWrapper.getToCopySize())); } } @@ -349,7 +293,7 @@ public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileExce } requestedLocations.put( lastUpdate.getId(), lastUpdate ); String node = lastUpdate.getLocation().getIdentifier(); - return new FileResponse( currentPath.toString(), node, getDaemonOnNode(node), node.equals(workflowEngineNode), symlinks, lastUpdate.getId() ); + return new FileResponse( currentPath.toString(), node, getDaemonIpOnNode(node), node.equals(workflowEngineNode), symlinks, lastUpdate.getId() ); } MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map availableByNode ){ @@ -379,6 +323,15 @@ MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map wrapperByPath = new HashMap<>(); task.getCopiedFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); - log.info( "Get daemon on node {}; daemons: {}", task.getNode().getNodeLocation().getIdentifier(), daemonByNode ); - final InputStream inputStream = getConnection(getDaemonOnNode(task.getNode().getNodeLocation().getIdentifier())).retrieveFileStream(file); + log.info( "Get daemon on node {}; daemons: {}", task.getNode().getNodeLocation().getIdentifier(), daemonHolder ); + final InputStream inputStream = getConnection( getDaemonIpOnNode(task.getNode().getNodeLocation().getIdentifier())).retrieveFileStream(file); if (inputStream == null) { //Init has not even started return; @@ -424,7 +377,7 @@ private void handleProblematicInit( Task task ){ } } - private FTPClient getConnection( String daemon ){ + FTPClient getConnection( String daemon ){ int trial = 0; while ( true ) { try { @@ -464,11 +417,6 @@ private void podWasInitialized( Pod pod ){ task.setCopyingToNode( null ); } - @Override - public boolean canSchedulePodOnNode( Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { - return this.getDaemonOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); - } - /** * Remove all Nodes with a location contained in taskInputs.excludedNodes * @param matchingNodes @@ -485,6 +433,18 @@ void filterNotMatchingNodesForTask(Set matchingNodes, TaskInputs } } + void addAlignmentToPlanned ( CurrentlyCopying planedToCopy, final Map nodeFileAlignment, + Task task, NodeWithAlloc node ) { + for (Map.Entry entry : nodeFileAlignment.entrySet()) { + final CurrentlyCopyingOnNode map = planedToCopy.get(node.getNodeLocation()); + for (FilePath filePath : entry.getValue().getFilesToCopy()) { + if ( entry.getKey() != node.getNodeLocation() ) { + map.add( filePath.getPath(), task, entry.getKey() ); + } + } + } + } + public void taskHasFinishedCopyTask( String name ){ final Task task = tasksByPodName.get( name ); task.getNode().startingTaskCopyingDataFinished( task ); @@ -507,7 +467,7 @@ void podEventReceived(Watcher.Action action, Pod pod){ log.info( "WorkflowEngineNode was set to {}", workflowEngineNode ); } //noinspection LoopConditionNotUpdatedInsideLoop - while ( daemonByNode == null ){ + while ( daemonHolder == null ){ //The Watcher can be started before the class is initialized try { Thread.sleep(20); @@ -518,18 +478,18 @@ void podEventReceived(Watcher.Action action, Pod pod){ if( ( "mount-" + this.getExecution().replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ final String nodeName = pod.getSpec().getNodeName(); if ( nodeName != null ){ - synchronized ( daemonByNode ) { + synchronized ( daemonHolder ) { final String podName = pod.getMetadata().getName(); - final boolean podIsCurrentDaemon = podName.equals(daemonByNode.get(nodeName)); + final boolean podIsCurrentDaemon = pod.getStatus().getPodIP() != null && pod.getStatus().getPodIP().equals(daemonHolder.getDaemonIp(nodeName)); if ( action == Watcher.Action.DELETED ) { if (podIsCurrentDaemon) { - daemonByNode.remove(nodeName); + daemonHolder.removeDaemon(nodeName); } } else if ( pod.getStatus().getPhase().equals("Running") ) { - daemonByNode.put( nodeName, pod.getStatus().getPodIP() ); + daemonHolder.addDaemon( nodeName, podName, pod.getStatus().getPodIP() ); informResourceChange(); } else if ( podIsCurrentDaemon ) { - daemonByNode.remove(nodeName); + daemonHolder.removeDaemon(nodeName); if( !pod.getStatus().getPhase().equals("Failed") ){ log.info( "Unexpected phase {} for daemon: {}", pod.getStatus().getPhase(), podName ); } diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java index 96649596..7fe171f5 100644 --- a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java @@ -1,6 +1,7 @@ package fonda.scheduler.scheduler; import fonda.scheduler.model.Task; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.util.LinkedList; @@ -8,14 +9,16 @@ import java.util.function.Function; @Slf4j +@RequiredArgsConstructor public class TaskprocessingThread extends Thread { private final List unprocessedTasks; private final Function, Integer> function; - public TaskprocessingThread(List unprocessedTasks, Function, Integer> function ) { - this.unprocessedTasks = unprocessedTasks; - this.function = function; + private boolean otherResourceChange = false; + + public void otherResourceChange() { + otherResourceChange = true; } @Override @@ -26,12 +29,13 @@ public void run() { LinkedList tasks; synchronized (unprocessedTasks) { do { - if (unscheduled == unprocessedTasks.size()) { + if ( !otherResourceChange && unscheduled == unprocessedTasks.size()) { unprocessedTasks.wait( 10000 ); } if( Thread.interrupted() ) { return; } + otherResourceChange = false; } while ( unprocessedTasks.isEmpty() ); tasks = new LinkedList<>(unprocessedTasks); } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index 6f5a02bc..47160cf3 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -12,6 +12,7 @@ import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -59,8 +60,8 @@ public GreedyAlignment(CostFunction cf) { public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, - Map> currentlyCopying, - Map> currentlyPlanedToCopy, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy, double maxCost) { inputsOfTask.sort(); return super.getInputAlignment( task, inputsOfTask, node, currentlyCopying, currentlyPlanedToCopy, maxCost ); diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java index ecaf7d57..32a64245 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java @@ -6,6 +6,7 @@ import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import org.jetbrains.annotations.NotNull; import java.util.Map; @@ -23,8 +24,8 @@ public interface InputAlignment { FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, - Map> currentlyCopying, - Map> currentlyPlanedToCopy, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy, double maxCost ); /** @@ -37,8 +38,8 @@ FileAlignment getInputAlignment( @NotNull Task task, default FileAlignment getInputAlignment( @NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, - Map> currentlyCopying, - Map> currentlyPlanedToCopy + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy ){ return getInputAlignment( task, inputsOfTask, node, currentlyCopying, currentlyPlanedToCopy, Double.MAX_VALUE ); } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java index 0b0e5a9f..d2b0c123 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -12,12 +12,16 @@ import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePathWithTask; import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CopySource; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import java.util.HashMap; import java.util.List; import java.util.Map; +@Slf4j abstract class InputAlignmentClass implements InputAlignment { /** @@ -28,22 +32,24 @@ abstract class InputAlignmentClass implements InputAlignment { * @param locations * @return */ - private Tuple alreadyCopying(Map> currentlyCopying, String path, List locations, String debug ){ - if ( currentlyCopying != null && currentlyCopying.containsKey( path ) ){ - final Tuple taskLocationTuple = currentlyCopying.get(path); - final Location copyFrom = taskLocationTuple.getB(); + private Tuple alreadyCopying( CurrentlyCopyingOnNode currentlyCopying, String path, List locations, String debug ){ + if ( currentlyCopying != null && currentlyCopying.isCurrentlyCopying( path ) ){ + final CopySource copySource = currentlyCopying.getCopySource( path ); + final Location copyFrom = copySource.getLocation(); for ( LocationWrapper locationWrapper : locations ) { if ( locationWrapper.getLocation() == copyFrom ) { - return new Tuple(locationWrapper,taskLocationTuple.getA()); + log.info( "Checking if {} is already copying: {} (Found)", path, currentlyCopying.getAllFilesCurrentlyCopying() ); + return new Tuple(locationWrapper,copySource.getTask()); } } throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); } + log.info( "Checking if {} is already copying: {} (Not found)", path, currentlyCopying == null ? null : currentlyCopying.getAllFilesCurrentlyCopying() ); return null; } private boolean canUseFileFromOtherTask ( - Map> currentlyCopying, + CurrentlyCopyingOnNode currentlyCopying, PathFileLocationTriple pathFileLocationTriple, Map map, String debug @@ -66,15 +72,15 @@ private boolean canUseFileFromOtherTask ( public FileAlignment getInputAlignment(@NotNull Task task, @NotNull TaskInputs inputsOfTask, @NotNull NodeWithAlloc node, - Map> currentlyCopying, - Map> currentlyPlanedToCopy, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy, double maxCost) { final HashMap map = new HashMap<>(); double cost = 0; for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) && - ! canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) + !canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) ) { final double newCost = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); if ( newCost > maxCost ) { diff --git a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java index 22d8b16d..f50b6ec6 100644 --- a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java +++ b/src/main/java/fonda/scheduler/util/AlignmentWrapper.java @@ -1,11 +1,13 @@ package fonda.scheduler.util; import lombok.Getter; +import lombok.ToString; import java.util.LinkedList; import java.util.List; @Getter +@ToString public class AlignmentWrapper { private final List filesToCopy = new LinkedList<>(); diff --git a/src/main/java/fonda/scheduler/util/CopyTask.java b/src/main/java/fonda/scheduler/util/CopyTask.java new file mode 100644 index 00000000..faaffb11 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/CopyTask.java @@ -0,0 +1,40 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.TaskInputFileLocationWrapper; +import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.scheduler.schedulingstrategy.Inputs; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +@Getter +public class CopyTask { + + @Getter(AccessLevel.NONE) + private static AtomicLong copyTaskId = new AtomicLong(0); + private long id = copyTaskId.getAndIncrement(); + private final Inputs inputs; + private final LinkedList inputFiles; + private final CurrentlyCopyingOnNode filesForCurrentNode; + + @Setter + private List allLocationWrapper; + + @Setter + private NodeLocation nodeLocation; + + public CopyTask( Inputs inputs, LinkedList inputFiles, CurrentlyCopyingOnNode filesForCurrentNode ) { + this.inputs = inputs; + this.inputFiles = inputFiles; + this.filesForCurrentNode = filesForCurrentNode; + } +} diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 8cc68026..3daa0237 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -1,16 +1,19 @@ package fonda.scheduler.util; import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.SymlinkInput; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @Getter +@ToString public class FileAlignment { /* @@ -33,6 +36,18 @@ public double getWorth() { return cost / weight; } + /** + * Check if data is copied from at least one node + * @param node this node is not checked + * @return + */ + public boolean copyFromSomewhere( Location node ) { + return nodeFileAlignment + .entrySet() + .stream() + .anyMatch( a -> a.getKey() != node && a.getValue().getFilesToCopy().size() > 0 ); + } + public List getAllLocationWrappers(){ return nodeFileAlignment .entrySet() diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/fonda/scheduler/util/FilePath.java index e8ffe061..29ff8b19 100644 --- a/src/main/java/fonda/scheduler/util/FilePath.java +++ b/src/main/java/fonda/scheduler/util/FilePath.java @@ -25,4 +25,9 @@ public RealHierarchyFile getFile() { public LocationWrapper getLocationWrapper() { return locationWrapper; } + + @Override + public String toString() { + return getPath(); + } } diff --git a/src/main/java/fonda/scheduler/util/LogCopyTask.java b/src/main/java/fonda/scheduler/util/LogCopyTask.java new file mode 100644 index 00000000..de6827f7 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/LogCopyTask.java @@ -0,0 +1,50 @@ +package fonda.scheduler.util; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; + +public class LogCopyTask { + + private final BufferedWriter writer; + + public LogCopyTask() { + try { + this.writer = new BufferedWriter( new FileWriter( "/pvcdata/scheduler/copytasks.csv" ) ); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + + public void log( String text ) { + try { + writer.write( text ); + writer.newLine(); + writer.flush(); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + + public void copy( String task, String node, int filesToCopy, String additional ) { + String str = "\"" + task + "\";\"" + node + "\";\"" + filesToCopy+ "\";\"" + additional + "\";" + System.currentTimeMillis(); + synchronized ( writer ) { + try { + writer.write( str ); + writer.newLine(); + writer.flush(); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + } + + public void close() { + try { + writer.close(); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/util/MyExecListner.java b/src/main/java/fonda/scheduler/util/MyExecListner.java new file mode 100644 index 00000000..bfdf4f52 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/MyExecListner.java @@ -0,0 +1,15 @@ +package fonda.scheduler.util; + +import io.fabric8.kubernetes.client.dsl.ExecListener; +import io.fabric8.kubernetes.client.dsl.ExecWatch; + +import java.io.ByteArrayOutputStream; + +public interface MyExecListner extends ExecListener { + + void setExec( ExecWatch exec); + void setError( ByteArrayOutputStream error ); + void setOut( ByteArrayOutputStream out ); + + +} diff --git a/src/main/java/fonda/scheduler/util/Tuple.java b/src/main/java/fonda/scheduler/util/Tuple.java index ec67bc01..129761ac 100644 --- a/src/main/java/fonda/scheduler/util/Tuple.java +++ b/src/main/java/fonda/scheduler/util/Tuple.java @@ -1,8 +1,10 @@ package fonda.scheduler.util; import lombok.Getter; +import lombok.ToString; @Getter +@ToString public class Tuple { private final S a; diff --git a/src/main/java/fonda/scheduler/util/copying/CopySource.java b/src/main/java/fonda/scheduler/util/copying/CopySource.java new file mode 100644 index 00000000..42510b8c --- /dev/null +++ b/src/main/java/fonda/scheduler/util/copying/CopySource.java @@ -0,0 +1,15 @@ +package fonda.scheduler.util.copying; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class CopySource { + + private final Task task; + private final Location location; + +} diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java new file mode 100644 index 00000000..cacc212b --- /dev/null +++ b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java @@ -0,0 +1,48 @@ +package fonda.scheduler.util.copying; + +import fonda.scheduler.model.location.NodeLocation; +import lombok.ToString; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@ToString +public class CurrentlyCopying { + + private final Map copyingToNode = new ConcurrentHashMap<>(); + + public void add( NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { + if ( currentlyCopyingOnNode == null || currentlyCopyingOnNode.isEmpty() ) { + return; + } + copyingToNode.compute( nodeLocation, ( node, currentlyCopying ) -> { + if ( currentlyCopying == null ) { + return currentlyCopyingOnNode; + } else { + currentlyCopying.add( currentlyCopyingOnNode ); + return currentlyCopying; + } + } ); + } + + public CurrentlyCopyingOnNode get( NodeLocation nodeLocation ) { + return copyingToNode.computeIfAbsent( nodeLocation, node -> new CurrentlyCopyingOnNode() ); + } + + public void remove( NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { + if ( currentlyCopyingOnNode == null || currentlyCopyingOnNode.isEmpty() ) { + return; + } + copyingToNode.compute( nodeLocation, ( node, currentlyCopying ) -> { + if ( currentlyCopying == null ) { + return null; + } else { + currentlyCopying.remove( currentlyCopyingOnNode ); + return currentlyCopying; + } + } ); + } + + + +} diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java new file mode 100644 index 00000000..8a012f0d --- /dev/null +++ b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java @@ -0,0 +1,65 @@ +package fonda.scheduler.util.copying; + +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; +import lombok.ToString; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class CurrentlyCopyingOnNode { + + private final Map< String, CopySource> currentlyCopying = new HashMap<>(); + + public void add( String path, Task task, Location location ) { + synchronized ( this.currentlyCopying ) { + if ( !this.currentlyCopying.containsKey( path ) ) { + this.currentlyCopying.put( path, new CopySource( task, location ) ); + } else { + throw new IllegalStateException( "Already copying " + path ); + } + } + } + + public boolean isCurrentlyCopying( String path ) { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.containsKey( path ); + } + } + + public CopySource getCopySource( String path ) { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.get( path ); + } + } + + void add ( CurrentlyCopyingOnNode currentlyCopying ) { + synchronized ( this.currentlyCopying ) { + this.currentlyCopying.putAll( currentlyCopying.currentlyCopying ); + } + } + + void remove ( CurrentlyCopyingOnNode currentlyCopying ) { + synchronized ( this.currentlyCopying ) { + this.currentlyCopying.keySet().removeAll( currentlyCopying.currentlyCopying.keySet() ); + } + } + + public Set getAllFilesCurrentlyCopying() { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.keySet(); + } + } + + public boolean isEmpty() { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.isEmpty(); + } + } + + @Override + public String toString() { + return getAllFilesCurrentlyCopying().toString(); + } +} From 96903c84bf90da3a21d1c6cb9cdceb8676d5915b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 23 Dec 2022 12:39:42 +0100 Subject: [PATCH 307/443] First version of separate copy tasks (LAv2) Signed-off-by: Lehmann_Fabian --- .../model/taskinputs/TaskInputs.java | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java index 30d4b5dd..07f16b24 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -1,14 +1,19 @@ package fonda.scheduler.model.taskinputs; import fonda.scheduler.model.location.Location; +import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.util.Tuple; import lombok.Getter; import lombok.ToString; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; @ToString @Getter +@Slf4j public class TaskInputs { private final List symlinks; @@ -23,14 +28,42 @@ public TaskInputs(List symlinks, List file this.excludedNodes = excludedNodes; } + public boolean allFilesAreOnLocationAndNotOverwritten( Location loc, Set pathCurrentlyCopying ){ + for (PathFileLocationTriple file : files) { + if ( !file.locatedOnLocation(loc) || (pathCurrentlyCopying != null && pathCurrentlyCopying.contains(file.path.toString())) ) { + return false; + } + } + return true; + } + + public List allLocationWrapperOnLocation( Location loc){ + return files + .parallelStream() + .map( file -> file.locationWrapperOnLocation( loc ) ) + .collect( Collectors.toList() ); + } + public long calculateDataOnNode( Location loc ) { + return calculateDataOnNodeAdditionalInfo( loc ).getB(); + } + + /** + * Calculates the data on a node and returns whether all data is on the location + * @param loc + * @return boolean: true if all files are on location, Long: data on location + */ + public Tuple calculateDataOnNodeAdditionalInfo( Location loc ) { long size = 0; + boolean allOnNode = true; for ( PathFileLocationTriple fileLocation : files ) { if (fileLocation.locatedOnLocation(loc)) { size += fileLocation.getSizeInBytes(); + } else { + allOnNode = false; } } - return size; + return new Tuple<>( allOnNode, size ); } public long calculateAvgSize() { From cb0344ca503c52365094e29356c160b654bf3bb5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 11:03:12 +0100 Subject: [PATCH 308/443] Added optimalAlignment Signed-off-by: Lehmann_Fabian --- Dockerfile | 25 ++-- Dockerfile-development | 2 +- pom.xml | 6 + src/main/java/fonda/scheduler/model/Task.java | 5 + .../rest/SchedulerRestController.java | 5 +- .../scheduler/LocationAwareSchedulerV2.java | 35 +++-- .../internal/OptimalReadyToRunToNode.java | 123 ++++++++++++++++++ .../RandomReadyToRunToNode.java} | 32 +++-- .../scheduler/internal/ReadyToRunToNode.java | 24 ++++ .../scheduler/util/score/CalculateScore.java | 13 ++ .../util/score/FileSizeRankScore.java | 17 +++ .../scheduler/util/score/FileSizeScore.java | 31 +++++ 12 files changed, 274 insertions(+), 44 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/internal/OptimalReadyToRunToNode.java rename src/main/java/fonda/scheduler/scheduler/{LocationAwareSchedulerV2Simple.java => internal/RandomReadyToRunToNode.java} (72%) create mode 100644 src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java create mode 100644 src/main/java/fonda/scheduler/util/score/CalculateScore.java create mode 100644 src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java create mode 100644 src/main/java/fonda/scheduler/util/score/FileSizeScore.java diff --git a/Dockerfile b/Dockerfile index ee6d817b..0091d513 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,23 @@ -FROM maven:3.8.3-jdk-11-slim AS build +FROM maven:3-openjdk-18-slim AS build WORKDIR /build COPY pom.xml pom.xml RUN mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository COPY src/ src/ -RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository +#RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository + +RUN mvn package -f /build/pom.xml -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository + +ENTRYPOINT ["/bin/sh","-c", "mvn spring-boot:run -f pom.xml -Dmaven.repo.local=/mvn/.m2nrepo/repository"] # # Package stage # -FROM openjdk:17-alpine -WORKDIR /app -RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data -COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar -RUN chown -R javauser:javagroup /app -USER javauser -EXPOSE 8080 -ENTRYPOINT ["java","-jar","/app/k8s-scheduler.jar"] \ No newline at end of file +#FROM openjdk:18-alpine +#WORKDIR /app +#RUN apk add --no-cache libstdc++ +#RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data +#COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar +#RUN chown -R javauser:javagroup /app +#USER javauser +#EXPOSE 8080 +#ENTRYPOINT ["java","-jar","/app/k8s-scheduler.jar"] \ No newline at end of file diff --git a/Dockerfile-development b/Dockerfile-development index 445f1b5a..ecf73f63 100644 --- a/Dockerfile-development +++ b/Dockerfile-development @@ -1,4 +1,4 @@ -FROM maven:3-openjdk-17-slim AS build +FROM maven:3-openjdk-18-slim AS build WORKDIR /build COPY pom.xml pom.xml RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository diff --git a/pom.xml b/pom.xml index 4e561fb3..8bf9554a 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,12 @@ 1.6.4 + + + com.google.ortools + ortools-java + 9.4.1874 + diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index b6ec6c75..be9a99f9 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -13,6 +13,7 @@ import java.io.File; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; @Slf4j public class Task { @@ -118,6 +119,10 @@ public long getInputSize(){ return inputSize; } + public Requirements getRequest() { + return pod.getRequest(); + } + @Override public String toString() { return "Task{" + diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index af9b5f05..0a01b179 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -11,12 +11,13 @@ import fonda.scheduler.scheduler.filealignment.GreedyAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; +import fonda.scheduler.scheduler.internal.OptimalReadyToRunToNode; import fonda.scheduler.scheduler.nodeassign.FairAssign; import fonda.scheduler.scheduler.nodeassign.NodeAssign; import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; import fonda.scheduler.scheduler.nodeassign.RoundRobinAssign; import fonda.scheduler.scheduler.prioritize.*; -import fonda.scheduler.scheduler.LocationAwareSchedulerV2Simple; +import fonda.scheduler.util.score.FileSizeScore; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -134,7 +135,7 @@ ResponseEntity registerScheduler( if ( costFunction == null ) { costFunction = new MinSizeCost( 0 ); } - scheduler = new LocationAwareSchedulerV2Simple( execution, client, namespace, config, new GreedyAlignment(costFunction) ); + scheduler = new LocationAwareSchedulerV2( execution, client, namespace, config, new GreedyAlignment(costFunction), new OptimalReadyToRunToNode() ); break; default: { final String[] split = strategy.split( "-" ); diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 27bf5d88..3602ab64 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -13,13 +13,14 @@ import fonda.scheduler.scheduler.data.TaskInputsNodes; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import fonda.scheduler.scheduler.internal.ReadyToRunToNode; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.*; import fonda.scheduler.util.copying.CurrentlyCopying; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import fonda.scheduler.util.score.FileSizeRankScore; import io.fabric8.kubernetes.api.model.Status; -import io.fabric8.kubernetes.client.dsl.ExecListener; import io.fabric8.kubernetes.client.dsl.ExecWatch; import lombok.*; import lombok.extern.slf4j.Slf4j; @@ -29,7 +30,7 @@ import java.util.stream.Collectors; @Slf4j -public abstract class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { +public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { @Getter(AccessLevel.PACKAGE) private final InputAlignment inputAlignment; @@ -43,6 +44,8 @@ public abstract class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final LogCopyTask logCopyTask = new LogCopyTask(); + private final ReadyToRunToNode readyToRunToNode; + /** * This lock is used to syncronize the creation of the copy tasks and the finishing. * Otherwise, it could happen that: @@ -57,18 +60,17 @@ public LocationAwareSchedulerV2( KubernetesClient client, String namespace, SchedulerConfig config, - InputAlignment inputAlignment) { + InputAlignment inputAlignment, + ReadyToRunToNode readyToRunToNode ) { super( name, client, namespace, config ); this.inputAlignment = inputAlignment; this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; + this.readyToRunToNode = readyToRunToNode; + this.readyToRunToNode.init( new FileSizeRankScore( hierarchyWrapper ) ); + readyToRunToNode.setLogger( logCopyTask ); } - protected abstract List createAlignmentForTasksWithAllDataOnNode( - List taskWithAllData, - Map availableByNode - ); - @Override public ScheduleObject getTaskNodeAlignment( final List unscheduledTasks, @@ -102,7 +104,7 @@ && canSchedulePodOnNode( task.getPod(), node ) .stream() .filter( td -> !td.getNodesWithAllData().isEmpty() ) .collect( Collectors.toList() ); - final List alignment = createAlignmentForTasksWithAllDataOnNode(taskWithAllData, availableByNode); + final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode(taskWithAllData, availableByNode); final ScheduleObject scheduleObject = new ScheduleObject( (List) alignment ); scheduleObject.setCheckStillPossible( true ); scheduleObject.setStopSubmitIfOneFails( true ); @@ -119,8 +121,7 @@ void postScheduling( List unscheduledTasks ) { .map( task -> { final TaskInputs inputsOfTask = extractInputsOfData( task ); if ( inputsOfTask == null ) return null; - final DataOnNode dataOnNode = getDataOnNode( task, inputsOfTask, allNodes ); - return dataOnNode; + return getDataOnNode( task, inputsOfTask, allNodes ); } ) .filter( Objects::nonNull ) .collect( Collectors.toList() ); @@ -146,11 +147,7 @@ private void startCopyTasks( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { command[2] += "/code/ftp.py false " + nodeTaskFilesAlignment.node.getName() + " \"" + json + "\""; String name = nodeTaskFilesAlignment.task.getConfig().getName() + "-copy-" + nodeTaskFilesAlignment.node.getName(); log.info( "Starting {} to node {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); - String files = ""; - for (TaskInputFileLocationWrapper t : copyTask.getInputFiles()) { - files += t.getPath() + ", "; - } - logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start (" + files + ")" ); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start" ); client.execCommand( getDaemonNameOnNode( copyTask.getNodeLocation().getIdentifier() ), getNamespace(), command, new LaListener( copyTask, name, nodeTaskFilesAlignment ) ); } @@ -387,7 +384,7 @@ Tuple calculateBestNode( final DataOnNode taskData return null; } - logCopyTask.log( taskData.getTask().getConfig().getName() + " " + bestAlignment ); + //logCopyTask.log( taskData.getTask().getConfig().getName() + " " + bestAlignment ); return new Tuple<>( bestNode, bestAlignment ); } @@ -413,7 +410,7 @@ private DataOnNode getDataOnNode( Task task, TaskInputs inputsOfTask, List booleanLongTuple = inputsOfTask.calculateDataOnNodeAdditionalInfo( node.getNodeLocation() ); if ( booleanLongTuple.getA() ) { dataOnNode.allData( node ); @@ -421,7 +418,7 @@ private DataOnNode getDataOnNode( Task task, TaskInputs inputsOfTask, List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ) { + + long start = System.currentTimeMillis(); + final LinkedList taskNodeBoolVars = new LinkedList<>(); + + CpModel model = new CpModel(); + + Map memUsed = new HashMap<>(); + Map cpuUsed = new HashMap<>(); + for ( NodeWithAlloc node : availableByNode.keySet() ) { + memUsed.put( node, LinearExpr.newBuilder() ); + cpuUsed.put( node, LinearExpr.newBuilder() ); + } + + LinearExprBuilder objective = LinearExpr.newBuilder(); + + int index = 0; + for ( TaskInputsNodes taskInputsNodes : taskWithAllData ) { + List onlyOnOneNode = new ArrayList<>(); + for ( NodeWithAlloc node : taskInputsNodes.getNodesWithAllData() ) { + final Requirements availableOnNode = availableByNode.get( node ); + //Can schedule task on node? + if ( availableOnNode != null ) { + final Requirements request = taskInputsNodes.getTask().getRequest(); + if ( availableOnNode.higherOrEquals( request ) ) { + final BoolVar boolVar = model.newBoolVar( "x_" + index + "_" + node ); + onlyOnOneNode.add( boolVar ); + memUsed.get(node).addTerm( boolVar, request.getRam().longValue() ); + cpuUsed.get(node).addTerm( boolVar, request.getCpu().multiply( MILLION ).longValue() ); + objective.addTerm( boolVar, calculateScore.getScore( taskInputsNodes ) ); + taskNodeBoolVars.add( new TaskNodeBoolVar( taskInputsNodes, node, boolVar ) ); + } + } + } + if ( !onlyOnOneNode.isEmpty() ) { + model.addAtMostOne(onlyOnOneNode); + } + index++; + } + for ( NodeWithAlloc node : memUsed.keySet() ) { + model.addLessOrEqual(memUsed.get( node ), availableByNode.get( node ).getRam().longValue() ); + model.addLessOrEqual(cpuUsed.get( node ), availableByNode.get( node ).getCpu().multiply( MILLION ).longValue() ); + } + + model.maximize( objective ); + CpSolver solver = new CpSolver(); + final CpSolverStatus solve = solver.solve( model ); + final String message = "Solved in " + (System.currentTimeMillis() - start) + "ms ( " + taskNodeBoolVars.size() + " vars )"; + log.info( message ); + logger.log( message ); + log.info("Total packed value: " + solver.objectiveValue()); + log.info( String.valueOf( solve ) ); + + return taskNodeBoolVars.stream() + .filter( taskNodeBoolVar -> solver.booleanValue( taskNodeBoolVar.getBoolVar() ) ) + .map( TaskNodeBoolVar::createAlignment ) + .collect( Collectors.toList() ); + } + + @Getter + @RequiredArgsConstructor + private class TaskNodeBoolVar { + private final TaskInputsNodes taskInputsNodes; + private final NodeWithAlloc node; + private final BoolVar boolVar; + + NodeTaskLocalFilesAlignment createAlignment(){ + return new NodeTaskLocalFilesAlignment( + node, + taskInputsNodes.getTask(), + taskInputsNodes.getInputsOfTask().getSymlinks(), + taskInputsNodes.getInputsOfTask().allLocationWrapperOnLocation( node.getNodeLocation() ) + ); + } + + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java b/src/main/java/fonda/scheduler/scheduler/internal/RandomReadyToRunToNode.java similarity index 72% rename from src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java rename to src/main/java/fonda/scheduler/scheduler/internal/RandomReadyToRunToNode.java index 0e405ed6..2d723a43 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2Simple.java +++ b/src/main/java/fonda/scheduler/scheduler/internal/RandomReadyToRunToNode.java @@ -1,26 +1,29 @@ -package fonda.scheduler.scheduler; +package fonda.scheduler.scheduler.internal; -import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.scheduler.LocationAwareSchedulerV2; import fonda.scheduler.scheduler.data.TaskInputsNodes; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.LogCopyTask; import fonda.scheduler.util.NodeTaskLocalFilesAlignment; +import fonda.scheduler.util.score.CalculateScore; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -public class LocationAwareSchedulerV2Simple extends LocationAwareSchedulerV2 { +@Slf4j +public class RandomReadyToRunToNode implements ReadyToRunToNode { + + + @Setter + private LogCopyTask logger; - public LocationAwareSchedulerV2Simple( String name, KubernetesClient client, String namespace, SchedulerConfig config, InputAlignment inputAlignment ) { - super( name, client, namespace, config, inputAlignment ); - } + + @Override + public void init( CalculateScore calculateScore ) {} /** * Select the first node that fits the requirements. @@ -29,10 +32,11 @@ public LocationAwareSchedulerV2Simple( String name, KubernetesClient client, Str * @return */ @Override - protected List createAlignmentForTasksWithAllDataOnNode( + public List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, Map availableByNode ) { + long start = System.currentTimeMillis(); final List alignment = new LinkedList<>(); final Iterator iterator = taskWithAllData.iterator(); while( iterator.hasNext() ) { @@ -56,6 +60,10 @@ protected List createAlignmentForTasksWithAllDataOn } } } + final String message = "Solved in " + (System.currentTimeMillis() - start) + "ms ( " + taskWithAllData.size() + " vars )"; + log.info( message ); + logger.log( message ); return alignment; } + } diff --git a/src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java new file mode 100644 index 00000000..307df094 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java @@ -0,0 +1,24 @@ +package fonda.scheduler.scheduler.internal; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.scheduler.data.TaskInputsNodes; +import fonda.scheduler.util.LogCopyTask; +import fonda.scheduler.util.NodeTaskLocalFilesAlignment; +import fonda.scheduler.util.score.CalculateScore; + +import java.util.List; +import java.util.Map; + +public interface ReadyToRunToNode { + + void init( CalculateScore calculateScore ); + + void setLogger( LogCopyTask logger ); + + List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ); + +} diff --git a/src/main/java/fonda/scheduler/util/score/CalculateScore.java b/src/main/java/fonda/scheduler/util/score/CalculateScore.java new file mode 100644 index 00000000..33861cf4 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/score/CalculateScore.java @@ -0,0 +1,13 @@ +package fonda.scheduler.util.score; + +import fonda.scheduler.scheduler.data.TaskInputsNodes; + +public interface CalculateScore { + + /** + * Score must be higher than 0 + * @return + */ + long getScore( TaskInputsNodes taskInputsNodes ); + +} diff --git a/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java b/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java new file mode 100644 index 00000000..47930997 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java @@ -0,0 +1,17 @@ +package fonda.scheduler.util.score; + +import fonda.scheduler.model.location.hierachy.HierarchyWrapper; +import fonda.scheduler.scheduler.data.TaskInputsNodes; + +public class FileSizeRankScore extends FileSizeScore { + + public FileSizeRankScore( HierarchyWrapper hierarchyWrapper ) { + super( hierarchyWrapper ); + } + + @Override + public long getScore( TaskInputsNodes taskInputsNodes ) { + return super.getScore( taskInputsNodes ) + taskInputsNodes.getTask().getProcess().getRank() * 100_000_000_000L; + } + +} diff --git a/src/main/java/fonda/scheduler/util/score/FileSizeScore.java b/src/main/java/fonda/scheduler/util/score/FileSizeScore.java new file mode 100644 index 00000000..36b15880 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/score/FileSizeScore.java @@ -0,0 +1,31 @@ +package fonda.scheduler.util.score; + +import fonda.scheduler.model.location.hierachy.HierarchyWrapper; +import fonda.scheduler.scheduler.data.TaskInputsNodes; +import lombok.RequiredArgsConstructor; + +import java.io.File; +import java.nio.file.Path; + +@RequiredArgsConstructor +public class FileSizeScore implements CalculateScore { + + private final HierarchyWrapper hierarchyWrapper; + + @Override + public long getScore( TaskInputsNodes taskInputsNodes ) { + + final long filesInSharedFS = taskInputsNodes.getTask().getConfig() + .getInputs() + .fileInputs + .parallelStream() + .filter( x -> { + final Path path = Path.of( x.value.sourceObj ); + return !this.hierarchyWrapper.isInScope( path ); + } ) + .mapToLong( input -> new File( input.value.sourceObj ).length() ) + .sum(); + + return filesInSharedFS + taskInputsNodes.getInputsOfTask().calculateAvgSize() + 1; + } +} From 1cc1d84768923483e95cd9df10dd91896ee75d81 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 11:05:14 +0100 Subject: [PATCH 309/443] Removed unused imports Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/WriteConfigResult.java | 2 -- .../java/fonda/scheduler/scheduler/RandomLAScheduler.java | 2 -- .../scheduler/scheduler/filealignment/GreedyAlignment.java | 2 -- .../scheduler/scheduler/filealignment/InputAlignment.java | 4 ---- .../fonda/scheduler/scheduler/outlabel/InternalHolder.java | 1 - .../fonda/scheduler/util/NodeTaskLocalFilesAlignment.java | 2 -- .../fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java | 1 - 7 files changed, 14 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java index ee69cb5f..da630904 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -1,7 +1,5 @@ package fonda.scheduler.model; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.util.Tuple; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java index 2d9ed7c3..13c810d4 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java @@ -5,8 +5,6 @@ import fonda.scheduler.model.Requirements; import fonda.scheduler.model.SchedulerConfig; import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.scheduler.data.NodeDataTuple; import fonda.scheduler.scheduler.data.TaskData; import fonda.scheduler.scheduler.filealignment.InputAlignment; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index 47160cf3..f7604a34 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -11,13 +11,11 @@ import fonda.scheduler.util.AlignmentWrapper; import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.Tuple; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import java.util.HashMap; -import java.util.Map; @Slf4j public class GreedyAlignment extends InputAlignmentClass { diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java index 32a64245..c30c631e 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java @@ -2,15 +2,11 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.Tuple; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import org.jetbrains.annotations.NotNull; -import java.util.Map; - public interface InputAlignment { /** diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java b/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java index 67a4f548..7059bb8e 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java +++ b/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java @@ -12,7 +12,6 @@ @Slf4j abstract class InternalHolder { - private final Object syncHelperUpdateBestValue = new Object(); private double bestValue = Double.MIN_VALUE; private final Set currentlyBestNodes = new HashSet<>(); private final Map> tasksByNode = new HashMap<>(); diff --git a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java index a154ce8d..375de2d1 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java @@ -4,8 +4,6 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.SymlinkInput; -import lombok.Getter; -import lombok.Setter; import java.util.List; diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java index 8a012f0d..19936bae 100644 --- a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java +++ b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java @@ -2,7 +2,6 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.location.Location; -import lombok.ToString; import java.util.HashMap; import java.util.Map; From 596a9d32e44bb96e19fb19ec1906a06e3120e83c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 11:06:04 +0100 Subject: [PATCH 310/443] Code improvements Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/LocationAwareScheduler.java | 2 +- .../fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java | 2 +- src/main/java/fonda/scheduler/util/FileAlignment.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index b230c94f..aa893faf 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -143,7 +143,7 @@ private boolean onlyAllowXCopyToNodeTasks( NodeTaskFilesAlignment nodeTaskAlignm //Does this task needs to copy data to the node? boolean copyDataToNode = false; for ( Map.Entry locationAlignmentWrapperEntry : nodeFileAlignment.entrySet() ) { - if ( locationAlignmentWrapperEntry.getValue().getFilesToCopy().size() > 0 ) { + if ( !locationAlignmentWrapperEntry.getValue().getFilesToCopy().isEmpty() ) { copyDataToNode = true; break; } diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java index 52992985..9fa6120b 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java @@ -53,7 +53,7 @@ public void informResourceChange() {} @Override public void newNode( NodeWithAlloc node ) { synchronized ( this ) { - if ( nodes != null & !nodes.contains( node ) ) { + if ( nodes != null && !nodes.contains( node ) ) { nodes.add( node ); } } diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index 3daa0237..ceb66109 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -45,7 +45,7 @@ public boolean copyFromSomewhere( Location node ) { return nodeFileAlignment .entrySet() .stream() - .anyMatch( a -> a.getKey() != node && a.getValue().getFilesToCopy().size() > 0 ); + .anyMatch( a -> a.getKey() != node && !a.getValue().getFilesToCopy().isEmpty() ); } public List getAllLocationWrappers(){ From 0a50067d226da6b4c3218ed344616425e37fa30a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 15:32:12 +0100 Subject: [PATCH 311/443] Refactoring: extracted copy strategy, package for la2 extensions Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 2 - .../rest/SchedulerRestController.java | 3 +- .../scheduler/LocationAwareSchedulerV2.java | 103 +++++------------- .../la2/copystrategy/CopyRunner.java | 8 ++ .../la2/copystrategy/LaListener.java | 80 ++++++++++++++ .../scheduler/la2/copystrategy/ShellCopy.java | 42 +++++++ .../ready2run}/OptimalReadyToRunToNode.java | 11 +- .../ready2run}/RandomReadyToRunToNode.java | 2 +- .../ready2run}/ReadyToRunToNode.java | 2 +- 9 files changed, 171 insertions(+), 82 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java rename src/main/java/fonda/scheduler/scheduler/{internal => la2/ready2run}/OptimalReadyToRunToNode.java (94%) rename src/main/java/fonda/scheduler/scheduler/{internal => la2/ready2run}/RandomReadyToRunToNode.java (98%) rename src/main/java/fonda/scheduler/scheduler/{internal => la2/ready2run}/ReadyToRunToNode.java (93%) diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/fonda/scheduler/client/KubernetesClient.java index 22b00dc3..62ccb134 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/fonda/scheduler/client/KubernetesClient.java @@ -2,13 +2,11 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.scheduler.LocationAwareSchedulerV2; import fonda.scheduler.util.MyExecListner; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; -import io.fabric8.kubernetes.client.dsl.ExecListener; import io.fabric8.kubernetes.client.dsl.ExecWatch; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 0a01b179..512c65f3 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -11,13 +11,12 @@ import fonda.scheduler.scheduler.filealignment.GreedyAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; -import fonda.scheduler.scheduler.internal.OptimalReadyToRunToNode; +import fonda.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; import fonda.scheduler.scheduler.nodeassign.FairAssign; import fonda.scheduler.scheduler.nodeassign.NodeAssign; import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; import fonda.scheduler.scheduler.nodeassign.RoundRobinAssign; import fonda.scheduler.scheduler.prioritize.*; -import fonda.scheduler.util.score.FileSizeScore; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 3602ab64..9db83bb5 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -13,15 +13,16 @@ import fonda.scheduler.scheduler.data.TaskInputsNodes; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.scheduler.internal.ReadyToRunToNode; +import fonda.scheduler.scheduler.la2.copystrategy.CopyRunner; +import fonda.scheduler.scheduler.la2.copystrategy.LaListener; +import fonda.scheduler.scheduler.la2.copystrategy.ShellCopy; +import fonda.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; import fonda.scheduler.util.*; import fonda.scheduler.util.copying.CurrentlyCopying; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import fonda.scheduler.util.score.FileSizeRankScore; -import io.fabric8.kubernetes.api.model.Status; -import io.fabric8.kubernetes.client.dsl.ExecWatch; import lombok.*; import lombok.extern.slf4j.Slf4j; @@ -46,6 +47,8 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final ReadyToRunToNode readyToRunToNode; + private final CopyRunner copyRunner; + /** * This lock is used to syncronize the creation of the copy tasks and the finishing. * Otherwise, it could happen that: @@ -69,6 +72,7 @@ public LocationAwareSchedulerV2( this.readyToRunToNode = readyToRunToNode; this.readyToRunToNode.init( new FileSizeRankScore( hierarchyWrapper ) ); readyToRunToNode.setLogger( logCopyTask ); + this.copyRunner = new ShellCopy( client, this, logCopyTask ); } @Override @@ -116,6 +120,7 @@ void postScheduling( List unscheduledTasks ) { final List allNodes = client.getAllNodes(); final List nodeTaskFilesAlignments; synchronized ( copyLock ) { + //Calculate the fraction of available data for each task and node. final List tasksAndData = unscheduledTasks .parallelStream() .map( task -> { @@ -125,75 +130,13 @@ void postScheduling( List unscheduledTasks ) { } ) .filter( Objects::nonNull ) .collect( Collectors.toList() ); - nodeTaskFilesAlignments = createCopyTasks( tasksAndData ); - } - nodeTaskFilesAlignments.parallelStream().forEach( this::startCopyTasks ); - } - - private void startCopyTasks( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { - final CopyTask copyTask = initializeCopyTask( nodeTaskFilesAlignment ); - String[] command = new String[3]; - command[0] = "/bin/bash"; - command[1] = "-c"; - command[2] = "cd " + nodeTaskFilesAlignment.task.getWorkingDir() + " && "; - final String json; - try { - json = new ObjectMapper() - .writeValueAsString( copyTask.getInputs() ) - .replace( "\"", "\\\"" ); - } catch ( JsonProcessingException e ) { - throw new RuntimeException( e ); - } - command[2] += "/code/ftp.py false " + nodeTaskFilesAlignment.node.getName() + " \"" + json + "\""; - String name = nodeTaskFilesAlignment.task.getConfig().getName() + "-copy-" + nodeTaskFilesAlignment.node.getName(); - log.info( "Starting {} to node {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); - logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start" ); - client.execCommand( getDaemonNameOnNode( copyTask.getNodeLocation().getIdentifier() ), getNamespace(), command, new LaListener( copyTask, name, nodeTaskFilesAlignment ) ); - } - @RequiredArgsConstructor - private class LaListener implements MyExecListner { - - @Setter - private ExecWatch exec; - private final CopyTask copyTask; - private final String name; - @Setter - private ByteArrayOutputStream out = new ByteArrayOutputStream(); - @Setter - private ByteArrayOutputStream error = new ByteArrayOutputStream(); - private boolean finished = false; - - private final NodeTaskFilesAlignment nodeTaskFilesAlignment; - - @Override - public void onClose( int exitCode, String reason ) { - if ( !finished ) { - log.error( "Copy task was not finished, but closed. ExitCode: " + exitCode + " Reason: " + reason ); - copyTaskFinished( copyTask, exitCode == 0 ); - } - informResourceChange(); - } - - @Override - public void onFailure( Throwable t, Response failureResponse ) { - log.info( name + " failed, output: ", t ); - log.info( name + " Exec Output: {} ", out ); - log.info( name + " Exec Error Output: {} ", error ); - exec.close(); - logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "failed" ); - } - - @Override - public void onExit( int exitCode, Status reason ) { - finished = true; - log.info( name + " was finished exitCode = {}, reason = {}", exitCode, reason ); - log.debug( name + " Exec Output: {} ", out ); - log.debug( name + " Exec Error Output: {} ", error ); - copyTaskFinished( copyTask, exitCode == 0 ); - exec.close(); - logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "finished(" + exitCode + ")" ); + //Use the fraction to calculate the alignment + //TODO align tasks that can start as soon as the data is available + //TODO align tasks that should start next + nodeTaskFilesAlignments = createCopyTasks( tasksAndData ); } + nodeTaskFilesAlignments.parallelStream().forEach( n -> copyRunner.startCopyTasks( initializeCopyTask( n ), n ) ); } /** @@ -208,14 +151,14 @@ CopyTask initializeCopyTask( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); copyTask.setAllLocationWrapper( allLocationWrappers ); - log.info( "addToCopyingToNode task: {}, node: {}, data: {}, currentlyCopying: {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getFilesForCurrentNode(), getCurrentlyCopying().get( nodeTaskFilesAlignment.node.getNodeLocation() ) ); + log.info( "addToCopyingToNode task: {}, node: {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); //Files that will be copied addToCopyingToNode( nodeTaskFilesAlignment.node.getNodeLocation(), copyTask.getFilesForCurrentNode() ); useLocations( allLocationWrappers ); return copyTask; } - private void copyTaskFinished( CopyTask copyTask, boolean success ) { + public void copyTaskFinished( CopyTask copyTask, boolean success ) { if( success ){ synchronized ( copyLock ) { copyTask.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); @@ -225,12 +168,12 @@ private void copyTaskFinished( CopyTask copyTask, boolean success ) { } else { synchronized ( copyLock ) { removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); - handleProblematicInit( copyTask ); + handleProblematicCopy( copyTask ); } } } - private void handleProblematicInit( CopyTask copyTask ){ + private void handleProblematicCopy( CopyTask copyTask ){ String file = this.localWorkDir + "/sync/" + copyTask.getInputs().execution; try { Map wrapperByPath = new HashMap<>(); @@ -406,18 +349,28 @@ private TaskInputs extractInputsOfData( Task task ) { } + /** + * Calculate the fraction of data available on each node. + * @param task + * @param inputsOfTask + * @param allNodes + * @return A wrapper containing the fraction of data available on each node, the nodes where all data is available, the inputs and the task. + */ private DataOnNode getDataOnNode( Task task, TaskInputs inputsOfTask, List allNodes ) { final DataOnNode dataOnNode = new DataOnNode( task, inputsOfTask ); + final long avgSize = inputsOfTask.calculateAvgSize(); for ( NodeWithAlloc node : allNodes ) { - final long avgSize = inputsOfTask.calculateAvgSize(); if ( !inputsOfTask.getExcludedNodes().contains( node.getNodeLocation() ) && affinitiesMatch( task.getPod(), node ) ) { final Tuple booleanLongTuple = inputsOfTask.calculateDataOnNodeAdditionalInfo( node.getNodeLocation() ); + //All data is on the node if ( booleanLongTuple.getA() ) { dataOnNode.allData( node ); + // special case, more data on the node than in avg } else if ( booleanLongTuple.getB() >= avgSize ) { //As we only calculate the average size of all files, it can happen that more than 100% are on the node dataOnNode.addData( node, 0.999 ); } else { + //Calculate the fraction of data on the node dataOnNode.addData( node, booleanLongTuple.getB() / (double) avgSize ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java new file mode 100644 index 00000000..c5b2c799 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java @@ -0,0 +1,8 @@ +package fonda.scheduler.scheduler.la2.copystrategy; + +import fonda.scheduler.util.CopyTask; +import fonda.scheduler.util.NodeTaskFilesAlignment; + +public interface CopyRunner { + void startCopyTasks( CopyTask copyTask, NodeTaskFilesAlignment nodeTaskFilesAlignment ); +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java new file mode 100644 index 00000000..549ce0bc --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java @@ -0,0 +1,80 @@ +package fonda.scheduler.scheduler.la2.copystrategy; + +import fonda.scheduler.scheduler.LocationAwareSchedulerV2; +import fonda.scheduler.util.CopyTask; +import fonda.scheduler.util.LogCopyTask; +import fonda.scheduler.util.MyExecListner; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import io.fabric8.kubernetes.api.model.Status; +import io.fabric8.kubernetes.client.dsl.ExecWatch; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; + +@Slf4j +@RequiredArgsConstructor +public class LaListener implements MyExecListner { + + @Setter + private ExecWatch exec; + private final CopyTask copyTask; + private final String name; + @Setter + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + @Setter + private ByteArrayOutputStream error = new ByteArrayOutputStream(); + private boolean finished = false; + + private final NodeTaskFilesAlignment nodeTaskFilesAlignment; + + private final LocationAwareSchedulerV2 scheduler; + + private final LogCopyTask logCopyTask; + + private void close() { + //Maybe exec was not yet set + int trial = 0; + while( exec == null && trial < 5 ) { + try { + Thread.sleep( (long) (100 * Math.pow( 2, trial )) ); + } catch ( InterruptedException e ) { + e.printStackTrace(); + } + trial++; + } + if ( exec != null ) { + exec.close(); + } + } + + @Override + public void onClose( int exitCode, String reason ) { + if ( !finished ) { + log.error( "Copy task was not finished, but closed. ExitCode: " + exitCode + " Reason: " + reason ); + scheduler.copyTaskFinished( copyTask, exitCode == 0 ); + } + scheduler.informResourceChange(); + } + + @Override + public void onFailure( Throwable t, Response failureResponse ) { + log.info( name + " failed, output: ", t ); + log.info( name + " Exec Output: {} ", out ); + log.info( name + " Exec Error Output: {} ", error ); + close(); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "failed" ); + } + + @Override + public void onExit( int exitCode, Status reason ) { + finished = true; + log.info( name + " was finished exitCode = {}, reason = {}", exitCode, reason ); + log.debug( name + " Exec Output: {} ", out ); + log.debug( name + " Exec Error Output: {} ", error ); + scheduler.copyTaskFinished( copyTask, exitCode == 0 ); + close(); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "finished(" + exitCode + ")" ); + } +} \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java new file mode 100644 index 00000000..ad7cf14b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java @@ -0,0 +1,42 @@ +package fonda.scheduler.scheduler.la2.copystrategy; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import fonda.scheduler.client.KubernetesClient; +import fonda.scheduler.scheduler.LocationAwareSchedulerV2; +import fonda.scheduler.util.CopyTask; +import fonda.scheduler.util.LogCopyTask; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class ShellCopy implements CopyRunner { + + private final KubernetesClient client; + private final LocationAwareSchedulerV2 scheduler; + private final LogCopyTask logCopyTask; + + @Override + public void startCopyTasks( final CopyTask copyTask, final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + String[] command = new String[3]; + command[0] = "/bin/bash"; + command[1] = "-c"; + command[2] = "cd " + nodeTaskFilesAlignment.task.getWorkingDir() + " && "; + final String json; + try { + json = new ObjectMapper() + .writeValueAsString( copyTask.getInputs() ) + .replace( "\"", "\\\"" ); + } catch ( JsonProcessingException e ) { + throw new RuntimeException( e ); + } + command[2] += "/code/ftp.py false " + nodeTaskFilesAlignment.node.getName() + " \"" + json + "\""; + String name = nodeTaskFilesAlignment.task.getConfig().getName() + "-copy-" + nodeTaskFilesAlignment.node.getName(); + log.info( "Starting {} to node {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start" ); + client.execCommand( scheduler.getDaemonNameOnNode( copyTask.getNodeLocation().getIdentifier() ), scheduler.getNamespace(), command, new LaListener( copyTask, name, nodeTaskFilesAlignment, scheduler, logCopyTask ) ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/internal/OptimalReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java similarity index 94% rename from src/main/java/fonda/scheduler/scheduler/internal/OptimalReadyToRunToNode.java rename to src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index fedfde08..e1ac967c 100644 --- a/src/main/java/fonda/scheduler/scheduler/internal/OptimalReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.internal; +package fonda.scheduler.scheduler.la2.ready2run; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Requirements; @@ -45,6 +45,10 @@ public List createAlignmentForTasksWithAllDataOnNod Map availableByNode ) { + if ( taskWithAllData.isEmpty() || availableByNode.isEmpty() ){ + return Collections.emptyList(); + } + long start = System.currentTimeMillis(); final LinkedList taskNodeBoolVars = new LinkedList<>(); @@ -82,6 +86,11 @@ public List createAlignmentForTasksWithAllDataOnNod } index++; } + + if ( taskNodeBoolVars.isEmpty() ) { + return Collections.emptyList(); + } + for ( NodeWithAlloc node : memUsed.keySet() ) { model.addLessOrEqual(memUsed.get( node ), availableByNode.get( node ).getRam().longValue() ); model.addLessOrEqual(cpuUsed.get( node ), availableByNode.get( node ).getCpu().multiply( MILLION ).longValue() ); diff --git a/src/main/java/fonda/scheduler/scheduler/internal/RandomReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java similarity index 98% rename from src/main/java/fonda/scheduler/scheduler/internal/RandomReadyToRunToNode.java rename to src/main/java/fonda/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java index 2d723a43..b1d8a2c7 100644 --- a/src/main/java/fonda/scheduler/scheduler/internal/RandomReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.internal; +package fonda.scheduler.scheduler.la2.ready2run; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Requirements; diff --git a/src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java rename to src/main/java/fonda/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java index 307df094..992ade58 100644 --- a/src/main/java/fonda/scheduler/scheduler/internal/ReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.internal; +package fonda.scheduler.scheduler.la2.ready2run; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Requirements; From dc37414c5def317d0b60ef2f76e40b4df7709b58 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 16:51:31 +0100 Subject: [PATCH 312/443] added a space Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/ready2run/OptimalReadyToRunToNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index e1ac967c..85703a13 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -92,8 +92,8 @@ public List createAlignmentForTasksWithAllDataOnNod } for ( NodeWithAlloc node : memUsed.keySet() ) { - model.addLessOrEqual(memUsed.get( node ), availableByNode.get( node ).getRam().longValue() ); - model.addLessOrEqual(cpuUsed.get( node ), availableByNode.get( node ).getCpu().multiply( MILLION ).longValue() ); + model.addLessOrEqual( memUsed.get( node ), availableByNode.get( node ).getRam().longValue() ); + model.addLessOrEqual( cpuUsed.get( node ), availableByNode.get( node ).getCpu().multiply( MILLION ).longValue() ); } model.maximize( objective ); From 8d7bb035cecbcc0c23c2b1e27478aaac4f3c3df4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 16:51:59 +0100 Subject: [PATCH 313/443] synchronized toString Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java index 19936bae..6c9ff0a8 100644 --- a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java +++ b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java @@ -59,6 +59,8 @@ public boolean isEmpty() { @Override public String toString() { - return getAllFilesCurrentlyCopying().toString(); + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.keySet().toString(); + } } } From c005be25b8874572d64901473652bdb5f4d11873 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 3 Jan 2023 16:56:32 +0100 Subject: [PATCH 314/443] Store the copy tasks and consider them Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 50 ++++++++++++++----- .../java/fonda/scheduler/util/CopyTask.java | 8 ++- .../scheduler/util/CopyToNodeManager.java | 19 +++---- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 9db83bb5..5d6513b9 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -1,7 +1,5 @@ package fonda.scheduler.scheduler; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import fonda.scheduler.client.KubernetesClient; import fonda.scheduler.model.*; import fonda.scheduler.model.location.Location; @@ -14,7 +12,6 @@ import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import fonda.scheduler.scheduler.la2.copystrategy.CopyRunner; -import fonda.scheduler.scheduler.la2.copystrategy.LaListener; import fonda.scheduler.scheduler.la2.copystrategy.ShellCopy; import fonda.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; @@ -50,7 +47,7 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final CopyRunner copyRunner; /** - * This lock is used to syncronize the creation of the copy tasks and the finishing. + * This lock is used to synchronize the creation of the copy tasks and the finishing. * Otherwise, it could happen that: * 1. Copy tasks checks for data already on the node * 2. Copy tasks finished, data is removed from currently copying and added to onNode @@ -136,9 +133,23 @@ void postScheduling( List unscheduledTasks ) { //TODO align tasks that should start next nodeTaskFilesAlignments = createCopyTasks( tasksAndData ); } - nodeTaskFilesAlignments.parallelStream().forEach( n -> copyRunner.startCopyTasks( initializeCopyTask( n ), n ) ); + nodeTaskFilesAlignments.parallelStream().forEach( this::startCopyTask ); } + private void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + final CopyTask copyTask = initializeCopyTask( nodeTaskFilesAlignment ); + //Files that will be copied + reserveCopyTask( copyTask ); + try { + copyRunner.startCopyTasks( copyTask, nodeTaskFilesAlignment ); + } catch ( Exception e ) { + log.error( "Could not start copy task", e ); + undoReserveCopyTask( copyTask ); + } + + } + + /** * Creates config and reserves files * @@ -150,15 +161,28 @@ CopyTask initializeCopyTask( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { copyTask.setNodeLocation( nodeTaskFilesAlignment.node.getNodeLocation() ); final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); copyTask.setAllLocationWrapper( allLocationWrappers ); - log.info( "addToCopyingToNode task: {}, node: {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); - //Files that will be copied - addToCopyingToNode( nodeTaskFilesAlignment.node.getNodeLocation(), copyTask.getFilesForCurrentNode() ); - useLocations( allLocationWrappers ); return copyTask; } + private void reserveCopyTask( CopyTask copyTask ) { + //Store files to copy + addToCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + //Copy tasks on nodes + copyToNodeManager.copyToNode( copyTask.getTask(), copyTask.getNodeLocation() ); + useLocations( copyTask.getAllLocationWrapper() ); + } + + private void undoReserveCopyTask( CopyTask copyTask ) { + //Store files to copy + removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + //Copy tasks on nodes + copyToNodeManager.finishedCopyToNode( copyTask.getTask(), copyTask.getNodeLocation() ); + freeLocations( copyTask.getAllLocationWrapper() ); + } + public void copyTaskFinished( CopyTask copyTask, boolean success ) { + copyToNodeManager.finishedCopyToNode( copyTask.getTask(), copyTask.getNodeLocation() ); if( success ){ synchronized ( copyLock ) { copyTask.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); @@ -248,20 +272,20 @@ CopyTask createCopyTask( NodeTaskFilesAlignment alignment ) { } inputs.sortData(); - return new CopyTask( inputs, inputFiles, filesForCurrentNode); + return new CopyTask( inputs, inputFiles, filesForCurrentNode, alignment.task ); } List createCopyTasks( List tasksAndData ){ final CurrentlyCopying planedToCopy = new CurrentlyCopying(); tasksAndData.sort( Comparator.comparing( DataOnNode::getNodesWithAllData ).reversed() ); - final Map assignedPodsByNode = copyToNodeManager.getCurrentlyCopyingTasksOnNode(); + final Map assignedPodsByNode = copyToNodeManager.getCurrentlyCopyingTasksOnNode(); List nodeTaskAlignments = new LinkedList<>(); for ( DataOnNode dataOnNode : tasksAndData ) { final Tuple result = calculateBestNode( dataOnNode, planedToCopy ); - if ( result != null && assignedPodsByNode.getOrDefault( result.getA(), 0 ) < getMaxCopyTasksPerNode() ) { + if ( result != null && assignedPodsByNode.getOrDefault( result.getA().getNodeLocation(), 0 ) < getMaxCopyTasksPerNode() ) { addAlignmentToPlanned( planedToCopy, result.getB().getNodeFileAlignment(), dataOnNode.getTask(), result.getA() ); nodeTaskAlignments.add( new NodeTaskFilesAlignment( result.getA(), dataOnNode.getTask(), result.getB() ) ); - assignedPodsByNode.put( result.getA(), assignedPodsByNode.getOrDefault( result.getA(), 0 ) + 1 ); + assignedPodsByNode.put( result.getA().getNodeLocation(), assignedPodsByNode.getOrDefault( result.getA().getNodeLocation(), 0 ) + 1 ); } } return nodeTaskAlignments; diff --git a/src/main/java/fonda/scheduler/util/CopyTask.java b/src/main/java/fonda/scheduler/util/CopyTask.java index faaffb11..48a95141 100644 --- a/src/main/java/fonda/scheduler/util/CopyTask.java +++ b/src/main/java/fonda/scheduler/util/CopyTask.java @@ -9,6 +9,7 @@ import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.AccessLevel; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.Setter; import java.util.HashMap; @@ -17,6 +18,7 @@ import java.util.concurrent.atomic.AtomicLong; @Getter +@RequiredArgsConstructor public class CopyTask { @Getter(AccessLevel.NONE) @@ -25,6 +27,7 @@ public class CopyTask { private final Inputs inputs; private final LinkedList inputFiles; private final CurrentlyCopyingOnNode filesForCurrentNode; + private final Task task; @Setter private List allLocationWrapper; @@ -32,9 +35,4 @@ public class CopyTask { @Setter private NodeLocation nodeLocation; - public CopyTask( Inputs inputs, LinkedList inputFiles, CurrentlyCopyingOnNode filesForCurrentNode ) { - this.inputs = inputs; - this.inputFiles = inputFiles; - this.filesForCurrentNode = filesForCurrentNode; - } } diff --git a/src/main/java/fonda/scheduler/util/CopyToNodeManager.java b/src/main/java/fonda/scheduler/util/CopyToNodeManager.java index 69541a48..8423ce79 100644 --- a/src/main/java/fonda/scheduler/util/CopyToNodeManager.java +++ b/src/main/java/fonda/scheduler/util/CopyToNodeManager.java @@ -2,6 +2,7 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; import java.util.HashMap; import java.util.LinkedList; @@ -10,10 +11,10 @@ public class CopyToNodeManager { - private final Map> taskOnNodes = new HashMap<>(); - private final Map> nodesForTask = new HashMap<>(); + private final Map> taskOnNodes = new HashMap<>(); + private final Map> nodesForTask = new HashMap<>(); - public void copyToNode( Task task, NodeWithAlloc node, double part ){ + public void copyToNode( Task task, NodeLocation node ){ synchronized ( this ) { List tasks = taskOnNodes.get( node ); if ( tasks == null ) { @@ -22,7 +23,7 @@ public void copyToNode( Task task, NodeWithAlloc node, double part ){ } tasks.add( task ); - List nodes = nodesForTask.get( task ); + List nodes = nodesForTask.get( task ); if ( nodes == null ) { nodes = new LinkedList<>(); nodesForTask.put( task, nodes ); @@ -31,23 +32,23 @@ public void copyToNode( Task task, NodeWithAlloc node, double part ){ } } - public void finishedCopyToNode( Task task, NodeWithAlloc node ){ + public void finishedCopyToNode( Task task, NodeLocation node ){ synchronized ( this ) { List tasks = taskOnNodes.get( node ); if ( tasks != null ) { tasks.remove( task ); } - List nodes = nodesForTask.get( task ); + List nodes = nodesForTask.get( task ); if ( nodes != null ) { nodes.remove( node ); } } } - public Map getCurrentlyCopyingTasksOnNode() { - Map result = new HashMap<>(); - for ( Map.Entry> entry : taskOnNodes.entrySet() ) { + public Map getCurrentlyCopyingTasksOnNode() { + Map result = new HashMap<>(); + for ( Map.Entry> entry : taskOnNodes.entrySet() ) { result.put( entry.getKey(), entry.getValue().size() ); } return result; From c7f9b0e36d141b0b5ee3f767597213542e8faee9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 14:54:45 +0100 Subject: [PATCH 315/443] Fix problem that Node's hash changed Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/NodeWithAlloc.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java index 702b8af0..c3227b2b 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/fonda/scheduler/model/NodeWithAlloc.java @@ -179,11 +179,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (getMaxResources() != null ? getMaxResources().hashCode() : 0); - result = 31 * result + (getAssignedPods() != null ? getAssignedPods().hashCode() : 0); - result = 31 * result + (getNodeLocation() != null ? getNodeLocation().hashCode() : 0); - return result; + return this.getName().hashCode(); } public boolean isReady(){ From 6d56b2d75a151f34b345e6b1d6dbf5a98d32eeb5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 14:55:21 +0100 Subject: [PATCH 316/443] Path into variable Signed-off-by: Lehmann_Fabian --- .../scheduler/util/CopyToNodeManager.java | 57 ------------------- .../fonda/scheduler/util/LogCopyTask.java | 5 +- 2 files changed, 4 insertions(+), 58 deletions(-) delete mode 100644 src/main/java/fonda/scheduler/util/CopyToNodeManager.java diff --git a/src/main/java/fonda/scheduler/util/CopyToNodeManager.java b/src/main/java/fonda/scheduler/util/CopyToNodeManager.java deleted file mode 100644 index 8423ce79..00000000 --- a/src/main/java/fonda/scheduler/util/CopyToNodeManager.java +++ /dev/null @@ -1,57 +0,0 @@ -package fonda.scheduler.util; - -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class CopyToNodeManager { - - private final Map> taskOnNodes = new HashMap<>(); - private final Map> nodesForTask = new HashMap<>(); - - public void copyToNode( Task task, NodeLocation node ){ - synchronized ( this ) { - List tasks = taskOnNodes.get( node ); - if ( tasks == null ) { - tasks = new LinkedList<>(); - taskOnNodes.put( node, tasks ); - } - tasks.add( task ); - - List nodes = nodesForTask.get( task ); - if ( nodes == null ) { - nodes = new LinkedList<>(); - nodesForTask.put( task, nodes ); - } - nodes.add( node ); - } - } - - public void finishedCopyToNode( Task task, NodeLocation node ){ - synchronized ( this ) { - List tasks = taskOnNodes.get( node ); - if ( tasks != null ) { - tasks.remove( task ); - } - - List nodes = nodesForTask.get( task ); - if ( nodes != null ) { - nodes.remove( node ); - } - } - } - - public Map getCurrentlyCopyingTasksOnNode() { - Map result = new HashMap<>(); - for ( Map.Entry> entry : taskOnNodes.entrySet() ) { - result.put( entry.getKey(), entry.getValue().size() ); - } - return result; - } - -} diff --git a/src/main/java/fonda/scheduler/util/LogCopyTask.java b/src/main/java/fonda/scheduler/util/LogCopyTask.java index de6827f7..35370f97 100644 --- a/src/main/java/fonda/scheduler/util/LogCopyTask.java +++ b/src/main/java/fonda/scheduler/util/LogCopyTask.java @@ -1,6 +1,7 @@ package fonda.scheduler.util; import java.io.BufferedWriter; +import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -10,7 +11,9 @@ public class LogCopyTask { public LogCopyTask() { try { - this.writer = new BufferedWriter( new FileWriter( "/pvcdata/scheduler/copytasks.csv" ) ); + final String pathname = "/input/data/scheduler/"; + new File( pathname ).mkdirs(); + this.writer = new BufferedWriter( new FileWriter( pathname + "copytasks.csv" ) ); } catch ( IOException e ) { throw new RuntimeException( e ); } From 9a53c0382f7ef6eef236f7872a70c388747e01e1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 14:55:44 +0100 Subject: [PATCH 317/443] added comment Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java index 6c9ff0a8..bc74425f 100644 --- a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java +++ b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java @@ -7,6 +7,10 @@ import java.util.Map; import java.util.Set; +/** + * Keeps track of what is currently being copied to a node. + * This is used to avoid copying the same data to a node multiple times. + */ public class CurrentlyCopyingOnNode { private final Map< String, CopySource> currentlyCopying = new HashMap<>(); From 323d5eeda68e699339b1ad3f996c1cc7e4378c89 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 14:58:28 +0100 Subject: [PATCH 318/443] Check affinities and that the node has generally enough resources Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/Scheduler.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index cdc88ced..9ad754ea 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -344,6 +344,12 @@ public boolean canSchedulePodOnNode(PodWithAge pod, NodeWithAlloc node ) { } boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ + + final boolean nodeCouldRunThisPod = node.getMaxResources().higherOrEquals( pod.getRequest() ); + if ( !nodeCouldRunThisPod ){ + return false; + } + final Map podsNodeSelector = pod.getSpec().getNodeSelector(); final Map nodesLabels = node.getMetadata().getLabels(); if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) { From b20131d3b231f785886c808db459f919673e55d1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 14:59:05 +0100 Subject: [PATCH 319/443] Make Requirements clonable Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Requirements.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/Requirements.java b/src/main/java/fonda/scheduler/model/Requirements.java index 3e6d95fc..b599e780 100644 --- a/src/main/java/fonda/scheduler/model/Requirements.java +++ b/src/main/java/fonda/scheduler/model/Requirements.java @@ -7,7 +7,7 @@ import java.math.BigDecimal; @ToString -public class Requirements implements Serializable { +public class Requirements implements Serializable, Cloneable { private static final long serialVersionUID = 1L; @@ -61,4 +61,8 @@ public boolean higherOrEquals( Requirements requirements ){ && this.ram.compareTo( requirements.ram ) >= 0; } + @Override + public Requirements clone() { + return new Requirements( this.cpu, this.ram ); + } } From 57c2c3b4d4688213fab552415627ff770844a135 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 14:59:22 +0100 Subject: [PATCH 320/443] SortedList + test Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/util/SortedList.java | 63 ++++++++ .../fonda/scheduler/util/SortedListTest.java | 138 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 src/main/java/fonda/scheduler/util/SortedList.java create mode 100644 src/test/java/fonda/scheduler/util/SortedListTest.java diff --git a/src/main/java/fonda/scheduler/util/SortedList.java b/src/main/java/fonda/scheduler/util/SortedList.java new file mode 100644 index 00000000..7a566487 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/SortedList.java @@ -0,0 +1,63 @@ +package fonda.scheduler.util; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.ListIterator; + +public class SortedList> extends LinkedList { + + + public SortedList( Collection collection ) { + super( collection ); + this.sort( Comparable::compareTo ); + } + + + @Override + public boolean add( T elem ) { + final ListIterator iterator = this.listIterator(); + while ( iterator.hasNext() ) { + final Comparable next = iterator.next(); + if ( next.compareTo( elem ) > 0 ) { + if ( iterator.hasPrevious() ) { + iterator.previous(); + iterator.add( elem ); + } else { + //first element of list + super.addFirst( elem ); + } + return true; + } + } + iterator.add( elem ); + return true; + } + + @Override + public void add( int index, T element ) { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public boolean addAll( Collection c ) { + for ( T t : c ) { + this.add( t ); + } + return true; + } + + @Override + public boolean addAll( int index, Collection c ) { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public void addFirst( T t ) { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public void addLast( T t ) { + throw new UnsupportedOperationException( "Not supported" ); + } +} diff --git a/src/test/java/fonda/scheduler/util/SortedListTest.java b/src/test/java/fonda/scheduler/util/SortedListTest.java new file mode 100644 index 00000000..7f44d43b --- /dev/null +++ b/src/test/java/fonda/scheduler/util/SortedListTest.java @@ -0,0 +1,138 @@ +package fonda.scheduler.util; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +class SortedListTest { + + @Test + void constructorTest() { + final List integers = Arrays.asList( 8, 4, 1, 5 ); + final SortedList sortedList = new SortedList<>( integers ); + assertEquals( 1, sortedList.get( 0 ) ); + assertEquals( 4, sortedList.get( 1 ) ); + assertEquals( 5, sortedList.get( 2 ) ); + assertEquals( 8, sortedList.get( 3 ) ); + } + + @Test + void add() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + log.info( sortedList.toString() ); + sortedList.add( 8 ); + log.info( sortedList.toString() ); + assertEquals( 8, sortedList.get( 3 ) ); + sortedList.add( 3 ); + log.info( sortedList.toString() ); + assertEquals( 3, sortedList.get( 1 ) ); + sortedList.add( 1 ); + log.info( sortedList.toString() ); + assertEquals( 1, sortedList.get( 1 ) ); + sortedList.add( 5 ); + log.info( sortedList.toString() ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 3, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + + } + + @Test + void addWithDuplicates() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + sortedList.add( 8 ); + assertEquals( 8, sortedList.get( 3 ) ); + sortedList.add( 4 ); + assertEquals( 4, sortedList.get( 1 ) ); + assertEquals( 4, sortedList.get( 2 ) ); + sortedList.add( 1 ); + assertEquals( 1, sortedList.get( 1 ) ); + sortedList.add( 5 ); + log.info( sortedList.toString() ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 4, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + + } + + @Test + void addToEmptyList() { + final SortedList sortedList = new SortedList<>( new LinkedList< Integer >() ); + sortedList.add( 4 ); + log.info( sortedList.toString() ); + assertEquals( 4, sortedList.get( 0 ) ); + } + + @Test + void addAtEnd() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 5 ) ); + sortedList.add( 6 ); + log.info( sortedList.toString() ); + assertEquals( 5, sortedList.get( 0 ) ); + assertEquals( 6, sortedList.get( 1 ) ); + } + + @Test + void addAtBeginning() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 5 ) ); + sortedList.add( 4 ); + log.info( sortedList.toString() ); + assertEquals( 4, sortedList.get( 0 ) ); + assertEquals( 5, sortedList.get( 1 ) ); + } + + + @Test + void addAtMiddle() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 4, 6 ) ); + sortedList.add( 5 ); + log.info( sortedList.toString() ); + assertEquals( 4, sortedList.get( 0 ) ); + assertEquals( 5, sortedList.get( 1 ) ); + assertEquals( 6, sortedList.get( 2 ) ); + } + + @Test + void addAll() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + sortedList.addAll( Arrays.asList( 8, 3, 1, 5 ) ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 3, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + } + + @Test + void addAllWithDuplicates() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + sortedList.addAll( Arrays.asList( 8, 4, 1, 5 ) ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 4, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + } +} \ No newline at end of file From a768aebd49459f4e2da6756cef5b2e26058bc7d7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 15:00:37 +0100 Subject: [PATCH 321/443] added GetMinSizeInBytes to get more realistic prediction Signed-off-by: Lehmann_Fabian --- .../model/taskinputs/PathFileLocationTriple.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index ac48d116..191a6975 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -36,6 +36,19 @@ public long getSizeInBytes() { return this.size; } + public long getMinSizeInBytes() { + if ( locations.isEmpty() ) { + throw new IllegalStateException("No locations for file " + path); + } + long minSize = Long.MAX_VALUE; + for (LocationWrapper location : locations) { + if ( location.getSizeInBytes() < minSize ) { + minSize = location.getSizeInBytes(); + } + } + return minSize; + } + public LocationWrapper locationWrapperOnLocation(Location loc){ for (LocationWrapper location : locations) { if ( location.getLocation().equals(loc) ) { From 93b3ea62fcf61efdfe564858bf0eb19248d95433 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 15:02:06 +0100 Subject: [PATCH 322/443] Calculate the missing data for TaskInputs Signed-off-by: Lehmann_Fabian --- .../model/taskinputs/TaskInputs.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java index 07f16b24..733c6aae 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -2,7 +2,10 @@ import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import fonda.scheduler.util.TaskNodeStats; import fonda.scheduler.util.Tuple; +import fonda.scheduler.util.copying.CopySource; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @@ -66,6 +69,39 @@ public Tuple calculateDataOnNodeAdditionalInfo( Location loc ) { return new Tuple<>( allOnNode, size ); } + /** + * Calculates the data on a node and returns whether all data is on the location + * @param loc + * @return the size remaining and the amount of data currently copying. Null if the task cannot run on this node. + */ + public TaskNodeStats calculateMissingData( Location loc, CurrentlyCopyingOnNode currentlyCopying ) { + long sizeRemaining = 0; + long sizeCurrentlyCopying = 0; + long sizeOnNode = 0; + for ( PathFileLocationTriple fileLocation : files ) { + final long minSizeInBytes = fileLocation.getMinSizeInBytes(); + //Is the file already on the node? + if ( fileLocation.locatedOnLocation(loc) ) { + sizeOnNode += minSizeInBytes; + } else { + //is the file currently copying? + final CopySource copySource = currentlyCopying.getCopySource( fileLocation.path.toString() ); + if ( copySource != null ) { + //Is this file compatible with the task? + if ( fileLocation.locatedOnLocation( copySource.getLocation() ) ) { + sizeCurrentlyCopying += minSizeInBytes; + } else { + //currently copying file is incompatible with this task + return null; + } + } else { + sizeRemaining += minSizeInBytes; + } + } + } + return new TaskNodeStats( sizeRemaining, sizeCurrentlyCopying, sizeOnNode ); + } + public long calculateAvgSize() { long size = 0; for ( PathFileLocationTriple file : files ) { From f1896da321775a3b32ef90cb2af0cc594ae2332c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 15:22:13 +0100 Subject: [PATCH 323/443] added second scheduling phase Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareScheduler.java | 4 +- .../scheduler/LocationAwareSchedulerV2.java | 95 +++++------ .../fonda/scheduler/scheduler/Scheduler.java | 4 +- .../scheduler/SchedulerWithDaemonSet.java | 24 +-- .../scheduler/la2/MinCopyingComparator.java | 22 +++ .../scheduler/la2/MinSizeComparator.java | 26 +++ .../scheduler/scheduler/la2/TaskStat.java | 94 +++++++++++ .../scheduler/la2/TaskStatComparator.java | 15 ++ .../CapacityAvailableToNode.java | 22 +++ .../SimpleCapacityAvailableToNode.java | 150 ++++++++++++++++++ .../fonda/scheduler/util/TaskNodeStats.java | 26 +++ .../java/fonda/scheduler/util/TaskStats.java | 25 +++ .../util/copying/CurrentlyCopying.java | 77 ++++++++- 13 files changed, 514 insertions(+), 70 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java create mode 100644 src/main/java/fonda/scheduler/util/TaskNodeStats.java create mode 100644 src/main/java/fonda/scheduler/util/TaskStats.java diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index aa893faf..158b0340 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -122,7 +122,7 @@ List createAlignment( if ( nodeAlignment != null && onlyAllowXCopyToNodeTasks( nodeAlignment, taskCountWhichCopyToNode, getMaxCopyTasksPerNode() ) ) { alignment.add(nodeAlignment); outLabelHolder.scheduleTaskOnNode( taskData.getTask(), nodeAlignment.node.getNodeLocation() ); - addAlignmentToPlanned( planedToCopy, nodeAlignment.fileAlignment.getNodeFileAlignment(), taskData.getTask(), nodeAlignment.node ); + planedToCopy.addAlignment( nodeAlignment.fileAlignment.getNodeFileAlignment(), taskData.getTask(), nodeAlignment.node ); } } return alignment; @@ -411,7 +411,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { } nodeTaskFilesAlignment.setRemoveInit( !writeConfigResult.isWroteConfig() ); alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); - addToCopyingToNode( alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); + addToCopyingToNode( alignment.task, alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); if ( writeConfigResult.isWroteConfig() ) { getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 5d6513b9..e74e84b4 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -11,6 +11,11 @@ import fonda.scheduler.scheduler.data.TaskInputsNodes; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import fonda.scheduler.scheduler.la2.MinCopyingComparator; +import fonda.scheduler.scheduler.la2.MinSizeComparator; +import fonda.scheduler.scheduler.la2.TaskStat; +import fonda.scheduler.scheduler.la2.TaskStatComparator; +import fonda.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; import fonda.scheduler.scheduler.la2.copystrategy.CopyRunner; import fonda.scheduler.scheduler.la2.copystrategy.ShellCopy; import fonda.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; @@ -37,15 +42,18 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { @Getter(AccessLevel.PACKAGE) private final int maxWaitingCopyTasksPerNode; - @Getter(AccessLevel.PACKAGE) - private final CopyToNodeManager copyToNodeManager = new CopyToNodeManager(); - private final LogCopyTask logCopyTask = new LogCopyTask(); private final ReadyToRunToNode readyToRunToNode; private final CopyRunner copyRunner; + private final SimpleCapacityAvailableToNode capacityAvailableToNode; + + private final TaskStatComparator phaseTwoComparator; + + private final int copySameTaskInParallel; + /** * This lock is used to synchronize the creation of the copy tasks and the finishing. * Otherwise, it could happen that: @@ -70,6 +78,9 @@ public LocationAwareSchedulerV2( this.readyToRunToNode.init( new FileSizeRankScore( hierarchyWrapper ) ); readyToRunToNode.setLogger( logCopyTask ); this.copyRunner = new ShellCopy( client, this, logCopyTask ); + this.copySameTaskInParallel = 2; + capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); } @Override @@ -113,12 +124,17 @@ && canSchedulePodOnNode( task.getPod(), node ) } @Override - void postScheduling( List unscheduledTasks ) { - final List allNodes = client.getAllNodes(); + void postScheduling( List unscheduledTasks, final Map availableByNode ) { + final Map currentlyCopyingTasksOnNode = getCurrentlyCopying().getCurrentlyCopyingTasksOnNode(); + final List allNodes = client.getAllNodes().stream().filter( + node -> currentlyCopyingTasksOnNode + .getOrDefault( node.getNodeLocation(), 0 ) < getMaxCopyTasksPerNode() + ).collect( Collectors.toList() ); final List nodeTaskFilesAlignments; synchronized ( copyLock ) { - //Calculate the fraction of available data for each task and node. - final List tasksAndData = unscheduledTasks + final TaskStats taskStats = new TaskStats(); + //Calculate the stats of available data for each task and node. + unscheduledTasks .parallelStream() .map( task -> { final TaskInputs inputsOfTask = extractInputsOfData( task ); @@ -126,12 +142,12 @@ void postScheduling( List unscheduledTasks ) { return getDataOnNode( task, inputsOfTask, allNodes ); } ) .filter( Objects::nonNull ) - .collect( Collectors.toList() ); + .sequential() + .forEach( taskStats::add ); - //Use the fraction to calculate the alignment - //TODO align tasks that can start as soon as the data is available - //TODO align tasks that should start next - nodeTaskFilesAlignments = createCopyTasks( tasksAndData ); + final CurrentlyCopying planedToCopy = new CurrentlyCopying(); + //Fill the currently available resources as fast as possible: start the tasks with the least data missing on a node. + nodeTaskFilesAlignments = capacityAvailableToNode.createAlignmentForTasksWithEnoughCapacity( taskStats, planedToCopy, availableByNode, allNodes, getMaxCopyTasksPerNode() ); } nodeTaskFilesAlignments.parallelStream().forEach( this::startCopyTask ); } @@ -151,7 +167,7 @@ private void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment /** - * Creates config and reserves files + * Creates config * * @param nodeTaskFilesAlignment * @return @@ -167,32 +183,25 @@ CopyTask initializeCopyTask( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { private void reserveCopyTask( CopyTask copyTask ) { //Store files to copy - addToCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); - //Copy tasks on nodes - copyToNodeManager.copyToNode( copyTask.getTask(), copyTask.getNodeLocation() ); + addToCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); useLocations( copyTask.getAllLocationWrapper() ); } private void undoReserveCopyTask( CopyTask copyTask ) { //Store files to copy - removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); - //Copy tasks on nodes - copyToNodeManager.finishedCopyToNode( copyTask.getTask(), copyTask.getNodeLocation() ); + removeFromCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); freeLocations( copyTask.getAllLocationWrapper() ); } public void copyTaskFinished( CopyTask copyTask, boolean success ) { - copyToNodeManager.finishedCopyToNode( copyTask.getTask(), copyTask.getNodeLocation() ); - if( success ){ - synchronized ( copyLock ) { - copyTask.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); - removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); - } + synchronized ( copyLock ) { freeLocations( copyTask.getAllLocationWrapper() ); - } else { - synchronized ( copyLock ) { - removeFromCopyingToNode( copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); - handleProblematicCopy( copyTask ); + if( success ){ + copyTask.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); + removeFromCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + } else { + removeFromCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + handleProblematicCopy( copyTask ); } } } @@ -374,32 +383,26 @@ private TaskInputs extractInputsOfData( Task task ) { /** - * Calculate the fraction of data available on each node. + * Calculate the remaining data on each node. * @param task * @param inputsOfTask * @param allNodes - * @return A wrapper containing the fraction of data available on each node, the nodes where all data is available, the inputs and the task. + * @return A wrapper containing the remaining data on each node, the nodes where all data is available, the inputs and the task. */ - private DataOnNode getDataOnNode( Task task, TaskInputs inputsOfTask, List allNodes ) { - final DataOnNode dataOnNode = new DataOnNode( task, inputsOfTask ); - final long avgSize = inputsOfTask.calculateAvgSize(); + private TaskStat getDataOnNode( Task task, TaskInputs inputsOfTask, List allNodes ) { + TaskStat taskStats = new TaskStat( task, inputsOfTask, phaseTwoComparator ); + final CurrentlyCopying currentlyCopying = getCurrentlyCopying(); for ( NodeWithAlloc node : allNodes ) { if ( !inputsOfTask.getExcludedNodes().contains( node.getNodeLocation() ) && affinitiesMatch( task.getPod(), node ) ) { - final Tuple booleanLongTuple = inputsOfTask.calculateDataOnNodeAdditionalInfo( node.getNodeLocation() ); - //All data is on the node - if ( booleanLongTuple.getA() ) { - dataOnNode.allData( node ); - // special case, more data on the node than in avg - } else if ( booleanLongTuple.getB() >= avgSize ) { - //As we only calculate the average size of all files, it can happen that more than 100% are on the node - dataOnNode.addData( node, 0.999 ); - } else { - //Calculate the fraction of data on the node - dataOnNode.addData( node, booleanLongTuple.getB() / (double) avgSize ); + final CurrentlyCopyingOnNode currentlyCopyingOnNode = currentlyCopying.get( node.getNodeLocation() ); + final TaskNodeStats taskNodeStats = inputsOfTask.calculateMissingData( node.getNodeLocation(), currentlyCopyingOnNode ); + if ( taskNodeStats != null ) { + taskStats.add( node, taskNodeStats ); } } } - return dataOnNode; + taskStats.finish(); + return taskStats; } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index 9ad754ea..e9fd048d 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -132,7 +132,7 @@ public int schedule( final List unscheduledTasks ) { scheduled++; } //Use instance object that does not contain yet scheduled tasks - postScheduling( unscheduledTasksCopy ); + postScheduling( unscheduledTasksCopy, getAvailableByNode() ); return unscheduledTasks.size() - taskNodeAlignment.size() + failure; } @@ -140,7 +140,7 @@ public int schedule( final List unscheduledTasks ) { * This method is called when a SchedulePlan was successfully executed. * @param unscheduledTasks */ - void postScheduling( final List unscheduledTasks ) {} + void postScheduling( final List unscheduledTasks, Map availableByNode ) {} /** * Call this method in case of any scheduling problems diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index ad050838..3513cd0a 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -121,7 +121,7 @@ void undoTaskScheduling( Task task ){ task.setInputFiles( null ); } if ( task.getCopyingToNode() != null ) { - removeFromCopyingToNode( task.getNode().getNodeLocation(), task.getCopyingToNode()); + removeFromCopyingToNode( task, task.getNode().getNodeLocation(), task.getCopyingToNode()); task.setCopyingToNode( null ); } task.setCopiedFiles( null ); @@ -174,11 +174,11 @@ int terminateTasks(List finishedTasks) { * @param nodeLocation * @param toAdd */ - void addToCopyingToNode( NodeLocation nodeLocation, CurrentlyCopyingOnNode toAdd ){ + void addToCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode toAdd ){ if ( nodeLocation == null ) { throw new IllegalArgumentException( "NodeLocation cannot be null" ); } - currentlyCopying.add( nodeLocation, toAdd ); + currentlyCopying.add( task, nodeLocation, toAdd ); } /** @@ -186,11 +186,11 @@ void addToCopyingToNode( NodeLocation nodeLocation, CurrentlyCopyingOnNode toAd * @param nodeLocation * @param toRemove */ - void removeFromCopyingToNode( NodeLocation nodeLocation, CurrentlyCopyingOnNode toRemove ) { + void removeFromCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode toRemove ) { if (nodeLocation == null) { throw new IllegalArgumentException("NodeLocation cannot be null"); } - currentlyCopying.remove( nodeLocation, toRemove ); + currentlyCopying.remove( task, nodeLocation, toRemove ); } /** @@ -406,7 +406,7 @@ private void podWasInitialized( Pod pod ){ final Task task = changeStateOfTask(pod, exitCode == 0 ? State.PREPARED : State.INIT_WITH_ERRORS); task.setPod( new PodWithAge( pod ) ); log.info( "Pod {}, Init Code: {}", pod.getMetadata().getName(), exitCode); - removeFromCopyingToNode( task.getNode().getNodeLocation(), task.getCopyingToNode() ); + removeFromCopyingToNode( task, task.getNode().getNodeLocation(), task.getCopyingToNode() ); if( exitCode == 0 ){ task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); } else { @@ -433,18 +433,6 @@ void filterNotMatchingNodesForTask(Set matchingNodes, TaskInputs } } - void addAlignmentToPlanned ( CurrentlyCopying planedToCopy, final Map nodeFileAlignment, - Task task, NodeWithAlloc node ) { - for (Map.Entry entry : nodeFileAlignment.entrySet()) { - final CurrentlyCopyingOnNode map = planedToCopy.get(node.getNodeLocation()); - for (FilePath filePath : entry.getValue().getFilesToCopy()) { - if ( entry.getKey() != node.getNodeLocation() ) { - map.add( filePath.getPath(), task, entry.getKey() ); - } - } - } - } - public void taskHasFinishedCopyTask( String name ){ final Task task = tasksByPodName.get( name ); task.getNode().startingTaskCopyingDataFinished( task ); diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java new file mode 100644 index 00000000..f11e933d --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java @@ -0,0 +1,22 @@ +package fonda.scheduler.scheduler.la2; + +import java.util.Comparator; + +public class MinCopyingComparator extends TaskStatComparator { + + public MinCopyingComparator( Comparator comparator ) { + super( comparator ); + } + + @Override + public int compare( TaskStat o1, TaskStat o2 ) { + if ( o1.getCompleteOnNodes() < o2.getCompleteOnNodes() ) { + return -1; + } else if ( o1.getCompleteOnNodes() > o2.getCompleteOnNodes() ) { + return 1; + } else { + return getComparator().compare( o1.getBestStats(), o2.getBestStats() ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java new file mode 100644 index 00000000..6c8dd263 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java @@ -0,0 +1,26 @@ +package fonda.scheduler.scheduler.la2; + +import fonda.scheduler.util.TaskNodeStats; +import lombok.NoArgsConstructor; + +import java.util.Comparator; + +@NoArgsConstructor +public class MinSizeComparator implements Comparator { + + public final static MinSizeComparator INSTANCE = new MinSizeComparator(); + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + if ( o1Stats.getSizeRemaining() < o2Stats.getSizeRemaining() ) { + return -1; + } else if ( o1Stats.getSizeRemaining() > o2Stats.getSizeRemaining() ) { + return 1; + } else { + return Long.compare( o1Stats.getSizeCurrentlyCopying(), o2Stats.getSizeCurrentlyCopying() ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java b/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java new file mode 100644 index 00000000..4e784b03 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java @@ -0,0 +1,94 @@ +package fonda.scheduler.scheduler.la2; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +@RequiredArgsConstructor +public class TaskStat implements Comparable { + + @Getter + private final Task task; + @Getter + private final TaskInputs inputsOfTask; + private final TaskStatComparator comparator; + private final List taskStats = new ArrayList<>(); + @Getter + private int completeOnNodes = 0; + @Getter + private int copyingToNodes = 0; + private boolean finished = false; + private int indexToCompare = 0; + + /** + * Call this to sort the taskStats list + */ + public void finish() { + finished = true; + taskStats.sort( comparator.getComparator() ); + } + + /** + * Increases the index to compare. + * False if no other opportunity exists + * @return + */ + public boolean increaseIndexToCompare() { + indexToCompare++; + return indexToCompare < taskStats.size(); + } + + private final AtomicInteger canStartAfterCopying = new AtomicInteger( 0 ); + + public void add( NodeWithAlloc node, TaskNodeStats taskNodeStats ) { + assert !finished; + this.taskStats.add( new NodeAndStatWrapper( node, taskNodeStats ) ); + if ( taskNodeStats.allOnNode() ) { + this.completeOnNodes++; + } else if ( taskNodeStats.allOnNodeOrCopying() ) { + this.copyingToNodes++; + } + } + + /** + * Mark this task as it is currently copying to a node with enough available resources. + */ + public void canStartAfterCopying() { + this.canStartAfterCopying.incrementAndGet(); + } + + public int getCanStartAfterCopying() { + return this.canStartAfterCopying.get(); + } + + public NodeAndStatWrapper getBestStats() { + assert finished; + assert indexToCompare < taskStats.size(); + return taskStats.get( indexToCompare ); + } + + @Override + public int compareTo( @NotNull TaskStat o ) { + if ( indexToCompare >= taskStats.size() || o.indexToCompare >= o.taskStats.size() ) { + throw new IllegalStateException( "Cannot compare tasks that have no stats" ); + } + return comparator.compare( this, o ); + } + + @Getter + @RequiredArgsConstructor(access = AccessLevel.PRIVATE) + public static class NodeAndStatWrapper { + private final NodeWithAlloc node; + private final TaskNodeStats taskNodeStats; + } + + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java new file mode 100644 index 00000000..bb95b21b --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java @@ -0,0 +1,15 @@ +package fonda.scheduler.scheduler.la2; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Comparator; + +@RequiredArgsConstructor +public abstract class TaskStatComparator implements Comparator { + + @Getter + private final Comparator comparator; + + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java new file mode 100644 index 00000000..b6b2f4f0 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java @@ -0,0 +1,22 @@ +package fonda.scheduler.scheduler.la2.capacityavailable; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.TaskStats; +import fonda.scheduler.util.copying.CurrentlyCopying; + +import java.util.List; +import java.util.Map; + +public interface CapacityAvailableToNode { + + List createAlignmentForTasksWithEnoughCapacity( + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final Map availableByNodes, + final List allNodes, + final int maxCopyingTaskPerNode + ); + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java new file mode 100644 index 00000000..45064dc3 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -0,0 +1,150 @@ +package fonda.scheduler.scheduler.la2.capacityavailable; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Requirements; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import fonda.scheduler.scheduler.la2.TaskStat; +import fonda.scheduler.util.*; +import fonda.scheduler.util.copying.CurrentlyCopying; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +@RequiredArgsConstructor +public class SimpleCapacityAvailableToNode implements CapacityAvailableToNode { + + private final CurrentlyCopying currentlyCopying; + private final InputAlignment inputAlignment; + private final int copySameTaskInParallel; + + public List createAlignmentForTasksWithEnoughCapacity( + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final Map availableByNodes, + final List allNodes, + final int maxCopyingTaskPerNode ) { + + + List nodeTaskAlignments = new LinkedList<>(); + + final Map currentlyCopyingTasksOnNode = currentlyCopying.getCurrentlyCopyingTasksOnNode(); + + //Remove available resources if a copy task is already running: this logic may not be optimal for more than 2 parallel copy tasks (unclear which task starts first) + removeAvailableResources( taskStats, availableByNodes, allNodes ); + //Sort tasks by missing data: prefer tasks where the least data is missing on the node + final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); + removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); + + final boolean cannotAdd; + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode + && + availableByNodes.get( node ).higherOrEquals( task.getRequest() ) ) { + cannotAdd = !createFileAlignment( planedToCopy, availableByNodes, nodeTaskAlignments, currentlyCopyingTasksOnNode, poll, task, node ); + } else { + cannotAdd = true; + } + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( cannotAdd && poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } + } + return nodeTaskAlignments; + } + + private void removeTasksThatAreCopiedMoreThanXTimeCurrently( SortedList taskStats, int maxParallelTasks ) { + taskStats.removeIf( elem -> currentlyCopying.getNumberOfNodesForTask( elem.getTask() ) == maxParallelTasks ); + } + + + /** + * + * @param planedToCopy + * @param availableByNodes + * @param nodeTaskAlignments + * @param currentlyCopyingTasksOnNode + * @param poll + * @param task + * @param node + * @return the success of this operation + */ + private boolean createFileAlignment( + CurrentlyCopying planedToCopy, + Map availableByNodes, + List nodeTaskAlignments, + Map currentlyCopyingTasksOnNode, + TaskStat poll, + Task task, + NodeWithAlloc node + ) { + final FileAlignment fileAlignmentForTaskAndNode = getFileAlignmentForTaskAndNode( node, task, poll.getInputsOfTask(), planedToCopy ); + if ( fileAlignmentForTaskAndNode != null ) { + planedToCopy.addAlignment( fileAlignmentForTaskAndNode.getNodeFileAlignment(), task, node ); + nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode ) ); + availableByNodes.get( node ).subFromThis( task.getRequest() ); + currentlyCopyingTasksOnNode.compute( node.getNodeLocation(), ( nodeLocation, value ) -> value == null ? 1 : value + 1 ); + return true; + } else { + return false; + } + } + + private FileAlignment getFileAlignmentForTaskAndNode( + final NodeWithAlloc node, + final Task task, + final TaskInputs inputsOfTask, + final CurrentlyCopying planedToCopy + ) { + final CurrentlyCopyingOnNode currentlyCopyingOnNode = this.currentlyCopying.get(node.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(node.getNodeLocation()); + try { + return inputAlignment.getInputAlignment( + task, + inputsOfTask, + node, + currentlyCopyingOnNode, + currentlyPlanedToCopy, + Double.MAX_VALUE + ); + } catch ( NoAligmentPossibleException e ){ + return null; + } + } + + private void removeAvailableResources( TaskStats taskStats, Map availableByNodes, List allNodes ) { + allNodes.parallelStream().forEach( node -> { + final Requirements availableOnNode = availableByNodes.get( node ); + final Requirements clone = availableOnNode.clone(); + for ( Task task : currentlyCopying.getTasksOnNode( node.getNodeLocation() ) ) { + final Requirements request = task.getRequest(); + if ( clone.higherOrEquals( request ) ) { + if ( availableOnNode.higherOrEquals( request ) ) { + //Here we remove the first tasks in the list. However, the ordering of copy tasks finished might be different. + availableOnNode.subFromThis( request ); + } + final TaskStat taskStat = taskStats.get( task ); + //We might still copy data for tasks that have already been started. + if ( taskStat != null ) { + taskStat.canStartAfterCopying(); + } + } + } + } ); + } + +} diff --git a/src/main/java/fonda/scheduler/util/TaskNodeStats.java b/src/main/java/fonda/scheduler/util/TaskNodeStats.java new file mode 100644 index 00000000..2c0f22ed --- /dev/null +++ b/src/main/java/fonda/scheduler/util/TaskNodeStats.java @@ -0,0 +1,26 @@ +package fonda.scheduler.util; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class TaskNodeStats { + + final long sizeRemaining; + final long sizeCurrentlyCopying; + final long sizeOnNode; + + public long getTaskSize() { + return sizeRemaining + sizeCurrentlyCopying + sizeOnNode; + } + + public boolean allOnNode() { + return sizeRemaining == 0 && sizeCurrentlyCopying == 0; + } + + public boolean allOnNodeOrCopying() { + return sizeRemaining == 0; + } + +} \ No newline at end of file diff --git a/src/main/java/fonda/scheduler/util/TaskStats.java b/src/main/java/fonda/scheduler/util/TaskStats.java new file mode 100644 index 00000000..c8c93705 --- /dev/null +++ b/src/main/java/fonda/scheduler/util/TaskStats.java @@ -0,0 +1,25 @@ +package fonda.scheduler.util; + +import fonda.scheduler.model.Task; +import fonda.scheduler.scheduler.la2.TaskStat; + +import java.util.*; + +public class TaskStats { + + private final Map taskStats = new HashMap<>(); + + public TaskStat get( Task task ) { + return this.taskStats.get( task ); + } + + + public void add( TaskStat taskStat ) { + this.taskStats.put( taskStat.getTask(), taskStat ); + } + + public Collection getTaskStats() { + return this.taskStats.values(); + } + +} diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java index cacc212b..1977f25f 100644 --- a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java +++ b/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java @@ -1,17 +1,31 @@ package fonda.scheduler.util.copying; +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.util.AlignmentWrapper; +import fonda.scheduler.util.FilePath; import lombok.ToString; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * Keeps track of what is currently being copied to all node and which copy tasks are running. + * This is used to avoid copying the same data to a node multiple times and to overload a single node with copy tasks. + */ @ToString public class CurrentlyCopying { private final Map copyingToNode = new ConcurrentHashMap<>(); + private final Map> taskOnNodes = new HashMap<>(); + private final Map> nodesForTask = new HashMap<>(); - public void add( NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { + public void add( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { if ( currentlyCopyingOnNode == null || currentlyCopyingOnNode.isEmpty() ) { return; } @@ -23,13 +37,14 @@ public void add( NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopy return currentlyCopying; } } ); + copyToNode( task, nodeLocation ); } public CurrentlyCopyingOnNode get( NodeLocation nodeLocation ) { return copyingToNode.computeIfAbsent( nodeLocation, node -> new CurrentlyCopyingOnNode() ); } - public void remove( NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { + public void remove( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { if ( currentlyCopyingOnNode == null || currentlyCopyingOnNode.isEmpty() ) { return; } @@ -41,6 +56,64 @@ public void remove( NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyC return currentlyCopying; } } ); + finishedCopyToNode( task, nodeLocation ); + } + + private void copyToNode( Task task, NodeLocation node ){ + synchronized ( this ) { + List tasks = taskOnNodes.computeIfAbsent( node, x -> new LinkedList<>() ); + tasks.add( task ); + + List nodes = nodesForTask.computeIfAbsent( task, x -> new LinkedList<>() ); + nodes.add( node ); + } + } + + private void finishedCopyToNode( Task task, NodeLocation node ){ + synchronized ( this ) { + List tasks = taskOnNodes.get( node ); + if ( tasks != null ) { + tasks.remove( task ); + } + + List nodes = nodesForTask.get( task ); + if ( nodes != null ) { + nodes.remove( node ); + } + } + } + + public List getTasksOnNode( NodeLocation nodeLocation ) { + synchronized ( this ) { + return taskOnNodes.getOrDefault( nodeLocation, new LinkedList<>() ); + } + } + + public int getNumberOfNodesForTask( Task task ) { + final List nodeLocations; + synchronized ( this ) { + nodeLocations = nodesForTask.get( task ); + } + return nodeLocations == null ? 0 : nodeLocations.size(); + } + + public Map getCurrentlyCopyingTasksOnNode() { + Map result = new HashMap<>(); + for ( Map.Entry> entry : taskOnNodes.entrySet() ) { + result.put( entry.getKey(), entry.getValue().size() ); + } + return result; + } + + public void addAlignment( final Map nodeFileAlignment, Task task, NodeWithAlloc node ) { + for (Map.Entry entry : nodeFileAlignment.entrySet()) { + final CurrentlyCopyingOnNode map = get(node.getNodeLocation()); + for ( FilePath filePath : entry.getValue().getFilesToCopy()) { + if ( entry.getKey() != node.getNodeLocation() ) { + map.add( filePath.getPath(), task, entry.getKey() ); + } + } + } } From 66dd040f17c7ab03d7c2851e8cff7e41768c771b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 15:27:58 +0100 Subject: [PATCH 324/443] removed unused imports Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/Task.java | 1 - src/main/java/fonda/scheduler/scheduler/Scheduler.java | 3 --- .../java/fonda/scheduler/scheduler/nodeassign/FairAssign.java | 3 --- .../fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java | 1 - .../fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java | 1 - src/main/java/fonda/scheduler/util/CopyTask.java | 2 -- src/main/java/fonda/scheduler/util/DataOnNode.java | 1 - src/main/java/fonda/scheduler/util/FileAlignment.java | 1 - src/test/java/fonda/scheduler/model/TaskResultParserTest.java | 1 - 9 files changed, 14 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index be9a99f9..3496bc19 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -13,7 +13,6 @@ import java.io.File; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Predicate; @Slf4j public class Task { diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index e9fd048d..af58c8a6 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -6,9 +6,6 @@ import fonda.scheduler.model.*; import fonda.scheduler.util.Batch; import fonda.scheduler.util.NodeTaskAlignment; -import io.fabric8.kubernetes.api.model.Binding; -import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.ObjectReference; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java index 2a38afb1..9aef70aa 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java @@ -1,6 +1,5 @@ package fonda.scheduler.scheduler.nodeassign; -import fonda.scheduler.client.Informable; import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.PodWithAge; import fonda.scheduler.model.Requirements; @@ -9,8 +8,6 @@ import lombok.extern.slf4j.Slf4j; import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java index 7bfd28ab..df829814 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java @@ -7,7 +7,6 @@ import fonda.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; -import java.math.BigDecimal; import java.util.*; @Slf4j diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java b/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java index 9fa6120b..50e5a853 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java +++ b/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java @@ -6,7 +6,6 @@ import fonda.scheduler.model.Requirements; import fonda.scheduler.model.Task; import fonda.scheduler.util.NodeTaskAlignment; -import io.swagger.v3.oas.models.links.Link; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/util/CopyTask.java b/src/main/java/fonda/scheduler/util/CopyTask.java index 48a95141..0ff3c76f 100644 --- a/src/main/java/fonda/scheduler/util/CopyTask.java +++ b/src/main/java/fonda/scheduler/util/CopyTask.java @@ -2,7 +2,6 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.TaskInputFileLocationWrapper; -import fonda.scheduler.model.location.Location; import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.scheduler.schedulingstrategy.Inputs; @@ -12,7 +11,6 @@ import lombok.RequiredArgsConstructor; import lombok.Setter; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; diff --git a/src/main/java/fonda/scheduler/util/DataOnNode.java b/src/main/java/fonda/scheduler/util/DataOnNode.java index 0793adc0..1e062121 100644 --- a/src/main/java/fonda/scheduler/util/DataOnNode.java +++ b/src/main/java/fonda/scheduler/util/DataOnNode.java @@ -2,7 +2,6 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index ceb66109..e2f01bf1 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -1,7 +1,6 @@ package fonda.scheduler.util; import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.SymlinkInput; import lombok.Getter; diff --git a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java index 825620da..5b085e14 100644 --- a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/fonda/scheduler/model/TaskResultParserTest.java @@ -13,7 +13,6 @@ import org.junit.Test; import java.io.BufferedWriter; -import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.nio.file.*; From 6b226d687bbca0b05aac3533b0cb6532d3a07803 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 16:17:10 +0100 Subject: [PATCH 325/443] Use Lombok for constructors Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/Edge.java | 11 ++----- .../java/fonda/scheduler/dag/InputEdge.java | 23 +++----------- src/main/java/fonda/scheduler/dag/Vertex.java | 7 ++--- .../fonda/scheduler/model/DateParser.java | 6 ++-- .../fonda/scheduler/model/FileHolder.java | 9 ++---- .../scheduler/model/InputFileCollector.java | 6 ++-- .../fonda/scheduler/model/InputParam.java | 8 ++--- .../java/fonda/scheduler/model/OutLabel.java | 9 ++---- .../fonda/scheduler/model/ScheduleObject.java | 6 ++-- .../scheduler/model/SchedulerConfig.java | 18 ++--------- .../java/fonda/scheduler/model/TaskInput.java | 31 ++++++------------- .../model/TaskInputFileLocationWrapper.java | 8 ++--- .../java/fonda/scheduler/model/TaskState.java | 7 ++--- .../scheduler/model/WriteConfigResult.java | 19 ++---------- .../model/location/NodeLocation.java | 7 ++--- .../model/location/hierachy/Folder.java | 5 +-- .../location/hierachy/LinkHierarchyFile.java | 6 ++-- .../scheduler/model/outfiles/OutputFile.java | 6 ++-- .../taskinputs/PathFileLocationTriple.java | 8 ++--- .../model/taskinputs/TaskInputs.java | 8 ++--- .../fonda/scheduler/rest/PathAttributes.java | 9 ++---- .../scheduler/MatchingFilesAndNodes.java | 7 ++--- .../scheduler/data/NodeDataTuple.java | 12 ++----- .../filealignment/GreedyAlignment.java | 6 ++-- .../costfunctions/MinSizeCost.java | 6 ++-- .../schedulingstrategy/InputEntry.java | 9 ++---- .../scheduler/schedulingstrategy/Inputs.java | 20 ++++-------- src/main/java/fonda/scheduler/util/Batch.java | 6 ++-- .../fonda/scheduler/util/FileAlignment.java | 8 ++--- .../java/fonda/scheduler/util/FilePath.java | 7 ++--- .../scheduler/util/NodeTaskAlignment.java | 6 ++-- src/main/java/fonda/scheduler/util/Tuple.java | 6 ++-- 32 files changed, 88 insertions(+), 222 deletions(-) diff --git a/src/main/java/fonda/scheduler/dag/Edge.java b/src/main/java/fonda/scheduler/dag/Edge.java index 5a88381a..f04425ad 100644 --- a/src/main/java/fonda/scheduler/dag/Edge.java +++ b/src/main/java/fonda/scheduler/dag/Edge.java @@ -1,12 +1,11 @@ package fonda.scheduler.dag; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; +import lombok.*; @Getter @ToString @EqualsAndHashCode +@RequiredArgsConstructor( access = AccessLevel.PACKAGE ) public class Edge { private final int uid; @@ -18,10 +17,4 @@ public class Edge { this(uid, null, from, to); } - Edge( int uid, String label, Vertex from, Vertex to) { - this.uid = uid; - this.label = label; - this.from = from; - this.to = to; - } } diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/fonda/scheduler/dag/InputEdge.java index 89e29e41..ff3c7179 100644 --- a/src/main/java/fonda/scheduler/dag/InputEdge.java +++ b/src/main/java/fonda/scheduler/dag/InputEdge.java @@ -1,31 +1,16 @@ package fonda.scheduler.dag; -import lombok.Getter; -import lombok.ToString; +import lombok.*; @Getter @ToString +@RequiredArgsConstructor +@NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) public class InputEdge { private final int uid; - private final String label; + private final String label = null; private final int from; private final int to; - @SuppressWarnings("unused") - private InputEdge() { - this(-1, -1, -1); - } - - /** - * Just for testing - * @param from - * @param to - */ - public InputEdge( int uid, int from, int to) { - this.uid = uid; - label = null; - this.from = from; - this.to = to; - } } diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/fonda/scheduler/dag/Vertex.java index 9611aedd..1fb92400 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/fonda/scheduler/dag/Vertex.java @@ -2,6 +2,7 @@ import lombok.AccessLevel; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.Setter; import java.util.Collection; @@ -10,6 +11,7 @@ import java.util.stream.Collectors; @Getter +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class Vertex { private final String label; @@ -20,11 +22,6 @@ public abstract class Vertex { @Setter(AccessLevel.PACKAGE) private int rank = 0; - Vertex( String label, int uid ) { - this.label = label; - this.uid = uid; - } - public abstract Type getType(); public abstract void addInbound( Edge e ); diff --git a/src/main/java/fonda/scheduler/model/DateParser.java b/src/main/java/fonda/scheduler/model/DateParser.java index a1a8cc80..dd3737bd 100644 --- a/src/main/java/fonda/scheduler/model/DateParser.java +++ b/src/main/java/fonda/scheduler/model/DateParser.java @@ -1,12 +1,14 @@ package fonda.scheduler.model; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + import java.nio.file.attribute.FileTime; import java.text.SimpleDateFormat; +@NoArgsConstructor(access = AccessLevel.PRIVATE) public final class DateParser { - private DateParser(){} - public static Long millisFromString( String date ) { if( date == null || date.isEmpty() || date.equals("-") || date.equals("w") ) { return null; diff --git a/src/main/java/fonda/scheduler/model/FileHolder.java b/src/main/java/fonda/scheduler/model/FileHolder.java index 305cf2fd..bd2cab0a 100644 --- a/src/main/java/fonda/scheduler/model/FileHolder.java +++ b/src/main/java/fonda/scheduler/model/FileHolder.java @@ -1,20 +1,17 @@ package fonda.scheduler.model; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import lombok.ToString; @ToString( exclude = {"stageName", "storePath"}) +@NoArgsConstructor(access = AccessLevel.NONE) public class FileHolder { public final String storePath; public final String sourceObj; public final String stageName; - private FileHolder() { - this.storePath = null; - this.sourceObj = null; - this.stageName = null; - } - /** * Only for testing * @param storePath diff --git a/src/main/java/fonda/scheduler/model/InputFileCollector.java b/src/main/java/fonda/scheduler/model/InputFileCollector.java index 0fc79d2c..6a9c7b9a 100644 --- a/src/main/java/fonda/scheduler/model/InputFileCollector.java +++ b/src/main/java/fonda/scheduler/model/InputFileCollector.java @@ -6,20 +6,18 @@ import fonda.scheduler.model.taskinputs.SymlinkInput; import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.util.Tuple; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; import java.util.*; @Slf4j +@RequiredArgsConstructor public class InputFileCollector { private final HierarchyWrapper hierarchyWrapper; - public InputFileCollector( HierarchyWrapper hierarchyWrapper ) { - this.hierarchyWrapper = hierarchyWrapper; - } - private void processNext( final LinkedList> toProcess, final List symlinks, diff --git a/src/main/java/fonda/scheduler/model/InputParam.java b/src/main/java/fonda/scheduler/model/InputParam.java index 3b638ed5..9492af4e 100644 --- a/src/main/java/fonda/scheduler/model/InputParam.java +++ b/src/main/java/fonda/scheduler/model/InputParam.java @@ -1,18 +1,16 @@ package fonda.scheduler.model; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import lombok.ToString; @ToString +@NoArgsConstructor(access = AccessLevel.NONE, force = true) public class InputParam { public final String name; public final T value; - private InputParam() { - this.name = null; - this.value = null; - } - /** * Only for testing * @param name diff --git a/src/main/java/fonda/scheduler/model/OutLabel.java b/src/main/java/fonda/scheduler/model/OutLabel.java index 49c040bf..85700dfc 100644 --- a/src/main/java/fonda/scheduler/model/OutLabel.java +++ b/src/main/java/fonda/scheduler/model/OutLabel.java @@ -1,22 +1,19 @@ package fonda.scheduler.model; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; @Getter @ToString +@RequiredArgsConstructor public class OutLabel { private final String label; private final double weight; private OutLabel() { - this.label = null; - this.weight = -1; + this(null,-1 ); } - public OutLabel(String label, double weight) { - this.label = label; - this.weight = weight; - } } diff --git a/src/main/java/fonda/scheduler/model/ScheduleObject.java b/src/main/java/fonda/scheduler/model/ScheduleObject.java index e8c2ef56..73bbb54d 100644 --- a/src/main/java/fonda/scheduler/model/ScheduleObject.java +++ b/src/main/java/fonda/scheduler/model/ScheduleObject.java @@ -2,20 +2,18 @@ import fonda.scheduler.util.NodeTaskAlignment; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.Setter; import java.util.List; @Setter @Getter +@RequiredArgsConstructor public class ScheduleObject { private final List taskAlignments; private boolean checkStillPossible = false; private boolean stopSubmitIfOneFails = false; - public ScheduleObject( List taskAlignments ) { - this.taskAlignments = taskAlignments; - } - } diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 28eb7b81..71e269f4 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -1,10 +1,13 @@ package fonda.scheduler.model; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import lombok.ToString; import java.util.List; @ToString +@NoArgsConstructor(access = AccessLevel.PRIVATE,force = true) public class SchedulerConfig { public final List localClaims; @@ -22,21 +25,6 @@ public class SchedulerConfig { public final Integer maxWaitingCopyTasksPerNode; - private SchedulerConfig() { - this.localClaims = null; - this.volumeClaims = null; - this.workDir = null; - this.dns = null; - this.copyStrategy = null; - this.locationAware = false; - this.traceEnabled = false; - this.costFunction = null; - this.namespace = null; - this.strategy = null; - this.maxCopyTasksPerNode = null; - this.maxWaitingCopyTasksPerNode = null; - } - @ToString public static class LocalClaim { public final String mountPath; diff --git a/src/main/java/fonda/scheduler/model/TaskInput.java b/src/main/java/fonda/scheduler/model/TaskInput.java index 8e662482..994463f8 100644 --- a/src/main/java/fonda/scheduler/model/TaskInput.java +++ b/src/main/java/fonda/scheduler/model/TaskInput.java @@ -1,7 +1,16 @@ package fonda.scheduler.model; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; + import java.util.List; +@NoArgsConstructor( access = AccessLevel.PRIVATE, force = true ) +/** + * Only for testing + */ +@RequiredArgsConstructor( access = AccessLevel.PACKAGE ) public class TaskInput { public final List> booleanInputs; @@ -9,27 +18,6 @@ public class TaskInput { public final List> stringInputs; public final List> fileInputs; - private TaskInput() { - this.booleanInputs = null; - this.numberInputs = null; - this.stringInputs = null; - this.fileInputs = null; - } - - /** - * Only for testing - * @param booleanInputs - * @param numberInputs - * @param stringInputs - * @param fileInputs - */ - TaskInput(List> booleanInputs, List> numberInputs, List> stringInputs, List> fileInputs) { - this.booleanInputs = booleanInputs; - this.numberInputs = numberInputs; - this.stringInputs = stringInputs; - this.fileInputs = fileInputs; - } - @Override public String toString() { return "TaskInput{" + @@ -39,4 +27,5 @@ public String toString() { ", fileInputs=#" + (fileInputs != null ? fileInputs.size() : 0) + '}'; } + } diff --git a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java index dc027654..8878a3f4 100644 --- a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java +++ b/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java @@ -3,20 +3,16 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealHierarchyFile; import lombok.Getter; +import lombok.RequiredArgsConstructor; @Getter +@RequiredArgsConstructor public class TaskInputFileLocationWrapper { private final String path; private final RealHierarchyFile file; private final LocationWrapper wrapper; - public TaskInputFileLocationWrapper( String path, RealHierarchyFile file, LocationWrapper wrapper) { - this.path = path; - this.file = file; - this.wrapper = wrapper; - } - public void success(){ file.addOrUpdateLocation( false, wrapper ); } diff --git a/src/main/java/fonda/scheduler/model/TaskState.java b/src/main/java/fonda/scheduler/model/TaskState.java index 99335ce4..2a2c6b56 100644 --- a/src/main/java/fonda/scheduler/model/TaskState.java +++ b/src/main/java/fonda/scheduler/model/TaskState.java @@ -7,18 +7,15 @@ public class TaskState { @Getter @Setter - private State state; + private State state = State.RECEIVED_CONFIG;; @Getter private String error; - public TaskState() { - this.state = State.RECEIVED_CONFIG; - } - public void error (String error){ this.state = State.ERROR; this.error = error; } + } diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/fonda/scheduler/model/WriteConfigResult.java index da630904..a229e5d8 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/fonda/scheduler/model/WriteConfigResult.java @@ -1,33 +1,20 @@ package fonda.scheduler.model; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.AllArgsConstructor; import lombok.Getter; import java.util.List; import java.util.Map; @Getter +@AllArgsConstructor public class WriteConfigResult { private final List inputFiles; private final Map waitForTask; private final CurrentlyCopyingOnNode copyingToNode; - - private boolean copyDataToNode; - private final boolean wroteConfig; + private boolean copyDataToNode; - public WriteConfigResult( - List inputFiles, - Map waitForTask, - CurrentlyCopyingOnNode copyingToNode, - boolean wroteConfig, - boolean copyDataToNode - ) { - this.inputFiles = inputFiles; - this.waitForTask = waitForTask; - this.copyingToNode = copyingToNode; - this.wroteConfig = wroteConfig; - this.copyDataToNode = copyDataToNode; - } } diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/fonda/scheduler/model/location/NodeLocation.java index 5f862e16..de69388b 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/fonda/scheduler/model/location/NodeLocation.java @@ -1,11 +1,14 @@ package fonda.scheduler.model.location; import io.fabric8.kubernetes.api.model.Node; +import lombok.AccessLevel; import lombok.Getter; +import lombok.RequiredArgsConstructor; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +@RequiredArgsConstructor( access = AccessLevel.PRIVATE ) public class NodeLocation extends Location { private static final long serialVersionUID = 1L; @@ -15,10 +18,6 @@ public class NodeLocation extends Location { @Getter private final String identifier; - private NodeLocation(String identifier) { - this.identifier = identifier; - } - public static NodeLocation getLocation( Node node ){ return getLocation( node.getMetadata().getName() ); } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java index 7b088d06..6b52d3d7 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java @@ -1,5 +1,7 @@ package fonda.scheduler.model.location.hierachy; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; @@ -9,12 +11,11 @@ import java.util.concurrent.ConcurrentMap; @Slf4j +@NoArgsConstructor(access = AccessLevel.PACKAGE) public class Folder extends HierarchyFile { private final ConcurrentMap children = new ConcurrentHashMap<>(); - Folder(){} - @Override public boolean isDirectory() { return true; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java b/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java index 3d3ddd17..ca8a5c93 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java +++ b/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java @@ -1,18 +1,16 @@ package fonda.scheduler.model.location.hierachy; import lombok.Getter; +import lombok.RequiredArgsConstructor; import java.nio.file.Path; +@RequiredArgsConstructor public class LinkHierarchyFile extends AbstractHierarchyFile { @Getter private final Path dst; - public LinkHierarchyFile(Path dst ) { - this.dst = dst; - } - @Override public boolean isDirectory() { throw new IllegalStateException("Call on link"); diff --git a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java b/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java index 79f8dfea..a161dc94 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java +++ b/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java @@ -1,19 +1,17 @@ package fonda.scheduler.model.outfiles; import lombok.Getter; +import lombok.RequiredArgsConstructor; import java.nio.file.Path; import java.util.Objects; @Getter +@RequiredArgsConstructor public class OutputFile { private final Path path; - public OutputFile(Path path) { - this.path = path; - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java index 191a6975..39601bb0 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -4,6 +4,7 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealHierarchyFile; import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; import lombok.ToString; import java.nio.file.Path; @@ -11,6 +12,7 @@ @ToString( exclude = "file" ) @EqualsAndHashCode +@RequiredArgsConstructor public class PathFileLocationTriple implements Input { public final Path path; @@ -18,12 +20,6 @@ public class PathFileLocationTriple implements Input { public final List locations; private long size = -1; - public PathFileLocationTriple(Path path, RealHierarchyFile file, List locations) { - this.path = path; - this.file = file; - this.locations = locations; - } - public long getSizeInBytes() { if ( this.size != -1 ) { return this.size; diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java index 733c6aae..483c1b95 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java @@ -7,6 +7,7 @@ import fonda.scheduler.util.copying.CopySource; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @@ -17,6 +18,7 @@ @ToString @Getter @Slf4j +@RequiredArgsConstructor public class TaskInputs { private final List symlinks; @@ -25,12 +27,6 @@ public class TaskInputs { private boolean sorted = false; - public TaskInputs(List symlinks, List files, Set excludedNodes) { - this.symlinks = symlinks; - this.files = files; - this.excludedNodes = excludedNodes; - } - public boolean allFilesAreOnLocationAndNotOverwritten( Location loc, Set pathCurrentlyCopying ){ for (PathFileLocationTriple file : files) { if ( !file.locatedOnLocation(loc) || (pathCurrentlyCopying != null && pathCurrentlyCopying.contains(file.path.toString())) ) { diff --git a/src/main/java/fonda/scheduler/rest/PathAttributes.java b/src/main/java/fonda/scheduler/rest/PathAttributes.java index d99dd472..24a80877 100644 --- a/src/main/java/fonda/scheduler/rest/PathAttributes.java +++ b/src/main/java/fonda/scheduler/rest/PathAttributes.java @@ -1,10 +1,13 @@ package fonda.scheduler.rest; +import lombok.AccessLevel; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.ToString; @Getter @ToString +@NoArgsConstructor( access = AccessLevel.PRIVATE, force = true ) public class PathAttributes { private final String path; @@ -12,10 +15,4 @@ public class PathAttributes { private final long timestamp; private final long locationWrapperID; - private PathAttributes() { - this.path = null; - this.size = -1; - this.timestamp = -1; - this.locationWrapperID = -1; - } } diff --git a/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java b/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java index 31e56da8..33717ac6 100644 --- a/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java +++ b/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java @@ -3,18 +3,15 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; +import lombok.RequiredArgsConstructor; import java.util.Set; @Getter +@RequiredArgsConstructor public class MatchingFilesAndNodes { private final Set nodes; private final TaskInputs inputsOfTask; - public MatchingFilesAndNodes( Set nodes, TaskInputs inputsOfTask ) { - this.nodes = nodes; - this.inputsOfTask = inputsOfTask; - } - } diff --git a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java index 45593a48..84818ad1 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java +++ b/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java @@ -1,14 +1,12 @@ package fonda.scheduler.scheduler.data; import fonda.scheduler.model.NodeWithAlloc; -import lombok.AccessLevel; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import org.jetbrains.annotations.NotNull; @Getter @EqualsAndHashCode +@AllArgsConstructor public class NodeDataTuple implements Comparable { private final NodeWithAlloc node; @@ -21,12 +19,6 @@ public NodeDataTuple( NodeWithAlloc node, long sizeInBytes ) { this( node, sizeInBytes, 1.0 ); } - public NodeDataTuple(NodeWithAlloc node, long sizeInBytes, double weight ) { - this.node = node; - this.sizeInBytes = sizeInBytes; - this.weight = weight; - } - /** * @return reduce the worth, if this is not the outLabelNode */ diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index f7604a34..f7359b04 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -12,20 +12,18 @@ import fonda.scheduler.util.FileAlignment; import fonda.scheduler.util.FilePath; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import java.util.HashMap; @Slf4j +@RequiredArgsConstructor public class GreedyAlignment extends InputAlignmentClass { private final CostFunction cf; - public GreedyAlignment(CostFunction cf) { - this.cf = cf; - } - double findAlignmentForFile ( PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java index 13dd11e3..191d0917 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java @@ -1,15 +1,13 @@ package fonda.scheduler.scheduler.filealignment.costfunctions; import fonda.scheduler.model.location.hierachy.LocationWrapper; +import lombok.RequiredArgsConstructor; +@RequiredArgsConstructor public class MinSizeCost implements CostFunction { private final double initCost; - public MinSizeCost(double initCost) { - this.initCost = initCost; - } - @Override public double getInitCost() { return initCost; diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java index 7eb2a07f..55e64c4a 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java @@ -1,9 +1,11 @@ package fonda.scheduler.scheduler.schedulingstrategy; +import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; import java.util.List; +@RequiredArgsConstructor public class InputEntry implements Comparable { public final String currentIP; @@ -11,13 +13,6 @@ public class InputEntry implements Comparable { public final List files; public final long size; - public InputEntry( String currentIP, String node, List files, long size ) { - this.currentIP = currentIP; - this.node = node; - this.files = files; - this.size = size; - } - @Override public int compareTo(@NotNull InputEntry o) { return Long.compare( size, o.size ); diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index 4ee5174f..1eaf1244 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -2,6 +2,7 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.taskinputs.SymlinkInput; +import lombok.RequiredArgsConstructor; import java.util.Collections; import java.util.LinkedList; @@ -9,25 +10,16 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +@RequiredArgsConstructor public class Inputs { public final String dns; public final String execution; - public final String hash; - public final List data; - public final List symlinks; + public final List data = new LinkedList<>(); + public final List symlinks = new LinkedList<>(); public final String syncDir; - public final Map> waitForFilesOfTask; - - public Inputs( String dns, String execution, String syncDir, String hash ) { - this.dns = dns; - this.execution = execution; - this.syncDir = syncDir; - this.hash = hash; - this.data = new LinkedList<>(); - this.symlinks = new LinkedList<>(); - waitForFilesOfTask = new ConcurrentHashMap<>(); - } + public final String hash; + public final Map> waitForFilesOfTask = new ConcurrentHashMap<>(); public void waitForTask( Map waitForTask ){ for (Map.Entry e : waitForTask.entrySet()) { diff --git a/src/main/java/fonda/scheduler/util/Batch.java b/src/main/java/fonda/scheduler/util/Batch.java index 0ccb9909..9d73daa9 100644 --- a/src/main/java/fonda/scheduler/util/Batch.java +++ b/src/main/java/fonda/scheduler/util/Batch.java @@ -3,12 +3,14 @@ import fonda.scheduler.model.Task; import fonda.scheduler.model.tracing.TraceRecord; import lombok.Getter; +import lombok.RequiredArgsConstructor; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; +@RequiredArgsConstructor public class Batch { @Getter @@ -23,10 +25,6 @@ public class Batch { private long closeTime; - public Batch(int id) { - this.id = id; - } - public void close( int tasksInBatch ){ this.closed = true; this.tasksInBatch = tasksInBatch; diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/fonda/scheduler/util/FileAlignment.java index e2f01bf1..d5c918ca 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/fonda/scheduler/util/FileAlignment.java @@ -4,6 +4,7 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.taskinputs.SymlinkInput; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.ToString; @@ -13,6 +14,7 @@ @Getter @ToString +@RequiredArgsConstructor public class FileAlignment { /* @@ -25,12 +27,6 @@ public class FileAlignment { @Setter private double weight = 1.0; - public FileAlignment(Map nodeFileAlignment, List symlinks, double cost) { - this.nodeFileAlignment = nodeFileAlignment; - this.symlinks = symlinks; - this.cost = cost; - } - public double getWorth() { return cost / weight; } diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/fonda/scheduler/util/FilePath.java index 29ff8b19..e994f6cd 100644 --- a/src/main/java/fonda/scheduler/util/FilePath.java +++ b/src/main/java/fonda/scheduler/util/FilePath.java @@ -3,17 +3,14 @@ import fonda.scheduler.model.location.hierachy.LocationWrapper; import fonda.scheduler.model.location.hierachy.RealHierarchyFile; import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import lombok.RequiredArgsConstructor; +@RequiredArgsConstructor public class FilePath { private final PathFileLocationTriple pathFileLocationTriple; private final LocationWrapper locationWrapper; - public FilePath( PathFileLocationTriple pathFileLocationTriple, LocationWrapper locationWrapper ) { - this.pathFileLocationTriple = pathFileLocationTriple; - this.locationWrapper = locationWrapper; - } - public String getPath() { return pathFileLocationTriple.path.toString(); } diff --git a/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java index 94867c5f..799660a6 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java @@ -2,14 +2,12 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Task; +import lombok.RequiredArgsConstructor; +@RequiredArgsConstructor public class NodeTaskAlignment { public final NodeWithAlloc node; public final Task task; - public NodeTaskAlignment( NodeWithAlloc node, Task task ) { - this.node = node; - this.task = task; - } } diff --git a/src/main/java/fonda/scheduler/util/Tuple.java b/src/main/java/fonda/scheduler/util/Tuple.java index 129761ac..1291d02d 100644 --- a/src/main/java/fonda/scheduler/util/Tuple.java +++ b/src/main/java/fonda/scheduler/util/Tuple.java @@ -1,17 +1,15 @@ package fonda.scheduler.util; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; @Getter @ToString +@RequiredArgsConstructor public class Tuple { private final S a; private final T b; - public Tuple( S a, T b ) { - this.a = a; - this.b = b; - } } From 3bd2c1f091380f1b503724200e444df989aa3537 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 16:17:41 +0100 Subject: [PATCH 326/443] Removed empty lines Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/model/outfiles/PathLocationWrapperPair.java | 1 - src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java | 1 - .../java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java | 1 - src/main/java/fonda/scheduler/util/SortedList.java | 2 -- 4 files changed, 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java index 289bdbac..aa5a2b74 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java +++ b/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java @@ -9,7 +9,6 @@ @Getter public class PathLocationWrapperPair extends OutputFile { - private final LocationWrapper locationWrapper; public PathLocationWrapperPair(Path path, LocationWrapper locationWrapper) { diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java index 0d2dd958..e4306efe 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java @@ -7,7 +7,6 @@ public class NodeTaskFilesAlignment extends NodeTaskAlignment { - public final FileAlignment fileAlignment; @Getter diff --git a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java index 375de2d1..56aea6b7 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java @@ -9,7 +9,6 @@ public class NodeTaskLocalFilesAlignment extends NodeTaskAlignment { - public final List symlinks; public final List locationWrappers; diff --git a/src/main/java/fonda/scheduler/util/SortedList.java b/src/main/java/fonda/scheduler/util/SortedList.java index 7a566487..0e96a884 100644 --- a/src/main/java/fonda/scheduler/util/SortedList.java +++ b/src/main/java/fonda/scheduler/util/SortedList.java @@ -6,13 +6,11 @@ public class SortedList> extends LinkedList { - public SortedList( Collection collection ) { super( collection ); this.sort( Comparable::compareTo ); } - @Override public boolean add( T elem ) { final ListIterator iterator = this.listIterator(); From 86b3f58fde6270b120cc20d48e6a71af1a0157b7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 16:25:41 +0100 Subject: [PATCH 327/443] Removed unused method Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index e74e84b4..de274eb3 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -284,22 +284,6 @@ CopyTask createCopyTask( NodeTaskFilesAlignment alignment ) { return new CopyTask( inputs, inputFiles, filesForCurrentNode, alignment.task ); } - List createCopyTasks( List tasksAndData ){ - final CurrentlyCopying planedToCopy = new CurrentlyCopying(); - tasksAndData.sort( Comparator.comparing( DataOnNode::getNodesWithAllData ).reversed() ); - final Map assignedPodsByNode = copyToNodeManager.getCurrentlyCopyingTasksOnNode(); - List nodeTaskAlignments = new LinkedList<>(); - for ( DataOnNode dataOnNode : tasksAndData ) { - final Tuple result = calculateBestNode( dataOnNode, planedToCopy ); - if ( result != null && assignedPodsByNode.getOrDefault( result.getA().getNodeLocation(), 0 ) < getMaxCopyTasksPerNode() ) { - addAlignmentToPlanned( planedToCopy, result.getB().getNodeFileAlignment(), dataOnNode.getTask(), result.getA() ); - nodeTaskAlignments.add( new NodeTaskFilesAlignment( result.getA(), dataOnNode.getTask(), result.getB() ) ); - assignedPodsByNode.put( result.getA().getNodeLocation(), assignedPodsByNode.getOrDefault( result.getA().getNodeLocation(), 0 ) + 1 ); - } - } - return nodeTaskAlignments; - } - Tuple calculateBestNode( final DataOnNode taskData, CurrentlyCopying planedToCopy ) { FileAlignment bestAlignment = null; NodeWithAlloc bestNode = null; From 0d1f678f570dc755488189c89e145d733beb97fd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 10 Jan 2023 18:05:41 +0100 Subject: [PATCH 328/443] Weight the individual and the overall cost Signed-off-by: Lehmann_Fabian --- .../rest/SchedulerRestController.java | 4 +-- .../filealignment/GreedyAlignment.java | 12 ++++++-- .../filealignment/InputAlignmentClass.java | 30 ++++++++++++++++--- .../filealignment/RandomAlignment.java | 8 +++-- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java index 512c65f3..17a5f8f2 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/fonda/scheduler/rest/SchedulerRestController.java @@ -125,7 +125,7 @@ ResponseEntity registerScheduler( if ( costFunction == null ) { costFunction = new MinSizeCost( 0 ); } - scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment(costFunction) ); + scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ) ); break; case "lav2" : if ( !config.locationAware ) { @@ -134,7 +134,7 @@ ResponseEntity registerScheduler( if ( costFunction == null ) { costFunction = new MinSizeCost( 0 ); } - scheduler = new LocationAwareSchedulerV2( execution, client, namespace, config, new GreedyAlignment(costFunction), new OptimalReadyToRunToNode() ); + scheduler = new LocationAwareSchedulerV2( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ), new OptimalReadyToRunToNode() ); break; default: { final String[] split = strategy.split( "-" ); diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index f7359b04..4bbea78b 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -19,17 +19,22 @@ import java.util.HashMap; @Slf4j -@RequiredArgsConstructor public class GreedyAlignment extends InputAlignmentClass { private final CostFunction cf; - double findAlignmentForFile ( + public GreedyAlignment( double weightForSingleSource, CostFunction cf ) { + super( weightForSingleSource ); + this.cf = cf; + } + + Costs findAlignmentForFile ( PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map ){ double minCost = Double.MAX_VALUE; + double individualCost = -1; Location bestLoc = null; LocationWrapper bestLocationWrapper = null; for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { @@ -45,11 +50,12 @@ public class GreedyAlignment extends InputAlignmentClass { bestLoc = currentLoc; minCost = calculatedCost; bestLocationWrapper = locationWrapper; + individualCost = cf.getCost( locationWrapper ); } } final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost, bestLocationWrapper.getSizeInBytes() ); - return minCost; + return new Costs( individualCost, minCost ); } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java index d2b0c123..b8313d16 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -14,6 +14,7 @@ import fonda.scheduler.util.Tuple; import fonda.scheduler.util.copying.CopySource; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -24,6 +25,17 @@ @Slf4j abstract class InputAlignmentClass implements InputAlignment { + /** + * This weight weights the cost of copying files from one node. 1 - this value weights the overall data copied. + * How much should the individual costs of one node be weighted compared to the overall data copied. + */ + private final double weightForSingleSource; + + public InputAlignmentClass( double weightForSingleSource ) { + assert weightForSingleSource >= 0 && weightForSingleSource <= 1; + this.weightForSingleSource = weightForSingleSource; + } + /** * Check if another scheduled task is already copying a required file * @@ -38,13 +50,11 @@ private Tuple alreadyCopying( CurrentlyCopyingOnNode curre final Location copyFrom = copySource.getLocation(); for ( LocationWrapper locationWrapper : locations ) { if ( locationWrapper.getLocation() == copyFrom ) { - log.info( "Checking if {} is already copying: {} (Found)", path, currentlyCopying.getAllFilesCurrentlyCopying() ); return new Tuple(locationWrapper,copySource.getTask()); } } throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); } - log.info( "Checking if {} is already copying: {} (Not found)", path, currentlyCopying == null ? null : currentlyCopying.getAllFilesCurrentlyCopying() ); return null; } @@ -77,23 +87,35 @@ public FileAlignment getInputAlignment(@NotNull Task task, double maxCost) { final HashMap map = new HashMap<>(); double cost = 0; + double overallCost = 0; for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) && !canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) ) { - final double newCost = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); + final Costs result = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); + final double newCost = (result.singleSourceCost * weightForSingleSource) + + (result.individualCost + overallCost ) * ( 1 - weightForSingleSource); if ( newCost > maxCost ) { return null; } if ( newCost > cost ) { cost = newCost; + overallCost += result.individualCost; } } } return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); } - abstract double findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map); + abstract Costs findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map); + + @RequiredArgsConstructor + class Costs { + + private final double individualCost; + private final double singleSourceCost; + + } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index 6d10bb72..f0b3f56c 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -15,8 +15,12 @@ public class RandomAlignment extends InputAlignmentClass { private final Random random = new Random(); + public RandomAlignment() { + super( 0 ); + } + @Override - double findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map) { + Costs findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map) { final Optional first = pathFileLocationTriple .locations .stream() @@ -28,7 +32,7 @@ public class RandomAlignment extends InputAlignmentClass { final Location location = locationWrapper.getLocation(); final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0, locationWrapper.getSizeInBytes() ); - return 0; + return new Costs( 0, locationWrapper.getSizeInBytes() ); } } From ba90baa986f42a9fdeee922bf276a2b4993b2d7d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 12 Jan 2023 10:38:42 +0100 Subject: [PATCH 329/443] Weight the individual and the overall cost moved into GreedyAlignment Signed-off-by: Lehmann_Fabian --- .../filealignment/GreedyAlignment.java | 22 ++++++--- .../filealignment/InputAlignmentClass.java | 47 ++++++++++++------- .../filealignment/RandomAlignment.java | 10 +++- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java index 4bbea78b..9e8e9879 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -31,31 +31,39 @@ public GreedyAlignment( double weightForSingleSource, CostFunction cf ) { Costs findAlignmentForFile ( PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, - HashMap map + HashMap map, + final Costs costs ){ double minCost = Double.MAX_VALUE; - double individualCost = -1; + double currentMaxCostForIndividualNode = costs.getMaxCostForIndividualNode(); + double currentSumOfCosts = costs.getSumOfCosts(); Location bestLoc = null; LocationWrapper bestLocationWrapper = null; for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { final Location currentLoc = locationWrapper.getLocation(); - final double calculatedCost; + final double maxCostOnIndividualNode; + final double tmpCurrentSumOfCosts; if ( currentLoc != scheduledNode) { final AlignmentWrapper alignmentWrapper = map.get(currentLoc); - calculatedCost = cf.calculateCost(alignmentWrapper, locationWrapper); + tmpCurrentSumOfCosts = costs.getSumOfCosts() + cf.getCost( locationWrapper ); + final double costOnIndividualNode = cf.calculateCost(alignmentWrapper, locationWrapper); + maxCostOnIndividualNode = Math.max( costOnIndividualNode, costs.getMaxCostForIndividualNode() ); } else { - calculatedCost = 0; + tmpCurrentSumOfCosts = costs.getSumOfCosts(); + maxCostOnIndividualNode = costs.getMaxCostForIndividualNode(); } + final double calculatedCost = calculateCost( maxCostOnIndividualNode, tmpCurrentSumOfCosts ); if ( calculatedCost < minCost ) { bestLoc = currentLoc; minCost = calculatedCost; bestLocationWrapper = locationWrapper; - individualCost = cf.getCost( locationWrapper ); + currentMaxCostForIndividualNode = maxCostOnIndividualNode; + currentSumOfCosts = tmpCurrentSumOfCosts; } } final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost, bestLocationWrapper.getSizeInBytes() ); - return new Costs( individualCost, minCost ); + return new Costs( currentMaxCostForIndividualNode, currentSumOfCosts, minCost ); } @Override diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java index b8313d16..b4a39ea3 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -14,6 +14,8 @@ import fonda.scheduler.util.Tuple; import fonda.scheduler.util.copying.CopySource; import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -29,11 +31,13 @@ abstract class InputAlignmentClass implements InputAlignment { * This weight weights the cost of copying files from one node. 1 - this value weights the overall data copied. * How much should the individual costs of one node be weighted compared to the overall data copied. */ - private final double weightForSingleSource; + private final double weightForIndividualNode; - public InputAlignmentClass( double weightForSingleSource ) { - assert weightForSingleSource >= 0 && weightForSingleSource <= 1; - this.weightForSingleSource = weightForSingleSource; + public InputAlignmentClass( double weightForIndividualNode ) { + if ( weightForIndividualNode < 0 && weightForIndividualNode > 1 ) { + throw new IllegalArgumentException( "Weight for single source must be between 0 and 1" ); + } + this.weightForIndividualNode = weightForIndividualNode; } /** @@ -86,36 +90,43 @@ public FileAlignment getInputAlignment(@NotNull Task task, CurrentlyCopyingOnNode currentlyPlanedToCopy, double maxCost) { final HashMap map = new HashMap<>(); - double cost = 0; - double overallCost = 0; + Costs costs = new Costs(); for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) && !canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) ) { - final Costs result = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map ); - final double newCost = (result.singleSourceCost * weightForSingleSource) - + (result.individualCost + overallCost ) * ( 1 - weightForSingleSource); - if ( newCost > maxCost ) { + costs = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map, costs ); + if ( costs.calculatedCost > maxCost ) { return null; } - if ( newCost > cost ) { - cost = newCost; - overallCost += result.individualCost; - } } } - return new FileAlignment( map, inputsOfTask.getSymlinks(), cost); + return new FileAlignment( map, inputsOfTask.getSymlinks(), costs.calculatedCost); + } + + protected double calculateCost( double maxIndividual, double sumOfCost ) { + return maxIndividual * weightForIndividualNode + sumOfCost * ( 1 - weightForIndividualNode ); } - abstract Costs findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map); + abstract Costs findAlignmentForFile( + PathFileLocationTriple pathFileLocationTriple, + NodeLocation scheduledNode, + HashMap map, + final Costs costs + ); + @Getter @RequiredArgsConstructor class Costs { - private final double individualCost; - private final double singleSourceCost; + private final double maxCostForIndividualNode; + private final double sumOfCosts; + private final double calculatedCost; + public Costs() { + this( 0, 0, 0 ); + } } } diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java index f0b3f56c..0ef71d41 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java @@ -20,7 +20,12 @@ public RandomAlignment() { } @Override - Costs findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLocation scheduledNode, HashMap map) { + Costs findAlignmentForFile ( + PathFileLocationTriple pathFileLocationTriple, + NodeLocation scheduledNode, + HashMap map, + final Costs costs + ){ final Optional first = pathFileLocationTriple .locations .stream() @@ -32,7 +37,8 @@ Costs findAlignmentForFile(PathFileLocationTriple pathFileLocationTriple, NodeLo final Location location = locationWrapper.getLocation(); final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0, locationWrapper.getSizeInBytes() ); - return new Costs( 0, locationWrapper.getSizeInBytes() ); + final double calculatedCost = costs.getSumOfCosts() + locationWrapper.getSizeInBytes(); + return new Costs( 0, calculatedCost, calculatedCost ); } } From 9f339bc9a8ae407f2727351ead0b7e738fe6ce64 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 12 Jan 2023 11:49:02 +0100 Subject: [PATCH 330/443] Use interface instead of concrete class for class instance Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/LocationAwareSchedulerV2.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index de274eb3..1934b6d8 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -15,6 +15,7 @@ import fonda.scheduler.scheduler.la2.MinSizeComparator; import fonda.scheduler.scheduler.la2.TaskStat; import fonda.scheduler.scheduler.la2.TaskStatComparator; +import fonda.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import fonda.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; import fonda.scheduler.scheduler.la2.copystrategy.CopyRunner; import fonda.scheduler.scheduler.la2.copystrategy.ShellCopy; @@ -48,7 +49,7 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final CopyRunner copyRunner; - private final SimpleCapacityAvailableToNode capacityAvailableToNode; + private final CapacityAvailableToNode capacityAvailableToNode; private final TaskStatComparator phaseTwoComparator; From f94a3b90c34a77bb0e338cadda55babd448d7d7c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 14:07:48 +0100 Subject: [PATCH 331/443] log available resources only once Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/Scheduler.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index af58c8a6..e2b2e3b0 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -35,11 +35,9 @@ public abstract class Scheduler implements Informable { private boolean close; @Getter private final DAG dag; - private final Object batchHelper = new Object(); private int currentBatch = 0; private Batch currentBatchInstance = null; - final KubernetesClient client; private final Set upcomingTasks = new HashSet<>(); private final List unscheduledTasks = new ArrayList<>( 100 ); @@ -88,7 +86,7 @@ public int schedule( final List unscheduledTasks ) { if( traceEnabled ) { unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule( startSchedule ) ); } - final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode()); + final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode( true )); final List taskNodeAlignment = scheduleObject.getTaskAlignments(); //check if still possible... @@ -129,7 +127,7 @@ public int schedule( final List unscheduledTasks ) { scheduled++; } //Use instance object that does not contain yet scheduled tasks - postScheduling( unscheduledTasksCopy, getAvailableByNode() ); + postScheduling( unscheduledTasksCopy, getAvailableByNode( false ) ); return unscheduledTasks.size() - taskNodeAlignment.size() + failure; } @@ -147,7 +145,7 @@ void undoTaskScheduling( Task task ){} public boolean validSchedulePlan( List taskNodeAlignment ){ - Map< NodeWithAlloc, Requirements> availableByNode = getAvailableByNode(); + Map< NodeWithAlloc, Requirements> availableByNode = getAvailableByNode( false ); for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { final Requirements requirements = availableByNode.get(nodeTaskAlignment.node); if ( requirements == null ) { @@ -486,20 +484,29 @@ Task getTaskByPod( Pod pod ) { return t; } - Map getAvailableByNode(){ + Map getAvailableByNode( boolean logging ){ Map availableByNode = new HashMap<>(); - List logInfo = new LinkedList<>(); - logInfo.add("------------------------------------"); + final List logInfo; + if ( logging ){ + logInfo = new LinkedList<>(); + logInfo.add("------------------------------------"); + } else { + logInfo = null; + } for (NodeWithAlloc item : getNodeList()) { if ( !item.isReady() ) { continue; } final Requirements availableResources = item.getAvailableResources(); availableByNode.put(item, availableResources); - logInfo.add("Node: " + item.getName() + " " + availableResources); + if ( logging ){ + logInfo.add("Node: " + item.getName() + " " + availableResources); + } + } + if ( logging ) { + logInfo.add("------------------------------------"); + log.info(String.join("\n", logInfo)); } - logInfo.add("------------------------------------"); - log.info(String.join("\n", logInfo)); return availableByNode; } From e0cbbc8f1a1155e5861b8d8860c03a2cba2c5400 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 14:09:38 +0100 Subject: [PATCH 332/443] Compare whether the task is already copied somewhere. Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/la2/MinCopyingComparator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java index f11e933d..8345e00e 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java @@ -14,6 +14,10 @@ public int compare( TaskStat o1, TaskStat o2 ) { return -1; } else if ( o1.getCompleteOnNodes() > o2.getCompleteOnNodes() ) { return 1; + } else if ( o1.getCopyingToNodes() < o2.getCopyingToNodes() ) { + return -1; + } else if ( o1.getCopyingToNodes() > o2.getCopyingToNodes() ) { + return 1; } else { return getComparator().compare( o1.getBestStats(), o2.getBestStats() ); } From c36608d5c273427712b21f4d19f26a0c37949bfc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 14:13:48 +0100 Subject: [PATCH 333/443] use currentlyCopyingTasksOnNode as method parameter Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/LocationAwareSchedulerV2.java | 10 +++++++++- .../la2/capacityavailable/CapacityAvailableToNode.java | 4 +++- .../SimpleCapacityAvailableToNode.java | 6 +++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 1934b6d8..7ce5a742 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -148,7 +148,15 @@ void postScheduling( List unscheduledTasks, final Map createAlignmentForTasksWithEnoughCapacity( final CurrentlyCopying planedToCopy, final Map availableByNodes, final List allNodes, - final int maxCopyingTaskPerNode + final int maxCopyingTaskPerNode, + final Map currentlyCopyingTasksOnNode ); } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java index 45064dc3..bd6ff332 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -29,13 +29,13 @@ public List createAlignmentForTasksWithEnoughCapacity( final CurrentlyCopying planedToCopy, final Map availableByNodes, final List allNodes, - final int maxCopyingTaskPerNode ) { + final int maxCopyingTaskPerNode, + final Map currentlyCopyingTasksOnNode + ) { List nodeTaskAlignments = new LinkedList<>(); - final Map currentlyCopyingTasksOnNode = currentlyCopying.getCurrentlyCopyingTasksOnNode(); - //Remove available resources if a copy task is already running: this logic may not be optimal for more than 2 parallel copy tasks (unclear which task starts first) removeAvailableResources( taskStats, availableByNodes, allNodes ); //Sort tasks by missing data: prefer tasks where the least data is missing on the node From ef6b0bf8602a6664f5af227991259a37b91341e7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 14:17:12 +0100 Subject: [PATCH 334/443] Ignore tasks in phase 3 that are currently copied to a node where they can run. Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/LocationAwareSchedulerV2.java | 2 ++ src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java | 7 +++++++ .../capacityavailable/SimpleCapacityAvailableToNode.java | 1 + src/main/java/fonda/scheduler/util/TaskStats.java | 7 +++++++ 4 files changed, 17 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 7ce5a742..cbc2bf60 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -157,6 +157,8 @@ void postScheduling( List unscheduledTasks, final Map { private boolean finished = false; private int indexToCompare = 0; + @Getter + private boolean copyToNodeWithAvailableResources = false; + + public void copyToNodeWithAvailableResources() { + this.copyToNodeWithAvailableResources = true; + } + /** * Call this to sort the taskStats list */ diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java index bd6ff332..2a7d1b44 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -98,6 +98,7 @@ private boolean createFileAlignment( nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode ) ); availableByNodes.get( node ).subFromThis( task.getRequest() ); currentlyCopyingTasksOnNode.compute( node.getNodeLocation(), ( nodeLocation, value ) -> value == null ? 1 : value + 1 ); + poll.copyToNodeWithAvailableResources(); return true; } else { return false; diff --git a/src/main/java/fonda/scheduler/util/TaskStats.java b/src/main/java/fonda/scheduler/util/TaskStats.java index c8c93705..a243cabf 100644 --- a/src/main/java/fonda/scheduler/util/TaskStats.java +++ b/src/main/java/fonda/scheduler/util/TaskStats.java @@ -22,4 +22,11 @@ public Collection getTaskStats() { return this.taskStats.values(); } + /** + * This method will remove all tasks, that have set copyToNodeWithAvailableResources + */ + public void removeTasksThatHaveBeenStarted() { + taskStats.entrySet().removeIf(entry -> entry.getValue().isCopyToNodeWithAvailableResources()); + } + } From 90d015cc00b83a55a40be2fee1245488b310dd8a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 14:18:14 +0100 Subject: [PATCH 335/443] Ignore tasks that have already all data on a node. Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 2 +- .../fonda/scheduler/scheduler/la2/TaskStat.java | 14 +++++++++++++- .../SimpleCapacityAvailableToNode.java | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index cbc2bf60..039716b0 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -142,7 +142,7 @@ void postScheduling( List unscheduledTasks, final Map Date: Fri, 13 Jan 2023 14:19:55 +0100 Subject: [PATCH 336/443] Do not filter nodes that cannot yet start a new copy task Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/LocationAwareSchedulerV2.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 039716b0..cb980fa7 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -127,10 +127,7 @@ && canSchedulePodOnNode( task.getPod(), node ) @Override void postScheduling( List unscheduledTasks, final Map availableByNode ) { final Map currentlyCopyingTasksOnNode = getCurrentlyCopying().getCurrentlyCopyingTasksOnNode(); - final List allNodes = client.getAllNodes().stream().filter( - node -> currentlyCopyingTasksOnNode - .getOrDefault( node.getNodeLocation(), 0 ) < getMaxCopyTasksPerNode() - ).collect( Collectors.toList() ); + final List allNodes = client.getAllNodes(); final List nodeTaskFilesAlignments; synchronized ( copyLock ) { final TaskStats taskStats = new TaskStats(); From bc5ab54f44603bc039290ef47c53f87055ca6d76 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 15:34:40 +0100 Subject: [PATCH 337/443] extract methods in SimpleCapacityAvailableToNode into super class to reuse in phase three Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/CreateCopyTasks.java | 76 ++++++++++++++++ .../CapacityAvailableToNode.java | 13 ++- .../SimpleCapacityAvailableToNode.java | 89 ++++--------------- 3 files changed, 106 insertions(+), 72 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java diff --git a/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java b/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java new file mode 100644 index 00000000..b3197d30 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java @@ -0,0 +1,76 @@ +package fonda.scheduler.scheduler.la2; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.model.taskinputs.TaskInputs; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import fonda.scheduler.util.FileAlignment; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.SortedList; +import fonda.scheduler.util.copying.CurrentlyCopying; +import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.RequiredArgsConstructor; + +import java.util.List; +import java.util.Map; + +@RequiredArgsConstructor +public abstract class CreateCopyTasks { + + protected final CurrentlyCopying currentlyCopying; + protected final InputAlignment inputAlignment; + protected final int copySameTaskInParallel; + + protected FileAlignment getFileAlignmentForTaskAndNode( + final NodeWithAlloc node, + final Task task, + final TaskInputs inputsOfTask, + final CurrentlyCopying planedToCopy + ) { + final CurrentlyCopyingOnNode currentlyCopyingOnNode = this.currentlyCopying.get(node.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(node.getNodeLocation()); + try { + return inputAlignment.getInputAlignment( + task, + inputsOfTask, + node, + currentlyCopyingOnNode, + currentlyPlanedToCopy, + Double.MAX_VALUE + ); + } catch ( NoAligmentPossibleException e ){ + return null; + } + } + + /** + * + * @return the success of this operation + */ + protected boolean createFileAlignment( + CurrentlyCopying planedToCopy, + List nodeTaskAlignments, + Map currentlyCopyingTasksOnNode, + TaskStat poll, + Task task, + NodeWithAlloc node + ) { + final FileAlignment fileAlignmentForTaskAndNode = getFileAlignmentForTaskAndNode( node, task, poll.getInputsOfTask(), planedToCopy ); + if ( fileAlignmentForTaskAndNode != null && fileAlignmentForTaskAndNode.copyFromSomewhere( node.getNodeLocation() ) ) { + planedToCopy.addAlignment( fileAlignmentForTaskAndNode.getNodeFileAlignment(), task, node ); + nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode ) ); + currentlyCopyingTasksOnNode.compute( node.getNodeLocation(), ( nodeLocation, value ) -> value == null ? 1 : value + 1 ); + poll.copyToNodeWithAvailableResources(); + return true; + } else { + return false; + } + } + + protected void removeTasksThatAreCopiedMoreThanXTimeCurrently( SortedList taskStats, int maxParallelTasks ) { + taskStats.removeIf( elem -> currentlyCopying.getNumberOfNodesForTask( elem.getTask() ) == maxParallelTasks ); + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java index 1405c638..b632e8dd 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java @@ -3,6 +3,8 @@ import fonda.scheduler.model.NodeWithAlloc; import fonda.scheduler.model.Requirements; import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.scheduler.la2.CreateCopyTasks; import fonda.scheduler.util.NodeTaskFilesAlignment; import fonda.scheduler.util.TaskStats; import fonda.scheduler.util.copying.CurrentlyCopying; @@ -10,9 +12,16 @@ import java.util.List; import java.util.Map; -public interface CapacityAvailableToNode { +public abstract class CapacityAvailableToNode extends CreateCopyTasks { - List createAlignmentForTasksWithEnoughCapacity( + public CapacityAvailableToNode( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + public abstract List createAlignmentForTasksWithEnoughCapacity( final TaskStats taskStats, final CurrentlyCopying planedToCopy, final Map availableByNodes, diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java index 9843f31c..21b1a83e 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -4,25 +4,27 @@ import fonda.scheduler.model.Requirements; import fonda.scheduler.model.Task; import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import fonda.scheduler.scheduler.la2.TaskStat; -import fonda.scheduler.util.*; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.SortedList; +import fonda.scheduler.util.TaskStats; import fonda.scheduler.util.copying.CurrentlyCopying; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; @Slf4j -@RequiredArgsConstructor -public class SimpleCapacityAvailableToNode implements CapacityAvailableToNode { +public class SimpleCapacityAvailableToNode extends CapacityAvailableToNode { - private final CurrentlyCopying currentlyCopying; - private final InputAlignment inputAlignment; - private final int copySameTaskInParallel; + public SimpleCapacityAvailableToNode( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } public List createAlignmentForTasksWithEnoughCapacity( final TaskStats taskStats, @@ -54,7 +56,12 @@ public List createAlignmentForTasksWithEnoughCapacity( if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode && availableByNodes.get( node ).higherOrEquals( task.getRequest() ) ) { - cannotAdd = !createFileAlignment( planedToCopy, availableByNodes, nodeTaskAlignments, currentlyCopyingTasksOnNode, poll, task, node ); + if ( createFileAlignment( planedToCopy, nodeTaskAlignments, currentlyCopyingTasksOnNode, poll, task, node ) ) { + cannotAdd = false; + availableByNodes.get( node ).subFromThis( task.getRequest() ); + } else { + cannotAdd = true; + } } else { cannotAdd = true; } @@ -67,65 +74,7 @@ public List createAlignmentForTasksWithEnoughCapacity( return nodeTaskAlignments; } - private void removeTasksThatAreCopiedMoreThanXTimeCurrently( SortedList taskStats, int maxParallelTasks ) { - taskStats.removeIf( elem -> currentlyCopying.getNumberOfNodesForTask( elem.getTask() ) == maxParallelTasks ); - } - - /** - * - * @param planedToCopy - * @param availableByNodes - * @param nodeTaskAlignments - * @param currentlyCopyingTasksOnNode - * @param poll - * @param task - * @param node - * @return the success of this operation - */ - private boolean createFileAlignment( - CurrentlyCopying planedToCopy, - Map availableByNodes, - List nodeTaskAlignments, - Map currentlyCopyingTasksOnNode, - TaskStat poll, - Task task, - NodeWithAlloc node - ) { - final FileAlignment fileAlignmentForTaskAndNode = getFileAlignmentForTaskAndNode( node, task, poll.getInputsOfTask(), planedToCopy ); - if ( fileAlignmentForTaskAndNode != null && fileAlignmentForTaskAndNode.copyFromSomewhere( node.getNodeLocation() ) ) { - planedToCopy.addAlignment( fileAlignmentForTaskAndNode.getNodeFileAlignment(), task, node ); - nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode ) ); - availableByNodes.get( node ).subFromThis( task.getRequest() ); - currentlyCopyingTasksOnNode.compute( node.getNodeLocation(), ( nodeLocation, value ) -> value == null ? 1 : value + 1 ); - poll.copyToNodeWithAvailableResources(); - return true; - } else { - return false; - } - } - - private FileAlignment getFileAlignmentForTaskAndNode( - final NodeWithAlloc node, - final Task task, - final TaskInputs inputsOfTask, - final CurrentlyCopying planedToCopy - ) { - final CurrentlyCopyingOnNode currentlyCopyingOnNode = this.currentlyCopying.get(node.getNodeLocation()); - final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(node.getNodeLocation()); - try { - return inputAlignment.getInputAlignment( - task, - inputsOfTask, - node, - currentlyCopyingOnNode, - currentlyPlanedToCopy, - Double.MAX_VALUE - ); - } catch ( NoAligmentPossibleException e ){ - return null; - } - } private void removeAvailableResources( TaskStats taskStats, Map availableByNodes, List allNodes ) { allNodes.parallelStream().forEach( node -> { From 9a6c63dd557d4a8a38fb68a133b35b44ad527325 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 13 Jan 2023 15:36:00 +0100 Subject: [PATCH 338/443] Make comparator of TaskStats configurable Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/LocationAwareSchedulerV2.java | 4 +++- src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java | 4 +++- src/main/java/fonda/scheduler/util/TaskStats.java | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index cb980fa7..18bf3e8c 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -143,6 +143,8 @@ void postScheduling( List unscheduledTasks, final Map allNodes ) { - TaskStat taskStats = new TaskStat( task, inputsOfTask, phaseTwoComparator ); + TaskStat taskStats = new TaskStat( task, inputsOfTask ); final CurrentlyCopying currentlyCopying = getCurrentlyCopying(); for ( NodeWithAlloc node : allNodes ) { if ( !inputsOfTask.getExcludedNodes().contains( node.getNodeLocation() ) && affinitiesMatch( task.getPod(), node ) ) { diff --git a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java b/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java index c5378b05..2c4800ca 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java @@ -7,6 +7,7 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.Setter; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -19,7 +20,8 @@ public class TaskStat implements Comparable { private final Task task; @Getter private final TaskInputs inputsOfTask; - private final TaskStatComparator comparator; + @Setter + private TaskStatComparator comparator; private final List taskStats = new ArrayList<>(); @Getter private int completeOnNodes = 0; diff --git a/src/main/java/fonda/scheduler/util/TaskStats.java b/src/main/java/fonda/scheduler/util/TaskStats.java index a243cabf..43412c26 100644 --- a/src/main/java/fonda/scheduler/util/TaskStats.java +++ b/src/main/java/fonda/scheduler/util/TaskStats.java @@ -2,6 +2,7 @@ import fonda.scheduler.model.Task; import fonda.scheduler.scheduler.la2.TaskStat; +import fonda.scheduler.scheduler.la2.TaskStatComparator; import java.util.*; @@ -29,4 +30,10 @@ public void removeTasksThatHaveBeenStarted() { taskStats.entrySet().removeIf(entry -> entry.getValue().isCopyToNodeWithAvailableResources()); } + public void setComparator( TaskStatComparator comparator ) { + for ( TaskStat taskStat : taskStats.values() ) { + taskStat.setComparator( comparator ); + } + } + } From bc4cd873f98fd2a790d8a7bd5da461fb452d27e5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:48:26 +0100 Subject: [PATCH 339/443] EntrySet instead of KeySet Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/ready2run/OptimalReadyToRunToNode.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index 85703a13..39eb8d54 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -91,9 +91,9 @@ public List createAlignmentForTasksWithAllDataOnNod return Collections.emptyList(); } - for ( NodeWithAlloc node : memUsed.keySet() ) { - model.addLessOrEqual( memUsed.get( node ), availableByNode.get( node ).getRam().longValue() ); - model.addLessOrEqual( cpuUsed.get( node ), availableByNode.get( node ).getCpu().multiply( MILLION ).longValue() ); + for ( Map.Entry entry : memUsed.entrySet() ) { + model.addLessOrEqual( entry.getValue(), availableByNode.get( entry.getKey() ).getRam().longValue() ); + model.addLessOrEqual( cpuUsed.get( entry.getKey() ), availableByNode.get( entry.getKey() ).getCpu().multiply( MILLION ).longValue() ); } model.maximize( objective ); From b08f268ce35b6cf2310a3f61903bb002dccd59ba Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:49:08 +0100 Subject: [PATCH 340/443] Do not throw an exception if the model is invalid Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/ready2run/OptimalReadyToRunToNode.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index 39eb8d54..a46f6658 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -104,6 +104,9 @@ public List createAlignmentForTasksWithAllDataOnNod logger.log( message ); log.info("Total packed value: " + solver.objectiveValue()); log.info( String.valueOf( solve ) ); + if ( solve == CpSolverStatus.MODEL_INVALID ) { + return Collections.emptyList(); + } return taskNodeBoolVars.stream() .filter( taskNodeBoolVar -> solver.booleanValue( taskNodeBoolVar.getBoolVar() ) ) From 7b890a194475497419524d8642ce11fe3ff0b940 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:49:57 +0100 Subject: [PATCH 341/443] Store config json file in pvc Signed-off-by: Lehmann_Fabian --- daemons/ftp/ftp.py | 8 ++++---- src/main/java/fonda/scheduler/model/Task.java | 6 ++++++ .../scheduler/la2/copystrategy/ShellCopy.java | 15 ++++++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/daemons/ftp/ftp.py b/daemons/ftp/ftp.py index f1f5691e..7898c3d9 100644 --- a/daemons/ftp/ftp.py +++ b/daemons/ftp/ftp.py @@ -218,10 +218,10 @@ def waitForFiles(syncFilePath, files, startTime): def loadConfig(): log.info("Load config") - log.info( "Parse: --" + sys.argv[3] + "--" ) - config = json.loads(sys.argv[3]) - os.makedirs(config["syncDir"], exist_ok=True) - return config + with open(sys.argv[3]) as jsonFile: + config = json.load(jsonFile) + os.makedirs(config["syncDir"], exist_ok=True) + return config def registerSignal(syncFile): diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/fonda/scheduler/model/Task.java index 3496bc19..6c1162ce 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/fonda/scheduler/model/Task.java @@ -62,11 +62,17 @@ public class Task { @Setter private boolean copiesDataToNode = false; + private final AtomicInteger copyTaskId = new AtomicInteger(0); + public Task( TaskConfig config, DAG dag ) { this.config = config; this.process = dag.getByProcess( config.getTask() ); } + public int getCurrentCopyTaskId() { + return copyTaskId.getAndIncrement(); + } + public String getWorkingDir(){ return config.getWorkDir(); } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java index ad7cf14b..7d707e51 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java @@ -10,6 +10,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.nio.file.Path; + @Slf4j @RequiredArgsConstructor public class ShellCopy implements CopyRunner { @@ -20,19 +23,21 @@ public class ShellCopy implements CopyRunner { @Override public void startCopyTasks( final CopyTask copyTask, final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + final String nodeName = nodeTaskFilesAlignment.node.getName().replace( " ", "_" ); + String copyTaskIdentifier = nodeName + "-" + copyTask.getTask().getCurrentCopyTaskId(); + String filename = ".command.init." + copyTaskIdentifier + ".json"; String[] command = new String[3]; command[0] = "/bin/bash"; command[1] = "-c"; command[2] = "cd " + nodeTaskFilesAlignment.task.getWorkingDir() + " && "; - final String json; try { - json = new ObjectMapper() - .writeValueAsString( copyTask.getInputs() ) - .replace( "\"", "\\\"" ); + new ObjectMapper().writeValue( Path.of( nodeTaskFilesAlignment.task.getWorkingDir(), filename ).toFile(), copyTask.getInputs() ); } catch ( JsonProcessingException e ) { throw new RuntimeException( e ); + } catch ( IOException e ) { + throw new RuntimeException( e ); } - command[2] += "/code/ftp.py false " + nodeTaskFilesAlignment.node.getName() + " \"" + json + "\""; + command[2] += "/code/ftp.py false \"" + copyTaskIdentifier + "\" \"" + filename + "\""; String name = nodeTaskFilesAlignment.task.getConfig().getName() + "-copy-" + nodeTaskFilesAlignment.node.getName(); log.info( "Starting {} to node {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start" ); From 5bda348a2a6339d709e068b3728e91281b7d7ba5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:50:13 +0100 Subject: [PATCH 342/443] Log error to console Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/copystrategy/LaListener.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java index 549ce0bc..8a295936 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java @@ -41,6 +41,7 @@ private void close() { Thread.sleep( (long) (100 * Math.pow( 2, trial )) ); } catch ( InterruptedException e ) { e.printStackTrace(); + Thread.currentThread().interrupt(); } trial++; } @@ -70,9 +71,15 @@ public void onFailure( Throwable t, Response failureResponse ) { @Override public void onExit( int exitCode, Status reason ) { finished = true; - log.info( name + " was finished exitCode = {}, reason = {}", exitCode, reason ); - log.debug( name + " Exec Output: {} ", out ); - log.debug( name + " Exec Error Output: {} ", error ); + if ( exitCode != 0 ) { + log.info( name + " was finished exitCode = {}, reason = {}", exitCode, reason ); + log.info( name + " Exec Output: {} ", out ); + log.info( name + " Exec Error Output: {} ", error ); + } else { + log.info( name + " was finished successfully" ); + log.debug( name + " Exec Output: {} ", out ); + log.debug( name + " Exec Error Output: {} ", error ); + } scheduler.copyTaskFinished( copyTask, exitCode == 0 ); close(); logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "finished(" + exitCode + ")" ); From ffd62789d60db671c49479c37e22027540e45c17 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:51:51 +0100 Subject: [PATCH 343/443] Simplified code Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/MinCopyingComparator.java | 16 ++++++---------- .../scheduler/la2/MinSizeComparator.java | 14 +++++++------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java index 8345e00e..ff13c228 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java @@ -10,17 +10,13 @@ public MinCopyingComparator( Comparator comparator @Override public int compare( TaskStat o1, TaskStat o2 ) { - if ( o1.getCompleteOnNodes() < o2.getCompleteOnNodes() ) { - return -1; - } else if ( o1.getCompleteOnNodes() > o2.getCompleteOnNodes() ) { - return 1; - } else if ( o1.getCopyingToNodes() < o2.getCopyingToNodes() ) { - return -1; - } else if ( o1.getCopyingToNodes() > o2.getCopyingToNodes() ) { - return 1; - } else { - return getComparator().compare( o1.getBestStats(), o2.getBestStats() ); + if ( o1.getCompleteOnNodes() != o2.getCompleteOnNodes() ) { + return Long.compare( o1.getCompleteOnNodes(), o2.getCompleteOnNodes() ); } + if ( o1.getCopyingToNodes() != o2.getCopyingToNodes() ) { + return Long.compare( o1.getCopyingToNodes(), o2.getCopyingToNodes() ); + } + return getComparator().compare( o1.getBestStats(), o2.getBestStats() ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java index 6c8dd263..156a7700 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java @@ -1,25 +1,25 @@ package fonda.scheduler.scheduler.la2; import fonda.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; import lombok.NoArgsConstructor; import java.util.Comparator; -@NoArgsConstructor +@NoArgsConstructor(access = AccessLevel.PRIVATE) public class MinSizeComparator implements Comparator { - public final static MinSizeComparator INSTANCE = new MinSizeComparator(); + public static final MinSizeComparator INSTANCE = new MinSizeComparator(); @Override public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { final TaskNodeStats o1Stats = o1.getTaskNodeStats(); final TaskNodeStats o2Stats = o2.getTaskNodeStats(); - if ( o1Stats.getSizeRemaining() < o2Stats.getSizeRemaining() ) { - return -1; - } else if ( o1Stats.getSizeRemaining() > o2Stats.getSizeRemaining() ) { - return 1; - } else { + if ( o1Stats.getSizeRemaining() == o2Stats.getSizeRemaining() ) { + //Prefer the one that is copying fewer data --> expected to finish faster return Long.compare( o1Stats.getSizeCurrentlyCopying(), o2Stats.getSizeCurrentlyCopying() ); + } else { + return Long.compare( o1Stats.getSizeRemaining(), o2Stats.getSizeRemaining() ); } } From a782c0c674c4c7d2920679d11b6fa3db54d6a057 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:52:12 +0100 Subject: [PATCH 344/443] Avoid to large numbers Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/util/score/FileSizeRankScore.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java b/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java index 47930997..422ca78b 100644 --- a/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java @@ -11,7 +11,10 @@ public FileSizeRankScore( HierarchyWrapper hierarchyWrapper ) { @Override public long getScore( TaskInputsNodes taskInputsNodes ) { - return super.getScore( taskInputsNodes ) + taskInputsNodes.getTask().getProcess().getRank() * 100_000_000_000L; + //Add one to avoid becoming zero + final int rank = taskInputsNodes.getTask().getProcess().getRank() + 1; + final long rankFactor = 100_000_000_000_000L * rank; // long would allow a rank of 92233 + return super.getScore( taskInputsNodes ) + rankFactor ; } } From c8ee4ce7cd3893b9105baa9a5cd90fcf48f9e638 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:52:39 +0100 Subject: [PATCH 345/443] More informative logging Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/ready2run/OptimalReadyToRunToNode.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index a46f6658..79f478ff 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -35,9 +35,6 @@ public void init( CalculateScore calculateScore ) { /** * Creates an optimal alignment for tasks with all data on node. - * @param taskWithAllData - * @param availableByNode - * @return */ @Override public List createAlignmentForTasksWithAllDataOnNode( @@ -46,6 +43,7 @@ public List createAlignmentForTasksWithAllDataOnNod ) { if ( taskWithAllData.isEmpty() || availableByNode.isEmpty() ){ + log.info( "No tasks can be scheduled on any node. (No node has all data)" ); return Collections.emptyList(); } @@ -88,6 +86,7 @@ public List createAlignmentForTasksWithAllDataOnNod } if ( taskNodeBoolVars.isEmpty() ) { + log.info( "No tasks can be scheduled on any node. Not enough resources available." ); return Collections.emptyList(); } From 8631c41b91be62f09a7d348ef0330ccdf697b2aa Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:54:31 +0100 Subject: [PATCH 346/443] New comparators Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/MaxPartSizeComparator.java | 35 +++++++++++++++++++ .../scheduler/la2/MaxSizeComparator.java | 28 +++++++++++++++ .../la2/RankAndMinCopyingComparator.java | 23 ++++++++++++ .../scheduler/scheduler/la2/TaskStat.java | 3 +- 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java new file mode 100644 index 00000000..2599789d --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java @@ -0,0 +1,35 @@ +package fonda.scheduler.scheduler.la2; + +import fonda.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.Comparator; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MaxPartSizeComparator implements Comparator { + + public static final MaxPartSizeComparator INSTANCE = new MaxPartSizeComparator(); + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + final double o1TaskSize = o1Stats.getTaskSize(); + final double o2TaskSize = o2Stats.getTaskSize(); + + //Prefer o2 if o1 is missing more percent of data + if ( o1Stats.getSizeRemaining() / o1TaskSize > o2Stats.getSizeRemaining() / o2TaskSize ) { + return 1; + } else if ( o1Stats.getSizeRemaining() / o1TaskSize < o2Stats.getSizeRemaining() / o2TaskSize ) { + return -1; + } else { + //Prefer o1 if o1 is copying more percent of data + return Double.compare( + o2Stats.getSizeCurrentlyCopying() / o2TaskSize, + o1Stats.getSizeCurrentlyCopying() / o1TaskSize + ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java new file mode 100644 index 00000000..9c9dcffd --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java @@ -0,0 +1,28 @@ +package fonda.scheduler.scheduler.la2; + +import fonda.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.Comparator; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MaxSizeComparator implements Comparator { + + public static final MaxSizeComparator INSTANCE = new MaxSizeComparator(); + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + //same task does not necessarily have the same size + if ( o1.getTask() == o2.getTask() || o1Stats.getTaskSize() == o2Stats.getTaskSize() ) { + //Prefer the one with fewer remaining data + return Long.compare( o1Stats.getSizeRemaining(), o2Stats.getSizeRemaining() ); + } else { + //Prefer the one with larger task size + return Long.compare( o2Stats.getTaskSize(), o1Stats.getTaskSize() ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java b/src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java new file mode 100644 index 00000000..745a7012 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java @@ -0,0 +1,23 @@ +package fonda.scheduler.scheduler.la2; + +import java.util.Comparator; + +public class RankAndMinCopyingComparator extends MinCopyingComparator { + + public RankAndMinCopyingComparator( Comparator comparator ) { + super( comparator ); + } + + @Override + public int compare( TaskStat o1, TaskStat o2 ) { + final int rankO1 = o1.getTask().getProcess().getRank(); + final int rankO2 = o2.getTask().getProcess().getRank(); + if ( rankO1 == rankO2 ) { + return super.compare( o1, o2 ); + } else { + //Prefer larger rank + return Integer.compare( rankO2, rankO1 ); + } + } + +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java b/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java index 2c4800ca..eb20dbae 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java @@ -75,7 +75,7 @@ public void add( NodeWithAlloc node, TaskNodeStats taskNodeStats ) { this.copyingToNodes++; } else { //Only add in the case that not all data is already on the node - this.taskStats.add( new NodeAndStatWrapper( node, taskNodeStats ) ); + this.taskStats.add( new NodeAndStatWrapper( node, taskNodeStats, task ) ); } } @@ -109,6 +109,7 @@ public int compareTo( @NotNull TaskStat o ) { public static class NodeAndStatWrapper { private final NodeWithAlloc node; private final TaskNodeStats taskNodeStats; + private final Task task; } From 523cae903bde5e0061364fe690c7fdb6a485de62 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 15:58:33 +0100 Subject: [PATCH 347/443] First version of phase 3 Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 28 +++++--- .../scheduler/scheduler/la2/TaskStat.java | 23 +++++- .../CopyInAdvanceNodeWithMostData.java | 72 +++++++++++++++++++ 3 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 18bf3e8c..f683782e 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -11,12 +11,10 @@ import fonda.scheduler.scheduler.data.TaskInputsNodes; import fonda.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.scheduler.la2.MinCopyingComparator; -import fonda.scheduler.scheduler.la2.MinSizeComparator; -import fonda.scheduler.scheduler.la2.TaskStat; -import fonda.scheduler.scheduler.la2.TaskStatComparator; +import fonda.scheduler.scheduler.la2.*; import fonda.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import fonda.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; +import fonda.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; import fonda.scheduler.scheduler.la2.copystrategy.CopyRunner; import fonda.scheduler.scheduler.la2.copystrategy.ShellCopy; import fonda.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; @@ -51,7 +49,10 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final CapacityAvailableToNode capacityAvailableToNode; + private final CopyInAdvanceNodeWithMostData copyInAdvance; + private final TaskStatComparator phaseTwoComparator; + private final TaskStatComparator phaseThreeComparator; private final int copySameTaskInParallel; @@ -80,8 +81,10 @@ public LocationAwareSchedulerV2( readyToRunToNode.setLogger( logCopyTask ); this.copyRunner = new ShellCopy( client, this, logCopyTask ); this.copySameTaskInParallel = 2; - capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); - phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); + this.capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + this.phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); + this.phaseThreeComparator = new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ); + this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); } @Override @@ -117,7 +120,7 @@ && canSchedulePodOnNode( task.getPod(), node ) .stream() .filter( td -> !td.getNodesWithAllData().isEmpty() ) .collect( Collectors.toList() ); - final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode(taskWithAllData, availableByNode); + final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode( taskWithAllData, availableByNode ); final ScheduleObject scheduleObject = new ScheduleObject( (List) alignment ); scheduleObject.setCheckStillPossible( true ); scheduleObject.setStopSubmitIfOneFails( true ); @@ -158,6 +161,16 @@ void postScheduling( List unscheduledTasks, final Map { private final Task task; @Getter private final TaskInputs inputsOfTask; - @Setter private TaskStatComparator comparator; private final List taskStats = new ArrayList<>(); @Getter @@ -33,6 +32,26 @@ public class TaskStat implements Comparable { @Getter private boolean copyToNodeWithAvailableResources = false; + /** + * @return number is the number of nodes, that are better for this task depending on the comparator. + */ + public int getCurrentIndex() { + return indexToCompare; + } + + /** + * @return number of nodes that have already all data or are copying remaining data to this node. + */ + public int dataOnNodes() { + return completeOnNodes + copyingToNodes; + } + + public void setComparator( TaskStatComparator comparator ) { + this.comparator = comparator; + this.indexToCompare = 0; + finish(); + } + public void copyToNodeWithAvailableResources() { this.copyToNodeWithAvailableResources = true; } @@ -44,7 +63,7 @@ public boolean missingDataOnAnyNode() { /** * Call this to sort the taskStats list */ - public void finish() { + private void finish() { finished = true; taskStats.sort( comparator.getComparator() ); } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java new file mode 100644 index 00000000..0bc5a9d2 --- /dev/null +++ b/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -0,0 +1,72 @@ +package fonda.scheduler.scheduler.la2.copyinadvance; + +import fonda.scheduler.model.NodeWithAlloc; +import fonda.scheduler.model.Task; +import fonda.scheduler.model.location.NodeLocation; +import fonda.scheduler.scheduler.filealignment.InputAlignment; +import fonda.scheduler.scheduler.la2.CreateCopyTasks; +import fonda.scheduler.scheduler.la2.TaskStat; +import fonda.scheduler.util.NodeTaskFilesAlignment; +import fonda.scheduler.util.SortedList; +import fonda.scheduler.util.TaskStats; +import fonda.scheduler.util.copying.CurrentlyCopying; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; + +@Slf4j +public class CopyInAdvanceNodeWithMostData extends CreateCopyTasks { + + public CopyInAdvanceNodeWithMostData( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + public void createAlignmentForTasksWithEnoughCapacity( + final List nodeTaskFilesAlignments, + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final List allNodes, + final int maxCopyingTaskPerNode, + final Map currentlyCopyingTasksOnNode ) + { + final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); + removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + removeTasksThatAreReadyOnXNodes( stats, 3 ); + + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); + + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); + + final boolean cannotAdd; + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { + if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node ) ) { + cannotAdd = false; + log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); + } else { + cannotAdd = true; + } + } else { + cannotAdd = true; + } + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( cannotAdd && poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } + } + } + + protected void removeTasksThatAreReadyOnXNodes( SortedList taskStats, int maxHeldReady ) { + taskStats.removeIf( elem -> elem.dataOnNodes() == maxHeldReady ); + } + +} From 89b09378e4de55e392bd135e4abc4c0e29f9481d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 16:00:57 +0100 Subject: [PATCH 348/443] Make maxHeldReady configurable Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/model/SchedulerConfig.java | 1 + .../fonda/scheduler/scheduler/LocationAwareSchedulerV2.java | 3 +++ .../la2/copyinadvance/CopyInAdvanceNodeWithMostData.java | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index 71e269f4..e5165fea 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -24,6 +24,7 @@ public class SchedulerConfig { public final Integer maxCopyTasksPerNode; public final Integer maxWaitingCopyTasksPerNode; + public final Integer maxHeldCopyTaskReady; @ToString public static class LocalClaim { diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index f683782e..0cb5fe10 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -55,6 +55,7 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final TaskStatComparator phaseThreeComparator; private final int copySameTaskInParallel; + private final int maxHeldCopyTaskReady; /** * This lock is used to synchronize the creation of the copy tasks and the finishing. @@ -85,6 +86,7 @@ public LocationAwareSchedulerV2( this.phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); this.phaseThreeComparator = new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ); this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + this.maxHeldCopyTaskReady = config.maxHeldCopyTaskReady == null ? 3 : config.maxHeldCopyTaskReady; } @Override @@ -169,6 +171,7 @@ void postScheduling( List unscheduledTasks, final Map allNodes, final int maxCopyingTaskPerNode, + final int maxHeldCopyTaskReady, final Map currentlyCopyingTasksOnNode ) { final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); - removeTasksThatAreReadyOnXNodes( stats, 3 ); + removeTasksThatAreReadyOnXNodes( stats, maxHeldCopyTaskReady ); while( !stats.isEmpty() ) { final TaskStat poll = stats.poll(); From f8ad33e64af727adfc6e0d997b5b912cd34a395f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 17 Jan 2023 16:03:50 +0100 Subject: [PATCH 349/443] Removed unused calculateBestNode method Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 0cb5fe10..026fd401 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -10,7 +10,6 @@ import fonda.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.scheduler.data.TaskInputsNodes; import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import fonda.scheduler.scheduler.la2.*; import fonda.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import fonda.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; @@ -310,71 +309,6 @@ CopyTask createCopyTask( NodeTaskFilesAlignment alignment ) { return new CopyTask( inputs, inputFiles, filesForCurrentNode, alignment.task ); } - Tuple calculateBestNode( final DataOnNode taskData, CurrentlyCopying planedToCopy ) { - FileAlignment bestAlignment = null; - NodeWithAlloc bestNode = null; - final Set matchingNodes = taskData.getNodes(); - int triedNodes = 0; - int noAlignmentFound = 0; - int couldStopFetching = 0; - final List costs = traceEnabled ? new LinkedList<>() : null; - for ( final NodeWithAlloc node : matchingNodes) { - - final CurrentlyCopyingOnNode currentlyCopying = getCurrentlyCopying().get(node.getNodeLocation()); - final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(node.getNodeLocation()); - FileAlignment fileAlignment = null; - try { - fileAlignment = inputAlignment.getInputAlignment( - taskData.getTask(), - taskData.getInputsOfTask(), - node, - currentlyCopying, - currentlyPlanedToCopy, - bestAlignment == null ? Double.MAX_VALUE : bestAlignment.getCost() - ); - - if ( fileAlignment == null ){ - couldStopFetching++; - } else { - log.info( "Task: {}, node: {}, bestWeight: {}, currentWeight: {}", - taskData.getTask().getConfig().getName(), - node.getNodeLocation(), - bestAlignment == null ? null : bestAlignment.getWorth(), - fileAlignment.getWorth() - ); - - if ( - //Anything to copy? - fileAlignment.copyFromSomewhere( node.getNodeLocation() ) && - //Not set or better than current best - ( bestAlignment == null || bestAlignment.getWorth() > fileAlignment.getWorth() ) ) { - bestAlignment = fileAlignment; - bestNode = node; - log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getRunName(), fileAlignment.getCost() ); - } - } - } catch ( NoAligmentPossibleException e ){ - noAlignmentFound++; - log.info( "Task: {} - {}", taskData.getTask().getConfig().getName() , e.getMessage() ); - } - if ( traceEnabled ) { - triedNodes++; - final Double thisRoundCost = fileAlignment == null - ? null - : fileAlignment.getCost(); - costs.add( thisRoundCost ); - } - } - - if ( bestAlignment == null ) { - return null; - } - - //logCopyTask.log( taskData.getTask().getConfig().getName() + " " + bestAlignment ); - - return new Tuple<>( bestNode, bestAlignment ); - } - private TaskInputs extractInputsOfData( Task task ) { if ( task == null ) return null; From 2b2566e2845f55d72e8b5ea2f927b717b16ffe07 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 19 Jan 2023 15:10:15 +0100 Subject: [PATCH 350/443] Do not filter tasks that are already ready to run on different nodes in phase two. Signed-off-by: Lehmann_Fabian --- .../la2/copyinadvance/CopyInAdvanceNodeWithMostData.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index 73a3745c..1a86da34 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -25,6 +25,10 @@ public CopyInAdvanceNodeWithMostData( super( currentlyCopying, inputAlignment, copySameTaskInParallel ); } + + /** + * Do not filter maxHeldCopyTaskReady, that could lead to starving if another node has resources. + */ public void createAlignmentForTasksWithEnoughCapacity( final List nodeTaskFilesAlignments, final TaskStats taskStats, @@ -36,7 +40,6 @@ public void createAlignmentForTasksWithEnoughCapacity( { final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); - removeTasksThatAreReadyOnXNodes( stats, maxHeldCopyTaskReady ); while( !stats.isEmpty() ) { final TaskStat poll = stats.poll(); @@ -66,8 +69,4 @@ public void createAlignmentForTasksWithEnoughCapacity( } } - protected void removeTasksThatAreReadyOnXNodes( SortedList taskStats, int maxHeldReady ) { - taskStats.removeIf( elem -> elem.dataOnNodes() == maxHeldReady ); - } - } From 43985ea5c32154dae5b2c0426156b54818597f39 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 19 Jan 2023 15:11:55 +0100 Subject: [PATCH 351/443] Added some prio to the ftp's download speed Signed-off-by: Lehmann_Fabian --- daemons/ftp/ftp.py | 48 ++++++++++++------- .../scheduler/model/SchedulerConfig.java | 1 + .../scheduler/LocationAwareScheduler.java | 2 +- .../scheduler/LocationAwareSchedulerV2.java | 16 +++++-- .../scheduler/SchedulerWithDaemonSet.java | 3 +- .../scheduler/la2/CreateCopyTasks.java | 7 +-- .../CapacityAvailableToNode.java | 3 +- .../SimpleCapacityAvailableToNode.java | 5 +- .../CopyInAdvanceNodeWithMostData.java | 7 +-- .../scheduler/schedulingstrategy/Inputs.java | 1 + .../util/NodeTaskFilesAlignment.java | 4 +- 11 files changed, 66 insertions(+), 31 deletions(-) diff --git a/daemons/ftp/ftp.py b/daemons/ftp/ftp.py index 7898c3d9..176ffc64 100644 --- a/daemons/ftp/ftp.py +++ b/daemons/ftp/ftp.py @@ -116,7 +116,7 @@ def closeFTP(ftp): log.exception(UNEXPECTED_ERROR) -def downloadFile(ftp, filename, size, index, node, syncFile): +def downloadFile(ftp, filename, size, index, node, syncFile, speed): global errors log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) try: @@ -124,7 +124,23 @@ def downloadFile(ftp, filename, size, index, node, syncFile): clearLocation(filename) Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) start = time.time() - ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) + with open(filename, 'wb') as file: + if speed == 100: + ftp.retrbinary('RETR %s' % filename, file.write, 102400) + else: + timer = {"t": time.time_ns()} + + def callback(data): + now = time.time_ns() + diff = now - timer["t"] + file.write(data) + timeToSleep = (diff * (100 / speed) - diff) / 1_000_000_000 + # sleep at least 10ms + if timeToSleep > 0.01: + time.sleep(timeToSleep) + timer["t"] = time.time_ns() + + ftp.retrbinary('RETR %s' % filename, callback, 102400) end = time.time() sizeInMB = os.path.getsize(filename) / 1000000 delta = (end - start) @@ -152,7 +168,7 @@ def downloadFile(ftp, filename, size, index, node, syncFile): return 0, 0 -def download(node, currentIP, files, dns, execution, syncFile): +def download(node, currentIP, files, dns, execution, syncFile, speed): ftp = None size = len(files) global CLOSE @@ -164,7 +180,7 @@ def download(node, currentIP, files, dns, execution, syncFile): currentIP = None filename = files[0] index = size - len(files) + 1 - result = downloadFile(ftp, filename, size, index, node, syncFile) + result = downloadFile(ftp, filename, size, index, node, syncFile, speed) if result is None: ftp = None continue @@ -246,7 +262,7 @@ def generateSymlinks(symlinks): log.warning("File exists: %s -> %s", src, dst) -def downloadAllData(data, dns, execution, syncFile): +def downloadAllData(data, dns, execution, syncFile, speed): global trace throughput = [] with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: @@ -255,7 +271,7 @@ def downloadAllData(data, dns, execution, syncFile): files = d["files"] node = d["node"] currentIP = d["currentIP"] - futures.append(executor.submit(download, node, currentIP, files, dns, execution, syncFile)) + futures.append(executor.submit(download, node, currentIP, files, dns, execution, syncFile, speed)) lastNum = -1 while len(futures) > 0: if lastNum != len(futures): @@ -293,7 +309,7 @@ def finishedDownload(dns, execution, taskname): try: dns = dns + "downloadtask/" + execution log.info("Request: %s", dns) - urllib.request.urlopen(dns, taskname.encode( "utf-8" )) + urllib.request.urlopen(dns, taskname.encode("utf-8")) except BaseException as err: log.exception(err) myExit(100) @@ -321,7 +337,7 @@ def run(): syncFile.write('##SYMLINKS##\n') syncFile.flush() startTimeDownload = time.time() - downloadAllData(data, dns, execution, syncFile) + downloadAllData(data, dns, execution, syncFile, config["speed"]) trace["scheduler_init_download_runtime"] = int((time.time() - startTimeDownload) * 1000) if CLOSE: log.debug("Closed with code %s", str(EXIT)) @@ -330,16 +346,16 @@ def run(): syncFile.write('##FINISHED##\n') registerSignal2() - #finishedDownload(dns, execution, taskname) + # finishedDownload(dns, execution, taskname) - #startTimeDependingTasks = time.time() - #waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) - #trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) - #log.info("Waited for all tasks") + # startTimeDependingTasks = time.time() + # waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) + # trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) + # log.info("Waited for all tasks") - #runtime = int((time.time() - startTime) * 1000) - #trace["scheduler_init_runtime"] = runtime - #writeTrace(trace) + # runtime = int((time.time() - startTime) * 1000) + # trace["scheduler_init_runtime"] = runtime + # writeTrace(trace) if __name__ == '__main__': diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/fonda/scheduler/model/SchedulerConfig.java index e5165fea..85bd6b28 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/fonda/scheduler/model/SchedulerConfig.java @@ -25,6 +25,7 @@ public class SchedulerConfig { public final Integer maxWaitingCopyTasksPerNode; public final Integer maxHeldCopyTaskReady; + public final Integer prioPhaseThree; @ToString public static class LocalClaim { diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java index 158b0340..4cbe4850 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java @@ -80,7 +80,7 @@ private NodeTaskFilesAlignment createNodeAlignment ( .sum() ); } - return new NodeTaskFilesAlignment(result.getA(), task, result.getB()); + return new NodeTaskFilesAlignment(result.getA(), task, result.getB(), 100 ); } /** diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 026fd401..0e12fdb1 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -56,6 +56,12 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final int copySameTaskInParallel; private final int maxHeldCopyTaskReady; + /** + * This value must be between 1 and 100. + * 100 means that the data will be copied with full speed. + */ + private final int prioPhaseThree; + /** * This lock is used to synchronize the creation of the copy tasks and the finishing. * Otherwise, it could happen that: @@ -86,6 +92,7 @@ public LocationAwareSchedulerV2( this.phaseThreeComparator = new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ); this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); this.maxHeldCopyTaskReady = config.maxHeldCopyTaskReady == null ? 3 : config.maxHeldCopyTaskReady; + this.prioPhaseThree = config.prioPhaseThree == null ? 70 : config.prioPhaseThree; } @Override @@ -158,7 +165,8 @@ void postScheduling( List unscheduledTasks, final Map unscheduledTasks, final Map entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 3513cd0a..8dda196c 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -211,7 +211,8 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { this.getDns(), getExecution(), this.localWorkDir + "/sync/", - alignment.task.getConfig().getRunName() + alignment.task.getConfig().getRunName(), + 100 ); for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { diff --git a/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java b/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java index b3197d30..eaa0ec42 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java @@ -55,12 +55,13 @@ protected boolean createFileAlignment( Map currentlyCopyingTasksOnNode, TaskStat poll, Task task, - NodeWithAlloc node + NodeWithAlloc node, + int prio ) { final FileAlignment fileAlignmentForTaskAndNode = getFileAlignmentForTaskAndNode( node, task, poll.getInputsOfTask(), planedToCopy ); if ( fileAlignmentForTaskAndNode != null && fileAlignmentForTaskAndNode.copyFromSomewhere( node.getNodeLocation() ) ) { planedToCopy.addAlignment( fileAlignmentForTaskAndNode.getNodeFileAlignment(), task, node ); - nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode ) ); + nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode, prio ) ); currentlyCopyingTasksOnNode.compute( node.getNodeLocation(), ( nodeLocation, value ) -> value == null ? 1 : value + 1 ); poll.copyToNodeWithAvailableResources(); return true; @@ -70,7 +71,7 @@ protected boolean createFileAlignment( } protected void removeTasksThatAreCopiedMoreThanXTimeCurrently( SortedList taskStats, int maxParallelTasks ) { - taskStats.removeIf( elem -> currentlyCopying.getNumberOfNodesForTask( elem.getTask() ) == maxParallelTasks ); + taskStats.removeIf( elem -> currentlyCopying.getNumberOfNodesForTask( elem.getTask() ) >= maxParallelTasks ); } } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java index b632e8dd..b0221579 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java @@ -27,7 +27,8 @@ public abstract List createAlignmentForTasksWithEnoughCa final Map availableByNodes, final List allNodes, final int maxCopyingTaskPerNode, - final Map currentlyCopyingTasksOnNode + final Map currentlyCopyingTasksOnNode, + int prio ); } diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java index 21b1a83e..b405430b 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -32,7 +32,8 @@ public List createAlignmentForTasksWithEnoughCapacity( final Map availableByNodes, final List allNodes, final int maxCopyingTaskPerNode, - final Map currentlyCopyingTasksOnNode + final Map currentlyCopyingTasksOnNode, + final int prio ) { @@ -56,7 +57,7 @@ public List createAlignmentForTasksWithEnoughCapacity( if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode && availableByNodes.get( node ).higherOrEquals( task.getRequest() ) ) { - if ( createFileAlignment( planedToCopy, nodeTaskAlignments, currentlyCopyingTasksOnNode, poll, task, node ) ) { + if ( createFileAlignment( planedToCopy, nodeTaskAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { cannotAdd = false; availableByNodes.get( node ).subFromThis( task.getRequest() ); } else { diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index 1a86da34..dc5e86dc 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -36,8 +36,9 @@ public void createAlignmentForTasksWithEnoughCapacity( final List allNodes, final int maxCopyingTaskPerNode, final int maxHeldCopyTaskReady, - final Map currentlyCopyingTasksOnNode ) - { + final Map currentlyCopyingTasksOnNode, + int prio + ) { final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); @@ -52,7 +53,7 @@ public void createAlignmentForTasksWithEnoughCapacity( //Check if the node has still enough resources to run the task if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { - if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node ) ) { + if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { cannotAdd = false; log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); } else { diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java index 1eaf1244..90e71f35 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -20,6 +20,7 @@ public class Inputs { public final String syncDir; public final String hash; public final Map> waitForFilesOfTask = new ConcurrentHashMap<>(); + public final int speed; public void waitForTask( Map waitForTask ){ for (Map.Entry e : waitForTask.entrySet()) { diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java index e4306efe..494af816 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java +++ b/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java @@ -8,13 +8,15 @@ public class NodeTaskFilesAlignment extends NodeTaskAlignment { public final FileAlignment fileAlignment; + public final int prio; @Getter @Setter private boolean removeInit = false; - public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, FileAlignment fileAlignment ) { + public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, FileAlignment fileAlignment, int prio ) { super(node, task); this.fileAlignment = fileAlignment; + this.prio = prio; } } From ccfa802860ab6b532152d41729f5a980de6bb892 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 11:25:34 +0100 Subject: [PATCH 352/443] Only process ready nodes Signed-off-by: Lehmann_Fabian --- .../java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index 0e12fdb1..fdd9b54c 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -139,6 +139,7 @@ && canSchedulePodOnNode( task.getPod(), node ) void postScheduling( List unscheduledTasks, final Map availableByNode ) { final Map currentlyCopyingTasksOnNode = getCurrentlyCopying().getCurrentlyCopyingTasksOnNode(); final List allNodes = client.getAllNodes(); + allNodes.removeIf( x -> !x.isReady() ); final List nodeTaskFilesAlignments; synchronized ( copyLock ) { final TaskStats taskStats = new TaskStats(); From be52a87f9a7e633216df5198f251203e75400f00 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 11:26:07 +0100 Subject: [PATCH 353/443] Do not crash because of symlink creation Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/LocationAwareSchedulerV2.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java index fdd9b54c..9f490363 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -367,11 +367,11 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ) { try ( BufferedWriter writer = new BufferedWriter( new FileWriter( alignment.task.getWorkingDir() + '/' + ".command.symlinks" ) ) ) { writer.write( "create_symlink() {" ); writer.newLine(); - writer.write( " rm -rf \"$2\"" ); + writer.write( " rm -rf \"$2\" || true" ); writer.newLine(); - writer.write( " mkdir -p \"$(dirname \"$2\")\"" ); + writer.write( " mkdir -p \"$(dirname \"$2\")\" || true" ); writer.newLine(); - writer.write( " ln -s \"$1\" \"$2\"" ); + writer.write( " ln -s \"$1\" \"$2\" || true" ); writer.newLine(); writer.write( "}" ); writer.newLine(); From 662a39237ff33eebec634f3deb1747a84aa98c12 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 11:26:34 +0100 Subject: [PATCH 354/443] More precise Origin.addInbound Exception Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/dag/Origin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fonda/scheduler/dag/Origin.java b/src/main/java/fonda/scheduler/dag/Origin.java index 21be9be2..6977a2b0 100644 --- a/src/main/java/fonda/scheduler/dag/Origin.java +++ b/src/main/java/fonda/scheduler/dag/Origin.java @@ -21,7 +21,7 @@ public Type getType() { @Override public void addInbound(Edge e) { - throw new IllegalStateException("Cannot add an inbound to an Origin"); + throw new IllegalStateException("Cannot add an Edge(uid: " + e.getUid() + "; " + e.getFrom().getUid() + " -> " + e.getTo().getUid() + ") inbound to an Origin (uid: " + getUid() + ")"); } public Set getAncestors() { From 4187339bb10b3b51704cde63fab7fe6fb8a7e0cd Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 12:22:46 +0100 Subject: [PATCH 355/443] Handle DeadlineExceeded Signed-off-by: Lehmann_Fabian --- .../fonda/scheduler/scheduler/Scheduler.java | 5 +- .../scheduler/SchedulerWithDaemonSet.java | 50 ++++++++++--------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/fonda/scheduler/scheduler/Scheduler.java index e2b2e3b0..d897d33d 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/fonda/scheduler/scheduler/Scheduler.java @@ -578,7 +578,10 @@ public void eventReceived(Action action, Pod pod) { } break; case MODIFIED: - if (!pod.getStatus().getContainerStatuses().isEmpty() && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { + if ( "DeadlineExceeded".equals( pod.getStatus().getReason() ) || //Task ran out of time + ( !pod.getStatus().getContainerStatuses().isEmpty() && + pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null ) + ) { scheduler.onPodTermination(pwa); } else { final Task task = scheduler.getTaskByPod(pwa); diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java index 8dda196c..119d3776 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -134,30 +134,32 @@ int terminateTasks(List finishedTasks) { finishedTasks.parallelStream().forEach( finishedTask -> { try{ freeLocations( finishedTask.getInputFiles() ); - final Integer exitCode = finishedTask.getPod().getStatus().getContainerStatuses().get(0).getState().getTerminated().getExitCode(); - log.info( "Pod finished with exitCode: {}", exitCode ); - //Init failure - final Path workdir = Paths.get(finishedTask.getWorkingDir()); - if ( exitCode == 123 && Files.exists( workdir.resolve(".command.init.failure") ) ) { - log.info( "Task " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and outfiles" ); - } else { - final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( - workdir, - finishedTask.getNode().getNodeLocation(), - !finishedTask.wasSuccessfullyExecuted(), - finishedTask - ); - for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { - if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { - hierarchyWrapper.addFile( - newAndUpdatedFile.getPath(), - ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() - ); - } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ - hierarchyWrapper.addSymlink( - newAndUpdatedFile.getPath(), - ((SymlinkOutput) newAndUpdatedFile).getDst() - ); + if ( !"DeadlineExceeded".equals( finishedTask.getPod().getStatus().getReason() ) ) { //If Deadline exceeded, task cannot write oufiles and containerStatuses.terminated is not available + final Integer exitCode = finishedTask.getPod().getStatus().getContainerStatuses().get(0).getState().getTerminated().getExitCode(); + log.info( "Pod finished with exitCode: {}", exitCode ); + //Init failure + final Path workdir = Paths.get(finishedTask.getWorkingDir()); + if ( exitCode == 123 && Files.exists( workdir.resolve(".command.init.failure") ) ) { + log.info( "Task " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and outfiles" ); + } else { + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + workdir, + finishedTask.getNode().getNodeLocation(), + !finishedTask.wasSuccessfullyExecuted(), + finishedTask + ); + for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { + if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { + hierarchyWrapper.addFile( + newAndUpdatedFile.getPath(), + ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() + ); + } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ + hierarchyWrapper.addSymlink( + newAndUpdatedFile.getPath(), + ((SymlinkOutput) newAndUpdatedFile).getDst() + ); + } } } } From 7ce83edf2e3f91cce5c417ab44caf82a45543ef8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 13:37:06 +0100 Subject: [PATCH 356/443] Change value Signed-off-by: Lehmann_Fabian --- src/main/java/fonda/scheduler/scheduler/data/TaskData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java index a11f3bf6..2f93d64d 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/fonda/scheduler/scheduler/data/TaskData.java @@ -120,9 +120,9 @@ private void calc() { if ( nodeDataTuples.isEmpty() ) { value = Double.MIN_VALUE; } else if ( size == 0 ) { - value = 1; + value = Double.MAX_VALUE; } else { - value = nodeDataTuples.get(0).getSizeInBytes() / size; + value = size;//nodeDataTuples.get(0).getSizeInBytes() / size; } } From 5d563ea88d0c45538a1b3f124e1bbf3790b3f909 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 13:40:14 +0100 Subject: [PATCH 357/443] update ftp.py Signed-off-by: Lehmann_Fabian --- daemons/ftp/ftp.py | 4 ++-- src/main/resources/copystrategies/ftp.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/daemons/ftp/ftp.py b/daemons/ftp/ftp.py index 176ffc64..4a779dde 100644 --- a/daemons/ftp/ftp.py +++ b/daemons/ftp/ftp.py @@ -142,9 +142,9 @@ def callback(data): ftp.retrbinary('RETR %s' % filename, callback, 102400) end = time.time() - sizeInMB = os.path.getsize(filename) / 1000000 + sizeInMB = os.path.getsize(filename) / 1048576 delta = (end - start) - log.info("Speed: %.3f Mbit/s", sizeInMB / delta) + log.info("Speed: %.3f Mb/s", sizeInMB / delta) return sizeInMB, delta except ftplib.error_perm as err: errors += 1 diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py index 0c711d8e..da9191a0 100644 --- a/src/main/resources/copystrategies/ftp.py +++ b/src/main/resources/copystrategies/ftp.py @@ -17,7 +17,14 @@ CLOSE = False UNEXPECTED_ERROR = "Unexpected error" EXIT = 0 -log.basicConfig(format='%(levelname)s: %(message)s', level=log.DEBUG) +log.basicConfig( + format='%(levelname)s: %(message)s', + level=log.DEBUG, + handlers=[ + log.FileHandler(".command.init.log"), + log.StreamHandler() + ] +) trace = {} traceFilePath = ".command.scheduler.trace" errors = 0 From f64244f7b67401409c225b8a4fd79fdee1370a02 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 22 Mar 2023 13:44:54 +0100 Subject: [PATCH 358/443] Renamed folder Signed-off-by: Lehmann_Fabian --- pom.xml | 4 +- .../{fonda => cws/k8s}/scheduler/Main.java | 2 +- .../k8s}/scheduler/client/Informable.java | 4 +- .../scheduler/client/KubernetesClient.java | 8 +-- .../k8s}/scheduler/config/Beans.java | 4 +- .../{fonda => cws/k8s}/scheduler/dag/DAG.java | 2 +- .../k8s}/scheduler/dag/Edge.java | 2 +- .../k8s}/scheduler/dag/InputEdge.java | 2 +- .../k8s}/scheduler/dag/Node.java | 2 +- .../k8s}/scheduler/dag/NotProcess.java | 2 +- .../k8s}/scheduler/dag/Operator.java | 2 +- .../k8s}/scheduler/dag/Origin.java | 2 +- .../k8s}/scheduler/dag/Process.java | 2 +- .../k8s}/scheduler/dag/Type.java | 2 +- .../k8s}/scheduler/dag/Vertex.java | 2 +- .../scheduler/dag/VertexDeserializer.java | 2 +- .../k8s}/scheduler/model/DateParser.java | 2 +- .../k8s}/scheduler/model/FileHolder.java | 2 +- .../scheduler/model/InputFileCollector.java | 13 ++--- .../k8s}/scheduler/model/InputParam.java | 2 +- .../k8s}/scheduler/model/NodeWithAlloc.java | 6 +-- .../k8s}/scheduler/model/OutLabel.java | 2 +- .../k8s}/scheduler/model/PodWithAge.java | 4 +- .../k8s}/scheduler/model/Requirements.java | 2 +- .../k8s}/scheduler/model/ScheduleObject.java | 4 +- .../k8s}/scheduler/model/SchedulerConfig.java | 2 +- .../k8s}/scheduler/model/State.java | 2 +- .../k8s}/scheduler/model/Task.java | 16 +++--- .../k8s}/scheduler/model/TaskConfig.java | 2 +- .../k8s}/scheduler/model/TaskInput.java | 2 +- .../model/TaskInputFileLocationWrapper.java | 6 +-- .../scheduler/model/TaskResultParser.java | 12 ++--- .../k8s}/scheduler/model/TaskState.java | 2 +- .../scheduler/model/WriteConfigResult.java | 4 +- .../scheduler/model/location/Location.java | 2 +- .../model/location/LocationType.java | 7 +++ .../model/location/NodeLocation.java | 2 +- .../hierachy/AbstractHierarchyFile.java | 2 +- .../model/location/hierachy/Folder.java | 2 +- .../location/hierachy/HierarchyFile.java | 2 +- .../location/hierachy/HierarchyWrapper.java | 2 +- .../location/hierachy/LinkHierarchyFile.java | 2 +- .../location/hierachy/LocationWrapper.java | 6 +-- .../hierachy/NoAlignmentFoundException.java | 2 +- .../location/hierachy/RealHierarchyFile.java | 10 ++-- .../scheduler/model/outfiles/OutputFile.java | 2 +- .../outfiles/PathLocationWrapperPair.java | 4 +- .../model/outfiles/SymlinkOutput.java | 2 +- .../k8s/scheduler/model/taskinputs/Input.java | 4 ++ .../taskinputs/PathFileLocationTriple.java | 8 +-- .../model/taskinputs/SymlinkInput.java | 2 +- .../model/taskinputs/TaskInputs.java | 14 +++--- .../scheduler/model/tracing/TraceRecord.java | 2 +- .../k8s}/scheduler/rest/PathAttributes.java | 2 +- .../rest/SchedulerRestController.java | 36 ++++++++------ .../exceptions/NotARealFileException.java | 2 +- .../rest/response/getfile/FileResponse.java | 4 +- .../scheduler/scheduler/LASchedulerV1.java | 8 +-- .../scheduler/LocationAwareScheduler.java | 28 ++++++----- .../scheduler/LocationAwareSchedulerV2.java | 47 +++++++++--------- .../scheduler/MatchingFilesAndNodes.java | 6 +-- .../scheduler/PrioritizeAssignScheduler.java | 13 ++--- .../scheduler/RandomLAScheduler.java | 26 +++++----- .../k8s}/scheduler/scheduler/Scheduler.java | 13 ++--- .../scheduler/SchedulerWithDaemonSet.java | 49 ++++++++++--------- .../scheduler/TaskprocessingThread.java | 4 +- .../scheduler/copystrategy/CopyStrategy.java | 4 +- .../scheduler/copystrategy/FTPstrategy.java | 2 +- .../scheduler/data/NodeDataTuple.java | 4 +- .../scheduler/scheduler/data/TaskData.java | 12 ++--- .../scheduler/data/TaskInputsNodes.java | 8 +-- .../filealignment/GreedyAlignment.java | 27 +++++----- .../filealignment/InputAlignment.java | 12 ++--- .../filealignment/InputAlignmentClass.java | 31 ++++++------ .../filealignment/RandomAlignment.java | 14 +++--- .../costfunctions/CostFunction.java | 6 +-- .../costfunctions/MinSizeCost.java | 4 +- .../NoAligmentPossibleException.java | 2 +- .../scheduler/la2/CreateCopyTasks.java | 24 ++++----- .../scheduler/la2/MaxPartSizeComparator.java | 4 +- .../scheduler/la2/MaxSizeComparator.java | 4 +- .../scheduler/la2/MinCopyingComparator.java | 2 +- .../scheduler/la2/MinSizeComparator.java | 4 +- .../la2/RankAndMinCopyingComparator.java | 2 +- .../scheduler/scheduler/la2/TaskStat.java | 11 ++--- .../scheduler/la2/TaskStatComparator.java | 2 +- .../CapacityAvailableToNode.java | 18 +++---- .../SimpleCapacityAvailableToNode.java | 22 ++++----- .../CopyInAdvanceNodeWithMostData.java | 22 ++++----- .../la2/copystrategy/CopyRunner.java | 8 +++ .../la2/copystrategy/LaListener.java | 12 ++--- .../scheduler/la2/copystrategy/ShellCopy.java | 12 ++--- .../ready2run/OptimalReadyToRunToNode.java | 16 +++--- .../la2/ready2run/RandomReadyToRunToNode.java | 14 +++--- .../la2/ready2run/ReadyToRunToNode.java | 14 +++--- .../scheduler/nodeassign/FairAssign.java | 12 ++--- .../scheduler/nodeassign/NodeAssign.java | 12 ++--- .../nodeassign/RandomNodeAssign.java | 12 ++--- .../nodeassign/RoundRobinAssign.java | 14 +++--- .../scheduler/outlabel/HolderMaxTasks.java | 4 +- .../scheduler/outlabel/InternalHolder.java | 6 +-- .../scheduler/outlabel/OutLabelHolder.java | 6 +-- .../scheduler/prioritize/FifoPrioritize.java | 4 +- .../prioritize/MaxInputPrioritize.java | 4 +- .../prioritize/MinInputPrioritize.java | 4 +- .../scheduler/prioritize/Prioritize.java | 4 +- .../prioritize/RandomPrioritize.java | 4 +- .../prioritize/RankMaxPrioritize.java | 4 +- .../prioritize/RankMinPrioritize.java | 4 +- .../scheduler/prioritize/RankPrioritize.java | 4 +- .../schedulingstrategy/InputEntry.java | 2 +- .../scheduler/schedulingstrategy/Inputs.java | 6 +-- .../k8s}/scheduler/util/AlignmentWrapper.java | 2 +- .../k8s}/scheduler/util/Batch.java | 6 +-- .../k8s}/scheduler/util/CopyTask.java | 14 +++--- .../k8s}/scheduler/util/DaemonHolder.java | 4 +- .../k8s}/scheduler/util/DataOnNode.java | 8 +-- .../k8s}/scheduler/util/FileAlignment.java | 8 +-- .../k8s}/scheduler/util/FilePath.java | 8 +-- .../k8s}/scheduler/util/FilePathWithTask.java | 8 +-- .../k8s}/scheduler/util/LogCopyTask.java | 2 +- .../k8s}/scheduler/util/MyExecListner.java | 2 +- .../scheduler/util/NodeTaskAlignment.java | 6 +-- .../util/NodeTaskFilesAlignment.java | 6 +-- .../util/NodeTaskLocalFilesAlignment.java | 10 ++-- .../k8s}/scheduler/util/PodPhase.java | 2 +- .../k8s}/scheduler/util/SortedList.java | 2 +- .../k8s}/scheduler/util/TaskNodeStats.java | 2 +- .../k8s}/scheduler/util/TaskStats.java | 8 +-- .../k8s}/scheduler/util/Tuple.java | 2 +- .../scheduler/util/copying/CopySource.java | 6 +-- .../util/copying/CurrentlyCopying.java | 16 +++--- .../util/copying/CurrentlyCopyingOnNode.java | 6 +-- .../scheduler/util/score/CalculateScore.java | 4 +- .../util/score/FileSizeRankScore.java | 6 +-- .../scheduler/util/score/FileSizeScore.java | 6 +-- .../model/location/LocationType.java | 7 --- .../scheduler/model/taskinputs/Input.java | 4 -- .../la2/copystrategy/CopyRunner.java | 8 --- .../k8s}/scheduler/dag/DAGTest.java | 2 +- .../scheduler/dag/VertexDeserializerTest.java | 2 +- .../model/InputFileCollectorTest.java | 26 +++++----- .../scheduler/model/TaskResultParserTest.java | 20 ++++---- .../hierachy/HierarchyWrapperTest.java | 19 +++---- .../hierachy/RealHierarchyFileTest.java | 20 ++++---- .../outlabel/OutLabelHolderMaxTasksTest.java | 18 +++---- .../prioritize/RankPrioritizeTest.java | 16 +++--- .../k8s}/scheduler/util/SortedListTest.java | 2 +- 148 files changed, 585 insertions(+), 570 deletions(-) rename src/main/java/{fonda => cws/k8s}/scheduler/Main.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/client/Informable.java (65%) rename src/main/java/{fonda => cws/k8s}/scheduler/client/KubernetesClient.java (98%) rename src/main/java/{fonda => cws/k8s}/scheduler/config/Beans.java (85%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/DAG.java (99%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Edge.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/InputEdge.java (89%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Node.java (85%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/NotProcess.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Operator.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Origin.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Process.java (99%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Type.java (69%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/Vertex.java (99%) rename src/main/java/{fonda => cws/k8s}/scheduler/dag/VertexDeserializer.java (98%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/DateParser.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/FileHolder.java (94%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/InputFileCollector.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/InputParam.java (92%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/NodeWithAlloc.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/OutLabel.java (89%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/PodWithAge.java (96%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/Requirements.java (98%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/ScheduleObject.java (80%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/SchedulerConfig.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/State.java (83%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/Task.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/TaskConfig.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/TaskInput.java (96%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/TaskInputFileLocationWrapper.java (71%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/TaskResultParser.java (94%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/TaskState.java (89%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/WriteConfigResult.java (81%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/Location.java (93%) create mode 100644 src/main/java/cws/k8s/scheduler/model/location/LocationType.java rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/NodeLocation.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/AbstractHierarchyFile.java (58%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/Folder.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/HierarchyFile.java (68%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/HierarchyWrapper.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/LinkHierarchyFile.java (88%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/LocationWrapper.java (96%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/NoAlignmentFoundException.java (54%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/RealHierarchyFile.java (98%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/outfiles/OutputFile.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/outfiles/PathLocationWrapperPair.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/outfiles/SymlinkOutput.java (95%) create mode 100644 src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java rename src/main/java/{fonda => cws/k8s}/scheduler/model/taskinputs/PathFileLocationTriple.java (88%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/taskinputs/SymlinkInput.java (89%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/taskinputs/TaskInputs.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/model/tracing/TraceRecord.java (99%) rename src/main/java/{fonda => cws/k8s}/scheduler/rest/PathAttributes.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/rest/SchedulerRestController.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/rest/exceptions/NotARealFileException.java (76%) rename src/main/java/{fonda => cws/k8s}/scheduler/rest/response/getfile/FileResponse.java (90%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/LASchedulerV1.java (65%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/LocationAwareScheduler.java (96%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/LocationAwareSchedulerV2.java (92%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/MatchingFilesAndNodes.java (64%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/PrioritizeAssignScheduler.java (83%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/RandomLAScheduler.java (80%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/Scheduler.java (98%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/SchedulerWithDaemonSet.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/TaskprocessingThread.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/copystrategy/CopyStrategy.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/copystrategy/FTPstrategy.java (73%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/data/NodeDataTuple.java (89%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/data/TaskData.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/data/TaskInputsNodes.java (69%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/GreedyAlignment.java (81%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/InputAlignment.java (83%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/InputAlignmentClass.java (85%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/RandomAlignment.java (78%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/costfunctions/CostFunction.java (67%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java (73%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java (70%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/CreateCopyTasks.java (79%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/MaxPartSizeComparator.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/MaxSizeComparator.java (92%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/MinCopyingComparator.java (94%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/MinSizeComparator.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/RankAndMinCopyingComparator.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/TaskStat.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/TaskStatComparator.java (87%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java (63%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java (88%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java (81%) create mode 100644 src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/copystrategy/LaListener.java (90%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/copystrategy/ShellCopy.java (87%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java (86%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java (51%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/nodeassign/FairAssign.java (87%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/nodeassign/NodeAssign.java (57%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/nodeassign/RandomNodeAssign.java (84%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/nodeassign/RoundRobinAssign.java (88%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/outlabel/HolderMaxTasks.java (82%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/outlabel/InternalHolder.java (89%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/outlabel/OutLabelHolder.java (88%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/FifoPrioritize.java (73%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/MaxInputPrioritize.java (75%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/MinInputPrioritize.java (74%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/Prioritize.java (53%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/RandomPrioritize.java (72%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/RankMaxPrioritize.java (84%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/RankMinPrioritize.java (84%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/RankPrioritize.java (83%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/schedulingstrategy/InputEntry.java (88%) rename src/main/java/{fonda => cws/k8s}/scheduler/scheduler/schedulingstrategy/Inputs.java (87%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/AlignmentWrapper.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/Batch.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/CopyTask.java (66%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/DaemonHolder.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/DataOnNode.java (85%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/FileAlignment.java (87%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/FilePath.java (70%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/FilePathWithTask.java (62%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/LogCopyTask.java (97%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/MyExecListner.java (91%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/NodeTaskAlignment.java (59%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/NodeTaskFilesAlignment.java (80%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/NodeTaskLocalFilesAlignment.java (66%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/PodPhase.java (95%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/SortedList.java (98%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/TaskNodeStats.java (94%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/TaskStats.java (83%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/Tuple.java (86%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/copying/CopySource.java (60%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/copying/CurrentlyCopying.java (93%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/copying/CurrentlyCopyingOnNode.java (94%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/score/CalculateScore.java (62%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/score/FileSizeRankScore.java (76%) rename src/main/java/{fonda => cws/k8s}/scheduler/util/score/FileSizeScore.java (84%) delete mode 100644 src/main/java/fonda/scheduler/model/location/LocationType.java delete mode 100644 src/main/java/fonda/scheduler/model/taskinputs/Input.java delete mode 100644 src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java rename src/test/java/{fonda => cws/k8s}/scheduler/dag/DAGTest.java (99%) rename src/test/java/{fonda => cws/k8s}/scheduler/dag/VertexDeserializerTest.java (96%) rename src/test/java/{fonda => cws/k8s}/scheduler/model/InputFileCollectorTest.java (94%) rename src/test/java/{fonda => cws/k8s}/scheduler/model/TaskResultParserTest.java (97%) rename src/test/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/HierarchyWrapperTest.java (92%) rename src/test/java/{fonda => cws/k8s}/scheduler/model/location/hierachy/RealHierarchyFileTest.java (98%) rename src/test/java/{fonda => cws/k8s}/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java (80%) rename src/test/java/{fonda => cws/k8s}/scheduler/scheduler/prioritize/RankPrioritizeTest.java (85%) rename src/test/java/{fonda => cws/k8s}/scheduler/util/SortedListTest.java (99%) diff --git a/pom.xml b/pom.xml index 8bf9554a..4b601aae 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 11 11 - fonda.scheduler.Main + cws.k8s.scheduler.Main @@ -139,7 +139,7 @@ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> - fonda.scheduler.Main + cws.k8s.scheduler.Main diff --git a/src/main/java/fonda/scheduler/Main.java b/src/main/java/cws/k8s/scheduler/Main.java similarity index 95% rename from src/main/java/fonda/scheduler/Main.java rename to src/main/java/cws/k8s/scheduler/Main.java index fd72a1c6..fb73a3f4 100644 --- a/src/main/java/fonda/scheduler/Main.java +++ b/src/main/java/cws/k8s/scheduler/Main.java @@ -1,4 +1,4 @@ -package fonda.scheduler; +package cws.k8s.scheduler; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; diff --git a/src/main/java/fonda/scheduler/client/Informable.java b/src/main/java/cws/k8s/scheduler/client/Informable.java similarity index 65% rename from src/main/java/fonda/scheduler/client/Informable.java rename to src/main/java/cws/k8s/scheduler/client/Informable.java index a5c2ec56..af5cc0f2 100644 --- a/src/main/java/fonda/scheduler/client/Informable.java +++ b/src/main/java/cws/k8s/scheduler/client/Informable.java @@ -1,6 +1,6 @@ -package fonda.scheduler.client; +package cws.k8s.scheduler.client; -import fonda.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.NodeWithAlloc; public interface Informable { diff --git a/src/main/java/fonda/scheduler/client/KubernetesClient.java b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java similarity index 98% rename from src/main/java/fonda/scheduler/client/KubernetesClient.java rename to src/main/java/cws/k8s/scheduler/client/KubernetesClient.java index 62ccb134..d83cdbeb 100644 --- a/src/main/java/fonda/scheduler/client/KubernetesClient.java +++ b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java @@ -1,8 +1,8 @@ -package fonda.scheduler.client; +package cws.k8s.scheduler.client; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.util.MyExecListner; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.PodWithAge; +import cws.k8s.scheduler.util.MyExecListner; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; diff --git a/src/main/java/fonda/scheduler/config/Beans.java b/src/main/java/cws/k8s/scheduler/config/Beans.java similarity index 85% rename from src/main/java/fonda/scheduler/config/Beans.java rename to src/main/java/cws/k8s/scheduler/config/Beans.java index 9bc53e73..ab6e6bc8 100644 --- a/src/main/java/fonda/scheduler/config/Beans.java +++ b/src/main/java/cws/k8s/scheduler/config/Beans.java @@ -1,6 +1,6 @@ -package fonda.scheduler.config; +package cws.k8s.scheduler.config; -import fonda.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.client.KubernetesClient; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/src/main/java/fonda/scheduler/dag/DAG.java b/src/main/java/cws/k8s/scheduler/dag/DAG.java similarity index 99% rename from src/main/java/fonda/scheduler/dag/DAG.java rename to src/main/java/cws/k8s/scheduler/dag/DAG.java index f64dba95..0d199e09 100644 --- a/src/main/java/fonda/scheduler/dag/DAG.java +++ b/src/main/java/cws/k8s/scheduler/dag/DAG.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/fonda/scheduler/dag/Edge.java b/src/main/java/cws/k8s/scheduler/dag/Edge.java similarity index 91% rename from src/main/java/fonda/scheduler/dag/Edge.java rename to src/main/java/cws/k8s/scheduler/dag/Edge.java index f04425ad..0b0232a9 100644 --- a/src/main/java/fonda/scheduler/dag/Edge.java +++ b/src/main/java/cws/k8s/scheduler/dag/Edge.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import lombok.*; diff --git a/src/main/java/fonda/scheduler/dag/InputEdge.java b/src/main/java/cws/k8s/scheduler/dag/InputEdge.java similarity index 89% rename from src/main/java/fonda/scheduler/dag/InputEdge.java rename to src/main/java/cws/k8s/scheduler/dag/InputEdge.java index ff3c7179..f85147b9 100644 --- a/src/main/java/fonda/scheduler/dag/InputEdge.java +++ b/src/main/java/cws/k8s/scheduler/dag/InputEdge.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import lombok.*; diff --git a/src/main/java/fonda/scheduler/dag/Node.java b/src/main/java/cws/k8s/scheduler/dag/Node.java similarity index 85% rename from src/main/java/fonda/scheduler/dag/Node.java rename to src/main/java/cws/k8s/scheduler/dag/Node.java index be3a0f57..4c34b697 100644 --- a/src/main/java/fonda/scheduler/dag/Node.java +++ b/src/main/java/cws/k8s/scheduler/dag/Node.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; public class Node extends Operator { diff --git a/src/main/java/fonda/scheduler/dag/NotProcess.java b/src/main/java/cws/k8s/scheduler/dag/NotProcess.java similarity index 97% rename from src/main/java/fonda/scheduler/dag/NotProcess.java rename to src/main/java/cws/k8s/scheduler/dag/NotProcess.java index 31150c22..e4b6ba80 100644 --- a/src/main/java/fonda/scheduler/dag/NotProcess.java +++ b/src/main/java/cws/k8s/scheduler/dag/NotProcess.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import java.util.Collection; import java.util.HashSet; diff --git a/src/main/java/fonda/scheduler/dag/Operator.java b/src/main/java/cws/k8s/scheduler/dag/Operator.java similarity index 97% rename from src/main/java/fonda/scheduler/dag/Operator.java rename to src/main/java/cws/k8s/scheduler/dag/Operator.java index 7cf03c4a..b8e4bc68 100644 --- a/src/main/java/fonda/scheduler/dag/Operator.java +++ b/src/main/java/cws/k8s/scheduler/dag/Operator.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import java.util.HashSet; import java.util.Set; diff --git a/src/main/java/fonda/scheduler/dag/Origin.java b/src/main/java/cws/k8s/scheduler/dag/Origin.java similarity index 95% rename from src/main/java/fonda/scheduler/dag/Origin.java rename to src/main/java/cws/k8s/scheduler/dag/Origin.java index 6977a2b0..e0e0ee44 100644 --- a/src/main/java/fonda/scheduler/dag/Origin.java +++ b/src/main/java/cws/k8s/scheduler/dag/Origin.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import java.util.HashSet; import java.util.Set; diff --git a/src/main/java/fonda/scheduler/dag/Process.java b/src/main/java/cws/k8s/scheduler/dag/Process.java similarity index 99% rename from src/main/java/fonda/scheduler/dag/Process.java rename to src/main/java/cws/k8s/scheduler/dag/Process.java index ec916c57..1e37ff59 100644 --- a/src/main/java/fonda/scheduler/dag/Process.java +++ b/src/main/java/cws/k8s/scheduler/dag/Process.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import java.util.Collection; import java.util.HashSet; diff --git a/src/main/java/fonda/scheduler/dag/Type.java b/src/main/java/cws/k8s/scheduler/dag/Type.java similarity index 69% rename from src/main/java/fonda/scheduler/dag/Type.java rename to src/main/java/cws/k8s/scheduler/dag/Type.java index edf4d6dc..759f4274 100644 --- a/src/main/java/fonda/scheduler/dag/Type.java +++ b/src/main/java/cws/k8s/scheduler/dag/Type.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; public enum Type { diff --git a/src/main/java/fonda/scheduler/dag/Vertex.java b/src/main/java/cws/k8s/scheduler/dag/Vertex.java similarity index 99% rename from src/main/java/fonda/scheduler/dag/Vertex.java rename to src/main/java/cws/k8s/scheduler/dag/Vertex.java index 1fb92400..35f85255 100644 --- a/src/main/java/fonda/scheduler/dag/Vertex.java +++ b/src/main/java/cws/k8s/scheduler/dag/Vertex.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import lombok.AccessLevel; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/dag/VertexDeserializer.java b/src/main/java/cws/k8s/scheduler/dag/VertexDeserializer.java similarity index 98% rename from src/main/java/fonda/scheduler/dag/VertexDeserializer.java rename to src/main/java/cws/k8s/scheduler/dag/VertexDeserializer.java index dd12183e..242e29a1 100644 --- a/src/main/java/fonda/scheduler/dag/VertexDeserializer.java +++ b/src/main/java/cws/k8s/scheduler/dag/VertexDeserializer.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.TreeNode; diff --git a/src/main/java/fonda/scheduler/model/DateParser.java b/src/main/java/cws/k8s/scheduler/model/DateParser.java similarity index 97% rename from src/main/java/fonda/scheduler/model/DateParser.java rename to src/main/java/cws/k8s/scheduler/model/DateParser.java index dd3737bd..87c099b7 100644 --- a/src/main/java/fonda/scheduler/model/DateParser.java +++ b/src/main/java/cws/k8s/scheduler/model/DateParser.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/FileHolder.java b/src/main/java/cws/k8s/scheduler/model/FileHolder.java similarity index 94% rename from src/main/java/fonda/scheduler/model/FileHolder.java rename to src/main/java/cws/k8s/scheduler/model/FileHolder.java index bd2cab0a..cfd38dad 100644 --- a/src/main/java/fonda/scheduler/model/FileHolder.java +++ b/src/main/java/cws/k8s/scheduler/model/FileHolder.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/InputFileCollector.java b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java similarity index 91% rename from src/main/java/fonda/scheduler/model/InputFileCollector.java rename to src/main/java/cws/k8s/scheduler/model/InputFileCollector.java index 6a9c7b9a..fc08f47f 100644 --- a/src/main/java/fonda/scheduler/model/InputFileCollector.java +++ b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java @@ -1,11 +1,12 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.*; import fonda.scheduler.model.location.hierachy.*; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.SymlinkInput; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.util.Tuple; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.Tuple; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/model/InputParam.java b/src/main/java/cws/k8s/scheduler/model/InputParam.java similarity index 92% rename from src/main/java/fonda/scheduler/model/InputParam.java rename to src/main/java/cws/k8s/scheduler/model/InputParam.java index 9492af4e..8651e66c 100644 --- a/src/main/java/fonda/scheduler/model/InputParam.java +++ b/src/main/java/cws/k8s/scheduler/model/InputParam.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java similarity index 97% rename from src/main/java/fonda/scheduler/model/NodeWithAlloc.java rename to src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java index c3227b2b..be4a70be 100644 --- a/src/main/java/fonda/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java @@ -1,7 +1,7 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; diff --git a/src/main/java/fonda/scheduler/model/OutLabel.java b/src/main/java/cws/k8s/scheduler/model/OutLabel.java similarity index 89% rename from src/main/java/fonda/scheduler/model/OutLabel.java rename to src/main/java/cws/k8s/scheduler/model/OutLabel.java index 85700dfc..86516ea8 100644 --- a/src/main/java/fonda/scheduler/model/OutLabel.java +++ b/src/main/java/cws/k8s/scheduler/model/OutLabel.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/PodWithAge.java b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java similarity index 96% rename from src/main/java/fonda/scheduler/model/PodWithAge.java rename to src/main/java/cws/k8s/scheduler/model/PodWithAge.java index 370cd967..fb989052 100644 --- a/src/main/java/fonda/scheduler/model/PodWithAge.java +++ b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java @@ -1,6 +1,6 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.util.PodPhase; +import cws.k8s.scheduler.util.PodPhase; import io.fabric8.kubernetes.api.model.*; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/model/Requirements.java b/src/main/java/cws/k8s/scheduler/model/Requirements.java similarity index 98% rename from src/main/java/fonda/scheduler/model/Requirements.java rename to src/main/java/cws/k8s/scheduler/model/Requirements.java index b599e780..95de4fda 100644 --- a/src/main/java/fonda/scheduler/model/Requirements.java +++ b/src/main/java/cws/k8s/scheduler/model/Requirements.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.Getter; import lombok.ToString; diff --git a/src/main/java/fonda/scheduler/model/ScheduleObject.java b/src/main/java/cws/k8s/scheduler/model/ScheduleObject.java similarity index 80% rename from src/main/java/fonda/scheduler/model/ScheduleObject.java rename to src/main/java/cws/k8s/scheduler/model/ScheduleObject.java index 73bbb54d..26b92ad8 100644 --- a/src/main/java/fonda/scheduler/model/ScheduleObject.java +++ b/src/main/java/cws/k8s/scheduler/model/ScheduleObject.java @@ -1,6 +1,6 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/model/SchedulerConfig.java b/src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java similarity index 97% rename from src/main/java/fonda/scheduler/model/SchedulerConfig.java rename to src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java index 85bd6b28..e2dcdbee 100644 --- a/src/main/java/fonda/scheduler/model/SchedulerConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/State.java b/src/main/java/cws/k8s/scheduler/model/State.java similarity index 83% rename from src/main/java/fonda/scheduler/model/State.java rename to src/main/java/cws/k8s/scheduler/model/State.java index 7e92ed77..4c7430e0 100644 --- a/src/main/java/fonda/scheduler/model/State.java +++ b/src/main/java/cws/k8s/scheduler/model/State.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; public enum State { diff --git a/src/main/java/fonda/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java similarity index 91% rename from src/main/java/fonda/scheduler/model/Task.java rename to src/main/java/cws/k8s/scheduler/model/Task.java index 6c1162ce..58434342 100644 --- a/src/main/java/fonda/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -1,11 +1,11 @@ -package fonda.scheduler.model; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.Process; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.tracing.TraceRecord; -import fonda.scheduler.util.Batch; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.tracing.TraceRecord; +import cws.k8s.scheduler.util.Batch; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/model/TaskConfig.java b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java similarity index 97% rename from src/main/java/fonda/scheduler/model/TaskConfig.java rename to src/main/java/cws/k8s/scheduler/model/TaskConfig.java index 6798cd8d..209dff5a 100644 --- a/src/main/java/fonda/scheduler/model/TaskConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.Getter; import lombok.ToString; diff --git a/src/main/java/fonda/scheduler/model/TaskInput.java b/src/main/java/cws/k8s/scheduler/model/TaskInput.java similarity index 96% rename from src/main/java/fonda/scheduler/model/TaskInput.java rename to src/main/java/cws/k8s/scheduler/model/TaskInput.java index 994463f8..7aac8b53 100644 --- a/src/main/java/fonda/scheduler/model/TaskInput.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskInput.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/TaskInputFileLocationWrapper.java similarity index 71% rename from src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java rename to src/main/java/cws/k8s/scheduler/model/TaskInputFileLocationWrapper.java index 8878a3f4..bbbb1811 100644 --- a/src/main/java/fonda/scheduler/model/TaskInputFileLocationWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskInputFileLocationWrapper.java @@ -1,7 +1,7 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealHierarchyFile; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/TaskResultParser.java b/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java similarity index 94% rename from src/main/java/fonda/scheduler/model/TaskResultParser.java rename to src/main/java/cws/k8s/scheduler/model/TaskResultParser.java index 032a1f14..971b8e3b 100644 --- a/src/main/java/fonda/scheduler/model/TaskResultParser.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java @@ -1,10 +1,10 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.outfiles.OutputFile; -import fonda.scheduler.model.outfiles.PathLocationWrapperPair; -import fonda.scheduler.model.outfiles.SymlinkOutput; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.outfiles.OutputFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; import java.io.File; diff --git a/src/main/java/fonda/scheduler/model/TaskState.java b/src/main/java/cws/k8s/scheduler/model/TaskState.java similarity index 89% rename from src/main/java/fonda/scheduler/model/TaskState.java rename to src/main/java/cws/k8s/scheduler/model/TaskState.java index 2a2c6b56..79ef48f1 100644 --- a/src/main/java/fonda/scheduler/model/TaskState.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskState.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/model/WriteConfigResult.java b/src/main/java/cws/k8s/scheduler/model/WriteConfigResult.java similarity index 81% rename from src/main/java/fonda/scheduler/model/WriteConfigResult.java rename to src/main/java/cws/k8s/scheduler/model/WriteConfigResult.java index a229e5d8..6f4246c4 100644 --- a/src/main/java/fonda/scheduler/model/WriteConfigResult.java +++ b/src/main/java/cws/k8s/scheduler/model/WriteConfigResult.java @@ -1,6 +1,6 @@ -package fonda.scheduler.model; +package cws.k8s.scheduler.model; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/model/location/Location.java b/src/main/java/cws/k8s/scheduler/model/location/Location.java similarity index 93% rename from src/main/java/fonda/scheduler/model/location/Location.java rename to src/main/java/cws/k8s/scheduler/model/location/Location.java index 3b057be6..ad4cc207 100644 --- a/src/main/java/fonda/scheduler/model/location/Location.java +++ b/src/main/java/cws/k8s/scheduler/model/location/Location.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location; +package cws.k8s.scheduler.model.location; import java.io.Serializable; import java.util.Objects; diff --git a/src/main/java/cws/k8s/scheduler/model/location/LocationType.java b/src/main/java/cws/k8s/scheduler/model/location/LocationType.java new file mode 100644 index 00000000..ed623a2f --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/LocationType.java @@ -0,0 +1,7 @@ +package cws.k8s.scheduler.model.location; + +public enum LocationType { + + NODE + +} diff --git a/src/main/java/fonda/scheduler/model/location/NodeLocation.java b/src/main/java/cws/k8s/scheduler/model/location/NodeLocation.java similarity index 97% rename from src/main/java/fonda/scheduler/model/location/NodeLocation.java rename to src/main/java/cws/k8s/scheduler/model/location/NodeLocation.java index de69388b..c97b585e 100644 --- a/src/main/java/fonda/scheduler/model/location/NodeLocation.java +++ b/src/main/java/cws/k8s/scheduler/model/location/NodeLocation.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location; +package cws.k8s.scheduler.model.location; import io.fabric8.kubernetes.api.model.Node; import lombok.AccessLevel; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/AbstractHierarchyFile.java similarity index 58% rename from src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/AbstractHierarchyFile.java index ab6abc30..3a431269 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/AbstractHierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/AbstractHierarchyFile.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; public abstract class AbstractHierarchyFile extends HierarchyFile { diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/Folder.java similarity index 95% rename from src/main/java/fonda/scheduler/model/location/hierachy/Folder.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/Folder.java index 6b52d3d7..ef4c0ae4 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/Folder.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/Folder.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyFile.java similarity index 68% rename from src/main/java/fonda/scheduler/model/location/hierachy/HierarchyFile.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyFile.java index 32090ef8..a32b6226 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyFile.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; public abstract class HierarchyFile { diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapper.java similarity index 95% rename from src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapper.java index 555d38ea..0d2069a4 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/HierarchyWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LinkHierarchyFile.java similarity index 88% rename from src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/LinkHierarchyFile.java index ca8a5c93..7b36b85c 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LinkHierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LinkHierarchyFile.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java similarity index 96% rename from src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java index c737775f..c59c8195 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java @@ -1,7 +1,7 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.Task; import lombok.Getter; import java.util.Objects; diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/NoAlignmentFoundException.java similarity index 54% rename from src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/NoAlignmentFoundException.java index 77e5b1b3..e73de13a 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/NoAlignmentFoundException.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/NoAlignmentFoundException.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; public class NoAlignmentFoundException extends Exception { } diff --git a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java similarity index 98% rename from src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java rename to src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java index cbd2bbd9..06d76268 100644 --- a/src/main/java/fonda/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -1,9 +1,9 @@ -package fonda.scheduler.model.location.hierachy; +package cws.k8s.scheduler.model.location.hierachy; -import fonda.scheduler.dag.Process; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.LocationType; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.LocationType; +import cws.k8s.scheduler.model.Task; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java b/src/main/java/cws/k8s/scheduler/model/outfiles/OutputFile.java similarity index 93% rename from src/main/java/fonda/scheduler/model/outfiles/OutputFile.java rename to src/main/java/cws/k8s/scheduler/model/outfiles/OutputFile.java index a161dc94..2e43afc4 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/OutputFile.java +++ b/src/main/java/cws/k8s/scheduler/model/outfiles/OutputFile.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.outfiles; +package cws.k8s.scheduler.model.outfiles; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java b/src/main/java/cws/k8s/scheduler/model/outfiles/PathLocationWrapperPair.java similarity index 91% rename from src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java rename to src/main/java/cws/k8s/scheduler/model/outfiles/PathLocationWrapperPair.java index aa5a2b74..1465f079 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/PathLocationWrapperPair.java +++ b/src/main/java/cws/k8s/scheduler/model/outfiles/PathLocationWrapperPair.java @@ -1,6 +1,6 @@ -package fonda.scheduler.model.outfiles; +package cws.k8s.scheduler.model.outfiles; -import fonda.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import lombok.Getter; import java.nio.file.Path; diff --git a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java b/src/main/java/cws/k8s/scheduler/model/outfiles/SymlinkOutput.java similarity index 95% rename from src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java rename to src/main/java/cws/k8s/scheduler/model/outfiles/SymlinkOutput.java index 9fac8935..d3345f73 100644 --- a/src/main/java/fonda/scheduler/model/outfiles/SymlinkOutput.java +++ b/src/main/java/cws/k8s/scheduler/model/outfiles/SymlinkOutput.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.outfiles; +package cws.k8s.scheduler.model.outfiles; import lombok.Getter; diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java new file mode 100644 index 00000000..a1b71c29 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java @@ -0,0 +1,4 @@ +package cws.k8s.scheduler.model.taskinputs; + +public interface Input { +} diff --git a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/PathFileLocationTriple.java similarity index 88% rename from src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java rename to src/main/java/cws/k8s/scheduler/model/taskinputs/PathFileLocationTriple.java index 39601bb0..aab7cdcf 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/PathFileLocationTriple.java +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -1,8 +1,8 @@ -package fonda.scheduler.model.taskinputs; +package cws.k8s.scheduler.model.taskinputs; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealHierarchyFile; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; import lombok.ToString; diff --git a/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/SymlinkInput.java similarity index 89% rename from src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java rename to src/main/java/cws/k8s/scheduler/model/taskinputs/SymlinkInput.java index 10dcf887..cc08e6dd 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/SymlinkInput.java +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/SymlinkInput.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.taskinputs; +package cws.k8s.scheduler.model.taskinputs; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java similarity index 91% rename from src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java rename to src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java index 483c1b95..71c91e3e 100644 --- a/src/main/java/fonda/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java @@ -1,11 +1,11 @@ -package fonda.scheduler.model.taskinputs; +package cws.k8s.scheduler.model.taskinputs; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.util.TaskNodeStats; -import fonda.scheduler.util.Tuple; -import fonda.scheduler.util.copying.CopySource; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.util.TaskNodeStats; +import cws.k8s.scheduler.util.Tuple; +import cws.k8s.scheduler.util.copying.CopySource; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; diff --git a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java similarity index 99% rename from src/main/java/fonda/scheduler/model/tracing/TraceRecord.java rename to src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java index 515c0c62..148a5191 100644 --- a/src/main/java/fonda/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java @@ -1,4 +1,4 @@ -package fonda.scheduler.model.tracing; +package cws.k8s.scheduler.model.tracing; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/rest/PathAttributes.java b/src/main/java/cws/k8s/scheduler/rest/PathAttributes.java similarity index 91% rename from src/main/java/fonda/scheduler/rest/PathAttributes.java rename to src/main/java/cws/k8s/scheduler/rest/PathAttributes.java index 24a80877..fafc03b7 100644 --- a/src/main/java/fonda/scheduler/rest/PathAttributes.java +++ b/src/main/java/cws/k8s/scheduler/rest/PathAttributes.java @@ -1,4 +1,4 @@ -package fonda.scheduler.rest; +package cws.k8s.scheduler.rest; import lombok.AccessLevel; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java similarity index 95% rename from src/main/java/fonda/scheduler/rest/SchedulerRestController.java rename to src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 17a5f8f2..3a5e3b73 100644 --- a/src/main/java/fonda/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -1,21 +1,25 @@ -package fonda.scheduler.rest; - -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.InputEdge; -import fonda.scheduler.dag.Vertex; +package cws.k8s.scheduler.rest; + +import cws.k8s.scheduler.model.SchedulerConfig; +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.scheduler.*; +import cws.k8s.scheduler.scheduler.filealignment.GreedyAlignment; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.InputEdge; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.scheduler.prioritize.*; import fonda.scheduler.model.*; -import fonda.scheduler.rest.exceptions.NotARealFileException; -import fonda.scheduler.rest.response.getfile.FileResponse; +import cws.k8s.scheduler.rest.exceptions.NotARealFileException; +import cws.k8s.scheduler.rest.response.getfile.FileResponse; import fonda.scheduler.scheduler.*; -import fonda.scheduler.scheduler.filealignment.GreedyAlignment; -import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; -import fonda.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; -import fonda.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; -import fonda.scheduler.scheduler.nodeassign.FairAssign; -import fonda.scheduler.scheduler.nodeassign.NodeAssign; -import fonda.scheduler.scheduler.nodeassign.RandomNodeAssign; -import fonda.scheduler.scheduler.nodeassign.RoundRobinAssign; +import cws.k8s.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; +import cws.k8s.scheduler.scheduler.nodeassign.FairAssign; +import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; +import cws.k8s.scheduler.scheduler.nodeassign.RandomNodeAssign; +import cws.k8s.scheduler.scheduler.nodeassign.RoundRobinAssign; import fonda.scheduler.scheduler.prioritize.*; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; diff --git a/src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java b/src/main/java/cws/k8s/scheduler/rest/exceptions/NotARealFileException.java similarity index 76% rename from src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java rename to src/main/java/cws/k8s/scheduler/rest/exceptions/NotARealFileException.java index 5b552ac2..98bb4cf8 100644 --- a/src/main/java/fonda/scheduler/rest/exceptions/NotARealFileException.java +++ b/src/main/java/cws/k8s/scheduler/rest/exceptions/NotARealFileException.java @@ -1,4 +1,4 @@ -package fonda.scheduler.rest.exceptions; +package cws.k8s.scheduler.rest.exceptions; public class NotARealFileException extends Exception { diff --git a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java b/src/main/java/cws/k8s/scheduler/rest/response/getfile/FileResponse.java similarity index 90% rename from src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java rename to src/main/java/cws/k8s/scheduler/rest/response/getfile/FileResponse.java index f499278f..b7bc9656 100644 --- a/src/main/java/fonda/scheduler/rest/response/getfile/FileResponse.java +++ b/src/main/java/cws/k8s/scheduler/rest/response/getfile/FileResponse.java @@ -1,6 +1,6 @@ -package fonda.scheduler.rest.response.getfile; +package cws.k8s.scheduler.rest.response.getfile; -import fonda.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import lombok.ToString; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java b/src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java similarity index 65% rename from src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java rename to src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java index bb729da0..c9d66911 100644 --- a/src/main/java/fonda/scheduler/scheduler/LASchedulerV1.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java @@ -1,8 +1,8 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.SchedulerConfig; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java similarity index 96% rename from src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java rename to src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java index 4cbe4850..4cde0871 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java @@ -1,19 +1,21 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.util.*; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.model.tracing.TraceRecord; -import fonda.scheduler.scheduler.data.NodeDataTuple; -import fonda.scheduler.scheduler.data.TaskData; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.model.tracing.TraceRecord; +import cws.k8s.scheduler.scheduler.data.NodeDataTuple; +import cws.k8s.scheduler.scheduler.data.TaskData; import fonda.scheduler.util.*; -import fonda.scheduler.util.copying.CurrentlyCopying; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java similarity index 92% rename from src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java rename to src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 9f490363..02d919fb 100644 --- a/src/main/java/fonda/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -1,28 +1,31 @@ -package fonda.scheduler.scheduler; - -import fonda.scheduler.client.KubernetesClient; +package cws.k8s.scheduler.scheduler; + +import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.scheduler.la2.*; +import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.util.*; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.NoAlignmentFoundException; -import fonda.scheduler.model.taskinputs.SymlinkInput; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.scheduler.data.TaskInputsNodes; -import fonda.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.NoAlignmentFoundException; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import fonda.scheduler.scheduler.la2.*; -import fonda.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; -import fonda.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; -import fonda.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; -import fonda.scheduler.scheduler.la2.copystrategy.CopyRunner; -import fonda.scheduler.scheduler.la2.copystrategy.ShellCopy; -import fonda.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; -import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; -import fonda.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; +import cws.k8s.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; +import cws.k8s.scheduler.scheduler.la2.copystrategy.CopyRunner; +import cws.k8s.scheduler.scheduler.la2.copystrategy.ShellCopy; +import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; import fonda.scheduler.util.*; -import fonda.scheduler.util.copying.CurrentlyCopying; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; -import fonda.scheduler.util.score.FileSizeRankScore; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.util.score.FileSizeRankScore; import lombok.*; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/MatchingFilesAndNodes.java similarity index 64% rename from src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java rename to src/main/java/cws/k8s/scheduler/scheduler/MatchingFilesAndNodes.java index 33717ac6..012872a9 100644 --- a/src/main/java/fonda/scheduler/scheduler/MatchingFilesAndNodes.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/MatchingFilesAndNodes.java @@ -1,7 +1,7 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java similarity index 83% rename from src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java rename to src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java index d466e685..0e5a91f0 100644 --- a/src/main/java/fonda/scheduler/scheduler/PrioritizeAssignScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -1,11 +1,12 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.client.Informable; -import fonda.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.client.Informable; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.*; import fonda.scheduler.model.*; -import fonda.scheduler.scheduler.nodeassign.NodeAssign; -import fonda.scheduler.scheduler.prioritize.Prioritize; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; +import cws.k8s.scheduler.scheduler.prioritize.Prioritize; +import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java similarity index 80% rename from src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java rename to src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java index 13c810d4..57f572e3 100644 --- a/src/main/java/fonda/scheduler/scheduler/RandomLAScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java @@ -1,17 +1,17 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.SchedulerConfig; -import fonda.scheduler.model.Task; -import fonda.scheduler.scheduler.data.NodeDataTuple; -import fonda.scheduler.scheduler.data.TaskData; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.Tuple; -import fonda.scheduler.util.copying.CurrentlyCopying; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.SchedulerConfig; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.scheduler.data.NodeDataTuple; +import cws.k8s.scheduler.scheduler.data.TaskData; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.Tuple; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java similarity index 98% rename from src/main/java/fonda/scheduler/scheduler/Scheduler.java rename to src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index d897d33d..ca25ad14 100644 --- a/src/main/java/fonda/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -1,11 +1,12 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.client.Informable; -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.dag.DAG; +import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.client.Informable; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.dag.DAG; import fonda.scheduler.model.*; -import fonda.scheduler.util.Batch; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.util.Batch; +import cws.k8s.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; diff --git a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java rename to src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 119d3776..eab242ac 100644 --- a/src/main/java/fonda/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -1,28 +1,31 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; import com.fasterxml.jackson.databind.ObjectMapper; -import fonda.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.model.location.hierachy.*; +import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.util.*; import fonda.scheduler.model.*; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.LocationType; -import fonda.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.LocationType; +import cws.k8s.scheduler.model.location.NodeLocation; import fonda.scheduler.model.location.hierachy.*; -import fonda.scheduler.model.outfiles.OutputFile; -import fonda.scheduler.model.outfiles.PathLocationWrapperPair; -import fonda.scheduler.model.outfiles.SymlinkOutput; -import fonda.scheduler.model.taskinputs.SymlinkInput; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.rest.exceptions.NotARealFileException; -import fonda.scheduler.rest.response.getfile.FileResponse; -import fonda.scheduler.scheduler.copystrategy.CopyStrategy; -import fonda.scheduler.scheduler.copystrategy.FTPstrategy; -import fonda.scheduler.scheduler.outlabel.OutLabelHolder; -import fonda.scheduler.scheduler.outlabel.HolderMaxTasks; -import fonda.scheduler.scheduler.schedulingstrategy.InputEntry; -import fonda.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.model.outfiles.OutputFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.outfiles.SymlinkOutput; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.rest.exceptions.NotARealFileException; +import cws.k8s.scheduler.rest.response.getfile.FileResponse; +import cws.k8s.scheduler.scheduler.copystrategy.CopyStrategy; +import cws.k8s.scheduler.scheduler.copystrategy.FTPstrategy; +import cws.k8s.scheduler.scheduler.outlabel.OutLabelHolder; +import cws.k8s.scheduler.scheduler.outlabel.HolderMaxTasks; import fonda.scheduler.util.*; -import fonda.scheduler.util.copying.CurrentlyCopying; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.Watcher; import lombok.AccessLevel; @@ -48,7 +51,7 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final CopyStrategy copyStrategy; final HierarchyWrapper hierarchyWrapper; private final InputFileCollector inputFileCollector; - private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); final String localWorkDir; protected final OutLabelHolder outLabelHolder = new HolderMaxTasks() ; @@ -203,7 +206,7 @@ void removeFromCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCop WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final File config = new File(alignment.task.getWorkingDir() + '/' + ".command.inputs.json"); - LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); + LinkedList inputFiles = new LinkedList<>(); Map waitForTask = new HashMap<>(); final CurrentlyCopyingOnNode filesForCurrentNode = new CurrentlyCopyingOnNode(); final NodeLocation currentNode = alignment.node.getNodeLocation(); @@ -224,7 +227,7 @@ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { final NodeLocation location = (NodeLocation) entry.getKey(); final AlignmentWrapper alignmentWrapper = entry.getValue(); - for (FilePathWithTask filePath : alignmentWrapper.getWaitFor()) { + for ( FilePathWithTask filePath : alignmentWrapper.getWaitFor()) { //Node copies currently from somewhere else! //May be problematic if the task depending on fails/is stopped before all files are downloaded waitForTask.put( filePath.getPath(), filePath.getTask() ); diff --git a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java similarity index 95% rename from src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java rename to src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java index 7fe171f5..270bcb9d 100644 --- a/src/main/java/fonda/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler; +package cws.k8s.scheduler.scheduler; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java similarity index 95% rename from src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java rename to src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java index fe166d29..cb30c029 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.copystrategy; +package cws.k8s.scheduler.scheduler.copystrategy; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; import java.io.*; diff --git a/src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/FTPstrategy.java similarity index 73% rename from src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java rename to src/main/java/cws/k8s/scheduler/scheduler/copystrategy/FTPstrategy.java index c7cf8263..4369efb8 100644 --- a/src/main/java/fonda/scheduler/scheduler/copystrategy/FTPstrategy.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/FTPstrategy.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.copystrategy; +package cws.k8s.scheduler.scheduler.copystrategy; public class FTPstrategy extends CopyStrategy { diff --git a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java similarity index 89% rename from src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java rename to src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java index 84818ad1..e24c8081 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/NodeDataTuple.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.data; +package cws.k8s.scheduler.scheduler.data; -import fonda.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.NodeWithAlloc; import lombok.*; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/data/TaskData.java rename to src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java index 2f93d64d..d33ea6a5 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskData.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java @@ -1,10 +1,10 @@ -package fonda.scheduler.scheduler.data; +package cws.k8s.scheduler.scheduler.data; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.scheduler.MatchingFilesAndNodes; +import cws.k8s.scheduler.scheduler.MatchingFilesAndNodes; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java similarity index 69% rename from src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java rename to src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java index 0880bd64..0cc9245d 100644 --- a/src/main/java/fonda/scheduler/scheduler/data/TaskInputsNodes.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java @@ -1,8 +1,8 @@ -package fonda.scheduler.scheduler.data; +package cws.k8s.scheduler.scheduler.data; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java similarity index 81% rename from src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java index 9e8e9879..a3c69bef 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -1,18 +1,17 @@ -package fonda.scheduler.scheduler.filealignment; +package cws.k8s.scheduler.scheduler.filealignment; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.scheduler.filealignment.costfunctions.CostFunction; -import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.FilePath; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; -import lombok.RequiredArgsConstructor; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.FilePath; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignment.java similarity index 83% rename from src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignment.java index c30c631e..5fbbeb36 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignment.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignment.java @@ -1,10 +1,10 @@ -package fonda.scheduler.scheduler.filealignment; +package cws.k8s.scheduler.scheduler.filealignment; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import org.jetbrains.annotations.NotNull; public interface InputAlignment { diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java similarity index 85% rename from src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java index b4a39ea3..27972298 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -1,21 +1,20 @@ -package fonda.scheduler.scheduler.filealignment; +package cws.k8s.scheduler.scheduler.filealignment; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.FilePathWithTask; -import fonda.scheduler.util.Tuple; -import fonda.scheduler.util.copying.CopySource; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.FilePathWithTask; +import cws.k8s.scheduler.util.Tuple; +import cws.k8s.scheduler.util.copying.CopySource; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; -import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/RandomAlignment.java similarity index 78% rename from src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/RandomAlignment.java index 0ef71d41..e8d919d0 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/RandomAlignment.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/RandomAlignment.java @@ -1,11 +1,11 @@ -package fonda.scheduler.scheduler.filealignment; +package cws.k8s.scheduler.scheduler.filealignment; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FilePath; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FilePath; import java.util.HashMap; import java.util.Optional; diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/CostFunction.java similarity index 67% rename from src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/CostFunction.java index a7737367..d7b8a40b 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/CostFunction.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/CostFunction.java @@ -1,7 +1,7 @@ -package fonda.scheduler.scheduler.filealignment.costfunctions; +package cws.k8s.scheduler.scheduler.filealignment.costfunctions; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.util.AlignmentWrapper; public interface CostFunction { diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java similarity index 73% rename from src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java index 191d0917..d20a709c 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.filealignment.costfunctions; +package cws.k8s.scheduler.scheduler.filealignment.costfunctions; -import fonda.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor diff --git a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java similarity index 70% rename from src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java rename to src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java index c8d0cb2d..812ff26c 100644 --- a/src/main/java/fonda/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.filealignment.costfunctions; +package cws.k8s.scheduler.scheduler.filealignment.costfunctions; public class NoAligmentPossibleException extends RuntimeException { diff --git a/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/CreateCopyTasks.java similarity index 79% rename from src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/CreateCopyTasks.java index eaa0ec42..458c4b8b 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/CreateCopyTasks.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/CreateCopyTasks.java @@ -1,16 +1,16 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import fonda.scheduler.util.FileAlignment; -import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.SortedList; -import fonda.scheduler.util.copying.CurrentlyCopying; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.RequiredArgsConstructor; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxPartSizeComparator.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/MaxPartSizeComparator.java index 2599789d..2ee7eb7a 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MaxPartSizeComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxPartSizeComparator.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; -import fonda.scheduler.util.TaskNodeStats; +import cws.k8s.scheduler.util.TaskNodeStats; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java similarity index 92% rename from src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java index 9c9dcffd..d77b9d91 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MaxSizeComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; -import fonda.scheduler.util.TaskNodeStats; +import cws.k8s.scheduler.util.TaskNodeStats; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java similarity index 94% rename from src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java index ff13c228..0cc46dfb 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MinCopyingComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; import java.util.Comparator; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinSizeComparator.java similarity index 91% rename from src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/MinSizeComparator.java index 156a7700..443bb012 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/MinSizeComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinSizeComparator.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; -import fonda.scheduler.util.TaskNodeStats; +import cws.k8s.scheduler.util.TaskNodeStats; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java index 745a7012..6098d440 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/RankAndMinCopyingComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; import java.util.Comparator; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java index 78813859..fbdc6994 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/TaskStat.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java @@ -1,13 +1,12 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.util.TaskNodeStats; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.TaskNodeStats; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.Setter; import org.jetbrains.annotations.NotNull; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStatComparator.java similarity index 87% rename from src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStatComparator.java index bb95b21b..5af7adc8 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/TaskStatComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStatComparator.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.la2; +package cws.k8s.scheduler.scheduler.la2; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java similarity index 63% rename from src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java index b0221579..b0c8a7d6 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java @@ -1,13 +1,13 @@ -package fonda.scheduler.scheduler.la2.capacityavailable; +package cws.k8s.scheduler.scheduler.la2.capacityavailable; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.la2.CreateCopyTasks; -import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.TaskStats; -import fonda.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; import java.util.List; import java.util.Map; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java similarity index 88% rename from src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java index b405430b..263dc4da 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -1,15 +1,15 @@ -package fonda.scheduler.scheduler.la2.capacityavailable; +package cws.k8s.scheduler.scheduler.la2.capacityavailable; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.la2.TaskStat; -import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.SortedList; -import fonda.scheduler.util.TaskStats; -import fonda.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; import lombok.extern.slf4j.Slf4j; import java.util.LinkedList; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java similarity index 81% rename from src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index dc5e86dc..cc7b6b2b 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -1,15 +1,15 @@ -package fonda.scheduler.scheduler.la2.copyinadvance; +package cws.k8s.scheduler.scheduler.la2.copyinadvance; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.la2.CreateCopyTasks; -import fonda.scheduler.scheduler.la2.TaskStat; -import fonda.scheduler.util.NodeTaskFilesAlignment; -import fonda.scheduler.util.SortedList; -import fonda.scheduler.util.TaskStats; -import fonda.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; import java.util.List; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java new file mode 100644 index 00000000..28503c12 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java @@ -0,0 +1,8 @@ +package cws.k8s.scheduler.scheduler.la2.copystrategy; + +import cws.k8s.scheduler.util.CopyTask; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; + +public interface CopyRunner { + void startCopyTasks( CopyTask copyTask, NodeTaskFilesAlignment nodeTaskFilesAlignment ); +} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java similarity index 90% rename from src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java index 8a295936..497f4575 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/LaListener.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java @@ -1,10 +1,10 @@ -package fonda.scheduler.scheduler.la2.copystrategy; +package cws.k8s.scheduler.scheduler.la2.copystrategy; -import fonda.scheduler.scheduler.LocationAwareSchedulerV2; -import fonda.scheduler.util.CopyTask; -import fonda.scheduler.util.LogCopyTask; -import fonda.scheduler.util.MyExecListner; -import fonda.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.scheduler.LocationAwareSchedulerV2; +import cws.k8s.scheduler.util.CopyTask; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.MyExecListner; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import io.fabric8.kubernetes.api.model.Status; import io.fabric8.kubernetes.client.dsl.ExecWatch; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/ShellCopy.java similarity index 87% rename from src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/ShellCopy.java index 7d707e51..12fd9d7a 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/ShellCopy.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/ShellCopy.java @@ -1,12 +1,12 @@ -package fonda.scheduler.scheduler.la2.copystrategy; +package cws.k8s.scheduler.scheduler.la2.copystrategy; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import fonda.scheduler.client.KubernetesClient; -import fonda.scheduler.scheduler.LocationAwareSchedulerV2; -import fonda.scheduler.util.CopyTask; -import fonda.scheduler.util.LogCopyTask; -import fonda.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.scheduler.LocationAwareSchedulerV2; +import cws.k8s.scheduler.util.CopyTask; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java similarity index 93% rename from src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index 79f478ff..fcbb00be 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -1,13 +1,13 @@ -package fonda.scheduler.scheduler.la2.ready2run; - -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.scheduler.data.TaskInputsNodes; -import fonda.scheduler.util.LogCopyTask; -import fonda.scheduler.util.NodeTaskLocalFilesAlignment; +package cws.k8s.scheduler.scheduler.la2.ready2run; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; +import cws.k8s.scheduler.util.score.CalculateScore; import com.google.ortools.Loader; import com.google.ortools.sat.*; -import fonda.scheduler.util.score.CalculateScore; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java similarity index 86% rename from src/main/java/fonda/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java index b1d8a2c7..7d698c0b 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java @@ -1,11 +1,11 @@ -package fonda.scheduler.scheduler.la2.ready2run; +package cws.k8s.scheduler.scheduler.la2.ready2run; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.scheduler.data.TaskInputsNodes; -import fonda.scheduler.util.LogCopyTask; -import fonda.scheduler.util.NodeTaskLocalFilesAlignment; -import fonda.scheduler.util.score.CalculateScore; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; +import cws.k8s.scheduler.util.score.CalculateScore; import lombok.Setter; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java similarity index 51% rename from src/main/java/fonda/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java rename to src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java index 992ade58..0ed60a39 100644 --- a/src/main/java/fonda/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java @@ -1,11 +1,11 @@ -package fonda.scheduler.scheduler.la2.ready2run; +package cws.k8s.scheduler.scheduler.la2.ready2run; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.scheduler.data.TaskInputsNodes; -import fonda.scheduler.util.LogCopyTask; -import fonda.scheduler.util.NodeTaskLocalFilesAlignment; -import fonda.scheduler.util.score.CalculateScore; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; +import cws.k8s.scheduler.util.score.CalculateScore; import java.util.List; import java.util.Map; diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/FairAssign.java similarity index 87% rename from src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java rename to src/main/java/cws/k8s/scheduler/scheduler/nodeassign/FairAssign.java index 9aef70aa..5479bc6b 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/FairAssign.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/FairAssign.java @@ -1,10 +1,10 @@ -package fonda.scheduler.scheduler.nodeassign; +package cws.k8s.scheduler.scheduler.nodeassign; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.Task; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.PodWithAge; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; import java.math.BigDecimal; diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java similarity index 57% rename from src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java rename to src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java index 616eeb83..1ef6f1de 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/NodeAssign.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java @@ -1,10 +1,10 @@ -package fonda.scheduler.scheduler.nodeassign; +package cws.k8s.scheduler.scheduler.nodeassign; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.Task; -import fonda.scheduler.scheduler.Scheduler; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.scheduler.Scheduler; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.util.NodeTaskAlignment; import java.util.List; import java.util.Map; diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RandomNodeAssign.java similarity index 84% rename from src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java rename to src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RandomNodeAssign.java index df829814..05dddcce 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/RandomNodeAssign.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RandomNodeAssign.java @@ -1,10 +1,10 @@ -package fonda.scheduler.scheduler.nodeassign; +package cws.k8s.scheduler.scheduler.nodeassign; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.Task; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.PodWithAge; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java similarity index 88% rename from src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java rename to src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java index 50e5a853..65929144 100644 --- a/src/main/java/fonda/scheduler/scheduler/nodeassign/RoundRobinAssign.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java @@ -1,11 +1,11 @@ -package fonda.scheduler.scheduler.nodeassign; +package cws.k8s.scheduler.scheduler.nodeassign; -import fonda.scheduler.client.Informable; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.PodWithAge; -import fonda.scheduler.model.Requirements; -import fonda.scheduler.model.Task; -import fonda.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.client.Informable; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.PodWithAge; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java similarity index 82% rename from src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java rename to src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java index 98073d10..9edc82d9 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/HolderMaxTasks.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.outlabel; +package cws.k8s.scheduler.scheduler.outlabel; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.Set; diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java similarity index 89% rename from src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java rename to src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java index 7059bb8e..88d7c997 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/InternalHolder.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java @@ -1,7 +1,7 @@ -package fonda.scheduler.scheduler.outlabel; +package cws.k8s.scheduler.scheduler.outlabel; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; import java.util.HashMap; diff --git a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java similarity index 88% rename from src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java rename to src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java index c63f61c0..dcc81dbd 100644 --- a/src/main/java/fonda/scheduler/scheduler/outlabel/OutLabelHolder.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java @@ -1,7 +1,7 @@ -package fonda.scheduler.scheduler.outlabel; +package cws.k8s.scheduler.scheduler.outlabel; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; import java.util.*; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritize.java similarity index 73% rename from src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritize.java index be0682c7..fb774714 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/FifoPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.Comparator; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritize.java similarity index 75% rename from src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritize.java index 188f088f..c0633e43 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/MaxInputPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.Comparator; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/MinInputPrioritize.java similarity index 74% rename from src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/MinInputPrioritize.java index c6e34a9f..e844b729 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/MinInputPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/MinInputPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.Comparator; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/Prioritize.java similarity index 53% rename from src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/Prioritize.java index 0c454ee6..b1272f32 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/Prioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/Prioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RandomPrioritize.java similarity index 72% rename from src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/RandomPrioritize.java index 9417a2b1..26f278b0 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/RandomPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RandomPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.Collections; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java similarity index 84% rename from src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java index 5c9e1d44..4c7af8c0 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/RankMaxPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMinPrioritize.java similarity index 84% rename from src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMinPrioritize.java index 351423cf..96d7994e 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/RankMinPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMinPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritize.java similarity index 83% rename from src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java rename to src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritize.java index 0b3376d7..e643c8de 100644 --- a/src/main/java/fonda/scheduler/scheduler/prioritize/RankPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritize.java @@ -1,6 +1,6 @@ -package fonda.scheduler.scheduler.prioritize; +package cws.k8s.scheduler.scheduler.prioritize; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.Task; import java.util.List; diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/InputEntry.java similarity index 88% rename from src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java rename to src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/InputEntry.java index 55e64c4a..469b1e3e 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/InputEntry.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/InputEntry.java @@ -1,4 +1,4 @@ -package fonda.scheduler.scheduler.schedulingstrategy; +package cws.k8s.scheduler.scheduler.schedulingstrategy; import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/Inputs.java similarity index 87% rename from src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java rename to src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/Inputs.java index 90e71f35..af266906 100644 --- a/src/main/java/fonda/scheduler/scheduler/schedulingstrategy/Inputs.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -1,7 +1,7 @@ -package fonda.scheduler.scheduler.schedulingstrategy; +package cws.k8s.scheduler.scheduler.schedulingstrategy; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import lombok.RequiredArgsConstructor; import java.util.Collections; diff --git a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java b/src/main/java/cws/k8s/scheduler/util/AlignmentWrapper.java similarity index 97% rename from src/main/java/fonda/scheduler/util/AlignmentWrapper.java rename to src/main/java/cws/k8s/scheduler/util/AlignmentWrapper.java index f50b6ec6..fcd6f67f 100644 --- a/src/main/java/fonda/scheduler/util/AlignmentWrapper.java +++ b/src/main/java/cws/k8s/scheduler/util/AlignmentWrapper.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import lombok.Getter; import lombok.ToString; diff --git a/src/main/java/fonda/scheduler/util/Batch.java b/src/main/java/cws/k8s/scheduler/util/Batch.java similarity index 95% rename from src/main/java/fonda/scheduler/util/Batch.java rename to src/main/java/cws/k8s/scheduler/util/Batch.java index 9d73daa9..e0fc2a13 100644 --- a/src/main/java/fonda/scheduler/util/Batch.java +++ b/src/main/java/cws/k8s/scheduler/util/Batch.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.tracing.TraceRecord; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.tracing.TraceRecord; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/util/CopyTask.java b/src/main/java/cws/k8s/scheduler/util/CopyTask.java similarity index 66% rename from src/main/java/fonda/scheduler/util/CopyTask.java rename to src/main/java/cws/k8s/scheduler/util/CopyTask.java index 0ff3c76f..a8e34090 100644 --- a/src/main/java/fonda/scheduler/util/CopyTask.java +++ b/src/main/java/cws/k8s/scheduler/util/CopyTask.java @@ -1,11 +1,11 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.TaskInputFileLocationWrapper; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.scheduler.schedulingstrategy.Inputs; -import fonda.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.model.TaskInputFileLocationWrapper; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/util/DaemonHolder.java b/src/main/java/cws/k8s/scheduler/util/DaemonHolder.java similarity index 93% rename from src/main/java/fonda/scheduler/util/DaemonHolder.java rename to src/main/java/cws/k8s/scheduler/util/DaemonHolder.java index 83a262d2..32c37cb0 100644 --- a/src/main/java/fonda/scheduler/util/DaemonHolder.java +++ b/src/main/java/cws/k8s/scheduler/util/DaemonHolder.java @@ -1,6 +1,6 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.NodeLocation; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; diff --git a/src/main/java/fonda/scheduler/util/DataOnNode.java b/src/main/java/cws/k8s/scheduler/util/DataOnNode.java similarity index 85% rename from src/main/java/fonda/scheduler/util/DataOnNode.java rename to src/main/java/cws/k8s/scheduler/util/DataOnNode.java index 1e062121..6d53b221 100644 --- a/src/main/java/fonda/scheduler/util/DataOnNode.java +++ b/src/main/java/cws/k8s/scheduler/util/DataOnNode.java @@ -1,8 +1,8 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; import java.util.HashMap; diff --git a/src/main/java/fonda/scheduler/util/FileAlignment.java b/src/main/java/cws/k8s/scheduler/util/FileAlignment.java similarity index 87% rename from src/main/java/fonda/scheduler/util/FileAlignment.java rename to src/main/java/cws/k8s/scheduler/util/FileAlignment.java index d5c918ca..2df400cd 100644 --- a/src/main/java/fonda/scheduler/util/FileAlignment.java +++ b/src/main/java/cws/k8s/scheduler/util/FileAlignment.java @@ -1,8 +1,8 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/util/FilePath.java b/src/main/java/cws/k8s/scheduler/util/FilePath.java similarity index 70% rename from src/main/java/fonda/scheduler/util/FilePath.java rename to src/main/java/cws/k8s/scheduler/util/FilePath.java index e994f6cd..ddb5437a 100644 --- a/src/main/java/fonda/scheduler/util/FilePath.java +++ b/src/main/java/cws/k8s/scheduler/util/FilePath.java @@ -1,8 +1,8 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.location.hierachy.RealHierarchyFile; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor diff --git a/src/main/java/fonda/scheduler/util/FilePathWithTask.java b/src/main/java/cws/k8s/scheduler/util/FilePathWithTask.java similarity index 62% rename from src/main/java/fonda/scheduler/util/FilePathWithTask.java rename to src/main/java/cws/k8s/scheduler/util/FilePathWithTask.java index 4b5cedee..060672f9 100644 --- a/src/main/java/fonda/scheduler/util/FilePathWithTask.java +++ b/src/main/java/cws/k8s/scheduler/util/FilePathWithTask.java @@ -1,8 +1,8 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; import lombok.Getter; public class FilePathWithTask extends FilePath { diff --git a/src/main/java/fonda/scheduler/util/LogCopyTask.java b/src/main/java/cws/k8s/scheduler/util/LogCopyTask.java similarity index 97% rename from src/main/java/fonda/scheduler/util/LogCopyTask.java rename to src/main/java/cws/k8s/scheduler/util/LogCopyTask.java index 35370f97..3add5a4d 100644 --- a/src/main/java/fonda/scheduler/util/LogCopyTask.java +++ b/src/main/java/cws/k8s/scheduler/util/LogCopyTask.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import java.io.BufferedWriter; import java.io.File; diff --git a/src/main/java/fonda/scheduler/util/MyExecListner.java b/src/main/java/cws/k8s/scheduler/util/MyExecListner.java similarity index 91% rename from src/main/java/fonda/scheduler/util/MyExecListner.java rename to src/main/java/cws/k8s/scheduler/util/MyExecListner.java index bfdf4f52..c6767d6b 100644 --- a/src/main/java/fonda/scheduler/util/MyExecListner.java +++ b/src/main/java/cws/k8s/scheduler/util/MyExecListner.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import io.fabric8.kubernetes.client.dsl.ExecListener; import io.fabric8.kubernetes.client.dsl.ExecWatch; diff --git a/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java b/src/main/java/cws/k8s/scheduler/util/NodeTaskAlignment.java similarity index 59% rename from src/main/java/fonda/scheduler/util/NodeTaskAlignment.java rename to src/main/java/cws/k8s/scheduler/util/NodeTaskAlignment.java index 799660a6..34788649 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskAlignment.java +++ b/src/main/java/cws/k8s/scheduler/util/NodeTaskAlignment.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor diff --git a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/cws/k8s/scheduler/util/NodeTaskFilesAlignment.java similarity index 80% rename from src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java rename to src/main/java/cws/k8s/scheduler/util/NodeTaskFilesAlignment.java index 494af816..a3b0bc30 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskFilesAlignment.java +++ b/src/main/java/cws/k8s/scheduler/util/NodeTaskFilesAlignment.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java b/src/main/java/cws/k8s/scheduler/util/NodeTaskLocalFilesAlignment.java similarity index 66% rename from src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java rename to src/main/java/cws/k8s/scheduler/util/NodeTaskLocalFilesAlignment.java index 56aea6b7..3d773ab4 100644 --- a/src/main/java/fonda/scheduler/util/NodeTaskLocalFilesAlignment.java +++ b/src/main/java/cws/k8s/scheduler/util/NodeTaskLocalFilesAlignment.java @@ -1,9 +1,9 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import java.util.List; diff --git a/src/main/java/fonda/scheduler/util/PodPhase.java b/src/main/java/cws/k8s/scheduler/util/PodPhase.java similarity index 95% rename from src/main/java/fonda/scheduler/util/PodPhase.java rename to src/main/java/cws/k8s/scheduler/util/PodPhase.java index 59f2d1c9..e6a4453b 100644 --- a/src/main/java/fonda/scheduler/util/PodPhase.java +++ b/src/main/java/cws/k8s/scheduler/util/PodPhase.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; public enum PodPhase { diff --git a/src/main/java/fonda/scheduler/util/SortedList.java b/src/main/java/cws/k8s/scheduler/util/SortedList.java similarity index 98% rename from src/main/java/fonda/scheduler/util/SortedList.java rename to src/main/java/cws/k8s/scheduler/util/SortedList.java index 0e96a884..000ade14 100644 --- a/src/main/java/fonda/scheduler/util/SortedList.java +++ b/src/main/java/cws/k8s/scheduler/util/SortedList.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import java.util.Collection; import java.util.LinkedList; diff --git a/src/main/java/fonda/scheduler/util/TaskNodeStats.java b/src/main/java/cws/k8s/scheduler/util/TaskNodeStats.java similarity index 94% rename from src/main/java/fonda/scheduler/util/TaskNodeStats.java rename to src/main/java/cws/k8s/scheduler/util/TaskNodeStats.java index 2c0f22ed..2773cd0b 100644 --- a/src/main/java/fonda/scheduler/util/TaskNodeStats.java +++ b/src/main/java/cws/k8s/scheduler/util/TaskNodeStats.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/util/TaskStats.java b/src/main/java/cws/k8s/scheduler/util/TaskStats.java similarity index 83% rename from src/main/java/fonda/scheduler/util/TaskStats.java rename to src/main/java/cws/k8s/scheduler/util/TaskStats.java index 43412c26..85919749 100644 --- a/src/main/java/fonda/scheduler/util/TaskStats.java +++ b/src/main/java/cws/k8s/scheduler/util/TaskStats.java @@ -1,8 +1,8 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; -import fonda.scheduler.model.Task; -import fonda.scheduler.scheduler.la2.TaskStat; -import fonda.scheduler.scheduler.la2.TaskStatComparator; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.scheduler.la2.TaskStatComparator; +import cws.k8s.scheduler.model.Task; import java.util.*; diff --git a/src/main/java/fonda/scheduler/util/Tuple.java b/src/main/java/cws/k8s/scheduler/util/Tuple.java similarity index 86% rename from src/main/java/fonda/scheduler/util/Tuple.java rename to src/main/java/cws/k8s/scheduler/util/Tuple.java index 1291d02d..3eaa73b9 100644 --- a/src/main/java/fonda/scheduler/util/Tuple.java +++ b/src/main/java/cws/k8s/scheduler/util/Tuple.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/util/copying/CopySource.java b/src/main/java/cws/k8s/scheduler/util/copying/CopySource.java similarity index 60% rename from src/main/java/fonda/scheduler/util/copying/CopySource.java rename to src/main/java/cws/k8s/scheduler/util/copying/CopySource.java index 42510b8c..109a527a 100644 --- a/src/main/java/fonda/scheduler/util/copying/CopySource.java +++ b/src/main/java/cws/k8s/scheduler/util/copying/CopySource.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util.copying; +package cws.k8s.scheduler.util.copying; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopying.java similarity index 93% rename from src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java rename to src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopying.java index 1977f25f..a306e915 100644 --- a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopying.java +++ b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopying.java @@ -1,11 +1,11 @@ -package fonda.scheduler.util.copying; - -import fonda.scheduler.model.NodeWithAlloc; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.util.AlignmentWrapper; -import fonda.scheduler.util.FilePath; +package cws.k8s.scheduler.util.copying; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FilePath; import lombok.ToString; import java.util.HashMap; diff --git a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopyingOnNode.java similarity index 94% rename from src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java rename to src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopyingOnNode.java index bc74425f..d98c669d 100644 --- a/src/main/java/fonda/scheduler/util/copying/CurrentlyCopyingOnNode.java +++ b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopyingOnNode.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util.copying; +package cws.k8s.scheduler.util.copying; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.location.Location; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/fonda/scheduler/util/score/CalculateScore.java b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java similarity index 62% rename from src/main/java/fonda/scheduler/util/score/CalculateScore.java rename to src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java index 33861cf4..a3b41183 100644 --- a/src/main/java/fonda/scheduler/util/score/CalculateScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java @@ -1,6 +1,6 @@ -package fonda.scheduler.util.score; +package cws.k8s.scheduler.util.score; -import fonda.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; public interface CalculateScore { diff --git a/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java similarity index 76% rename from src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java rename to src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java index 422ca78b..5044a2df 100644 --- a/src/main/java/fonda/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util.score; +package cws.k8s.scheduler.util.score; -import fonda.scheduler.model.location.hierachy.HierarchyWrapper; -import fonda.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; public class FileSizeRankScore extends FileSizeScore { diff --git a/src/main/java/fonda/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java similarity index 84% rename from src/main/java/fonda/scheduler/util/score/FileSizeScore.java rename to src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index 36b15880..52fdbf0f 100644 --- a/src/main/java/fonda/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -1,7 +1,7 @@ -package fonda.scheduler.util.score; +package cws.k8s.scheduler.util.score; -import fonda.scheduler.model.location.hierachy.HierarchyWrapper; -import fonda.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import lombok.RequiredArgsConstructor; import java.io.File; diff --git a/src/main/java/fonda/scheduler/model/location/LocationType.java b/src/main/java/fonda/scheduler/model/location/LocationType.java deleted file mode 100644 index ff58776b..00000000 --- a/src/main/java/fonda/scheduler/model/location/LocationType.java +++ /dev/null @@ -1,7 +0,0 @@ -package fonda.scheduler.model.location; - -public enum LocationType { - - NODE - -} diff --git a/src/main/java/fonda/scheduler/model/taskinputs/Input.java b/src/main/java/fonda/scheduler/model/taskinputs/Input.java deleted file mode 100644 index 36779441..00000000 --- a/src/main/java/fonda/scheduler/model/taskinputs/Input.java +++ /dev/null @@ -1,4 +0,0 @@ -package fonda.scheduler.model.taskinputs; - -public interface Input { -} diff --git a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java b/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java deleted file mode 100644 index c5b2c799..00000000 --- a/src/main/java/fonda/scheduler/scheduler/la2/copystrategy/CopyRunner.java +++ /dev/null @@ -1,8 +0,0 @@ -package fonda.scheduler.scheduler.la2.copystrategy; - -import fonda.scheduler.util.CopyTask; -import fonda.scheduler.util.NodeTaskFilesAlignment; - -public interface CopyRunner { - void startCopyTasks( CopyTask copyTask, NodeTaskFilesAlignment nodeTaskFilesAlignment ); -} diff --git a/src/test/java/fonda/scheduler/dag/DAGTest.java b/src/test/java/cws/k8s/scheduler/dag/DAGTest.java similarity index 99% rename from src/test/java/fonda/scheduler/dag/DAGTest.java rename to src/test/java/cws/k8s/scheduler/dag/DAGTest.java index 4a12c5c7..95a29918 100644 --- a/src/test/java/fonda/scheduler/dag/DAGTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/DAGTest.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import lombok.extern.slf4j.Slf4j; import org.junit.Test; diff --git a/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java similarity index 96% rename from src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java rename to src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java index 133c3092..7b2ab3f4 100644 --- a/src/test/java/fonda/scheduler/dag/VertexDeserializerTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java @@ -1,4 +1,4 @@ -package fonda.scheduler.dag; +package cws.k8s.scheduler.dag; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; diff --git a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java similarity index 94% rename from src/test/java/fonda/scheduler/model/InputFileCollectorTest.java rename to src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java index 2c9164b7..fe6a52c3 100644 --- a/src/test/java/fonda/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java @@ -1,15 +1,17 @@ -package fonda.scheduler.model; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.InputEdge; -import fonda.scheduler.dag.Process; -import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.location.NodeLocation; +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.InputEdge; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.*; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; import fonda.scheduler.model.location.hierachy.*; -import fonda.scheduler.model.taskinputs.PathFileLocationTriple; -import fonda.scheduler.model.taskinputs.SymlinkInput; -import fonda.scheduler.model.taskinputs.TaskInputs; import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,7 +38,7 @@ public class InputFileCollectorTest { @Before public void init(){ - location11 = new LocationWrapper(NodeLocation.getLocation("Node1"), 1, 100); + location11 = new LocationWrapper( NodeLocation.getLocation("Node1"), 1, 100); location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 1, 101); location21 = new LocationWrapper(NodeLocation.getLocation("Node1"), 1, 102); location23 = new LocationWrapper(NodeLocation.getLocation("Node3"), 1, 103); @@ -177,7 +179,7 @@ public void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { assertEquals( file1, inputsOfTaskFiles.get(0) ); assertTrue( inputsOfTask.getSymlinks().isEmpty() ); assertEquals( 1, inputsOfTask.getExcludedNodes().size() ); - assertEquals( NodeLocation.getLocation( "Node1" ), inputsOfTask.getExcludedNodes().iterator().next() ); + Assert.assertEquals( NodeLocation.getLocation( "Node1" ), inputsOfTask.getExcludedNodes().iterator().next() ); } diff --git a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java b/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java similarity index 97% rename from src/test/java/fonda/scheduler/model/TaskResultParserTest.java rename to src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java index 5b085e14..a7cc37dc 100644 --- a/src/test/java/fonda/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java @@ -1,13 +1,13 @@ -package fonda.scheduler.model; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.Process; -import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.LocationWrapper; -import fonda.scheduler.model.outfiles.OutputFile; -import fonda.scheduler.model.outfiles.PathLocationWrapperPair; -import fonda.scheduler.model.outfiles.SymlinkOutput; +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.outfiles.OutputFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java similarity index 92% rename from src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java rename to src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java index 7abdd7d2..32abad3b 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -1,10 +1,11 @@ -package fonda.scheduler.model.location.hierachy; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.TaskConfig; -import fonda.scheduler.model.location.NodeLocation; +package cws.k8s.scheduler.model.location.hierachy; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; @@ -35,8 +36,8 @@ private LocationWrapper getLocationWrapper( String location ){ public void init() { dag = new DAG(); List vertexList = new LinkedList<>(); - vertexList.add(new fonda.scheduler.dag.Process("processA", 1)); - vertexList.add(new fonda.scheduler.dag.Process("processB", 2)); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); dag.registerVertices(vertexList); node1 = getLocationWrapper("Node1"); diff --git a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java similarity index 98% rename from src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java rename to src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java index 47cd4fd7..98054da0 100644 --- a/src/test/java/fonda/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -1,13 +1,13 @@ -package fonda.scheduler.model.location.hierachy; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.InputEdge; -import fonda.scheduler.dag.Process; -import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.TaskConfig; -import fonda.scheduler.model.location.LocationType; -import fonda.scheduler.model.location.NodeLocation; +package cws.k8s.scheduler.model.location.hierachy; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.InputEdge; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.model.location.LocationType; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.Task; import org.apache.commons.collections4.iterators.PermutationIterator; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java b/src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java similarity index 80% rename from src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java rename to src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java index de014661..64b76a6d 100644 --- a/src/test/java/fonda/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java @@ -1,12 +1,12 @@ -package fonda.scheduler.scheduler.outlabel; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.Process; -import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.OutLabel; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.TaskConfig; -import fonda.scheduler.model.location.NodeLocation; +package cws.k8s.scheduler.scheduler.outlabel; + +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.OutLabel; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; import org.junit.jupiter.api.Test; import java.util.LinkedList; diff --git a/src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java similarity index 85% rename from src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java rename to src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java index c6a8aca5..7571ce91 100644 --- a/src/test/java/fonda/scheduler/scheduler/prioritize/RankPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java @@ -1,11 +1,11 @@ -package fonda.scheduler.scheduler.prioritize; - -import fonda.scheduler.dag.DAG; -import fonda.scheduler.dag.InputEdge; -import fonda.scheduler.dag.Process; -import fonda.scheduler.dag.Vertex; -import fonda.scheduler.model.Task; -import fonda.scheduler.model.TaskConfig; +package cws.k8s.scheduler.scheduler.prioritize; + +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.InputEdge; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.Task; import org.junit.jupiter.api.Test; import java.util.Arrays; diff --git a/src/test/java/fonda/scheduler/util/SortedListTest.java b/src/test/java/cws/k8s/scheduler/util/SortedListTest.java similarity index 99% rename from src/test/java/fonda/scheduler/util/SortedListTest.java rename to src/test/java/cws/k8s/scheduler/util/SortedListTest.java index 7f44d43b..512d95c8 100644 --- a/src/test/java/fonda/scheduler/util/SortedListTest.java +++ b/src/test/java/cws/k8s/scheduler/util/SortedListTest.java @@ -1,4 +1,4 @@ -package fonda.scheduler.util; +package cws.k8s.scheduler.util; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; From 654bdef45d42f6171f0429a00e017b2a928baff5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:14:52 +0200 Subject: [PATCH 359/443] Remove old imports Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 02d919fb..52777104 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -6,7 +6,6 @@ import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.util.*; -import fonda.scheduler.model.*; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; @@ -15,14 +14,12 @@ import cws.k8s.scheduler.model.taskinputs.TaskInputs; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import fonda.scheduler.scheduler.la2.*; import cws.k8s.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import cws.k8s.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; import cws.k8s.scheduler.scheduler.la2.copystrategy.CopyRunner; import cws.k8s.scheduler.scheduler.la2.copystrategy.ShellCopy; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; -import fonda.scheduler.util.*; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import cws.k8s.scheduler.util.score.FileSizeRankScore; From 837905a8e41ca66563230061182019db3e9aafae Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:15:28 +0200 Subject: [PATCH 360/443] IDs and createtime final Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/Task.java | 2 +- src/main/java/cws/k8s/scheduler/util/Batch.java | 2 +- src/main/java/cws/k8s/scheduler/util/CopyTask.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index 58434342..d0f7f41e 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -17,7 +17,7 @@ @Slf4j public class Task { - private static AtomicInteger idCounter = new AtomicInteger(0); + private static final AtomicInteger idCounter = new AtomicInteger(0); @Getter private final int id = idCounter.getAndIncrement(); diff --git a/src/main/java/cws/k8s/scheduler/util/Batch.java b/src/main/java/cws/k8s/scheduler/util/Batch.java index e0fc2a13..54b04e1b 100644 --- a/src/main/java/cws/k8s/scheduler/util/Batch.java +++ b/src/main/java/cws/k8s/scheduler/util/Batch.java @@ -21,7 +21,7 @@ public class Batch { private Set unready = new HashSet<>(); private int tasksInBatch = -1; - private long createTime = System.currentTimeMillis(); + private final long createTime = System.currentTimeMillis(); private long closeTime; diff --git a/src/main/java/cws/k8s/scheduler/util/CopyTask.java b/src/main/java/cws/k8s/scheduler/util/CopyTask.java index a8e34090..a0730e25 100644 --- a/src/main/java/cws/k8s/scheduler/util/CopyTask.java +++ b/src/main/java/cws/k8s/scheduler/util/CopyTask.java @@ -20,8 +20,8 @@ public class CopyTask { @Getter(AccessLevel.NONE) - private static AtomicLong copyTaskId = new AtomicLong(0); - private long id = copyTaskId.getAndIncrement(); + private static final AtomicLong copyTaskId = new AtomicLong(0); + private final long id = copyTaskId.getAndIncrement(); private final Inputs inputs; private final LinkedList inputFiles; private final CurrentlyCopyingOnNode filesForCurrentNode; From c436db1bbc29ba98630e14e84f99851cd92d07ac Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:18:25 +0200 Subject: [PATCH 361/443] Fix check weightForIndividualNode Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/filealignment/InputAlignmentClass.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java index 27972298..4f512c16 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -33,7 +33,7 @@ abstract class InputAlignmentClass implements InputAlignment { private final double weightForIndividualNode; public InputAlignmentClass( double weightForIndividualNode ) { - if ( weightForIndividualNode < 0 && weightForIndividualNode > 1 ) { + if ( weightForIndividualNode < 0 || weightForIndividualNode > 1 ) { throw new IllegalArgumentException( "Weight for single source must be between 0 and 1" ); } this.weightForIndividualNode = weightForIndividualNode; From a2dff1e14482bfc895f96587704652bb5f97bf18 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:19:52 +0200 Subject: [PATCH 362/443] Fix warnings in InputAlignmentClass Signed-off-by: Lehmann_Fabian --- .../scheduler/filealignment/InputAlignmentClass.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java index 4f512c16..0f141268 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -42,10 +42,6 @@ public InputAlignmentClass( double weightForIndividualNode ) { /** * Check if another scheduled task is already copying a required file * - * @param currentlyCopying - * @param path - * @param locations - * @return */ private Tuple alreadyCopying( CurrentlyCopyingOnNode currentlyCopying, String path, List locations, String debug ){ if ( currentlyCopying != null && currentlyCopying.isCurrentlyCopying( path ) ){ @@ -53,7 +49,7 @@ private Tuple alreadyCopying( CurrentlyCopyingOnNode curre final Location copyFrom = copySource.getLocation(); for ( LocationWrapper locationWrapper : locations ) { if ( locationWrapper.getLocation() == copyFrom ) { - return new Tuple(locationWrapper,copySource.getTask()); + return new Tuple<>(locationWrapper,copySource.getTask()); } } throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); From ca185652e9b21daa8efedd8bfce99478d4934a62 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:21:31 +0200 Subject: [PATCH 363/443] Remove old imports Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/InputFileCollector.java | 1 - .../java/cws/k8s/scheduler/rest/SchedulerRestController.java | 3 --- .../cws/k8s/scheduler/scheduler/LocationAwareScheduler.java | 2 -- .../cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java | 1 - src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java | 1 - .../cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java | 3 --- .../java/cws/k8s/scheduler/model/InputFileCollectorTest.java | 1 - 7 files changed, 12 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java index fc08f47f..425de7a4 100644 --- a/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java +++ b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.hierachy.*; -import fonda.scheduler.model.location.hierachy.*; import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import cws.k8s.scheduler.model.taskinputs.TaskInputs; diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 3a5e3b73..e69216f8 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -11,16 +11,13 @@ import cws.k8s.scheduler.dag.InputEdge; import cws.k8s.scheduler.dag.Vertex; import cws.k8s.scheduler.scheduler.prioritize.*; -import fonda.scheduler.model.*; import cws.k8s.scheduler.rest.exceptions.NotARealFileException; import cws.k8s.scheduler.rest.response.getfile.FileResponse; -import fonda.scheduler.scheduler.*; import cws.k8s.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; import cws.k8s.scheduler.scheduler.nodeassign.FairAssign; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RandomNodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RoundRobinAssign; -import fonda.scheduler.scheduler.prioritize.*; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java index 4cde0871..1f17e643 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java @@ -5,7 +5,6 @@ import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.util.*; -import fonda.scheduler.model.*; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; @@ -13,7 +12,6 @@ import cws.k8s.scheduler.model.tracing.TraceRecord; import cws.k8s.scheduler.scheduler.data.NodeDataTuple; import cws.k8s.scheduler.scheduler.data.TaskData; -import fonda.scheduler.util.*; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.AccessLevel; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java index c4654a82..fb46bbf2 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -3,7 +3,6 @@ import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.model.*; -import fonda.scheduler.model.*; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.prioritize.Prioritize; import cws.k8s.scheduler.util.NodeTaskAlignment; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index ca25ad14..a553d5eb 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -4,7 +4,6 @@ import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.dag.DAG; -import fonda.scheduler.model.*; import cws.k8s.scheduler.util.Batch; import cws.k8s.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Pod; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index eab242ac..327bf20e 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -7,11 +7,9 @@ import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.util.*; -import fonda.scheduler.model.*; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.LocationType; import cws.k8s.scheduler.model.location.NodeLocation; -import fonda.scheduler.model.location.hierachy.*; import cws.k8s.scheduler.model.outfiles.OutputFile; import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; import cws.k8s.scheduler.model.outfiles.SymlinkOutput; @@ -23,7 +21,6 @@ import cws.k8s.scheduler.scheduler.copystrategy.FTPstrategy; import cws.k8s.scheduler.scheduler.outlabel.OutLabelHolder; import cws.k8s.scheduler.scheduler.outlabel.HolderMaxTasks; -import fonda.scheduler.util.*; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import io.fabric8.kubernetes.api.model.*; diff --git a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java index fe6a52c3..7ecb3470 100644 --- a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java @@ -9,7 +9,6 @@ import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import cws.k8s.scheduler.model.taskinputs.TaskInputs; -import fonda.scheduler.model.location.hierachy.*; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Before; From 7814b1450a1332a466de2084db8cc47344eb525d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:23:37 +0200 Subject: [PATCH 364/443] Fix warnings in SchedulerWithDaemonSet Signed-off-by: Lehmann_Fabian --- .../scheduler/SchedulerWithDaemonSet.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 327bf20e..9cb142e7 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -90,7 +90,6 @@ String getDaemonIpOnNode( Node node ){ /** * Mark all locationWrappers as used - * @param locationWrappers */ void useLocations( List locationWrappers ){ locationWrappers.parallelStream().forEach( LocationWrapper::use ); @@ -98,7 +97,6 @@ void useLocations( List locationWrappers ){ /** * Mark all locationWrappers as unused - * @param locationWrappers */ void freeLocations( List locationWrappers ){ locationWrappers.parallelStream().forEach( LocationWrapper::free ); @@ -134,13 +132,13 @@ int terminateTasks(List finishedTasks) { finishedTasks.parallelStream().forEach( finishedTask -> { try{ freeLocations( finishedTask.getInputFiles() ); - if ( !"DeadlineExceeded".equals( finishedTask.getPod().getStatus().getReason() ) ) { //If Deadline exceeded, task cannot write oufiles and containerStatuses.terminated is not available + if ( !"DeadlineExceeded".equals( finishedTask.getPod().getStatus().getReason() ) ) { //If Deadline exceeded, task cannot write out files and containerStatuses.terminated is not available final Integer exitCode = finishedTask.getPod().getStatus().getContainerStatuses().get(0).getState().getTerminated().getExitCode(); log.info( "Pod finished with exitCode: {}", exitCode ); //Init failure final Path workdir = Paths.get(finishedTask.getWorkingDir()); if ( exitCode == 123 && Files.exists( workdir.resolve(".command.init.failure") ) ) { - log.info( "Task " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and outfiles" ); + log.info( "Task " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and out files" ); } else { final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( workdir, @@ -173,8 +171,6 @@ int terminateTasks(List finishedTasks) { /** * Register that file is copied to node - * @param nodeLocation - * @param toAdd */ void addToCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode toAdd ){ if ( nodeLocation == null ) { @@ -185,8 +181,6 @@ void addToCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingO /** * Remove that file is copied to node - * @param nodeLocation - * @param toRemove */ void removeFromCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode toRemove ) { if (nodeLocation == null) { @@ -197,7 +191,6 @@ void removeFromCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCop /** * - * @param alignment * @return null if the task cannot be scheduled */ WriteConfigResult writeInitConfig( NodeTaskFilesAlignment alignment ) { @@ -328,12 +321,6 @@ MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map matchingNodes, TaskInputs taskInputs ){ final Iterator iterator = matchingNodes.iterator(); From d39e8b67509b5b50424409ac8242edf2e7ee7bad Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:27:13 +0200 Subject: [PATCH 365/443] Fix warnings in DAG Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/dag/DAG.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/dag/DAG.java b/src/main/java/cws/k8s/scheduler/dag/DAG.java index 0d199e09..0eec7e88 100644 --- a/src/main/java/cws/k8s/scheduler/dag/DAG.java +++ b/src/main/java/cws/k8s/scheduler/dag/DAG.java @@ -12,8 +12,6 @@ public class DAG { /** * not private for testing - * @param uid - * @return */ Vertex getByUid( int uid ){ final Vertex vertex = vertices.get(uid); @@ -59,7 +57,6 @@ public void registerEdges( List edges ){ /** * This method removes vertices from the DAG plus the edges. - * @param verticesIds */ public void removeVertices( int... verticesIds ){ synchronized ( this.vertices ) { From 5a24a1f775e6fbcb1a78ac10d833d1ce272667a5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:33:01 +0200 Subject: [PATCH 366/443] Fix warnings in Vertex Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/dag/Vertex.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/dag/Vertex.java b/src/main/java/cws/k8s/scheduler/dag/Vertex.java index 35f85255..001d385a 100644 --- a/src/main/java/cws/k8s/scheduler/dag/Vertex.java +++ b/src/main/java/cws/k8s/scheduler/dag/Vertex.java @@ -29,7 +29,7 @@ public abstract class Vertex { public void removeInbound( Edge e ) { final boolean remove = in.remove(e); if ( !remove ) { - throw new IllegalStateException("Edge " + e + " not found in " + this); + throw new IllegalStateException( notFoundWarning(e) ); } removeInboundIntern( e ); } @@ -37,7 +37,7 @@ public void removeInbound( Edge e ) { public void removeInboundIntern( Edge e ) { final boolean remove = e.getFrom().out.remove(e); if ( !remove ) { - throw new IllegalStateException("Edge " + e + " not found in " + this); + throw new IllegalStateException( notFoundWarning(e) ); } final Set ancestors = e.getFrom().getAncestors(); if ( e.getFrom().getType() == Type.PROCESS ) { @@ -63,7 +63,7 @@ public void removeInboundIntern( Edge e ) { public void removeOutbound( Edge e ) { final boolean remove = out.remove(e); if ( !remove ) { - throw new IllegalStateException("Edge " + e + " not found in " + this); + throw new IllegalStateException( notFoundWarning(e) ); } removeOutboundIntern( e ); } @@ -123,8 +123,6 @@ public int hashCode() { /** * For processes add one, otherwise return the input - * @param rank - * @return */ abstract int incRank( int rank ); @@ -138,22 +136,26 @@ void informNewDescendent( int rank ) { } void informDeletedDescendent() { - int rank = 0; + int newRank = 0; if ( !out.isEmpty() ) { for ( Edge edge : out ) { final Vertex to = edge.getTo(); final int toRank = to.incRank( to.getRank() ); - if ( toRank > rank ) { - rank = toRank; + if ( toRank > newRank ) { + newRank = toRank; } } } - if ( rank != getRank() ) { - setRank( rank ); + if ( newRank != rank ) { + rank = newRank; for ( Edge edge : in ) { edge.getFrom().informDeletedDescendent(); } } } + private String notFoundWarning( Edge e ) { + return "Edge " + e + " not found in " + this; + } + } From 451cece5a55631c5d2e2157c8afbf3f2cdea860d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:33:11 +0200 Subject: [PATCH 367/443] Fix warnings in Process Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/dag/Process.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/dag/Process.java b/src/main/java/cws/k8s/scheduler/dag/Process.java index 1e37ff59..aff6d6f1 100644 --- a/src/main/java/cws/k8s/scheduler/dag/Process.java +++ b/src/main/java/cws/k8s/scheduler/dag/Process.java @@ -83,8 +83,6 @@ void removeAncestor( Edge e, Collection pp ) { /** * Only public for tests - * @param label - * @param uid */ public Process(String label, int uid) { super(label, uid); From cdeb8003caca90cd8b3f1e7b9256852de3d607bb Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:33:21 +0200 Subject: [PATCH 368/443] Fix warnings in Origin Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/dag/Origin.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/dag/Origin.java b/src/main/java/cws/k8s/scheduler/dag/Origin.java index e0e0ee44..3536b2d1 100644 --- a/src/main/java/cws/k8s/scheduler/dag/Origin.java +++ b/src/main/java/cws/k8s/scheduler/dag/Origin.java @@ -7,8 +7,6 @@ public class Origin extends NotProcess { /** * Only public for tests - * @param label - * @param uid */ public Origin(String label, int uid) { super(label, uid); From a80eb120ecf1226ebf1b437f1b56691f0593c3e7 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:37:37 +0200 Subject: [PATCH 369/443] Fix spelling Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java index 148a5191..158eb285 100644 --- a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java @@ -111,7 +111,7 @@ public class TraceRecord { @Getter @Setter - /*Time delta between a task was submitted and the batch became scheduable*/ + /*Time delta between a task was submitted and the batch became schedulable*/ private Integer schedulerDeltaSubmittedBatchEnd = null; public void writeRecord( String tracePath ) throws IOException { From d37e42cd0bb2703ead0000821439dff304b12e9a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:37:48 +0200 Subject: [PATCH 370/443] Fix warnings in TaskInputs Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java index 71c91e3e..f3861695 100644 --- a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java @@ -49,7 +49,6 @@ public long calculateDataOnNode( Location loc ) { /** * Calculates the data on a node and returns whether all data is on the location - * @param loc * @return boolean: true if all files are on location, Long: data on location */ public Tuple calculateDataOnNodeAdditionalInfo( Location loc ) { @@ -67,7 +66,6 @@ public Tuple calculateDataOnNodeAdditionalInfo( Location loc ) { /** * Calculates the data on a node and returns whether all data is on the location - * @param loc * @return the size remaining and the amount of data currently copying. Null if the task cannot run on this node. */ public TaskNodeStats calculateMissingData( Location loc, CurrentlyCopyingOnNode currentlyCopying ) { From 8462f7e0ecda83b20dd367401ae26f0253b77ebc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:38:02 +0200 Subject: [PATCH 371/443] Fix warnings in RealHierachyFile Signed-off-by: Lehmann_Fabian --- .../scheduler/model/location/hierachy/RealHierarchyFile.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java index 06d76268..90811337 100644 --- a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -179,9 +179,7 @@ private LinkedList addAndCreateList(LinkedList /** * This method is used to find all possible LocationWrappers of a file for a specific task. - * @param task * @return a list of all LocationWrapper of this file that could be used and a list of all Locations that are in use and are not in a version that this task could use. - * @throws NoAlignmentFoundException */ public MatchingLocationsPair getFilesForTask( Task task ) throws NoAlignmentFoundException { LocationWrapper[] locationsRef = this.locations; From 17ec05c4af8ab0c89d9d406792014eed6cfc29c3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:38:14 +0200 Subject: [PATCH 372/443] Fix warnings in LocationWrapper Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/location/hierachy/LocationWrapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java index c59c8195..15cd6256 100644 --- a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java @@ -76,7 +76,6 @@ public void free(){ /** * Any task currently reading or writing to this file - * @return */ public boolean isInUse(){ return inUse > 0; From 2dab2b65bf49808fb9ae785dc1a7309cbd572e8a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:38:28 +0200 Subject: [PATCH 373/443] Fix warnings in TaskState Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/TaskState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/model/TaskState.java b/src/main/java/cws/k8s/scheduler/model/TaskState.java index 79ef48f1..05f8278a 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskState.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskState.java @@ -7,7 +7,7 @@ public class TaskState { @Getter @Setter - private State state = State.RECEIVED_CONFIG;; + private State state = State.RECEIVED_CONFIG; @Getter private String error; From 307df8f4393faa54c42f424f52a1b019c30b0db4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 4 Apr 2023 11:49:13 +0200 Subject: [PATCH 374/443] Fix spelling Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/DateParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/DateParser.java b/src/main/java/cws/k8s/scheduler/model/DateParser.java index 87c099b7..6ddda2ba 100644 --- a/src/main/java/cws/k8s/scheduler/model/DateParser.java +++ b/src/main/java/cws/k8s/scheduler/model/DateParser.java @@ -31,8 +31,8 @@ public static Long millisFromString( String date ) { } public static FileTime fileTimeFromString(String date ) { - final Long millisFromSring = millisFromString( date ); - return millisFromSring == null ? null : FileTime.fromMillis( millisFromSring ); + final Long millisFromString = millisFromString( date ); + return millisFromString == null ? null : FileTime.fromMillis( millisFromString ); } } From 7b622519fdc6427008283825a49b24c96c433078 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Apr 2023 17:05:05 +0200 Subject: [PATCH 375/443] add imports Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java index 1de9a140..fb46bbf2 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -2,6 +2,7 @@ import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.prioritize.Prioritize; import cws.k8s.scheduler.util.NodeTaskAlignment; From d23972414e1dd6830218768c1619830438ed9b54 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 15 May 2023 14:50:21 +0200 Subject: [PATCH 376/443] New CopyInAdvanced Strategy Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 6 +- .../la2/copyinadvance/CopyInAdvance.java | 31 ++++++++ .../CopyInAdvanceNodeWithMostData.java | 2 +- ...yInAdvanceNodeWithMostDataIntelligent.java | 75 +++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java create mode 100644 src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 52777104..b8e9129a 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -2,6 +2,8 @@ import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.scheduler.la2.*; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostDataIntelligent; import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; import cws.k8s.scheduler.client.KubernetesClient; @@ -48,7 +50,7 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final CapacityAvailableToNode capacityAvailableToNode; - private final CopyInAdvanceNodeWithMostData copyInAdvance; + private final CopyInAdvance copyInAdvance; private final TaskStatComparator phaseTwoComparator; private final TaskStatComparator phaseThreeComparator; @@ -90,7 +92,7 @@ public LocationAwareSchedulerV2( this.capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); this.phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); this.phaseThreeComparator = new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ); - this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + this.copyInAdvance = new CopyInAdvanceNodeWithMostDataIntelligent( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); this.maxHeldCopyTaskReady = config.maxHeldCopyTaskReady == null ? 3 : config.maxHeldCopyTaskReady; this.prioPhaseThree = config.prioPhaseThree == null ? 70 : config.prioPhaseThree; } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java new file mode 100644 index 00000000..270c345a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java @@ -0,0 +1,31 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; + +import java.util.List; +import java.util.Map; + +public abstract class CopyInAdvance extends CreateCopyTasks { + + public CopyInAdvance( CurrentlyCopying currentlyCopying, InputAlignment inputAlignment, int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + public abstract void createAlignmentForTasksWithEnoughCapacity( + final List nodeTaskFilesAlignments, + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final List allNodes, + final int maxCopyingTaskPerNode, + final int maxHeldCopyTaskReady, + final Map currentlyCopyingTasksOnNode, + int prio + ); + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index cc7b6b2b..41039006 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -16,7 +16,7 @@ import java.util.Map; @Slf4j -public class CopyInAdvanceNodeWithMostData extends CreateCopyTasks { +public class CopyInAdvanceNodeWithMostData extends CopyInAdvance { public CopyInAdvanceNodeWithMostData( CurrentlyCopying currentlyCopying, diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java new file mode 100644 index 00000000..e31570bf --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -0,0 +1,75 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import lombok.extern.slf4j.Slf4j; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +@Slf4j +public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { + + public CopyInAdvanceNodeWithMostDataIntelligent( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + + /** + * Do not filter maxHeldCopyTaskReady, that could lead to starving if another node has resources. + */ + public void createAlignmentForTasksWithEnoughCapacity( + final List nodeTaskFilesAlignments, + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final List allNodes, + final int maxCopyingTaskPerNode, + final int maxHeldCopyTaskReady, + final Map currentlyCopyingTasksOnNode, + int prio + ) { + final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); + removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); + + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); + + final boolean cannotAdd; + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { + if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { + cannotAdd = false; + log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); + } else { + cannotAdd = true; + } + } else { + cannotAdd = true; + } + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( cannotAdd && poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } + } + } + +} + From a6d250e907a269ef7d306fb38083a2ba91acf21f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 15 May 2023 14:51:42 +0200 Subject: [PATCH 377/443] Do not copy the same task everywhere Signed-off-by: Lehmann_Fabian --- ...yInAdvanceNodeWithMostDataIntelligent.java | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index e31570bf..cb56a3ef 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -19,6 +19,11 @@ @Slf4j public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { + /** + * Try to distribute the tasks evenly on the nodes. More important tasks can be on x more nodes. (x = TASKS_READY_FACTOR) + */ + private final int TASKS_READY_FACTOR = 1; + public CopyInAdvanceNodeWithMostDataIntelligent( CurrentlyCopying currentlyCopying, InputAlignment inputAlignment, @@ -43,31 +48,43 @@ public void createAlignmentForTasksWithEnoughCapacity( final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + int readyOnNodes = 0; + //Outer loop to only process tasks that are not yet ready on enough nodes while( !stats.isEmpty() ) { - final TaskStat poll = stats.poll(); + LinkedList tasksReadyOnMoreNodes = new LinkedList<>(); + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); - final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); - final Task task = poll.getTask(); - final NodeWithAlloc node = bestStats.getNode(); + if ( poll.dataOnNodes() + TASKS_READY_FACTOR > readyOnNodes ) { + tasksReadyOnMoreNodes.add( poll ); + continue; + } - final boolean cannotAdd; + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); - //Check if the node has still enough resources to run the task - if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { - if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { - cannotAdd = false; - log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); + final boolean cannotAdd; + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { + if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { + cannotAdd = false; + log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); + } else { + cannotAdd = true; + } } else { cannotAdd = true; } - } else { - cannotAdd = true; - } - //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list - if ( cannotAdd && poll.increaseIndexToCompare() ) { - //Only re-add if still other opportunities exist - stats.add( poll ); + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( cannotAdd && poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } } + stats.addAll( tasksReadyOnMoreNodes ); + readyOnNodes++; } } From 416d7f2095984ae0198a829e8cadcc64aabc4297 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 17 Jul 2023 10:57:00 +0200 Subject: [PATCH 378/443] Make Zero, task and node requirements immutable Signed-off-by: Lehmann_Fabian --- .../model/ImmutableRequirements.java | 42 ++++++++++++++++ .../k8s/scheduler/model/NodeWithAlloc.java | 2 +- .../cws/k8s/scheduler/model/PodWithAge.java | 4 +- .../cws/k8s/scheduler/model/Requirements.java | 48 ++++++++++++++++++- .../k8s/scheduler/scheduler/Scheduler.java | 2 +- 5 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java diff --git a/src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java b/src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java new file mode 100644 index 00000000..540187db --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java @@ -0,0 +1,42 @@ +package cws.k8s.scheduler.model; + +import java.math.BigDecimal; + +public class ImmutableRequirements extends Requirements { + + public static final Requirements ZERO = new ImmutableRequirements(); + + private static final String ERROR_MESSAGE = "ImmutableRequirements cannot be modified"; + + public ImmutableRequirements( BigDecimal cpu, BigDecimal ram ) { + super( cpu, ram ); + } + + public ImmutableRequirements(){ + super(); + } + + public ImmutableRequirements( Requirements requirements ){ + super( requirements.getCpu(), requirements.getRam() ); + } + + @Override + public Requirements addCPUtoThis( BigDecimal cpu ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } + + @Override + public Requirements addRAMtoThis( BigDecimal ram ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } + + @Override + public Requirements addToThis( Requirements requirements ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } + + @Override + public Requirements subFromThis( Requirements requirements ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java index e3db61bb..6f8465e9 100644 --- a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java @@ -55,7 +55,7 @@ public NodeWithAlloc( Node node, KubernetesClient kubernetesClient ) { BigDecimal maxCpu = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("cpu")); BigDecimal maxRam = Quantity.getAmountInBytes(this.getStatus().getAllocatable().get("memory")); - maxResources = new Requirements( maxCpu, maxRam); + maxResources = new ImmutableRequirements( maxCpu, maxRam); assignedPods = new HashMap<>(); diff --git a/src/main/java/cws/k8s/scheduler/model/PodWithAge.java b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java index fb989052..2e83ff9f 100644 --- a/src/main/java/cws/k8s/scheduler/model/PodWithAge.java +++ b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java @@ -27,7 +27,7 @@ public PodWithAge(Pod pod) { } public Requirements getRequest(){ - return this + return new ImmutableRequirements( this .getSpec().getContainers().stream() .filter( x -> x.getResources() != null && x.getResources().getRequests() != null ) @@ -36,7 +36,7 @@ public Requirements getRequest(){ x.getResources().getRequests().get("cpu") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("cpu")), x.getResources().getRequests().get("memory") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("memory")) ) - ).reduce( new Requirements(), Requirements::addToThis ); + ).reduce( new Requirements(), Requirements::addToThis ) ); } public String getName(){ diff --git a/src/main/java/cws/k8s/scheduler/model/Requirements.java b/src/main/java/cws/k8s/scheduler/model/Requirements.java index 95de4fda..17296b5c 100644 --- a/src/main/java/cws/k8s/scheduler/model/Requirements.java +++ b/src/main/java/cws/k8s/scheduler/model/Requirements.java @@ -11,8 +11,6 @@ public class Requirements implements Serializable, Cloneable { private static final long serialVersionUID = 1L; - public static final Requirements ZERO = new Requirements(); - @Getter private BigDecimal cpu; @Getter @@ -33,6 +31,13 @@ public Requirements addToThis( Requirements requirements ){ return this; } + public Requirements add( Requirements requirements ){ + return new Requirements( + this.cpu.add(requirements.cpu), + this.ram.add(requirements.ram) + ); + } + public Requirements addRAMtoThis( BigDecimal ram ){ this.ram = this.ram.add( ram ); return this; @@ -56,13 +61,52 @@ public Requirements sub( Requirements requirements ){ ); } + public Requirements multiply( BigDecimal factor ){ + return new Requirements( + this.cpu.multiply(factor), + this.ram.multiply(factor) + ); + } + + public Requirements multiplyToThis( BigDecimal factor ){ + this.cpu = this.cpu.multiply(factor); + this.ram = this.ram.multiply(factor); + return this; + } + public boolean higherOrEquals( Requirements requirements ){ return this.cpu.compareTo( requirements.cpu ) >= 0 && this.ram.compareTo( requirements.ram ) >= 0; } + /** + * Always returns a mutable copy of this object + * @return + */ @Override public Requirements clone() { return new Requirements( this.cpu, this.ram ); } + + public boolean smaller( Requirements request ) { + return this.cpu.compareTo( request.cpu ) < 0 + && this.ram.compareTo( request.ram ) < 0; + } + + public boolean smallerEquals( Requirements request ) { + return this.cpu.compareTo( request.cpu ) <= 0 + && this.ram.compareTo( request.ram ) <= 0; + } + + @Override + public boolean equals( Object obj ) { + if ( obj == null ) { + return false; + } + if ( !(obj instanceof Requirements) ) { + return false; + } + return this.getRam().equals( ((Requirements)obj).getRam() ) + && this.getCpu().equals( ((Requirements)obj).getCpu() ); + } } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index 6f28ebf8..c49f901b 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -152,7 +152,7 @@ public boolean validSchedulePlan( List taskNodeAlignment ){ requirements.subFromThis(nodeTaskAlignment.task.getPod().getRequest()); } for ( Map.Entry e : availableByNode.entrySet() ) { - if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) { + if ( ! e.getValue().higherOrEquals( ImmutableRequirements.ZERO ) ) { log.info( "Node {} has not enough resources. Available: {}, After assignment it would be: {}", e.getKey().getMetadata().getName(), e.getKey().getAvailableResources(), e.getValue() ); return false; } From f23a0b9a5edfc984b5339425c002aec700a119be Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 17 Jul 2023 10:59:48 +0200 Subject: [PATCH 379/443] Added a testclass for Requirements Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/RequirementsTest.java | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 src/test/java/cws/k8s/scheduler/model/RequirementsTest.java diff --git a/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java b/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java new file mode 100644 index 00000000..b119324b --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java @@ -0,0 +1,167 @@ +package cws.k8s.scheduler.model; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +class RequirementsTest { + + @Test + void ZERO() { + final Requirements zero = ImmutableRequirements.ZERO; + assertEquals( zero, new Requirements( BigDecimal.ZERO, BigDecimal.ZERO ) ); + assertEquals( BigDecimal.ZERO, zero.getCpu() ); + assertEquals( BigDecimal.ZERO, zero.getRam() ); + } + + @Test + void addToThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 7 ), BigDecimal.valueOf( 8 ) ); + Requirements c = a.addToThis( b ); + assertSame( a, c ); + assertEquals( BigDecimal.valueOf( 12 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 14 ), c.getRam() ); + } + + @Test + void add() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 7 ), BigDecimal.valueOf( 8 ) ); + Requirements c = a.add( b ); + assertNotSame( a, c ); + assertEquals( BigDecimal.valueOf( 12 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 14 ), c.getRam() ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + @Test + void addRAMtoThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + a.addRAMtoThis( BigDecimal.valueOf( 8 ) ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 14 ), a.getRam() ); + } + + @Test + void addCPUtoThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + a.addCPUtoThis( BigDecimal.valueOf( 8 ) ); + assertEquals( BigDecimal.valueOf( 13 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + @Test + void subFromThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + Requirements c = a.subFromThis( b ); + assertSame( a, c ); + assertEquals( BigDecimal.valueOf( 2 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 1 ), c.getRam() ); + } + + @Test + void sub() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + Requirements c = a.sub( b ); + assertNotSame( a, c ); + assertEquals( BigDecimal.valueOf( 2 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 1 ), c.getRam() ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + @Test + void multiply() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = a.multiply( BigDecimal.valueOf( 3 ) ); + assertNotSame( a, b ); + assertEquals( BigDecimal.valueOf( 15 ) , b.getCpu() ); + assertEquals( BigDecimal.valueOf( 18 ) , b.getRam() ); + assertEquals( BigDecimal.valueOf( 5 ) , a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ) , a.getRam() ); + } + + @Test + void multiplyToThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = a.multiplyToThis( BigDecimal.valueOf( 3 ) ); + assertSame( a, b ); + assertEquals( BigDecimal.valueOf( 15 ), b.getCpu() ); + assertEquals( BigDecimal.valueOf( 18 ), b.getRam() ); + } + + @Test + void testHigherOrEquals() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertTrue( a.higherOrEquals( b ) ); + assertFalse( b.higherOrEquals( a ) ); + assertTrue( a.higherOrEquals( a ) ); + assertTrue( b.higherOrEquals( b ) ); + } + + @Test + void testClone() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = a.clone(); + assertNotSame( a, b ); + assertEquals( a.getCpu(), b.getCpu() ); + assertEquals( a.getRam(), b.getRam() ); + } + + + @Test + void smaller() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smaller( b ) ); + assertTrue( b.smaller( a ) ); + assertFalse( a.smaller( a ) ); + assertFalse( b.smaller( b ) ); + + Requirements c = new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smaller( c ) ); + assertFalse( c.smaller( a ) ); + assertFalse( c.smaller( c ) ); + } + + @Test + void smallerEquals() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smallerEquals( b ) ); + assertTrue( b.smallerEquals( a ) ); + assertTrue( a.smallerEquals( a ) ); + assertTrue( b.smallerEquals( b ) ); + + Requirements c = new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smallerEquals( c ) ); + assertFalse( c.smallerEquals( a ) ); + assertTrue( c.smallerEquals( c ) ); + + Requirements d = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertTrue( a.smallerEquals( d ) ); + assertTrue( d.smallerEquals( a ) ); + assertTrue( d.smallerEquals( d ) ); + } + + @Test + void getCpu() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + } + + @Test + void getRam() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } +} \ No newline at end of file From df4d3f9288e6f9ac69f30a3162be4a84f627ea2f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 19 Jul 2023 15:57:25 +0200 Subject: [PATCH 380/443] Spelling Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/rest/SchedulerRestController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index e69216f8..f182cc95 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -121,7 +121,7 @@ ResponseEntity registerScheduler( switch ( strategy.toLowerCase() ){ case "lav1" : if ( !config.locationAware ) { - return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); + return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); } if ( costFunction == null ) { costFunction = new MinSizeCost( 0 ); @@ -130,7 +130,7 @@ ResponseEntity registerScheduler( break; case "lav2" : if ( !config.locationAware ) { - return new ResponseEntity<>( "LA scheduler only work if location aware", HttpStatus.BAD_REQUEST ); + return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); } if ( costFunction == null ) { costFunction = new MinSizeCost( 0 ); From bf7c05cd38080ce53fe43f95e1c6acdfaeead9d8 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 19 Jul 2023 16:05:01 +0200 Subject: [PATCH 381/443] Improved sorted list using binarySearch Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/util/SortedList.java | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/util/SortedList.java b/src/main/java/cws/k8s/scheduler/util/SortedList.java index 000ade14..6cd162a5 100644 --- a/src/main/java/cws/k8s/scheduler/util/SortedList.java +++ b/src/main/java/cws/k8s/scheduler/util/SortedList.java @@ -1,8 +1,8 @@ package cws.k8s.scheduler.util; import java.util.Collection; +import java.util.Collections; import java.util.LinkedList; -import java.util.ListIterator; public class SortedList> extends LinkedList { @@ -13,21 +13,11 @@ public SortedList( Collection collection ) { @Override public boolean add( T elem ) { - final ListIterator iterator = this.listIterator(); - while ( iterator.hasNext() ) { - final Comparable next = iterator.next(); - if ( next.compareTo( elem ) > 0 ) { - if ( iterator.hasPrevious() ) { - iterator.previous(); - iterator.add( elem ); - } else { - //first element of list - super.addFirst( elem ); - } - return true; - } + int insertionIndex = Collections.binarySearch(this, elem ); + if (insertionIndex < 0) { + insertionIndex = -(insertionIndex + 1); } - iterator.add( elem ); + super.add( insertionIndex, elem ); return true; } From 36f2534b1584e61c5b982ae697fd00b8a98be9c6 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 15:53:58 +0200 Subject: [PATCH 382/443] Added java docs Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java | 4 ++++ .../k8s/scheduler/scheduler/la2/MinCopyingComparator.java | 6 ++++++ .../scheduler/la2/RankAndMinCopyingComparator.java | 4 ++++ .../java/cws/k8s/scheduler/util/score/FileSizeScore.java | 4 ++++ 4 files changed, 18 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java index d77b9d91..fbc6c891 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java @@ -6,6 +6,10 @@ import java.util.Comparator; +/** + * This comparator prioritizes tasks with a larger size. Exception is when two tasks have the same size, then the + * option where fewer data is copied is preferred. + */ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class MaxSizeComparator implements Comparator { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java index 0cc46dfb..5c7ac5a0 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java @@ -2,6 +2,12 @@ import java.util.Comparator; +/** + * This comparator first prioritizes tasks that are on fewer nodes yet. + * If two tasks are on the same number of nodes, the task that is currently copying to fewer nodes is preferred. + * If two tasks are on the same number of nodes and are copying to the same number of nodes, the comparator + * that is passed to the constructor is used. + */ public class MinCopyingComparator extends TaskStatComparator { public MinCopyingComparator( Comparator comparator ) { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java index 6098d440..16f4303e 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java @@ -2,6 +2,10 @@ import java.util.Comparator; +/** + * This comparator first prioritizes tasks with a higher rank. + * If two tasks have the same rank, the comparator that is passed to the constructor is used. + */ public class RankAndMinCopyingComparator extends MinCopyingComparator { public RankAndMinCopyingComparator( Comparator comparator ) { diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index 52fdbf0f..c268c6eb 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -7,6 +7,10 @@ import java.io.File; import java.nio.file.Path; +/** + * This class calculates the sum if the file sizes to process. + * It sums up the data in the shared filesystem and the data on the local node. + */ @RequiredArgsConstructor public class FileSizeScore implements CalculateScore { From b1b202a995f2e07ea3e2306bef3432eadc83f6ba Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 15:57:19 +0200 Subject: [PATCH 383/443] Changed CopyInAdvance Interface Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/copyinadvance/CopyInAdvance.java | 5 +++-- .../la2/copyinadvance/CopyInAdvanceNodeWithMostData.java | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java index 270c345a..cd8b75e6 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java @@ -1,6 +1,7 @@ package cws.k8s.scheduler.scheduler.la2.copyinadvance; import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; @@ -25,7 +26,7 @@ public abstract void createAlignmentForTasksWithEnoughCapacity( final int maxCopyingTaskPerNode, final int maxHeldCopyTaskReady, final Map currentlyCopyingTasksOnNode, - int prio - ); + int prio, + Map> readyTasksPerNode ); } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index 41039006..46e22f1d 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; import cws.k8s.scheduler.scheduler.la2.TaskStat; import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import cws.k8s.scheduler.util.SortedList; @@ -37,13 +36,14 @@ public void createAlignmentForTasksWithEnoughCapacity( final int maxCopyingTaskPerNode, final int maxHeldCopyTaskReady, final Map currentlyCopyingTasksOnNode, - int prio - ) { + int prio, + Map> readyTasksPerNode ) { final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); while( !stats.isEmpty() ) { final TaskStat poll = stats.poll(); + long start = System.currentTimeMillis(); final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); final Task task = poll.getTask(); @@ -67,6 +67,7 @@ public void createAlignmentForTasksWithEnoughCapacity( //Only re-add if still other opportunities exist stats.add( poll ); } + task.getTraceRecord().addSchedulerTimeDeltaPhaseThree( (int) (System.currentTimeMillis() - start) ); } } From fb1e739f1a25a1cd9ea12327aa2b7bd3abb242ad Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 15:57:50 +0200 Subject: [PATCH 384/443] Added schedulerTimeDeltaPhaseThree to the TraceRecord Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/tracing/TraceRecord.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java index 158eb285..7c9f31c5 100644 --- a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java @@ -6,6 +6,7 @@ import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -114,6 +115,16 @@ public class TraceRecord { /*Time delta between a task was submitted and the batch became schedulable*/ private Integer schedulerDeltaSubmittedBatchEnd = null; + @Getter + private List schedulerTimeDeltaPhaseThree = null; + + public void addSchedulerTimeDeltaPhaseThree( Integer schedulerTimeDeltaPhaseThree ) { + if ( this.schedulerTimeDeltaPhaseThree == null ) { + this.schedulerTimeDeltaPhaseThree = new ArrayList<>(); + } + this.schedulerTimeDeltaPhaseThree.add( schedulerTimeDeltaPhaseThree ); + } + public void writeRecord( String tracePath ) throws IOException { try ( BufferedWriter bw = new BufferedWriter( new FileWriter( tracePath ) ) ) { @@ -143,7 +154,7 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_delta_batch_start_received", schedulerDeltaBatchStartReceived, bw); writeValue("scheduler_delta_batch_closed_batch_end", schedulerDeltaBatchClosedBatchEnd, bw); writeValue("scheduler_delta_submitted_batch_end", schedulerDeltaSubmittedBatchEnd, bw); - + writeValue("scheduler_time_delta_phase_three", schedulerTimeDeltaPhaseThree, bw); } } @@ -154,7 +165,7 @@ private void writeValue( String name, T value, BufferedWriter } } - private void writeValue( String name, List value, BufferedWriter bw ) throws IOException { + private void writeValue( String name, List value, BufferedWriter bw ) throws IOException { if ( value != null ) { final String collect = value.stream() .map( x -> x==null ? "null" : x.toString() ) From 5142b6940a3c7b8f50de23810bb78ff3bebe3229 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 16:27:52 +0200 Subject: [PATCH 385/443] Added readyTasksPerNode in LAV2 Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index b8e9129a..d4c972d5 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -30,6 +30,7 @@ import java.io.*; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @Slf4j @@ -143,6 +144,7 @@ void postScheduling( List unscheduledTasks, final Map allNodes = client.getAllNodes(); allNodes.removeIf( x -> !x.isReady() ); final List nodeTaskFilesAlignments; + Map< NodeWithAlloc, List > readyTasksPerNode = new ConcurrentHashMap<>(); synchronized ( copyLock ) { final TaskStats taskStats = new TaskStats(); //Calculate the stats of available data for each task and node. @@ -151,7 +153,7 @@ void postScheduling( List unscheduledTasks, final Map { final TaskInputs inputsOfTask = extractInputsOfData( task ); if ( inputsOfTask == null ) return null; - return getDataOnNode( task, inputsOfTask, allNodes ); + return getDataOnNode( task, inputsOfTask, allNodes, readyTasksPerNode ); } ) .filter( TaskStat::missingDataOnAnyNode ) .sequential() @@ -183,7 +185,8 @@ void postScheduling( List unscheduledTasks, final Map allNodes ) { + private TaskStat getDataOnNode( Task task, TaskInputs inputsOfTask, List allNodes, Map> readyTasksPerNode ) { TaskStat taskStats = new TaskStat( task, inputsOfTask ); final CurrentlyCopying currentlyCopying = getCurrentlyCopying(); for ( NodeWithAlloc node : allNodes ) { @@ -355,6 +360,13 @@ private TaskStat getDataOnNode( Task task, TaskInputs inputsOfTask, List { + final List tasks = value == null ? new LinkedList<>() : value; + tasks.add(task); + return tasks; + }); + } } } } From a7cf6f36eb88f38f877a05f45ff13c012112b515 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 16:33:32 +0200 Subject: [PATCH 386/443] First version of a more intelligent strategy for phase 3 Signed-off-by: Lehmann_Fabian --- ...yInAdvanceNodeWithMostDataIntelligent.java | 119 ++++++++++++++---- 1 file changed, 98 insertions(+), 21 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index cb56a3ef..e4dcd91d 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -1,10 +1,10 @@ package cws.k8s.scheduler.scheduler.la2.copyinadvance; import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; import cws.k8s.scheduler.scheduler.la2.TaskStat; import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import cws.k8s.scheduler.util.SortedList; @@ -12,9 +12,8 @@ import cws.k8s.scheduler.util.copying.CurrentlyCopying; import lombok.extern.slf4j.Slf4j; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; @Slf4j public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { @@ -22,7 +21,8 @@ public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { /** * Try to distribute the tasks evenly on the nodes. More important tasks can be on x more nodes. (x = TASKS_READY_FACTOR) */ - private final int TASKS_READY_FACTOR = 1; + private static final int TASKS_READY_FACTOR = 2; + private static final RequirementsComparator requirementsComparator = new RequirementsComparator(); public CopyInAdvanceNodeWithMostDataIntelligent( CurrentlyCopying currentlyCopying, @@ -43,8 +43,11 @@ public void createAlignmentForTasksWithEnoughCapacity( final int maxCopyingTaskPerNode, final int maxHeldCopyTaskReady, final Map currentlyCopyingTasksOnNode, - int prio + int prio, + Map> readyTasksPerNode ) { + Map cache = new HashMap<>(); + long startMethod = System.currentTimeMillis(); final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); @@ -54,8 +57,9 @@ public void createAlignmentForTasksWithEnoughCapacity( LinkedList tasksReadyOnMoreNodes = new LinkedList<>(); while( !stats.isEmpty() ) { final TaskStat poll = stats.poll(); + long start = System.currentTimeMillis(); - if ( poll.dataOnNodes() + TASKS_READY_FACTOR > readyOnNodes ) { + if ( poll.dataOnNodes() > readyOnNodes + TASKS_READY_FACTOR ) { tasksReadyOnMoreNodes.add( poll ); continue; } @@ -63,30 +67,103 @@ public void createAlignmentForTasksWithEnoughCapacity( final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); final Task task = poll.getTask(); final NodeWithAlloc node = bestStats.getNode(); - - final boolean cannotAdd; + final NodeCache nodeCache = cache.computeIfAbsent( node, + n -> new NodeCache( + //Only consider workflow tasks, other will not finish soon + n.getAssignedPods().entrySet().stream().filter( x -> x.getKey().contains( "||nf-" ) ).map( x -> x.getValue() ).collect( Collectors.toList() ), + readyTasksPerNode.get( n ), + planedToCopy.getTasksOnNode( n.getNodeLocation() ) + ) + ); //Check if the node has still enough resources to run the task - if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { - if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { - cannotAdd = false; + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode + && reasonableToCopyData( node, task, nodeCache ) + && createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) + { log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); - } else { - cannotAdd = true; - } + nodeCache.addPlaned( task.getRequest() ); } else { - cannotAdd = true; - } - //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list - if ( cannotAdd && poll.increaseIndexToCompare() ) { - //Only re-add if still other opportunities exist - stats.add( poll ); + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } } + task.getTraceRecord().addSchedulerTimeDeltaPhaseThree( (int) (System.currentTimeMillis() - start) ); } stats.addAll( tasksReadyOnMoreNodes ); readyOnNodes++; } + log.info( "Time to create alignment for tasks with enough capacity: {}", System.currentTimeMillis() - startMethod ); } + private boolean reasonableToCopyData( NodeWithAlloc node, Task task, NodeCache cache ) { + return true; + } + + private static class NodeCache { + + /** + * A sorted list of tasks that are currently running on the node. + */ + private final List running; + /** + * A sorted list of tasks that are currently waiting on the node. + */ + private final ArrayList waiting; + + NodeCache( final Collection running, final List tasksAlreadyReady, final List tasksPlanedToCopy ) { + + this.running = new LinkedList<>(running); + this.running.sort( requirementsComparator ); + + ArrayList waitingTemp = null; + + // A list of tasks that are ready to run, or data is copied so that they can start soon. + if ( tasksAlreadyReady != null ) { + waitingTemp = tasksAlreadyReady.stream().map( Task::getRequest ).collect( Collectors.toCollection(ArrayList::new) ); + } + if ( tasksPlanedToCopy != null && !tasksPlanedToCopy.isEmpty() ) { + final ArrayList planned = tasksPlanedToCopy.stream().map( Task::getRequest ).collect( Collectors.toCollection(ArrayList::new) ); + if ( waitingTemp == null ) { + waitingTemp = planned; + } else { + waitingTemp.addAll( planned ); + } + } else if ( waitingTemp == null ) { + waitingTemp = new ArrayList<>(); + } + + waiting = waitingTemp; + waiting.sort( requirementsComparator ); + + } + + void addPlaned( Requirements requirement ) { + int insertionIndex = Collections.binarySearch( waiting, requirement, requirementsComparator ); + if (insertionIndex < 0) { + insertionIndex = -(insertionIndex + 1); + } + waiting.add(insertionIndex, requirement); + } + + } + + private static class RequirementsComparator implements Comparator { + + @Override + public int compare(Requirements r1, Requirements r2) { + int cpuComparison = r1.getCpu().compareTo(r2.getCpu()); + if (cpuComparison != 0) { + return cpuComparison; + } else { + return r1.getRam().compareTo(r2.getRam()); + } + } + + } + + } From 0fc108946a8179e1fc1d58b75b7e54c36063ac47 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 16:56:17 +0200 Subject: [PATCH 387/443] Reduce calculations in loop Signed-off-by: Lehmann_Fabian --- .../la2/ready2run/OptimalReadyToRunToNode.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index fcbb00be..ff3ef8ae 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -64,19 +64,20 @@ public List createAlignmentForTasksWithAllDataOnNod int index = 0; for ( TaskInputsNodes taskInputsNodes : taskWithAllData ) { List onlyOnOneNode = new ArrayList<>(); + final long score = calculateScore.getScore( taskInputsNodes ); + final Requirements request = taskInputsNodes.getTask().getRequest(); + final long ram = request.getRam().longValue(); + final long cpu = request.getCpu().multiply( MILLION ).longValue(); for ( NodeWithAlloc node : taskInputsNodes.getNodesWithAllData() ) { final Requirements availableOnNode = availableByNode.get( node ); //Can schedule task on node? - if ( availableOnNode != null ) { - final Requirements request = taskInputsNodes.getTask().getRequest(); - if ( availableOnNode.higherOrEquals( request ) ) { + if ( availableOnNode != null && availableOnNode.higherOrEquals( request ) ) { final BoolVar boolVar = model.newBoolVar( "x_" + index + "_" + node ); onlyOnOneNode.add( boolVar ); - memUsed.get(node).addTerm( boolVar, request.getRam().longValue() ); - cpuUsed.get(node).addTerm( boolVar, request.getCpu().multiply( MILLION ).longValue() ); - objective.addTerm( boolVar, calculateScore.getScore( taskInputsNodes ) ); + memUsed.get(node).addTerm( boolVar, ram ); + cpuUsed.get(node).addTerm( boolVar, cpu ); + objective.addTerm( boolVar, score ); taskNodeBoolVars.add( new TaskNodeBoolVar( taskInputsNodes, node, boolVar ) ); - } } } if ( !onlyOnOneNode.isEmpty() ) { From e22a8f7f0c95e5e40d18565ca8a1c46b24eccb90 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 20 Jul 2023 17:31:19 +0200 Subject: [PATCH 388/443] Refactored calculateScore to use it also in other places Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 2 +- .../scheduler/data/TaskInputsNodes.java | 18 +++++++++++ .../ready2run/OptimalReadyToRunToNode.java | 6 ++-- .../la2/ready2run/RandomReadyToRunToNode.java | 6 +++- .../la2/ready2run/ReadyToRunToNode.java | 4 ++- .../scheduler/util/score/CalculateScore.java | 4 +-- .../util/score/FileSizeRankScore.java | 11 +++---- .../scheduler/util/score/FileSizeScore.java | 30 ++----------------- 8 files changed, 40 insertions(+), 41 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index d4c972d5..4c15ba80 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -131,7 +131,7 @@ && canSchedulePodOnNode( task.getPod(), node ) .stream() .filter( td -> !td.getNodesWithAllData().isEmpty() ) .collect( Collectors.toList() ); - final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode( taskWithAllData, availableByNode ); + final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode( taskWithAllData, availableByNode, hierarchyWrapper ); final ScheduleObject scheduleObject = new ScheduleObject( (List) alignment ); scheduleObject.setCheckStillPossible( true ); scheduleObject.setStopSubmitIfOneFails( true ); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java index 0cc9245d..c891bf9b 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java @@ -2,10 +2,13 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; import lombok.RequiredArgsConstructor; +import java.io.File; +import java.nio.file.Path; import java.util.List; /** @@ -17,4 +20,19 @@ public class TaskInputsNodes { private final Task task; private final List nodesWithAllData; private final TaskInputs inputsOfTask; + + public long getTaskSize( final HierarchyWrapper hierarchyWrapper ) { + final long filesInSharedFS = task.getConfig() + .getInputs() + .fileInputs + .parallelStream() + .filter( x -> { + final Path path = Path.of( x.value.sourceObj ); + return !hierarchyWrapper.isInScope( path ); + } ) + .mapToLong( input -> new File( input.value.sourceObj ).length() ) + .sum(); + + return filesInSharedFS + inputsOfTask.calculateAvgSize() + 1; + } } \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index ff3ef8ae..fdd21397 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -2,6 +2,7 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; @@ -39,7 +40,8 @@ public void init( CalculateScore calculateScore ) { @Override public List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, - Map availableByNode + Map availableByNode, + HierarchyWrapper hierarchyWrapper ) { if ( taskWithAllData.isEmpty() || availableByNode.isEmpty() ){ @@ -64,7 +66,7 @@ public List createAlignmentForTasksWithAllDataOnNod int index = 0; for ( TaskInputsNodes taskInputsNodes : taskWithAllData ) { List onlyOnOneNode = new ArrayList<>(); - final long score = calculateScore.getScore( taskInputsNodes ); + final long score = calculateScore.getScore( taskInputsNodes.getTask(), taskInputsNodes.getTaskSize( hierarchyWrapper ) ); final Requirements request = taskInputsNodes.getTask().getRequest(); final long ram = request.getRam().longValue(); final long cpu = request.getCpu().multiply( MILLION ).longValue(); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java index 7d698c0b..44ce34d1 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java @@ -2,6 +2,7 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; @@ -27,14 +28,17 @@ public void init( CalculateScore calculateScore ) {} /** * Select the first node that fits the requirements. + * * @param taskWithAllData * @param availableByNode + * @param hierarchyWrapper * @return */ @Override public List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, - Map availableByNode + Map availableByNode, + HierarchyWrapper hierarchyWrapper ) { long start = System.currentTimeMillis(); final List alignment = new LinkedList<>(); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java index 0ed60a39..a8d34642 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java @@ -2,6 +2,7 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; @@ -18,7 +19,8 @@ public interface ReadyToRunToNode { List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, - Map availableByNode + Map availableByNode, + HierarchyWrapper hierarchyWrapper ); } diff --git a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java index a3b41183..c9d3ccce 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java @@ -1,6 +1,6 @@ package cws.k8s.scheduler.util.score; -import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.model.Task; public interface CalculateScore { @@ -8,6 +8,6 @@ public interface CalculateScore { * Score must be higher than 0 * @return */ - long getScore( TaskInputsNodes taskInputsNodes ); + long getScore( Task task, long inputSize ); } diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java index 5044a2df..51742275 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -1,20 +1,17 @@ package cws.k8s.scheduler.util.score; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; public class FileSizeRankScore extends FileSizeScore { - public FileSizeRankScore( HierarchyWrapper hierarchyWrapper ) { - super( hierarchyWrapper ); - } - @Override - public long getScore( TaskInputsNodes taskInputsNodes ) { + public long getScore( Task task, long size ) { //Add one to avoid becoming zero - final int rank = taskInputsNodes.getTask().getProcess().getRank() + 1; + final int rank = task.getProcess().getRank() + 1; final long rankFactor = 100_000_000_000_000L * rank; // long would allow a rank of 92233 - return super.getScore( taskInputsNodes ) + rankFactor ; + return super.getScore( task, size ) + rankFactor ; } } diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index c268c6eb..8192d1ee 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -1,35 +1,11 @@ package cws.k8s.scheduler.util.score; -import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; -import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; -import lombok.RequiredArgsConstructor; +import cws.k8s.scheduler.model.Task; -import java.io.File; -import java.nio.file.Path; - -/** - * This class calculates the sum if the file sizes to process. - * It sums up the data in the shared filesystem and the data on the local node. - */ -@RequiredArgsConstructor public class FileSizeScore implements CalculateScore { - private final HierarchyWrapper hierarchyWrapper; - @Override - public long getScore( TaskInputsNodes taskInputsNodes ) { - - final long filesInSharedFS = taskInputsNodes.getTask().getConfig() - .getInputs() - .fileInputs - .parallelStream() - .filter( x -> { - final Path path = Path.of( x.value.sourceObj ); - return !this.hierarchyWrapper.isInScope( path ); - } ) - .mapToLong( input -> new File( input.value.sourceObj ).length() ) - .sum(); - - return filesInSharedFS + taskInputsNodes.getInputsOfTask().calculateAvgSize() + 1; + public long getScore( Task task, long size ) { + return size; } } From 074b7e0e315c9fa8aef59efdeeefc107dd5f8e4f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 21 Jul 2023 10:10:43 +0200 Subject: [PATCH 389/443] Force NoArgsConstructor Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/FileHolder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/model/FileHolder.java b/src/main/java/cws/k8s/scheduler/model/FileHolder.java index e7d45a5a..6b7ff6f9 100644 --- a/src/main/java/cws/k8s/scheduler/model/FileHolder.java +++ b/src/main/java/cws/k8s/scheduler/model/FileHolder.java @@ -6,7 +6,7 @@ import lombok.ToString; @ToString( exclude = {"stageName", "storePath"}) -@NoArgsConstructor(access = AccessLevel.NONE) +@NoArgsConstructor(access = AccessLevel.NONE, force = true) @RequiredArgsConstructor public class FileHolder { From 4a494cd30fbc479a7166e9b9a38c65ed61d6b833 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 21 Jul 2023 10:13:13 +0200 Subject: [PATCH 390/443] Task.getInputSize() also for LA scheduling Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/Task.java | 26 +++++++++++++++++-- .../scheduler/LocationAwareSchedulerV2.java | 10 +++++-- .../k8s/scheduler/scheduler/Scheduler.java | 7 +++-- .../scheduler/data/TaskInputsNodes.java | 18 +++---------- .../ready2run/OptimalReadyToRunToNode.java | 5 ++-- .../la2/ready2run/RandomReadyToRunToNode.java | 4 +-- .../la2/ready2run/ReadyToRunToNode.java | 3 +-- 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index d0f7f41e..62c3a037 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -2,6 +2,7 @@ import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import cws.k8s.scheduler.model.tracing.TraceRecord; import cws.k8s.scheduler.util.Batch; @@ -11,8 +12,10 @@ import lombok.extern.slf4j.Slf4j; import java.io.File; +import java.nio.file.Path; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; @Slf4j public class Task { @@ -64,9 +67,16 @@ public class Task { private final AtomicInteger copyTaskId = new AtomicInteger(0); + private final HierarchyWrapper hierarchyWrapper; + public Task( TaskConfig config, DAG dag ) { + this( config, dag, null ); + } + + public Task( TaskConfig config, DAG dag, HierarchyWrapper hierarchyWrapper ) { this.config = config; this.process = dag.getByProcess( config.getTask() ); + this.hierarchyWrapper = hierarchyWrapper; } public int getCurrentCopyTaskId() { @@ -108,14 +118,26 @@ public String getOutLabel(){ private long inputSize = -1; + /** + * Calculates the size of all input files in bytes in the shared filesystem. + * @return The sum of all input files in bytes. + */ public long getInputSize(){ synchronized ( this ) { if ( inputSize == -1 ) { //calculate - inputSize = getConfig() + Stream> inputParamStream = getConfig() .getInputs() .fileInputs - .parallelStream() + .parallelStream(); + //If LA Scheduling, filter out files that are not in sharedFS + if ( hierarchyWrapper != null ) { + inputParamStream = inputParamStream.filter( x -> { + final Path path = Path.of( x.value.sourceObj ); + return !hierarchyWrapper.isInScope( path ); + } ); + } + inputSize = inputParamStream .mapToLong( input -> new File(input.value.sourceObj).length() ) .sum(); } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 4c15ba80..1f618597 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -86,7 +86,8 @@ public LocationAwareSchedulerV2( this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; this.readyToRunToNode = readyToRunToNode; - this.readyToRunToNode.init( new FileSizeRankScore( hierarchyWrapper ) ); + final FileSizeRankScore calculateScore = new FileSizeRankScore(); + this.readyToRunToNode.init( calculateScore ); readyToRunToNode.setLogger( logCopyTask ); this.copyRunner = new ShellCopy( client, this, logCopyTask ); this.copySameTaskInParallel = 2; @@ -131,7 +132,7 @@ && canSchedulePodOnNode( task.getPod(), node ) .stream() .filter( td -> !td.getNodesWithAllData().isEmpty() ) .collect( Collectors.toList() ); - final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode( taskWithAllData, availableByNode, hierarchyWrapper ); + final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode( taskWithAllData, availableByNode ); final ScheduleObject scheduleObject = new ScheduleObject( (List) alignment ); scheduleObject.setCheckStillPossible( true ); scheduleObject.setStopSubmitIfOneFails( true ); @@ -409,4 +410,9 @@ public void close() { logCopyTask.close(); super.close(); } + + Task createTask( TaskConfig conf ){ + return new Task( conf, getDag(), hierarchyWrapper ); + } + } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index c49f901b..4c309c2d 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -254,10 +254,13 @@ public void markPodAsDeleted( PodWithAge pod ) { task.setPod( pod ); } - /* External access to Tasks */ + Task createTask( TaskConfig conf ){ + return new Task( conf, dag ); + } + /* External access to Tasks */ public void addTask( int id, TaskConfig conf ) { - final Task task = new Task( conf, dag ); + final Task task = createTask( conf ); synchronized ( tasksByPodName ) { if ( !tasksByPodName.containsKey( conf.getRunName() ) ) { tasksByPodName.put( conf.getRunName(), task ); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java index c891bf9b..e565e9d0 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java @@ -7,8 +7,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -import java.io.File; -import java.nio.file.Path; import java.util.List; /** @@ -21,18 +19,8 @@ public class TaskInputsNodes { private final List nodesWithAllData; private final TaskInputs inputsOfTask; - public long getTaskSize( final HierarchyWrapper hierarchyWrapper ) { - final long filesInSharedFS = task.getConfig() - .getInputs() - .fileInputs - .parallelStream() - .filter( x -> { - final Path path = Path.of( x.value.sourceObj ); - return !hierarchyWrapper.isInScope( path ); - } ) - .mapToLong( input -> new File( input.value.sourceObj ).length() ) - .sum(); - - return filesInSharedFS + inputsOfTask.calculateAvgSize() + 1; + public long getTaskSize() { + final long filesInSharedFS = task.getInputSize(); + return filesInSharedFS + inputsOfTask.calculateAvgSize(); } } \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index fdd21397..7fa63d1b 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -40,8 +40,7 @@ public void init( CalculateScore calculateScore ) { @Override public List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, - Map availableByNode, - HierarchyWrapper hierarchyWrapper + Map availableByNode ) { if ( taskWithAllData.isEmpty() || availableByNode.isEmpty() ){ @@ -66,7 +65,7 @@ public List createAlignmentForTasksWithAllDataOnNod int index = 0; for ( TaskInputsNodes taskInputsNodes : taskWithAllData ) { List onlyOnOneNode = new ArrayList<>(); - final long score = calculateScore.getScore( taskInputsNodes.getTask(), taskInputsNodes.getTaskSize( hierarchyWrapper ) ); + final long score = calculateScore.getScore( taskInputsNodes.getTask(), taskInputsNodes.getTaskSize() ); final Requirements request = taskInputsNodes.getTask().getRequest(); final long ram = request.getRam().longValue(); final long cpu = request.getCpu().multiply( MILLION ).longValue(); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java index 44ce34d1..8e2a0438 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java @@ -31,14 +31,12 @@ public void init( CalculateScore calculateScore ) {} * * @param taskWithAllData * @param availableByNode - * @param hierarchyWrapper * @return */ @Override public List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, - Map availableByNode, - HierarchyWrapper hierarchyWrapper + Map availableByNode ) { long start = System.currentTimeMillis(); final List alignment = new LinkedList<>(); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java index a8d34642..5eafb7e5 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java @@ -19,8 +19,7 @@ public interface ReadyToRunToNode { List createAlignmentForTasksWithAllDataOnNode( List taskWithAllData, - Map availableByNode, - HierarchyWrapper hierarchyWrapper + Map availableByNode ); } From 07ce88a81f19822146bc4d29c9a44d840c2c821e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 21 Jul 2023 10:43:55 +0200 Subject: [PATCH 391/443] Use Lombok for equalsAndHash Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/Requirements.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/Requirements.java b/src/main/java/cws/k8s/scheduler/model/Requirements.java index 17296b5c..e9f79a50 100644 --- a/src/main/java/cws/k8s/scheduler/model/Requirements.java +++ b/src/main/java/cws/k8s/scheduler/model/Requirements.java @@ -1,5 +1,6 @@ package cws.k8s.scheduler.model; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @@ -7,6 +8,7 @@ import java.math.BigDecimal; @ToString +@EqualsAndHashCode public class Requirements implements Serializable, Cloneable { private static final long serialVersionUID = 1L; @@ -98,15 +100,4 @@ public boolean smallerEquals( Requirements request ) { && this.ram.compareTo( request.ram ) <= 0; } - @Override - public boolean equals( Object obj ) { - if ( obj == null ) { - return false; - } - if ( !(obj instanceof Requirements) ) { - return false; - } - return this.getRam().equals( ((Requirements)obj).getRam() ) - && this.getCpu().equals( ((Requirements)obj).getCpu() ); - } } From e9dab4ce189b4f619dbbadef0f636fd9038936b6 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 16:34:08 +0200 Subject: [PATCH 392/443] Store how often the task was copied Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/tracing/TraceRecord.java | 7 +++++++ .../k8s/scheduler/scheduler/LocationAwareSchedulerV2.java | 1 + 2 files changed, 8 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java index 7c9f31c5..9e331397 100644 --- a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java @@ -118,6 +118,8 @@ public class TraceRecord { @Getter private List schedulerTimeDeltaPhaseThree = null; + private int schedulerCopyTasks = 0; + public void addSchedulerTimeDeltaPhaseThree( Integer schedulerTimeDeltaPhaseThree ) { if ( this.schedulerTimeDeltaPhaseThree == null ) { this.schedulerTimeDeltaPhaseThree = new ArrayList<>(); @@ -155,6 +157,7 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_delta_batch_closed_batch_end", schedulerDeltaBatchClosedBatchEnd, bw); writeValue("scheduler_delta_submitted_batch_end", schedulerDeltaSubmittedBatchEnd, bw); writeValue("scheduler_time_delta_phase_three", schedulerTimeDeltaPhaseThree, bw); + writeValue("scheduler_copy_tasks", schedulerCopyTasks, bw); } } @@ -195,5 +198,9 @@ public void tryToSchedule( long startSchedule ){ schedulerTriedToSchedule++; } + public void copyTask(){ + schedulerCopyTasks++; + } + } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 1f618597..dcb5d2d9 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -194,6 +194,7 @@ void postScheduling( List unscheduledTasks, final Map Date: Tue, 25 Jul 2023 16:36:01 +0200 Subject: [PATCH 393/443] Log time of post scheduling Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index dcb5d2d9..05d09947 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -190,6 +190,7 @@ void postScheduling( List unscheduledTasks, final Map Date: Tue, 25 Jul 2023 16:37:00 +0200 Subject: [PATCH 394/443] Log time of post scheduling Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 05d09947..e1c5e5d7 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -141,6 +141,7 @@ && canSchedulePodOnNode( task.getPod(), node ) @Override void postScheduling( List unscheduledTasks, final Map availableByNode ) { + long start = System.currentTimeMillis(); final Map currentlyCopyingTasksOnNode = getCurrentlyCopying().getCurrentlyCopyingTasksOnNode(); final List allNodes = client.getAllNodes(); allNodes.removeIf( x -> !x.isReady() ); From 629442deecd3e4b47e5e10db108860111cb653b1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 16:39:06 +0200 Subject: [PATCH 395/443] Filter tasks not in LA2 Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/LocationAwareSchedulerV2.java | 3 --- .../capacityavailable/SimpleCapacityAvailableToNode.java | 3 ++- .../la2/copyinadvance/CopyInAdvanceNodeWithMostData.java | 3 +++ .../CopyInAdvanceNodeWithMostDataIntelligent.java | 2 +- src/main/java/cws/k8s/scheduler/util/TaskStats.java | 9 +-------- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index e1c5e5d7..63b67063 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -157,7 +157,6 @@ void postScheduling( List unscheduledTasks, final Map unscheduledTasks, final Map createAlignmentForTasksWithEnoughCapacity( //Remove available resources if a copy task is already running: this logic may not be optimal for more than 2 parallel copy tasks (unclear which task starts first) removeAvailableResources( taskStats, availableByNodes, allNodes ); //Sort tasks by missing data: prefer tasks where the least data is missing on the node - final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); + final SortedList stats = new SortedList<>( taskStats.getTaskStats().stream().filter( TaskStat::missingDataOnAnyNode ).collect( Collectors.toList() ) ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); while( !stats.isEmpty() ) { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index 46e22f1d..8027ea04 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -43,6 +43,9 @@ public void createAlignmentForTasksWithEnoughCapacity( while( !stats.isEmpty() ) { final TaskStat poll = stats.poll(); + if ( !poll.missingDataOnAnyNode() || poll.isCopyToNodeWithAvailableResources() ) { + continue; + } long start = System.currentTimeMillis(); final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index e4dcd91d..79c4dc22 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -48,7 +48,7 @@ public void createAlignmentForTasksWithEnoughCapacity( ) { Map cache = new HashMap<>(); long startMethod = System.currentTimeMillis(); - final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); + final SortedList stats = new SortedList<>( taskStats.getTaskStats().stream().filter( x -> x.missingDataOnAnyNode() && !x.isCopyToNodeWithAvailableResources() ).collect( Collectors.toList()) ); removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); int readyOnNodes = 0; diff --git a/src/main/java/cws/k8s/scheduler/util/TaskStats.java b/src/main/java/cws/k8s/scheduler/util/TaskStats.java index 85919749..ded8e0eb 100644 --- a/src/main/java/cws/k8s/scheduler/util/TaskStats.java +++ b/src/main/java/cws/k8s/scheduler/util/TaskStats.java @@ -5,6 +5,7 @@ import cws.k8s.scheduler.model.Task; import java.util.*; +import java.util.stream.Collectors; public class TaskStats { @@ -14,7 +15,6 @@ public TaskStat get( Task task ) { return this.taskStats.get( task ); } - public void add( TaskStat taskStat ) { this.taskStats.put( taskStat.getTask(), taskStat ); } @@ -23,13 +23,6 @@ public Collection getTaskStats() { return this.taskStats.values(); } - /** - * This method will remove all tasks, that have set copyToNodeWithAvailableResources - */ - public void removeTasksThatHaveBeenStarted() { - taskStats.entrySet().removeIf(entry -> entry.getValue().isCopyToNodeWithAvailableResources()); - } - public void setComparator( TaskStatComparator comparator ) { for ( TaskStat taskStat : taskStats.values() ) { taskStat.setComparator( comparator ); From 97cd90c95d76871ea3f2aa34434dd83637c017a3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 16:39:32 +0200 Subject: [PATCH 396/443] Minimize jar to save storage Signed-off-by: Lehmann_Fabian --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 58280d3a..16975cf8 100644 --- a/pom.xml +++ b/pom.xml @@ -146,6 +146,7 @@ shade + true From aa1d21ca671339017624e40184c2a9fd7db0b4f5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 16:53:18 +0200 Subject: [PATCH 397/443] Added new Constructor and added atLeastOneBigger Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/model/Requirements.java | 15 ++++++ .../k8s/scheduler/model/RequirementsTest.java | 53 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/model/Requirements.java b/src/main/java/cws/k8s/scheduler/model/Requirements.java index e9f79a50..9353ac9d 100644 --- a/src/main/java/cws/k8s/scheduler/model/Requirements.java +++ b/src/main/java/cws/k8s/scheduler/model/Requirements.java @@ -23,6 +23,16 @@ public Requirements( BigDecimal cpu, BigDecimal ram ) { this.ram = ram == null ? BigDecimal.ZERO : ram; } + /** + * Basically used for testing + * @param cpu + * @param ram + */ + public Requirements( int cpu, int ram ) { + this.cpu = BigDecimal.valueOf(cpu); + this.ram = BigDecimal.valueOf(ram); + } + public Requirements(){ this( BigDecimal.ZERO, BigDecimal.ZERO ); } @@ -100,4 +110,9 @@ public boolean smallerEquals( Requirements request ) { && this.ram.compareTo( request.ram ) <= 0; } + public boolean atLeastOneBigger( Requirements request ) { + return this.cpu.compareTo( request.cpu ) > 0 + || this.ram.compareTo( request.ram ) > 0; + } + } diff --git a/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java b/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java index b119324b..9f0716bf 100644 --- a/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java +++ b/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test; import java.math.BigDecimal; +import java.util.HashSet; import static org.junit.jupiter.api.Assertions.*; @@ -164,4 +165,56 @@ void getRam() { Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); } + + + @Test + void atLeastOneBigger() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertTrue( a.atLeastOneBigger( b ) ); + assertFalse( b.atLeastOneBigger( a ) ); + assertFalse( a.atLeastOneBigger( a ) ); + assertFalse( b.atLeastOneBigger( b ) ); + + Requirements c = new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 5 ) ); + assertTrue( a.atLeastOneBigger( c ) ); + assertTrue( c.atLeastOneBigger( a ) ); + assertFalse( c.atLeastOneBigger( c ) ); + + Requirements d = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertFalse( a.atLeastOneBigger( d ) ); + assertFalse( d.atLeastOneBigger( a ) ); + assertFalse( d.atLeastOneBigger( d ) ); + } + + @Test + void hashSetTest() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + final HashSet requirements = new HashSet<>(); + requirements.add( a ); + requirements.add( b ); + assertEquals( 1, requirements.size() ); + requirements.add( a.add( b ) ); + assertEquals( 2, requirements.size() ); + requirements.add( a.add( b ).sub( new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ) ) ); + assertEquals( 2, requirements.size() ); + } + + @Test + void equalsTest() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertEquals( a, b ); + assertEquals( b, a ); + assertEquals( a, a ); + assertEquals( b, b ); + assertNotEquals( null, a ); + assertNotEquals( a, new Object() ); + assertNotEquals( a, new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 7 ) ) ); + assertNotEquals( a, new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 6 ) ) ); + assertNotSame( a, b ); + } + + } \ No newline at end of file From 90c7d4795977682e7e8ca6c74960adf76f3c0335 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 16:58:02 +0200 Subject: [PATCH 398/443] Switch back to normal CopyInAdvance Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/scheduler/LocationAwareSchedulerV2.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 63b67063..64ac77e9 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -3,7 +3,7 @@ import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.scheduler.la2.*; import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; -import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostDataIntelligent; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; import cws.k8s.scheduler.client.KubernetesClient; @@ -18,7 +18,6 @@ import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import cws.k8s.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import cws.k8s.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; -import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; import cws.k8s.scheduler.scheduler.la2.copystrategy.CopyRunner; import cws.k8s.scheduler.scheduler.la2.copystrategy.ShellCopy; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; @@ -94,7 +93,7 @@ public LocationAwareSchedulerV2( this.capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); this.phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); this.phaseThreeComparator = new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ); - this.copyInAdvance = new CopyInAdvanceNodeWithMostDataIntelligent( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); this.maxHeldCopyTaskReady = config.maxHeldCopyTaskReady == null ? 3 : config.maxHeldCopyTaskReady; this.prioPhaseThree = config.prioPhaseThree == null ? 70 : config.prioPhaseThree; } From 63f862a2fc75fb295c6ba953a3c7e4924e609695 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 17:00:02 +0200 Subject: [PATCH 399/443] Change CopyInAdvanceMoreIntelligent Signed-off-by: Lehmann_Fabian --- ...yInAdvanceNodeWithMostDataIntelligent.java | 125 ++++++++++++------ .../la2/copyinadvance/ShouldCopyChecker.java | 75 +++++++++++ 2 files changed, 160 insertions(+), 40 deletions(-) create mode 100644 src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index 79c4dc22..5c4164a0 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -4,17 +4,28 @@ import cws.k8s.scheduler.model.Requirements; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import cws.k8s.scheduler.scheduler.la2.TaskStat; import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import cws.k8s.scheduler.util.SortedList; import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.Tuple; import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.score.CalculateScore; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; +/** + * This strategy does not perform better than the normal CopyInAdvanceNodeWithMostData. + */ @Slf4j public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { @@ -22,16 +33,23 @@ public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { * Try to distribute the tasks evenly on the nodes. More important tasks can be on x more nodes. (x = TASKS_READY_FACTOR) */ private static final int TASKS_READY_FACTOR = 2; - private static final RequirementsComparator requirementsComparator = new RequirementsComparator(); + + CalculateScore calculateScore; + + HierarchyWrapper hierarchyWrapper; public CopyInAdvanceNodeWithMostDataIntelligent( CurrentlyCopying currentlyCopying, InputAlignment inputAlignment, - int copySameTaskInParallel ) { + int copySameTaskInParallel , + CalculateScore calculateScore, + HierarchyWrapper hierarchyWrapper + ) { super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + this.calculateScore = calculateScore; + this.hierarchyWrapper = hierarchyWrapper; } - /** * Do not filter maxHeldCopyTaskReady, that could lead to starving if another node has resources. */ @@ -57,22 +75,23 @@ public void createAlignmentForTasksWithEnoughCapacity( LinkedList tasksReadyOnMoreNodes = new LinkedList<>(); while( !stats.isEmpty() ) { final TaskStat poll = stats.poll(); - long start = System.currentTimeMillis(); - if ( poll.dataOnNodes() > readyOnNodes + TASKS_READY_FACTOR ) { tasksReadyOnMoreNodes.add( poll ); continue; } + long start = System.currentTimeMillis(); final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); final Task task = poll.getTask(); final NodeWithAlloc node = bestStats.getNode(); final NodeCache nodeCache = cache.computeIfAbsent( node, n -> new NodeCache( //Only consider workflow tasks, other will not finish soon - n.getAssignedPods().entrySet().stream().filter( x -> x.getKey().contains( "||nf-" ) ).map( x -> x.getValue() ).collect( Collectors.toList() ), + n.getAssignedPods().entrySet().stream().filter( x -> x.getKey().contains( "||nf-" ) ).map( Map.Entry::getValue ).collect( Collectors.toList() ), readyTasksPerNode.get( n ), - planedToCopy.getTasksOnNode( n.getNodeLocation() ) + planedToCopy.getTasksOnNode( n.getNodeLocation() ), + taskStats, + node ) ); @@ -82,7 +101,7 @@ && reasonableToCopyData( node, task, nodeCache ) && createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); - nodeCache.addPlaned( task.getRequest() ); + nodeCache.addPlaned( task ); } else { //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list if ( poll.increaseIndexToCompare() ) { @@ -98,72 +117,98 @@ && createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingT log.info( "Time to create alignment for tasks with enough capacity: {}", System.currentTimeMillis() - startMethod ); } + /** + * Check if a single task or two tasks could replace one running task + * @param node + * @param task + * @param cache + * @return + */ private boolean reasonableToCopyData( NodeWithAlloc node, Task task, NodeCache cache ) { - return true; + final Requirements availableResources = node.getAvailableResources(); + final ShouldCopyChecker shouldCopyChecker = new ShouldCopyChecker( + cache.getTaskWithScore( task, "Problem in reasonableToCopyData" ).score, + cache.waiting, + task.getRequest() + ); + return cache.running + .parallelStream() + .anyMatch( r -> shouldCopyChecker.couldBeStarted( r.add( availableResources ) ) ); } - private static class NodeCache { + private class NodeCache { /** * A sorted list of tasks that are currently running on the node. */ - private final List running; + private final Set running; /** * A sorted list of tasks that are currently waiting on the node. */ - private final ArrayList waiting; + private final List waiting; + + private final NodeWithAlloc node; - NodeCache( final Collection running, final List tasksAlreadyReady, final List tasksPlanedToCopy ) { + private final TaskStats taskStats; - this.running = new LinkedList<>(running); - this.running.sort( requirementsComparator ); + NodeCache( final Collection running, final List tasksAlreadyReady, final List tasksPlanedToCopy, TaskStats taskStats, NodeWithAlloc node ) { + this.node = node; + this.taskStats = taskStats; + /** + * remove tasks with similar requirements, as replacement would be the same. + */ + this.running = new HashSet<>(running); - ArrayList waitingTemp = null; + Stream waitingTemp = null; // A list of tasks that are ready to run, or data is copied so that they can start soon. if ( tasksAlreadyReady != null ) { - waitingTemp = tasksAlreadyReady.stream().map( Task::getRequest ).collect( Collectors.toCollection(ArrayList::new) ); + waitingTemp = tasksAlreadyReady + .stream() + .map( x -> getTaskWithScore( x, "Problem with tasksAlreadyReady" ) ); } if ( tasksPlanedToCopy != null && !tasksPlanedToCopy.isEmpty() ) { - final ArrayList planned = tasksPlanedToCopy.stream().map( Task::getRequest ).collect( Collectors.toCollection(ArrayList::new) ); + final Stream planned = tasksPlanedToCopy.stream() + .map( x -> getTaskWithScore( x, "Problem with tasksPlanedToCopy" ) ); if ( waitingTemp == null ) { waitingTemp = planned; } else { - waitingTemp.addAll( planned ); + waitingTemp = Stream.concat( waitingTemp, planned ); } } else if ( waitingTemp == null ) { - waitingTemp = new ArrayList<>(); + waitingTemp = Stream.empty(); } - waiting = waitingTemp; - waiting.sort( requirementsComparator ); + waiting = waitingTemp + .collect( Collectors.toList() ); } - void addPlaned( Requirements requirement ) { - int insertionIndex = Collections.binarySearch( waiting, requirement, requirementsComparator ); - if (insertionIndex < 0) { - insertionIndex = -(insertionIndex + 1); + @NotNull + private TaskWithScore getTaskWithScore( Task task, String error ) { + if ( taskStats.get( task ) == null ) { + throw new RuntimeException( error ); } - waiting.add(insertionIndex, requirement); + final long dataOnNode = taskStats + .get( task ) + .getInputsOfTask() + .calculateDataOnNode( node.getNodeLocation() ); + final long dataInSharedFS = task.getInputSize(); + //The input should be similar to the first scheduling phase + return new TaskWithScore( task, calculateScore.getScore( task, dataOnNode + dataInSharedFS ) ); } - } - - private static class RequirementsComparator implements Comparator { - - @Override - public int compare(Requirements r1, Requirements r2) { - int cpuComparison = r1.getCpu().compareTo(r2.getCpu()); - if (cpuComparison != 0) { - return cpuComparison; - } else { - return r1.getRam().compareTo(r2.getRam()); - } + void addPlaned( Task task ) { + waiting.add( getTaskWithScore( task, "Problem in addPlanned" ) ); } } - + + @RequiredArgsConstructor + static class TaskWithScore { + final Task task; + final long score; + } } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java new file mode 100644 index 00000000..d828768c --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java @@ -0,0 +1,75 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.Requirements; +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; + +import java.util.List; +import java.util.Optional; + +@RequiredArgsConstructor +public class ShouldCopyChecker { + + private final long taskScore; + private final List waiting; + private final Requirements taskRequest; + + boolean couldBeStarted( Requirements becomesAvailable ) { + //True if the task to check would be started in the current case + Result results = new Result( true, taskScore ); + final Optional reduce = waiting.parallelStream().map( waitingTask -> { + final Result result = new Result( true, taskScore ); + final Requirements waitingRequest = waitingTask.task.getRequest(); + //In all cases waitingRequest is used and accordingly the request must always be smaller than the available resources + if ( waitingRequest.smallerEquals( becomesAvailable ) ) { + checkSingle( becomesAvailable, result, waitingRequest.add( taskRequest ), waitingTask.score ); +// checkPair( becomesAvailable, result, waitingTask ); + } + return result; + } ).reduce( ( a, b ) -> a.bestScore > b.bestScore ? a : b ); + return reduce.map( result -> result.couldBeStarted ).orElseGet( () -> results.couldBeStarted ); + } + + /** + * + * @param becomesAvailable resources available + * @param results wrapper for the result + * @param requestAndTaskRequest sum of task to check and waiting task + * @param score score of the waiting task + */ + private void checkSingle( Requirements becomesAvailable, Result results, Requirements requestAndTaskRequest, long score ) { + if ( results.bestScore < score ) { + results.bestScore = score; + results.couldBeStarted = false; + } + if ( results.bestScore < score + taskScore && requestAndTaskRequest.smallerEquals( becomesAvailable ) ) { + results.bestScore = score + taskScore; + results.couldBeStarted = true; + } + } + + private void checkPair( + Requirements becomesAvailable, + Result results, + CopyInAdvanceNodeWithMostDataIntelligent.TaskWithScore waitingTask + ) { + for ( CopyInAdvanceNodeWithMostDataIntelligent.TaskWithScore taskWithScore2 : waiting ) { + if ( waitingTask == taskWithScore2 ) { + continue; + } + final Requirements cRequest2 = taskWithScore2.task.getRequest(); + final Requirements waitingAndTask2Request = waitingTask.task.getRequest().add( cRequest2 ); + if ( waitingAndTask2Request.smallerEquals( becomesAvailable ) ) { + final long waitingScoreAndScore2 = waitingTask.score + taskWithScore2.score; + checkSingle( becomesAvailable, results, waitingAndTask2Request.add( taskRequest ), waitingScoreAndScore2 ); + } + } + } + + @AllArgsConstructor + private static class Result { + boolean couldBeStarted; + long bestScore; + } + +} From 1a1064e17f65646f24bdbff8b77f593d3ae110a1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jul 2023 17:00:25 +0200 Subject: [PATCH 400/443] prefer two tasks which sum up to the same score otherwise Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index 8192d1ee..c416dffe 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -6,6 +6,7 @@ public class FileSizeScore implements CalculateScore { @Override public long getScore( Task task, long size ) { - return size; + //add one to prefer two tasks which sum up to the same score otherwise + return size + 1; } } From 247f562bc87bf5c55eff58c36933ef6d63f1818e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 Oct 2023 12:45:56 +0200 Subject: [PATCH 401/443] Filter tasks that can not start on any node Signed-off-by: Lehmann_Fabian --- .../scheduler/scheduler/LocationAwareSchedulerV2.java | 1 + .../cws/k8s/scheduler/scheduler/la2/TaskStat.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 64ac77e9..9129596a 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -156,6 +156,7 @@ void postScheduling( List unscheduledTasks, final Map { @Getter private boolean copyToNodeWithAvailableResources = false; + public boolean canStartSomewhere() { + return !taskStats.isEmpty(); + } + /** * @return number is the number of nodes, that are better for this task depending on the comparator. */ @@ -116,8 +120,11 @@ public NodeAndStatWrapper getBestStats() { @Override public int compareTo( @NotNull TaskStat o ) { - if ( indexToCompare >= taskStats.size() || o.indexToCompare >= o.taskStats.size() ) { - throw new IllegalStateException( "Cannot compare tasks that have no stats" ); + if ( indexToCompare >= taskStats.size() ) { + throw new IllegalStateException( "Cannot compare task " + task.getConfig().getName() + " size = " + taskStats.size() + " indexToCompare = " + indexToCompare); + } + if ( o.indexToCompare >= o.taskStats.size() ) { + throw new IllegalStateException( "Cannot compare task " + o.task.getConfig().getName() + " size = " + o.taskStats.size() + " indexToCompare = " + o.indexToCompare ); } return comparator.compare( this, o ); } From badbfb31ad6c6830b61c1ea4e9dea35e72218b75 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 Oct 2023 12:48:00 +0200 Subject: [PATCH 402/443] Change Watcher to Handler Signed-off-by: Lehmann_Fabian --- .../scheduler/client/KubernetesClient.java | 102 +++++++++--------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java index d83cdbeb..9eb63084 100644 --- a/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java +++ b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java @@ -6,8 +6,9 @@ import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.WatcherException; import io.fabric8.kubernetes.client.dsl.ExecWatch; +import io.fabric8.kubernetes.client.informers.ResourceEventHandler; +import io.fabric8.kubernetes.client.informers.SharedIndexInformer; import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; @@ -25,8 +26,10 @@ public KubernetesClient(){ for( Node node : this.nodes().list().getItems() ){ nodeHolder.put( node.getMetadata().getName(), new NodeWithAlloc(node,this) ); } - this.pods().inAnyNamespace().watch( new PodWatcher( this ) ); - this.nodes().watch( new NodeWatcher( this ) ); + final SharedIndexInformer podSharedIndexInformer = this.pods().inAnyNamespace().inform( new PodHandler( this ) ); + podSharedIndexInformer.start(); + final SharedIndexInformer nodeSharedIndexInformer = this.nodes().inform( new NodeHandler( this ) ); + nodeSharedIndexInformer.start(); } public void addInformable( Informable informable ){ @@ -144,73 +147,63 @@ public void execCommand( String podName, String namespace, String[] command, MyE listener.setOut( out ); } - - static class NodeWatcher implements Watcher{ + static class NodeHandler implements ResourceEventHandler{ private final KubernetesClient kubernetesClient; - public NodeWatcher(KubernetesClient kubernetesClient) { + public NodeHandler(KubernetesClient kubernetesClient) { this.kubernetesClient = kubernetesClient; } @Override - public void eventReceived(Action action, Node node) { + public void onAdd( Node node ) { boolean change = false; NodeWithAlloc processedNode = null; - switch (action) { - case ADDED: - log.info("New Node {} was added", node.getMetadata().getName()); - synchronized ( kubernetesClient.nodeHolder ){ - if ( ! kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ - processedNode = new NodeWithAlloc(node,kubernetesClient); - kubernetesClient.nodeHolder.put( node.getMetadata().getName(), processedNode ); - change = true; - } - } - if ( change ) { - kubernetesClient.informAllNewNode( processedNode ); - } - break; - case DELETED: - log.info("Node {} was deleted", node.getMetadata().getName()); - synchronized ( kubernetesClient.nodeHolder ){ - if ( kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ - processedNode = kubernetesClient.nodeHolder.remove( node.getMetadata().getName() ); - change = true; - } - } - if ( change ) { - kubernetesClient.informAllRemovedNode( processedNode ); - } - break; - case ERROR: - log.info("Node {} has an error", node.getMetadata().getName()); - //todo deal with error - break; - case MODIFIED: - log.info("Node {} was modified", node.getMetadata().getName()); - //todo deal with changed state - break; - default: log.warn("No implementation for {}", action); + log.info("New Node {} was added", node.getMetadata().getName()); + synchronized ( kubernetesClient.nodeHolder ){ + if ( ! kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ + processedNode = new NodeWithAlloc(node,kubernetesClient); + kubernetesClient.nodeHolder.put( node.getMetadata().getName(), processedNode ); + change = true; + } + } + if ( change ) { + kubernetesClient.informAllNewNode( processedNode ); } } @Override - public void onClose(WatcherException cause) { - log.info( "Watcher was closed" ); + public void onUpdate( Node oldObj, Node newObj ) { + //todo deal with changed state + } + + @Override + public void onDelete( Node node, boolean deletedFinalStateUnknown ) { + boolean change = false; + NodeWithAlloc processedNode = null; + log.info("Node {} was deleted", node.getMetadata().getName()); + synchronized ( kubernetesClient.nodeHolder ){ + if ( kubernetesClient.nodeHolder.containsKey( node.getMetadata().getName() ) ){ + processedNode = kubernetesClient.nodeHolder.remove( node.getMetadata().getName() ); + change = true; + } + } + if ( change ) { + kubernetesClient.informAllRemovedNode( processedNode ); + } } + } - static class PodWatcher implements Watcher { + static class PodHandler implements ResourceEventHandler { private final KubernetesClient kubernetesClient; - public PodWatcher(KubernetesClient kubernetesClient) { + public PodHandler(KubernetesClient kubernetesClient) { this.kubernetesClient = kubernetesClient; } - @Override - public void eventReceived(Action action, Pod pod) { + public void eventReceived( Watcher.Action action, Pod pod) { String nodeName = pod.getSpec().getNodeName(); if( nodeName != null ){ NodeWithAlloc node = kubernetesClient.nodeHolder.get( pod.getSpec().getNodeName() ); @@ -240,10 +233,19 @@ public void eventReceived(Action action, Pod pod) { } } + @Override + public void onAdd( Pod pod ) { + eventReceived( Watcher.Action.ADDED, pod ); + } + + @Override + public void onUpdate( Pod oldPod, Pod newPod ) { + eventReceived( Watcher.Action.MODIFIED, newPod ); + } @Override - public void onClose(WatcherException cause) { - log.info( "Watcher was closed" ); + public void onDelete( Pod pod, boolean deletedFinalStateUnknown ) { + eventReceived( Watcher.Action.DELETED, pod ); } } From 8e9fb91301d941222f5bf6164eae59a7123b92f1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 6 Oct 2023 12:49:43 +0200 Subject: [PATCH 403/443] Change Watcher to Handler in Scheduler Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/scheduler/Scheduler.java | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index 4c309c2d..dccada0f 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -7,9 +7,9 @@ import cws.k8s.scheduler.util.Batch; import cws.k8s.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.informers.ResourceEventHandler; +import io.fabric8.kubernetes.client.informers.SharedIndexInformer; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -44,7 +44,7 @@ public abstract class Scheduler implements Informable { private final List unfinishedTasks = new ArrayList<>( 100 ); final Map tasksByPodName = new HashMap<>(); final Map tasksById = new HashMap<>(); - private final Watch watcher; + private final SharedIndexInformer podHandler; private final TaskprocessingThread schedulingThread; private final TaskprocessingThread finishThread; @@ -60,7 +60,7 @@ public abstract class Scheduler implements Informable { this.dag = new DAG(); this.traceEnabled = config.traceEnabled; - PodWatcher podWatcher = new PodWatcher(this); + PodHandler handler = new PodHandler(this ); schedulingThread = new TaskprocessingThread( unscheduledTasks, this::schedule ); schedulingThread.start(); @@ -68,8 +68,11 @@ public abstract class Scheduler implements Informable { finishThread = new TaskprocessingThread(unfinishedTasks, this::terminateTasks ); finishThread.start(); - log.info("Start watching"); - watcher = client.pods().inNamespace( this.namespace ).watch(podWatcher); + log.info("Start watching: {}", this.namespace ); + + this.podHandler = client.pods().inNamespace( this.namespace ).inform( handler ); + this.podHandler.start(); + log.info("Watching"); } @@ -518,21 +521,20 @@ LinkedList getUpcomingTasksCopy() { * Close used resources */ public void close(){ - watcher.close(); + podHandler.close(); schedulingThread.interrupt(); this.close = true; } - static class PodWatcher implements Watcher { + static class PodHandler implements ResourceEventHandler { private final Scheduler scheduler; - public PodWatcher(Scheduler scheduler) { + public PodHandler( Scheduler scheduler ) { this.scheduler = scheduler; } - @Override - public void eventReceived(Action action, Pod pod) { + public void eventReceived( Watcher.Action action, Pod pod) { scheduler.podEventReceived(action, pod); @@ -580,12 +582,20 @@ public void eventReceived(Action action, Pod pod) { } + @Override + public void onAdd( Pod pod ) { + eventReceived( Watcher.Action.ADDED, pod ); + } @Override - public void onClose(WatcherException cause) { - log.info( "Watcher was closed" ); + public void onUpdate( Pod oldPod, Pod newPod ) { + eventReceived( Watcher.Action.MODIFIED, newPod ); } + @Override + public void onDelete( Pod pod, boolean deletedFinalStateUnknown ) { + eventReceived( Watcher.Action.DELETED, pod ); + } } From c16cb2294c83bf3a391e9d2fbbad75ac043ade0d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Sat, 7 Oct 2023 19:21:05 +0200 Subject: [PATCH 404/443] Remove unused imports Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java | 1 - .../scheduler/la2/ready2run/RandomReadyToRunToNode.java | 1 - .../k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java | 1 - src/main/java/cws/k8s/scheduler/util/TaskStats.java | 1 - 4 files changed, 4 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java index e565e9d0..1b0115af 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java index 8e2a0438..c7d6b560 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; -import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java index 5eafb7e5..0ed60a39 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; -import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; diff --git a/src/main/java/cws/k8s/scheduler/util/TaskStats.java b/src/main/java/cws/k8s/scheduler/util/TaskStats.java index ded8e0eb..0db95707 100644 --- a/src/main/java/cws/k8s/scheduler/util/TaskStats.java +++ b/src/main/java/cws/k8s/scheduler/util/TaskStats.java @@ -5,7 +5,6 @@ import cws.k8s.scheduler.model.Task; import java.util.*; -import java.util.stream.Collectors; public class TaskStats { From d778be82d3cee4e89cc0984db3b1216cb770305f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Sat, 7 Oct 2023 19:22:04 +0200 Subject: [PATCH 405/443] Remove unused imports Signed-off-by: Lehmann_Fabian --- .../CopyInAdvanceNodeWithMostDataIntelligent.java | 3 --- .../java/cws/k8s/scheduler/util/score/FileSizeRankScore.java | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index 5c4164a0..82f1237f 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -10,11 +10,8 @@ import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import cws.k8s.scheduler.util.SortedList; import cws.k8s.scheduler.util.TaskStats; -import cws.k8s.scheduler.util.Tuple; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.score.CalculateScore; -import lombok.AllArgsConstructor; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java index 51742275..f9227b82 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -1,8 +1,6 @@ package cws.k8s.scheduler.util.score; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; -import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; public class FileSizeRankScore extends FileSizeScore { From 731d401945544b77c4acee5b4f8051aa1943a57f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 11 Oct 2023 10:42:08 +0200 Subject: [PATCH 406/443] OptimalReadyToRunToNode optimizer sops after 10 seconds Signed-off-by: Lehmann_Fabian --- .../scheduler/la2/ready2run/OptimalReadyToRunToNode.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index 7fa63d1b..c7256cbf 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; -import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; @@ -97,15 +96,16 @@ public List createAlignmentForTasksWithAllDataOnNod model.addLessOrEqual( cpuUsed.get( entry.getKey() ), availableByNode.get( entry.getKey() ).getCpu().multiply( MILLION ).longValue() ); } + log.info( "Model created in " + (System.currentTimeMillis() - start) + "ms ( " + taskNodeBoolVars.size() + " vars )" ); model.maximize( objective ); CpSolver solver = new CpSolver(); + solver.getParameters().setMaxTimeInSeconds( 10 ); final CpSolverStatus solve = solver.solve( model ); - final String message = "Solved in " + (System.currentTimeMillis() - start) + "ms ( " + taskNodeBoolVars.size() + " vars )"; + final String message = "Solved in " + (System.currentTimeMillis() - start) + "ms ( " + taskNodeBoolVars.size() + " vars ) + solution is: " + solve; log.info( message ); logger.log( message ); log.info("Total packed value: " + solver.objectiveValue()); - log.info( String.valueOf( solve ) ); - if ( solve == CpSolverStatus.MODEL_INVALID ) { + if ( solve != CpSolverStatus.OPTIMAL && solve != CpSolverStatus.FEASIBLE ) { return Collections.emptyList(); } From 5cfb40d7d027e6221725d9343bd290afebcea295 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Sat, 28 Oct 2023 10:56:46 +0200 Subject: [PATCH 407/443] Update versions Signed-off-by: Lehmann_Fabian --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 6e4e9610..8fe8cbb4 100644 --- a/pom.xml +++ b/pom.xml @@ -58,13 +58,13 @@ org.springframework.boot spring-boot-starter-web - 3.0.4 + 3.1.4 org.springframework.boot spring-boot-starter-test - 3.0.4 + 3.1.4 test @@ -92,13 +92,13 @@ commons-net commons-net - 3.8.0 + 3.10.0 org.springdoc springdoc-openapi-ui - 1.6.15 + 1.7.0 From d84c9b91f8ffd3e3e45aa5642251f9d9670cdb92 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Sat, 28 Oct 2023 10:56:55 +0200 Subject: [PATCH 408/443] Added comment Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/location/hierachy/LocationWrapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java index 15cd6256..793c8583 100644 --- a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java @@ -19,6 +19,7 @@ public class LocationWrapper { private long createTime = System.currentTimeMillis(); private Task createdByTask; private LocationWrapper copyOf; + //Deactivated if file was maybe not copied completely or if one file was changed by the workflow engine. private boolean active = true; private int inUse = 0; From fd3098771d3f28870a483c638b5f61f11f2bd99d Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Sat, 20 Jan 2024 12:16:29 +0100 Subject: [PATCH 409/443] Filter null in post scheduling Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 9129596a..f5cb3152 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -156,12 +156,14 @@ void postScheduling( List unscheduledTasks, final Map Date: Fri, 21 Jun 2024 12:11:30 +0200 Subject: [PATCH 410/443] Removed unused code and rewrote outLabel Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/model/OutLabel.java | 19 - .../java/cws/k8s/scheduler/model/Task.java | 5 +- .../cws/k8s/scheduler/model/TaskConfig.java | 2 +- .../rest/SchedulerRestController.java | 9 - .../scheduler/scheduler/LASchedulerV1.java | 21 - .../scheduler/LocationAwareScheduler.java | 469 ------------------ .../scheduler/LocationAwareSchedulerV2.java | 1 + .../scheduler/RandomLAScheduler.java | 85 ---- .../scheduler/SchedulerWithDaemonSet.java | 3 - .../scheduler/data/NodeDataTuple.java | 33 -- .../scheduler/scheduler/data/TaskData.java | 138 ------ .../scheduler/outlabel/HolderMaxTasks.java | 22 - .../scheduler/outlabel/InternalHolder.java | 43 -- .../scheduler/outlabel/OutLabelHolder.java | 41 -- .../outlabel/OutLabelHolderMaxTasksTest.java | 48 -- 15 files changed, 4 insertions(+), 935 deletions(-) delete mode 100644 src/main/java/cws/k8s/scheduler/model/OutLabel.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java delete mode 100644 src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java delete mode 100644 src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java diff --git a/src/main/java/cws/k8s/scheduler/model/OutLabel.java b/src/main/java/cws/k8s/scheduler/model/OutLabel.java deleted file mode 100644 index 86516ea8..00000000 --- a/src/main/java/cws/k8s/scheduler/model/OutLabel.java +++ /dev/null @@ -1,19 +0,0 @@ -package cws.k8s.scheduler.model; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.ToString; - -@Getter -@ToString -@RequiredArgsConstructor -public class OutLabel { - - private final String label; - private final double weight; - - private OutLabel() { - this(null,-1 ); - } - -} diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index 62c3a037..6da9055f 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -111,9 +111,8 @@ public void submitted(){ traceRecord.setSchedulerTimeInQueue( System.currentTimeMillis() - timeAddedToQueue ); } - public String getOutLabel(){ - final OutLabel outLabel = config.getOutLabel(); - return outLabel == null ? null : outLabel.getLabel(); + public String[] getOutLabel(){ + return config.getOutLabel(); } private long inputSize = -1; diff --git a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java index 493211ea..49bb984a 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java @@ -20,7 +20,7 @@ public class TaskConfig { private final String runName; private final float cpus; private final long memoryInBytes; - private final OutLabel outLabel; + private final String[] outLabel; private final String workDir; private TaskConfig() { diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index f182cc95..ef9a2548 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -119,15 +119,6 @@ ResponseEntity registerScheduler( } switch ( strategy.toLowerCase() ){ - case "lav1" : - if ( !config.locationAware ) { - return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); - } - if ( costFunction == null ) { - costFunction = new MinSizeCost( 0 ); - } - scheduler = new LASchedulerV1( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ) ); - break; case "lav2" : if ( !config.locationAware ) { return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java b/src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java deleted file mode 100644 index c9d66911..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/LASchedulerV1.java +++ /dev/null @@ -1,21 +0,0 @@ -package cws.k8s.scheduler.scheduler; - -import cws.k8s.scheduler.client.KubernetesClient; -import cws.k8s.scheduler.model.SchedulerConfig; -import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class LASchedulerV1 extends LocationAwareScheduler { - - public LASchedulerV1( - String name, - KubernetesClient client, - String namespace, - SchedulerConfig config, - InputAlignment inputAlignment - ) { - super(name, client, namespace, config, inputAlignment); - } - -} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java deleted file mode 100644 index 1f17e643..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareScheduler.java +++ /dev/null @@ -1,469 +0,0 @@ -package cws.k8s.scheduler.scheduler; - -import cws.k8s.scheduler.model.*; -import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; -import cws.k8s.scheduler.client.KubernetesClient; -import cws.k8s.scheduler.util.*; -import cws.k8s.scheduler.model.location.Location; -import cws.k8s.scheduler.model.location.NodeLocation; -import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; -import cws.k8s.scheduler.model.taskinputs.TaskInputs; -import cws.k8s.scheduler.model.tracing.TraceRecord; -import cws.k8s.scheduler.scheduler.data.NodeDataTuple; -import cws.k8s.scheduler.scheduler.data.TaskData; -import cws.k8s.scheduler.util.copying.CurrentlyCopying; -import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class LocationAwareScheduler extends SchedulerWithDaemonSet { - - @Getter(AccessLevel.PACKAGE) - private final InputAlignment inputAlignment; - @Getter(AccessLevel.PACKAGE) - private final int maxCopyTasksPerNode; - @Getter(AccessLevel.PACKAGE) - private final int maxWaitingCopyTasksPerNode; - - public LocationAwareScheduler ( - String name, - KubernetesClient client, - String namespace, - SchedulerConfig config, - InputAlignment inputAlignment) { - super( name, client, namespace, config ); - this.inputAlignment = inputAlignment; - this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; - this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; - } - - - /** - * Find the best alignment for one task - * - * @param taskData - * @param availableByNode - * @param planedToCopy - * @param index - * @return the alignment containing the node and the file alignments - */ - private NodeTaskFilesAlignment createNodeAlignment ( - final TaskData taskData, - final Map availableByNode, - final Map assignedPodsByNode, - CurrentlyCopying planedToCopy, - int index - ) { - long startTime = System.nanoTime(); - log.info( "Task: {} has a value of: {}", taskData.getTask().getConfig().getRunName(), taskData.getValue() ); - final Tuple result = calculateBestNode(taskData, planedToCopy, availableByNode, assignedPodsByNode); - if ( result == null ) { - return null; - } - final Task task = taskData.getTask(); - availableByNode.get(result.getA()).subFromThis(task.getPod().getRequest()); - assignedPodsByNode.put(result.getA(), assignedPodsByNode.getOrDefault(result.getA(),0) + 1); - taskData.addNs( System.nanoTime()- startTime ); - if ( traceEnabled ){ - task.getTraceRecord().setSchedulerTimeToSchedule((int) (taskData.getTimeInNs() / 1_000_000)); - task.getTraceRecord().setSchedulerPlaceInQueue( index ); - task.getTraceRecord().setSchedulerLocationCount( - taskData.getMatchingFilesAndNodes().getInputsOfTask().getFiles() - .parallelStream() - .mapToInt( x -> x.locations.size() ) - .sum() - ); - } - return new NodeTaskFilesAlignment(result.getA(), task, result.getB(), 100 ); - } - - /** - * Align all tasks to the best node - * @param unscheduledTasksSorted - * @param availableByNode - * @return a list with alignments - */ - List createAlignment( - final PriorityQueue unscheduledTasksSorted, - final Map availableByNode - ){ - int index = 0; - final List alignment = new LinkedList<>(); - final CurrentlyCopying planedToCopy = new CurrentlyCopying(); - final Map assignedPodsByNode = new HashMap<>(); - Map taskCountWhichCopyToNode = new HashMap<>(); - while( !unscheduledTasksSorted.isEmpty() ){ - TaskData taskData = unscheduledTasksSorted.poll(); - boolean changed = false; - log.trace( "TaskData: {}", taskData.getTask().getPod().getName() ); - if ( !taskData.isWeightWasSet() ) { - log.info( "TaskData: {} weight was not set", taskData.getTask().getPod().getName() ); - final NodeLocation nodeForLabel = outLabelHolder.getNodeForLabel(taskData.getTask().getOutLabel()); - if ( nodeForLabel != null ) { - changed = true; - taskData.setNodeAndWeight( nodeForLabel, taskData.getTask().getConfig().getOutLabel().getWeight() ); - } - } - //Calculate the available nodes for this task + their weight. Re-add it to the queue, if it changed - if ( taskData.calculate( availableByNode ) || changed ){ - if ( !taskData.getNodeDataTuples().isEmpty() ) { - unscheduledTasksSorted.add(taskData); - } - continue; - } - - final NodeTaskFilesAlignment nodeAlignment = createNodeAlignment(taskData, availableByNode, assignedPodsByNode, planedToCopy, ++index); - if ( nodeAlignment != null && onlyAllowXCopyToNodeTasks( nodeAlignment, taskCountWhichCopyToNode, getMaxCopyTasksPerNode() ) ) { - alignment.add(nodeAlignment); - outLabelHolder.scheduleTaskOnNode( taskData.getTask(), nodeAlignment.node.getNodeLocation() ); - planedToCopy.addAlignment( nodeAlignment.fileAlignment.getNodeFileAlignment(), taskData.getTask(), nodeAlignment.node ); - } - } - return alignment; - } - - - /** - * Check if the node is already used by x tasks which copy data to it - * @param nodeTaskAlignments - * @param taskCountWhichCopyToNode - * @param maxCopyToNodeTasks - * @return true if the task can be scheduled on that node and no if already maxCopyToNodeTasks tasks copy to that node - */ - private boolean onlyAllowXCopyToNodeTasks( NodeTaskFilesAlignment nodeTaskAlignments, Map taskCountWhichCopyToNode, int maxCopyToNodeTasks ){ - - final Map nodeFileAlignment = nodeTaskAlignments.fileAlignment.getNodeFileAlignment(); - - //Does this task needs to copy data to the node? - boolean copyDataToNode = false; - for ( Map.Entry locationAlignmentWrapperEntry : nodeFileAlignment.entrySet() ) { - if ( !locationAlignmentWrapperEntry.getValue().getFilesToCopy().isEmpty() ) { - copyDataToNode = true; - break; - } - } - - //Only keep if there are no more than maxCopyToNodeTasks - if ( copyDataToNode ) { - final int alreadyAssignedToNode = taskCountWhichCopyToNode.computeIfAbsent( nodeTaskAlignments.node, NodeWithAlloc::getStartingPods ); - if ( alreadyAssignedToNode >= maxCopyToNodeTasks ) { - return false; - } else { - taskCountWhichCopyToNode.put( nodeTaskAlignments.node, alreadyAssignedToNode + 1 ); - } - } - return true; - } - - @Override - public ScheduleObject getTaskNodeAlignment( - final List unscheduledTasks, - final Map availableByNode - ){ - final PriorityQueue unscheduledTasksSorted = unscheduledTasks - .parallelStream() - .map(task -> { - long startTime = System.nanoTime(); - final TaskData taskData = calculateTaskData(task, availableByNode); - if (taskData != null) { - taskData.addNs(System.nanoTime() - startTime); - } - return taskData; - }) - .filter(Objects::nonNull) - .collect(Collectors.toCollection(() -> new PriorityQueue<>(Collections.reverseOrder()))); - final List alignment = createAlignment(unscheduledTasksSorted, availableByNode); - final ScheduleObject scheduleObject = new ScheduleObject(alignment); - scheduleObject.setCheckStillPossible( true ); - scheduleObject.setStopSubmitIfOneFails( true ); - return scheduleObject; - } - - /** - * Performs a stalemate between two possible alignments. - * First check for available cpu, then for the number of running tasks, than for the memory. - * Decide randomly if all three are equal. - * @return True if the new alignment is better - */ - protected boolean stalemate( - NodeWithAlloc oldNode, - NodeWithAlloc newNode, - Map availableByNode, - Map assignedPodsByNode, - Requirements taskRequest ) - { - - //Resources if this node executes the task - final Requirements availableNodeOld = availableByNode.get( oldNode ).sub( taskRequest ); - final Requirements availableNodeNew = availableByNode.get( newNode ).sub( taskRequest ); - - //Calculate percentage of resources that are available - final double availableCpuOld = availableNodeOld.getCpu().doubleValue() / oldNode.getMaxResources().getCpu().doubleValue(); - final double availableCpuNew = availableNodeNew.getCpu().doubleValue() / newNode.getMaxResources().getCpu().doubleValue(); - - double threshold = 0.05; - - if ( availableCpuOld + threshold < availableCpuNew ) { - log.trace( "Node {} has less available CPU than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); - return true; - } else if ( availableCpuOld - threshold > availableCpuNew ) { - log.trace( "Node {} has more available CPU than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); - return false; - } else { - - //CPU load comparable, compare number of running tasks - - final int podsOnNewNode = newNode.getRunningPods() + assignedPodsByNode.getOrDefault( newNode, 0 ); - final int podsOnOldNode = oldNode.getRunningPods() + assignedPodsByNode.getOrDefault( oldNode, 0 ); - - if ( podsOnNewNode < podsOnOldNode ) { - log.trace( "Node {} has less running pods than node {}", newNode.getNodeLocation(), oldNode.getNodeLocation() ); - return true; - } else if ( podsOnNewNode > podsOnOldNode ) { - log.trace( "Node {} has more running pods than node {}", newNode.getNodeLocation(), oldNode.getNodeLocation() ); - return false; - } else { - - //CPU and number of running tasks are comparable, compare memory - - final double availableRamOld = availableNodeOld.getRam().doubleValue() / oldNode.getMaxResources().getRam().doubleValue(); - final double availableRamNew = availableNodeNew.getRam().doubleValue() / newNode.getMaxResources().getRam().doubleValue(); - - if ( availableRamOld + threshold < availableRamNew ) { - log.trace( "Node {} has less available RAM than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); - return true; - } else if ( availableRamOld - threshold > availableRamNew ) { - log.trace( "Node {} has more available RAM than node {}", oldNode.getNodeLocation(), newNode.getNodeLocation() ); - return false; - } else { - - //Everything is comparable, decide randomly - log.trace( "Node {} and node {} are equally comparable -> decide randomly", oldNode.getNodeLocation(), newNode.getNodeLocation() ); - return Math.random() > 0.5; - - } - - } - - } - - } - - Tuple calculateBestNode( - final TaskData taskData, - CurrentlyCopying planedToCopy, - Map availableByNode, - Map assignedPodsByNode - ) { - FileAlignment bestAlignment = null; - NodeWithAlloc bestNode = null; - boolean bestNodeHasOutLabel = false; - final List nodeDataTuples = taskData.getNodeDataTuples(); - int triedNodes = 0; - int noAlignmentFound = 0; - int couldStopFetching = 0; - final List costs = traceEnabled ? new LinkedList<>() : null; - for (NodeDataTuple nodeDataTuple : nodeDataTuples) { - - final NodeWithAlloc currentNode = nodeDataTuple.getNode(); - final CurrentlyCopyingOnNode currentlyCopying = getCurrentlyCopying().get(currentNode.getNodeLocation()); - final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(currentNode.getNodeLocation()); - FileAlignment fileAlignment = null; - try { - fileAlignment = inputAlignment.getInputAlignment( - taskData.getTask(), - taskData.getMatchingFilesAndNodes().getInputsOfTask(), - currentNode, - currentlyCopying, - currentlyPlanedToCopy, - bestAlignment == null ? Double.MAX_VALUE : bestAlignment.getCost() - ); - - if ( fileAlignment == null ){ - couldStopFetching++; - } else { - final boolean isOnOutLabelNode = nodeDataTuple.getNode().getNodeLocation() == taskData.getOutLabelNode(); - if ( isOnOutLabelNode ) { - fileAlignment.setWeight( taskData.getWeight() ); - } - log.info( "Task: {}, outLabelNode: {}, currentNode: {}, bestWeight: {}, currentWeight: {}", - taskData.getTask().getConfig().getName(), - taskData.getOutLabelNode(), - nodeDataTuple.getNode().getNodeLocation(), - bestAlignment == null ? null : bestAlignment.getWorth(), - fileAlignment.getWorth() - ); - - if ( //Not set - bestAlignment == null - || - // better - bestAlignment.getWorth() > fileAlignment.getWorth() - || - //Alignment is comparable - ( bestAlignment.getWorth() + 1e8 > fileAlignment.getWorth() - && - //This is the outLabel node - ( isOnOutLabelNode - || - //Previous node was not the out label node and this alignment wins in stalemate - ( !bestNodeHasOutLabel && stalemate( bestNode, currentNode, availableByNode, assignedPodsByNode, taskData.getTask().getPod().getRequest() ) ) ) - ) - ) { - bestNodeHasOutLabel = isOnOutLabelNode; - bestAlignment = fileAlignment; - bestNode = currentNode; - log.info( "Best alignment for task: {} costs: {}", taskData.getTask().getConfig().getRunName(), fileAlignment.getCost() ); - } - } - } catch ( NoAligmentPossibleException e ){ - noAlignmentFound++; - log.info( "Task: {} - {}", taskData.getTask().getConfig().getName() , e.getMessage() ); - } - if ( traceEnabled ) { - triedNodes++; - final Double thisRoundCost = fileAlignment == null - ? null - : fileAlignment.getCost(); - costs.add( thisRoundCost ); - } - } - - if ( bestAlignment == null ) { - return null; - } - storeTraceData( - taskData.getTask().getTraceRecord(), - triedNodes, - costs, - couldStopFetching, - bestAlignment.getCost(), - noAlignmentFound - ); - return new Tuple<>( bestNode, bestAlignment ); - } - - private void storeTraceData(final TraceRecord traceRecord, int triedNodes, List costs, int couldStopFetching, double bestCost, int noAlignmentFound){ - if ( !traceEnabled ) { - return; - } - traceRecord.setSchedulerNodesTried( triedNodes ); - traceRecord.setSchedulerNodesCost( costs ); - traceRecord.setSchedulerCouldStopFetching( couldStopFetching ); - traceRecord.setSchedulerBestCost( bestCost ); - traceRecord.setSchedulerNoAlignmentFound( noAlignmentFound ); - traceRecord.foundAlignment(); - } - - - /** - * Create a TaskData object for the input Task - * @param task - * @param availableByNode - * @return null if no nodes available - */ - TaskData calculateTaskData( - final Task task, - final Map availableByNode - ) { - final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) { - return null; - } - final TaskInputs inputsOfTask = matchingFilesAndNodes.getInputsOfTask(); - final long size = inputsOfTask.calculateAvgSize(); - final OutLabel outLabel = task.getConfig().getOutLabel(); - final NodeLocation nodeForLabel = outLabel == null ? null : outLabelHolder.getNodeForLabel(outLabel.getLabel()); - final boolean weightWasSet = outLabel == null || nodeForLabel != null; - final boolean nodeForLabelNotNull = nodeForLabel != null; - final double outWeight = outLabel != null ? outLabel.getWeight() : 1.0; - final List nodeDataTuples = matchingFilesAndNodes - .getNodes() - .parallelStream() - .map(node -> { - double weight = (nodeForLabelNotNull && node.getNodeLocation() != nodeForLabel) ? outWeight : 1.0; - return new NodeDataTuple(node, inputsOfTask.calculateDataOnNode(node.getNodeLocation()), weight ); - } ) - .sorted(Comparator.reverseOrder()) - .collect(Collectors.toList()); - final double antiStarvingFactor = 0; - final TaskData taskData = new TaskData(size, task, nodeDataTuples, matchingFilesAndNodes, antiStarvingFactor, weightWasSet); - taskData.setOutLabelNode( nodeForLabel ); - taskData.setWeight( outWeight ); - return taskData; - } - - @Override - boolean assignTaskToNode( NodeTaskAlignment alignment ) { - final NodeTaskFilesAlignment nodeTaskFilesAlignment = (NodeTaskFilesAlignment) alignment; - final WriteConfigResult writeConfigResult = writeInitConfig(nodeTaskFilesAlignment); - if ( writeConfigResult == null ) { - return false; - } - if ( traceEnabled ) { - traceAlignment( nodeTaskFilesAlignment, writeConfigResult ); - } - nodeTaskFilesAlignment.setRemoveInit( !writeConfigResult.isWroteConfig() ); - alignment.task.setCopiedFiles( writeConfigResult.getInputFiles() ); - addToCopyingToNode( alignment.task, alignment.node.getNodeLocation(), writeConfigResult.getCopyingToNode() ); - alignment.task.setCopyingToNode( writeConfigResult.getCopyingToNode() ); - if ( writeConfigResult.isWroteConfig() ) { - getCopyStrategy().generateCopyScript( alignment.task, writeConfigResult.isWroteConfig() ); - } - alignment.task.setCopiesDataToNode( writeConfigResult.isCopyDataToNode() ); - final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); - alignment.task.setInputFiles( allLocationWrappers ); - useLocations( allLocationWrappers ); - return super.assignTaskToNode( alignment ); - } - - private void traceAlignment( NodeTaskFilesAlignment alignment, WriteConfigResult writeConfigResult ) { - final TraceRecord traceRecord = alignment.task.getTraceRecord(); - - int filesOnNodeOtherTask = 0; - int filesNotOnNode = 0; - long filesOnNodeOtherTaskByte = 0; - long filesNotOnNodeByte = 0; - - final NodeLocation currentNode = alignment.node.getNodeLocation(); - for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { - final AlignmentWrapper alignmentWrapper = entry.getValue(); - if( entry.getKey() == currentNode) { - traceRecord.setSchedulerFilesNode( alignmentWrapper.getFilesToCopy().size() + alignmentWrapper.getWaitFor().size() ); - traceRecord.setSchedulerFilesNodeBytes( alignmentWrapper.getToCopySize() + alignmentWrapper.getToWaitSize() ); - } else { - filesOnNodeOtherTask += alignmentWrapper.getWaitFor().size(); - filesOnNodeOtherTaskByte += alignmentWrapper.getToWaitSize(); - filesNotOnNodeByte += alignmentWrapper.getToCopySize(); - filesNotOnNode += alignmentWrapper.getFilesToCopy().size(); - } - } - if (traceRecord.getSchedulerFilesNode() == null) { - traceRecord.setSchedulerFilesNode(0); - } - if (traceRecord.getSchedulerFilesNodeBytes() == null) { - traceRecord.setSchedulerFilesNodeBytes(0l); - } - traceRecord.setSchedulerFilesNodeOtherTask(filesOnNodeOtherTask); - traceRecord.setSchedulerFilesNodeOtherTaskBytes(filesOnNodeOtherTaskByte); - final int schedulerFilesNode = traceRecord.getSchedulerFilesNode() == null ? 0 : traceRecord.getSchedulerFilesNode(); - traceRecord.setSchedulerFiles(schedulerFilesNode + filesOnNodeOtherTask + filesNotOnNode); - final long schedulerFilesNodeBytes = traceRecord.getSchedulerFilesNodeBytes() == null ? 0 : traceRecord.getSchedulerFilesNodeBytes(); - traceRecord.setSchedulerFilesBytes(schedulerFilesNodeBytes + filesOnNodeOtherTaskByte + filesNotOnNodeByte); - traceRecord.setSchedulerDependingTask( (int) writeConfigResult.getWaitForTask().values().stream().distinct().count() ); - traceRecord.setSchedulerNodesToCopyFrom( alignment.fileAlignment.getNodeFileAlignment().size() - (schedulerFilesNode > 0 ? 1 : 0) ); - } - - @Override - public boolean canSchedulePodOnNode( Requirements availableByNode, PodWithAge pod, NodeWithAlloc node ) { - return this.getDaemonIpOnNode( node ) != null && super.canSchedulePodOnNode( availableByNode, pod, node ); - } - - -} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index f5cb3152..b2d80eda 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -413,6 +413,7 @@ public void close() { super.close(); } + @Override Task createTask( TaskConfig conf ){ return new Task( conf, getDag(), hierarchyWrapper ); } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java deleted file mode 100644 index 57f572e3..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/RandomLAScheduler.java +++ /dev/null @@ -1,85 +0,0 @@ -package cws.k8s.scheduler.scheduler; - -import cws.k8s.scheduler.client.KubernetesClient; -import cws.k8s.scheduler.model.NodeWithAlloc; -import cws.k8s.scheduler.model.Requirements; -import cws.k8s.scheduler.model.SchedulerConfig; -import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.scheduler.data.NodeDataTuple; -import cws.k8s.scheduler.scheduler.data.TaskData; -import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import cws.k8s.scheduler.util.FileAlignment; -import cws.k8s.scheduler.util.Tuple; -import cws.k8s.scheduler.util.copying.CurrentlyCopying; -import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class RandomLAScheduler extends LocationAwareScheduler { - - private final Random random = new Random(); - - public RandomLAScheduler( - String name, - KubernetesClient client, - String namespace, - SchedulerConfig config, - InputAlignment inputAlignment - ) { - super(name, client, namespace, config, inputAlignment); - } - - private Optional selectNode( List matchingNodes, Task task ){ - if (matchingNodes.isEmpty()) return Optional.empty(); - else { - return Optional.of( new LinkedList<>(matchingNodes).get(random.nextInt(matchingNodes.size())).getNode()); - } - } - - @Override - Tuple calculateBestNode( - final TaskData taskData, - CurrentlyCopying planedToCopy, - Map availableByNode, - Map assignedPodsByNode - ){ - final Optional nodeWithAlloc = selectNode(taskData.getNodeDataTuples(), taskData.getTask()); - if (nodeWithAlloc.isEmpty()) { - return null; - } - final NodeWithAlloc node = nodeWithAlloc.get(); - final CurrentlyCopyingOnNode currentlyCopying = getCurrentlyCopying().get(node.getNodeLocation()); - final CurrentlyCopyingOnNode currentlyPlanetToCopy = planedToCopy.get(node.getNodeLocation()); - final FileAlignment fileAlignment = getInputAlignment().getInputAlignment( - taskData.getTask(), - taskData.getMatchingFilesAndNodes().getInputsOfTask(), - node, - currentlyCopying, - currentlyPlanetToCopy - ); - return new Tuple<>( node, fileAlignment ); - } - - @Override - TaskData calculateTaskData( - final Task task, - final Map availableByNode - ) { - final MatchingFilesAndNodes matchingFilesAndNodes = getMatchingFilesAndNodes(task, availableByNode); - if ( matchingFilesAndNodes == null || matchingFilesAndNodes.getNodes().isEmpty() ) { - return null; - } - final List nodeDataTuples = matchingFilesAndNodes - .getNodes() - .parallelStream() - .map(node -> new NodeDataTuple(node, 0 ) ) - .collect(Collectors.toList()); - //Do not set any weights, as it is randomly scheduled - return new TaskData( 0, task, nodeDataTuples, matchingFilesAndNodes, 0, true ); - } - - -} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 9cb142e7..8dd045c3 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -19,8 +19,6 @@ import cws.k8s.scheduler.rest.response.getfile.FileResponse; import cws.k8s.scheduler.scheduler.copystrategy.CopyStrategy; import cws.k8s.scheduler.scheduler.copystrategy.FTPstrategy; -import cws.k8s.scheduler.scheduler.outlabel.OutLabelHolder; -import cws.k8s.scheduler.scheduler.outlabel.HolderMaxTasks; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import io.fabric8.kubernetes.api.model.*; @@ -50,7 +48,6 @@ public abstract class SchedulerWithDaemonSet extends Scheduler { private final InputFileCollector inputFileCollector; private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); final String localWorkDir; - protected final OutLabelHolder outLabelHolder = new HolderMaxTasks() ; /** * Which node is currently copying files from which node diff --git a/src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java b/src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java deleted file mode 100644 index e24c8081..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/data/NodeDataTuple.java +++ /dev/null @@ -1,33 +0,0 @@ -package cws.k8s.scheduler.scheduler.data; - -import cws.k8s.scheduler.model.NodeWithAlloc; -import lombok.*; -import org.jetbrains.annotations.NotNull; - -@Getter -@EqualsAndHashCode -@AllArgsConstructor -public class NodeDataTuple implements Comparable { - - private final NodeWithAlloc node; - private final long sizeInBytes; - @Getter(AccessLevel.NONE) - @Setter - private double weight; - - public NodeDataTuple( NodeWithAlloc node, long sizeInBytes ) { - this( node, sizeInBytes, 1.0 ); - } - - /** - * @return reduce the worth, if this is not the outLabelNode - */ - public double getWorth() { - return getSizeInBytes() / weight; - } - - @Override - public int compareTo(@NotNull NodeDataTuple o) { - return Double.compare( getWorth(), o.getWorth() ); - } -} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java deleted file mode 100644 index d33ea6a5..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskData.java +++ /dev/null @@ -1,138 +0,0 @@ -package cws.k8s.scheduler.scheduler.data; - -import cws.k8s.scheduler.scheduler.MatchingFilesAndNodes; -import cws.k8s.scheduler.model.NodeWithAlloc; -import cws.k8s.scheduler.model.Requirements; -import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; -import lombok.AccessLevel; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import org.jetbrains.annotations.NotNull; - -import java.util.Iterator; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; - -@Getter -@EqualsAndHashCode -public class TaskData implements Comparable { - - private final double size; - - private final Task task; - - private final List nodeDataTuples; - - private final MatchingFilesAndNodes matchingFilesAndNodes; - - private long timeInNs = 0; - - private final double antiStarvingFactor; - - @Getter(AccessLevel.NONE) - private NodeDataTuple calulatedFor = null; - - private double value = 0; - - private boolean weightWasSet; - - @Setter - private NodeLocation outLabelNode = null; - - @Setter - private double weight = 1.0; - - public TaskData( - double size, - Task task, - List nodeDataTuples, - MatchingFilesAndNodes matchingFilesAndNodes, - double antiStarvingFactor, - boolean weightWasSet ) { - this.size = size; - this.task = task; - this.nodeDataTuples = nodeDataTuples; - this.matchingFilesAndNodes = matchingFilesAndNodes; - this.antiStarvingFactor = antiStarvingFactor; - this.weightWasSet = weightWasSet; - calc(); - } - - - /** - * After a location for a label was found, adjust the nodes. - * @param nodeForLabel - * @param weight - */ - public void setNodeAndWeight(NodeLocation nodeForLabel, double weight ) { - this.outLabelNode = nodeForLabel; - this.weight = weight; - final Iterator iterator = nodeDataTuples.iterator(); - NodeDataTuple onOutLabelNode = null; - while ( iterator.hasNext() ) { - final NodeDataTuple next = iterator.next(); - if (next.getNode().getNodeLocation() != nodeForLabel ) { - next.setWeight( weight ); - } else { - onOutLabelNode = next; - iterator.remove(); - } - } - //Insert the one element manuel. Sorting would be too expensive. - if ( onOutLabelNode != null ) { - final ListIterator listIterator = nodeDataTuples.listIterator(); - while ( listIterator.hasNext() ) { - if ( listIterator.next().getWorth() <= onOutLabelNode.getWorth() ) { - listIterator.previous(); - listIterator.add( onOutLabelNode ); - break; - } - } - } - weightWasSet = true; - } - - /** - * Recalculate the value. New value is smaller or equal old value - * Delete all nodes that have not enough unassigned resources for the task. - * @param availableByNode - * @return if the value has changed - */ - public boolean calculate( Map availableByNode ) { - final Iterator iterator = nodeDataTuples.iterator(); - boolean changed = false; - while (iterator.hasNext()) { - if ( !availableByNode.get( iterator.next().getNode() ).higherOrEquals( task.getPod().getRequest() ) ) { - iterator.remove(); - changed = true; - } else { - break; - } - } - calc(); - return changed; - } - - private void calc() { - if ( nodeDataTuples.isEmpty() ) { - value = Double.MIN_VALUE; - } else if ( size == 0 ) { - value = Double.MAX_VALUE; - } else { - value = size;//nodeDataTuples.get(0).getSizeInBytes() / size; - } - } - - @Override - public int compareTo(@NotNull TaskData o) { - return Double.compare(getValue(), o.getValue()); - } - - public void addNs( long timeInNs ){ - this.timeInNs += timeInNs; - } - -} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java deleted file mode 100644 index 9edc82d9..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/outlabel/HolderMaxTasks.java +++ /dev/null @@ -1,22 +0,0 @@ -package cws.k8s.scheduler.scheduler.outlabel; - -import cws.k8s.scheduler.model.Task; - -import java.util.Set; - -public class HolderMaxTasks extends OutLabelHolder { - - @Override - protected InternalHolderMaxTasks create() { - return new InternalHolderMaxTasks(); - } - - private class InternalHolderMaxTasks extends InternalHolder { - - @Override - protected double calculateValue( Set input ) { - return input.size(); - } - - } -} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java deleted file mode 100644 index 88d7c997..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/outlabel/InternalHolder.java +++ /dev/null @@ -1,43 +0,0 @@ -package cws.k8s.scheduler.scheduler.outlabel; - -import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -@Slf4j -abstract class InternalHolder { - - private double bestValue = Double.MIN_VALUE; - private final Set currentlyBestNodes = new HashSet<>(); - private final Map> tasksByNode = new HashMap<>(); - - public void addTask(Task task, NodeLocation node) { - synchronized ( tasksByNode ) { - final Set tasks = tasksByNode.computeIfAbsent(node, key -> new HashSet<>()); - tasks.add(task); - final double calculatedValue = calculateValue(tasks); - if (calculatedValue > bestValue) { - bestValue = calculatedValue; - currentlyBestNodes.clear(); - } - if (calculatedValue >= bestValue) { - currentlyBestNodes.add(node); - } - } - } - - protected abstract double calculateValue( Set input ); - - public Set getBestNode() { - synchronized ( tasksByNode ){ - log.info("Best nodes: {}", currentlyBestNodes); - return currentlyBestNodes; - } - } - -} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java b/src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java deleted file mode 100644 index dcc81dbd..00000000 --- a/src/main/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolder.java +++ /dev/null @@ -1,41 +0,0 @@ -package cws.k8s.scheduler.scheduler.outlabel; - -import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; - -import java.util.*; - -/** - * The holder cannot be updated, if scheduling for a task fails. - */ -public abstract class OutLabelHolder { - - protected HashMap internalHolder = new HashMap<>(); - - /** - * Get Node with most Tasks running on it - * @param outLabel - * @return Null if no node was determined until now - */ - public NodeLocation getNodeForLabel(String outLabel) { - if ( outLabel == null ) { - return null; - } - final InternalHolder holder = internalHolder.get(outLabel); - return holder == null ? null : holder.getBestNode().stream().findFirst().orElse(null); - } - - public void scheduleTaskOnNode(Task task, NodeLocation node ){ - final String outLabel; - if ( (outLabel = task.getOutLabel()) == null ) { - return; - } - internalHolder.computeIfAbsent( outLabel, key -> create() ).addTask( task, node ); - } - - /** - * @return new Instance of the required Holder - */ - protected abstract InternalHolder create(); - -} diff --git a/src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java b/src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java deleted file mode 100644 index 64b76a6d..00000000 --- a/src/test/java/cws/k8s/scheduler/scheduler/outlabel/OutLabelHolderMaxTasksTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package cws.k8s.scheduler.scheduler.outlabel; - -import cws.k8s.scheduler.model.TaskConfig; -import cws.k8s.scheduler.dag.DAG; -import cws.k8s.scheduler.dag.Process; -import cws.k8s.scheduler.dag.Vertex; -import cws.k8s.scheduler.model.OutLabel; -import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; -import org.junit.jupiter.api.Test; - -import java.util.LinkedList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -class OutLabelHolderMaxTasksTest { - - private class TaskConfigMock extends TaskConfig { - - public TaskConfigMock(String task) { - super(task); - } - - @Override - public OutLabel getOutLabel() { - return new OutLabel( "a", 2.0 ); - } - } - - @Test - void determineBestNode() { - final HolderMaxTasks outLabelHolderMaxTasks = new HolderMaxTasks(); - DAG dag = new DAG(); - List vertexList = new LinkedList<>(); - vertexList.add(new Process("processA", 1)); - dag.registerVertices(vertexList); - final TaskConfig taskConfig = new TaskConfigMock("processA"); - final NodeLocation nodeA = NodeLocation.getLocation("nodeA"); - final NodeLocation nodeB = NodeLocation.getLocation("nodeB"); - outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeA); - assertEquals( nodeA, outLabelHolderMaxTasks.getNodeForLabel( "a" ) ); - outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeB); - outLabelHolderMaxTasks.scheduleTaskOnNode( new Task( taskConfig, dag ), nodeB); - assertEquals( nodeB, outLabelHolderMaxTasks.getNodeForLabel( "a" ) ); - } - -} \ No newline at end of file From 603506369fcc1a4e9da3ba2ccd4630639941973b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 21 Jun 2024 14:57:36 +0200 Subject: [PATCH 411/443] Prepare Task grouping Signed-off-by: Lehmann_Fabian --- .../scheduler/model/cluster/GroupCluster.java | 13 ++++++++ .../LocationAwareSchedulerGroups.java | 33 +++++++++++++++++++ .../k8s/scheduler/scheduler/Scheduler.java | 14 ++++++++ 3 files changed, 60 insertions(+) create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java create mode 100644 src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java new file mode 100644 index 00000000..4eee2ecb --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java @@ -0,0 +1,13 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.Task; + +import java.util.List; + +public interface GroupCluster { + + void tasksBecameAvailable( List tasks ); + + void taskWasAssignedToNode( Task task ); + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java new file mode 100644 index 00000000..47bbcdbb --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java @@ -0,0 +1,33 @@ +package cws.k8s.scheduler.scheduler; + +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.SchedulerConfig; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.cluster.GroupCluster; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; + +import java.util.List; + +public class LocationAwareSchedulerGroups extends LocationAwareSchedulerV2 { + + GroupCluster groupCluster; + + public LocationAwareSchedulerGroups( String name, KubernetesClient client, String namespace, SchedulerConfig config, InputAlignment inputAlignment, ReadyToRunToNode readyToRunToNode ) { + super( name, client, namespace, config, inputAlignment, readyToRunToNode ); + } + + @Override + protected void tasksWhereAddedToQueue( List newTasks ){ + super.tasksWhereAddedToQueue( newTasks ); + groupCluster.tasksBecameAvailable( newTasks ); + } + + @Override + void taskWasScheduled(Task task ) { + super.taskWasScheduled( task ); + groupCluster.taskWasAssignedToNode( task ); + } + + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index 0c7bd951..7fcb8ed3 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -211,6 +211,7 @@ public void schedulePod(PodWithAge pod ) { if ( task.getBatch() == null ){ synchronized (unscheduledTasks){ unscheduledTasks.add( task ); + tasksWhereAddedToQueue( List.of(task) ); unscheduledTasks.notifyAll(); synchronized ( upcomingTasks ){ upcomingTasks.remove( task ); @@ -233,6 +234,7 @@ private void tryToScheduleBatch( Batch batch ){ synchronized (unscheduledTasks){ final List tasksToScheduleAndDestroy = batch.getTasksToScheduleAndDestroy(); unscheduledTasks.addAll(tasksToScheduleAndDestroy); + tasksWhereAddedToQueue( tasksToScheduleAndDestroy ); unscheduledTasks.notifyAll(); synchronized ( upcomingTasks ){ tasksToScheduleAndDestroy.forEach(upcomingTasks::remove); @@ -241,6 +243,18 @@ private void tryToScheduleBatch( Batch batch ){ } } + /** + * This methode is called whenever new task(s) are available to be scheduled. + * E.g. when a batch is full. + * The methode is called, before the scheduling is performed. + * @param newTasks the tasks that are now ready. + */ + protected void tasksWhereAddedToQueue( List newTasks ){} + + /** + * This method is called whenever a task was scheduled and the assignment is final. + * @param task the task that was scheduled + */ void taskWasScheduled(Task task ) { synchronized (unscheduledTasks){ unscheduledTasks.remove( task ); From 2f1c09f735aa66be8683469ad0741fcec75c09b0 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 25 Jun 2024 15:06:01 +0200 Subject: [PATCH 412/443] Move affinityMatch method from Scheduler to Node Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/NodeWithAlloc.java | 21 ++++++++++++++++++ .../scheduler/model/cluster/GroupCluster.java | 13 ----------- .../scheduler/LocationAwareSchedulerV2.java | 2 +- .../k8s/scheduler/scheduler/Scheduler.java | 22 +------------------ 4 files changed, 23 insertions(+), 35 deletions(-) delete mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java diff --git a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java index 6f8465e9..4c86f52f 100644 --- a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java @@ -181,4 +181,25 @@ public int hashCode() { public boolean isReady(){ return Readiness.isNodeReady(this); } + + public boolean affinitiesMatch( PodWithAge pod ){ + + final boolean nodeCouldRunThisPod = this.getMaxResources().higherOrEquals( pod.getRequest() ); + if ( !nodeCouldRunThisPod ){ + return false; + } + + final Map podsNodeSelector = pod.getSpec().getNodeSelector(); + final Map nodesLabels = this.getMetadata().getLabels(); + if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) { + return true; + } + //cannot be fulfilled if podsNodeSelector is not empty + if ( nodesLabels == null || nodesLabels.isEmpty() ) { + return false; + } + + return nodesLabels.entrySet().containsAll( podsNodeSelector.entrySet() ); + } + } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java deleted file mode 100644 index 4eee2ecb..00000000 --- a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java +++ /dev/null @@ -1,13 +0,0 @@ -package cws.k8s.scheduler.model.cluster; - -import cws.k8s.scheduler.model.Task; - -import java.util.List; - -public interface GroupCluster { - - void tasksBecameAvailable( List tasks ); - - void taskWasAssignedToNode( Task task ); - -} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index b2d80eda..7bbf1172 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -358,7 +358,7 @@ private TaskStat getDataOnNode( Task task, TaskInputs inputsOfTask, List podsNodeSelector = pod.getSpec().getNodeSelector(); - final Map nodesLabels = node.getMetadata().getLabels(); - if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) { - return true; - } - //cannot be fulfilled if podsNodeSelector is not empty - if ( nodesLabels == null || nodesLabels.isEmpty() ) { - return false; - } - - return nodesLabels.entrySet().containsAll( podsNodeSelector.entrySet() ); + return node.canScheduleNewPod() && node.affinitiesMatch( pod ); } public void newNode(NodeWithAlloc node) { From cf7a01af62d8473ec526292eb7f9594c471b5dbe Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 1 Jul 2024 13:26:11 +0200 Subject: [PATCH 413/443] change signature of calculateScore to also consider the node Signed-off-by: Lehmann_Fabian --- .../CopyInAdvanceNodeWithMostDataIntelligent.java | 2 +- .../scheduler/la2/ready2run/OptimalReadyToRunToNode.java | 2 +- .../java/cws/k8s/scheduler/util/score/CalculateScore.java | 3 ++- .../java/cws/k8s/scheduler/util/score/FileSizeRankScore.java | 5 +++-- .../java/cws/k8s/scheduler/util/score/FileSizeScore.java | 3 ++- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index 82f1237f..1e197166 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -192,7 +192,7 @@ private TaskWithScore getTaskWithScore( Task task, String error ) { .calculateDataOnNode( node.getNodeLocation() ); final long dataInSharedFS = task.getInputSize(); //The input should be similar to the first scheduling phase - return new TaskWithScore( task, calculateScore.getScore( task, dataOnNode + dataInSharedFS ) ); + return new TaskWithScore( task, calculateScore.getScore( task, node.getNodeLocation(), dataOnNode + dataInSharedFS ) ); } void addPlaned( Task task ) { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index c7256cbf..28a57adf 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -64,7 +64,6 @@ public List createAlignmentForTasksWithAllDataOnNod int index = 0; for ( TaskInputsNodes taskInputsNodes : taskWithAllData ) { List onlyOnOneNode = new ArrayList<>(); - final long score = calculateScore.getScore( taskInputsNodes.getTask(), taskInputsNodes.getTaskSize() ); final Requirements request = taskInputsNodes.getTask().getRequest(); final long ram = request.getRam().longValue(); final long cpu = request.getCpu().multiply( MILLION ).longValue(); @@ -76,6 +75,7 @@ public List createAlignmentForTasksWithAllDataOnNod onlyOnOneNode.add( boolVar ); memUsed.get(node).addTerm( boolVar, ram ); cpuUsed.get(node).addTerm( boolVar, cpu ); + final long score = calculateScore.getScore( taskInputsNodes.getTask(), node.getNodeLocation(), taskInputsNodes.getTaskSize() ); objective.addTerm( boolVar, score ); taskNodeBoolVars.add( new TaskNodeBoolVar( taskInputsNodes, node, boolVar ) ); } diff --git a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java index c9d3ccce..00015d27 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java @@ -1,6 +1,7 @@ package cws.k8s.scheduler.util.score; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; public interface CalculateScore { @@ -8,6 +9,6 @@ public interface CalculateScore { * Score must be higher than 0 * @return */ - long getScore( Task task, long inputSize ); + long getScore( Task task, NodeLocation location, long inputSize ); } diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java index f9227b82..cfdd0d74 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -1,15 +1,16 @@ package cws.k8s.scheduler.util.score; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; public class FileSizeRankScore extends FileSizeScore { @Override - public long getScore( Task task, long size ) { + public long getScore( Task task, NodeLocation location, long size ) { //Add one to avoid becoming zero final int rank = task.getProcess().getRank() + 1; final long rankFactor = 100_000_000_000_000L * rank; // long would allow a rank of 92233 - return super.getScore( task, size ) + rankFactor ; + return super.getScore( task, location, size ) + rankFactor ; } } diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index c416dffe..695c3b65 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -1,11 +1,12 @@ package cws.k8s.scheduler.util.score; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; public class FileSizeScore implements CalculateScore { @Override - public long getScore( Task task, long size ) { + public long getScore( Task task, NodeLocation location, long size ) { //add one to prefer two tasks which sum up to the same score otherwise return size + 1; } From dc61f33e91d2fe039992bb3aff24813b0586413e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Tue, 2 Jul 2024 11:49:28 +0200 Subject: [PATCH 414/443] Make outLabels a set Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/Task.java | 3 ++- src/main/java/cws/k8s/scheduler/model/TaskConfig.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index 6da9055f..9ddbcb2f 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -14,6 +14,7 @@ import java.io.File; import java.nio.file.Path; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -111,7 +112,7 @@ public void submitted(){ traceRecord.setSchedulerTimeInQueue( System.currentTimeMillis() - timeAddedToQueue ); } - public String[] getOutLabel(){ + public Set getOutLabel(){ return config.getOutLabel(); } diff --git a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java index 49bb984a..3e162f4c 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java @@ -8,6 +8,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; @Getter @ToString @@ -20,7 +21,7 @@ public class TaskConfig { private final String runName; private final float cpus; private final long memoryInBytes; - private final String[] outLabel; + private final Set outLabel; private final String workDir; private TaskConfig() { From da73c704e27389de710512c933f6e2e6a17afe57 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 3 Jul 2024 11:44:39 +0200 Subject: [PATCH 415/443] Refactor LAV2 Scheduler to set phaseTwo and phaseThreeComperator by inheriting class Signed-off-by: Lehmann_Fabian --- .../scheduler/LocationAwareSchedulerV2.java | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 7bbf1172..dffa7238 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -52,8 +52,10 @@ public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { private final CopyInAdvance copyInAdvance; - private final TaskStatComparator phaseTwoComparator; - private final TaskStatComparator phaseThreeComparator; + @Setter(AccessLevel.PACKAGE) + private TaskStatComparator phaseTwoComparator; + @Setter(AccessLevel.PACKAGE) + private TaskStatComparator phaseThreeComparator; private final int copySameTaskInParallel; private final int maxHeldCopyTaskReady; @@ -79,20 +81,42 @@ public LocationAwareSchedulerV2( String namespace, SchedulerConfig config, InputAlignment inputAlignment, - ReadyToRunToNode readyToRunToNode ) { + ReadyToRunToNode readyToRunToNode + ) { + this( + name, + client, + namespace, + config, + inputAlignment, + readyToRunToNode, + new FileSizeRankScore() + ); + setPhaseTwoComparator( new MinCopyingComparator( MinSizeComparator.INSTANCE ) ); + setPhaseThreeComparator( new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ) ); + } + + LocationAwareSchedulerV2( + String name, + KubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment, + ReadyToRunToNode readyToRunToNode, + FileSizeRankScore calculateScore + ) { super( name, client, namespace, config ); this.inputAlignment = inputAlignment; this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; this.readyToRunToNode = readyToRunToNode; - final FileSizeRankScore calculateScore = new FileSizeRankScore(); - this.readyToRunToNode.init( calculateScore ); + if ( calculateScore != null ) { + this.readyToRunToNode.init( calculateScore ); + } readyToRunToNode.setLogger( logCopyTask ); this.copyRunner = new ShellCopy( client, this, logCopyTask ); this.copySameTaskInParallel = 2; this.capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); - this.phaseTwoComparator = new MinCopyingComparator( MinSizeComparator.INSTANCE ); - this.phaseThreeComparator = new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ); this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); this.maxHeldCopyTaskReady = config.maxHeldCopyTaskReady == null ? 3 : config.maxHeldCopyTaskReady; this.prioPhaseThree = config.prioPhaseThree == null ? 70 : config.prioPhaseThree; From 5c305eb9d0193b7131430b1f1b2102384675535b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 3 Jul 2024 11:47:03 +0200 Subject: [PATCH 416/443] Make new LAGroups scheduler usable Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/rest/SchedulerRestController.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index ef9a2548..546ec2cf 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -128,6 +128,15 @@ ResponseEntity registerScheduler( } scheduler = new LocationAwareSchedulerV2( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ), new OptimalReadyToRunToNode() ); break; + case "lagroup" : + if ( !config.locationAware ) { + return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); + } + if ( costFunction == null ) { + costFunction = new MinSizeCost( 0 ); + } + scheduler = new LocationAwareSchedulerGroups( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ), new OptimalReadyToRunToNode() ); + break; default: { final String[] split = strategy.split( "-" ); Prioritize prioritize; From 2b5621efa24736c1056d73516522b832f107b876 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 8 Jul 2024 12:04:41 +0200 Subject: [PATCH 417/443] First version of OutLabel aware scheduling Signed-off-by: Lehmann_Fabian --- .../model/cluster/FileSizeRankScoreGroup.java | 23 ++++ .../scheduler/model/cluster/GroupCluster.java | 122 ++++++++++++++++++ .../scheduler/model/cluster/LabelCount.java | 73 +++++++++++ .../cluster/MostOutLabelsComparator.java | 34 +++++ .../model/cluster/SimpleGroupCluster.java | 116 +++++++++++++++++ .../model/cluster/TasksOnNodeWrapper.java | 27 ++++ .../LocationAwareSchedulerGroups.java | 21 ++- 7 files changed, 414 insertions(+), 2 deletions(-) create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java new file mode 100644 index 00000000..f0fc3315 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java @@ -0,0 +1,23 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.util.score.FileSizeRankScore; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class FileSizeRankScoreGroup extends FileSizeRankScore { + + private final GroupCluster groupCluster; + + @Override + public long getScore( Task task, NodeLocation location, long size ) { + final long score = super.getScore( task, location, size ); + final long newScore = (long) (score * groupCluster.getScoreForTaskOnNode( task, location )); + if ( newScore < 1 ) { + return 1; + } + return newScore; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java new file mode 100644 index 00000000..12347d32 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java @@ -0,0 +1,122 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; + +import java.util.*; + +public abstract class GroupCluster { + + protected final Set allLabels = new HashSet<>(); + protected final Map countPerLabel = new HashMap<>(); + protected final LinkedList unscheduledTasks = new LinkedList<>(); + protected final LinkedList scheduledTasks = new LinkedList<>(); + protected final LinkedList finishedTasks = new LinkedList<>(); + protected final Map labelToNode = new HashMap<>(); + protected final Map> nodeToLabel = new HashMap<>(); + + private void addAllOutLabels( List tasks ) { + for ( Task task : tasks ) { + if ( task.getOutLabel() != null ) { + allLabels.addAll( task.getOutLabel() ); + } + } + } + + public void tasksBecameAvailable( List tasks ) { + boolean anyWithLabel = false; + synchronized ( this ) { + unscheduledTasks.addAll( tasks ); + addAllOutLabels( tasks ); + for ( Task task : tasks ) { + if ( task.getOutLabel() == null || task.getOutLabel().isEmpty() ) { + continue; + } + for ( String label : task.getOutLabel() ) { + anyWithLabel = true; + countPerLabel.computeIfAbsent( label, k -> new LabelCount() ).addWaitingTask( task ); + final LabelCount labelCount; + if ( !countPerLabel.containsKey( label ) ) { + labelCount = new LabelCount(); + countPerLabel.put( label, labelCount ); + } else { + labelCount = countPerLabel.get( label ); + } + labelCount.addWaitingTask( task ); + } + } + if ( anyWithLabel ) { + recalculate(); + } + } + } + + public void taskWasAssignedToNode( Task task ) { + synchronized ( this ) { + unscheduledTasks.remove( task ); + scheduledTasks.add( task ); + if ( task.getOutLabel() != null ) { + for ( String label : task.getOutLabel() ) { + countPerLabel.get( label ).makeTaskRunning( task ); + } + recalculate(); + } + } + } + + public void tasksHaveFinished( List tasks ) { + synchronized ( this ) { + scheduledTasks.removeAll( tasks ); + finishedTasks.addAll( tasks ); + for ( Task task : tasks ) { + if ( task.getOutLabel() == null ) { + continue; + } + for ( String label : task.getOutLabel() ) { + if ( countPerLabel.get( label ) == null ) { + continue; + } + countPerLabel.get( label ).makeTaskFinished( task ); + } + } + } + } + + abstract void recalculate(); + + public double getScoreForTaskOnNode( Task task, NodeLocation nodeLocation ) { + if ( task.getOutLabel() == null || task.getOutLabel().isEmpty() ) { + return 1; + } + //Jaccard similarity coefficient + final Set outLabel = new HashSet<>( task.getOutLabel() ); + int outLabelSize = outLabel.size(); + final Set nodeLabels = nodeToLabel.get( nodeLocation ); + if ( nodeLabels == null ) { + return 0.01; + } + outLabel.retainAll( nodeLabels ); + if ( outLabelSize == 0 ) { + return 1; + } + return (double) outLabel.size() / outLabelSize + 0.01; // to avoid zero + } + + protected void addNodeToLabel( NodeLocation nodeLocation, String label ) { + final NodeLocation currentLocation = labelToNode.get( label ); + + if ( nodeLocation.equals( currentLocation ) ) { + return; + } + + Set labels = nodeToLabel.computeIfAbsent( nodeLocation, k -> new HashSet<>() ); + labels.add( label ); + + if ( currentLocation != null ) { + labels = nodeToLabel.get( currentLocation ); + labels.remove( label ); + } + labelToNode.put( label, nodeLocation ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java new file mode 100644 index 00000000..104dc393 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java @@ -0,0 +1,73 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.*; + +@NoArgsConstructor( access = AccessLevel.PACKAGE ) +public class LabelCount { + + @Getter + private int countFinished = 0; + @Getter + private int countStarted = 0; + @Getter + private int countWaiting = 0; + + @Getter + private final Set waitingTasks = new HashSet<>(); + private final Set runningTasks = new HashSet<>(); + private final Set finishedTasks = new HashSet<>(); + private final Set tasks = new HashSet<>(); + @Getter + private final Queue runningOrfinishedOnNodes = new PriorityQueue<>(); + private final Map nodeToShare = new HashMap<>(); + private final Map> taskHasDataOnNode = new HashMap<>(); + + /** + * Get the number of tasks with this label + * @return the number of tasks with this label + */ + public int getCount(){ + return countFinished + countStarted + countWaiting; + } + + public void addWaitingTask( Task task ){ + countWaiting++; + tasks.add(task); + waitingTasks.add(task); + } + + public void makeTaskRunning( Task task ) { + final NodeWithAlloc node = task.getNode(); + final TasksOnNodeWrapper tasksOnNodeWrapper; + if ( nodeToShare.containsKey( node ) ) { + tasksOnNodeWrapper = nodeToShare.get( node ); + } else { + tasksOnNodeWrapper = new TasksOnNodeWrapper( node.getNodeLocation() ); + nodeToShare.put( node, tasksOnNodeWrapper ); + runningOrfinishedOnNodes.add( tasksOnNodeWrapper ); + } + tasksOnNodeWrapper.addRunningTask(); + countWaiting--; + countStarted++; + waitingTasks.remove( task ); + runningTasks.add( task ); + } + + public void makeTaskFinished( Task task ) { + countStarted--; + countFinished++; + runningTasks.remove( task ); + finishedTasks.add( task ); + final HashSet locations = new HashSet<>(); + locations.add( task.getNode().getNodeLocation() ); + taskHasDataOnNode.put( task, locations ); + } + + } \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java b/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java new file mode 100644 index 00000000..aae7903f --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java @@ -0,0 +1,34 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.RequiredArgsConstructor; + +import java.util.Comparator; + +@RequiredArgsConstructor +public class MostOutLabelsComparator implements Comparator { + + private final GroupCluster groupCluster; + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + //same task does not necessarily have the same size + if ( o1.getTask() == o2.getTask() || o1Stats.getTaskSize() == o2Stats.getTaskSize() ) { + double ratingO1 = groupCluster.getScoreForTaskOnNode( o1.getTask(), o1.getNode().getNodeLocation() ); + double ratingO2 = groupCluster.getScoreForTaskOnNode( o2.getTask(), o2.getNode().getNodeLocation() ); + // Use the node with a higher rating + if ( ratingO1 != ratingO2 ) { + return Double.compare( ratingO2, ratingO1 ); + } + //Prefer the one with fewer remaining data + return Long.compare( o1Stats.getSizeRemaining(), o2Stats.getSizeRemaining() ); + } else { + //Prefer the one with larger task size + return Long.compare( o2Stats.getTaskSize(), o1Stats.getTaskSize() ); + } + } + +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java new file mode 100644 index 00000000..83c1e8f6 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java @@ -0,0 +1,116 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.location.NodeLocation; +import lombok.RequiredArgsConstructor; + +import java.util.*; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +public class SimpleGroupCluster extends GroupCluster { + + private final KubernetesClient client; + + @Override + void recalculate() { + final List allNodes = client.getAllNodes(); + + // Filter labels that have waiting tasks. + final List> labelsWithWaitingTasks = countPerLabel + .entrySet() + .stream() + // If nothing waiting, we do not need a suitable node, so we do not reassign it + // If there is only one task with a label it can run where ever resources are available. Do not force a location. + .filter( kv -> kv.getValue().getCountWaiting() > 0 && kv.getValue().getCount() > 1 ) + // Sort reverse by count + .sorted( Comparator.comparingInt( kv -> - kv.getValue().getCount() ) ) + .collect( Collectors.toList() ); + + // store how many tasks would have been executed on every with the current alignment + // this is an approximation since tasks can have multiple labels and would appear multiple times + Map tasksOnNode = new HashMap<>(); + + // For every label and node count how many tasks could run (affinities match), we ignore the current load completely + for ( Map.Entry labelWithWaitingTasks : labelsWithWaitingTasks ) { + Queue xTasksCanRunOnNode = new PriorityQueue<>(); + for ( NodeWithAlloc node : allNodes ) { + final long count = labelWithWaitingTasks + .getValue() + .getWaitingTasks() + .stream() + .filter( task -> node.affinitiesMatch( task.getPod() ) ) + .count(); + if ( count > 0 ) { + final TasksOnNodeWrapper tasksOnNodeWrapper = new TasksOnNodeWrapper( node.getNodeLocation(), (int) count ); + xTasksCanRunOnNode.add( tasksOnNodeWrapper ); + } + } + if ( !xTasksCanRunOnNode.isEmpty() ) { + final NodeLocation nodeLocation = calculateBestFittingNode( labelWithWaitingTasks.getKey(), xTasksCanRunOnNode, labelWithWaitingTasks.getValue(), tasksOnNode ); + if ( nodeLocation != null ) { + addNodeToLabel( nodeLocation, labelWithWaitingTasks.getKey() ); + tasksOnNode.put( nodeLocation, tasksOnNode.getOrDefault( nodeLocation, 0 ) + labelWithWaitingTasks.getValue().getCountWaiting() ); + } + } + } + } + + /** + * This method first looks which nodes ran the most tasks with the current label already and then selects the node where the most tasks can run. + * @param label + * @param xTasksCanRunOnNode + * @param labelCount + * @param tasksOnNode + * @return + */ + private NodeLocation calculateBestFittingNode( String label, Queue xTasksCanRunOnNode, LabelCount labelCount, Map tasksOnNode ) { + if ( xTasksCanRunOnNode.isEmpty() ) { + return null; + } + final Set bestFittingNodes = new HashSet<>(); + final TasksOnNodeWrapper bestFittingNode = xTasksCanRunOnNode.poll(); + bestFittingNodes.add( bestFittingNode.getNode() ); + while( !xTasksCanRunOnNode.isEmpty() && xTasksCanRunOnNode.peek().getShare() == bestFittingNode.getShare() ){ + bestFittingNodes.add( xTasksCanRunOnNode.poll().getNode() ); + } + + final List runningOrfinishedOnNodes = new ArrayList<>(labelCount.getRunningOrfinishedOnNodes()); + if ( runningOrfinishedOnNodes.isEmpty() ) { + // if tass with this label have not been executed + return findBestFittingNode( bestFittingNodes, tasksOnNode ); + } + + for ( TasksOnNodeWrapper tasksOnNodeWrapper : runningOrfinishedOnNodes ) { + if ( bestFittingNodes.contains( tasksOnNodeWrapper.getNode() ) ) { + return tasksOnNodeWrapper.getNode(); + } + } + + final NodeLocation nodeLocation = calculateBestFittingNode( label, xTasksCanRunOnNode, labelCount, tasksOnNode ); + if ( nodeLocation != null ) { + return nodeLocation; + } + // If no node is found, return a random one + return findBestFittingNode( bestFittingNodes, tasksOnNode ); + + } + + /** + * Assign to the node that has the least tasks to process yet + * @param bestFittingNodes list of potential nodes + * @param tasksOnNode map of tasks on each node + * @return + */ + private NodeLocation findBestFittingNode( final Set bestFittingNodes, Map tasksOnNode ) { + NodeLocation best = null; + for ( NodeLocation fittingNode : bestFittingNodes ) { + if ( best == null || tasksOnNode.getOrDefault( fittingNode, 0 ) < tasksOnNode.getOrDefault( best, 0 ) ) { + best = fittingNode; + } + } + return best; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java new file mode 100644 index 00000000..fd035a3c --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java @@ -0,0 +1,27 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.location.NodeLocation; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; + +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@AllArgsConstructor +public class TasksOnNodeWrapper implements Comparable { + + private final NodeLocation node; + private int share = 0; + + @Override + public int compareTo( @NotNull TasksOnNodeWrapper o ) { + return Integer.compare( o.share, share ); + } + + public void addRunningTask() { + share++; + } +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java index 47bbcdbb..d3604b71 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java @@ -3,9 +3,17 @@ import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.model.SchedulerConfig; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.cluster.FileSizeRankScoreGroup; import cws.k8s.scheduler.model.cluster.GroupCluster; +import cws.k8s.scheduler.model.cluster.MostOutLabelsComparator; +import cws.k8s.scheduler.model.cluster.SimpleGroupCluster; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.MaxSizeComparator; +import cws.k8s.scheduler.scheduler.la2.MinCopyingComparator; +import cws.k8s.scheduler.scheduler.la2.MinSizeComparator; +import cws.k8s.scheduler.scheduler.la2.RankAndMinCopyingComparator; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; +import cws.k8s.scheduler.util.score.FileSizeRankScore; import java.util.List; @@ -14,7 +22,11 @@ public class LocationAwareSchedulerGroups extends LocationAwareSchedulerV2 { GroupCluster groupCluster; public LocationAwareSchedulerGroups( String name, KubernetesClient client, String namespace, SchedulerConfig config, InputAlignment inputAlignment, ReadyToRunToNode readyToRunToNode ) { - super( name, client, namespace, config, inputAlignment, readyToRunToNode ); + super( name, client, namespace, config, inputAlignment, readyToRunToNode, null ); + groupCluster = new SimpleGroupCluster( client ); + readyToRunToNode.init( new FileSizeRankScoreGroup( groupCluster ) ); + setPhaseTwoComparator( new MinCopyingComparator( MinSizeComparator.INSTANCE ) ); + setPhaseThreeComparator( new RankAndMinCopyingComparator( new MostOutLabelsComparator( groupCluster ) ) ); } @Override @@ -29,5 +41,10 @@ void taskWasScheduled(Task task ) { groupCluster.taskWasAssignedToNode( task ); } - + @Override + int terminateTasks( List finishedTasks ) { + final int terminatedTasks = super.terminateTasks( finishedTasks ); + groupCluster.tasksHaveFinished( finishedTasks ); + return terminatedTasks; + } } From be8e0d7e05cb4c016fec6c79f6621ef6f0f575e4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 8 Jul 2024 17:00:18 +0200 Subject: [PATCH 418/443] Fix indent Signed-off-by: Lehmann_Fabian --- .../scheduler/model/cluster/LabelCount.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java index 104dc393..acafa56a 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java @@ -12,22 +12,22 @@ @NoArgsConstructor( access = AccessLevel.PACKAGE ) public class LabelCount { - @Getter - private int countFinished = 0; - @Getter - private int countStarted = 0; - @Getter - private int countWaiting = 0; + @Getter + private int countFinished = 0; + @Getter + private int countStarted = 0; + @Getter + private int countWaiting = 0; - @Getter - private final Set waitingTasks = new HashSet<>(); - private final Set runningTasks = new HashSet<>(); - private final Set finishedTasks = new HashSet<>(); - private final Set tasks = new HashSet<>(); - @Getter - private final Queue runningOrfinishedOnNodes = new PriorityQueue<>(); - private final Map nodeToShare = new HashMap<>(); - private final Map> taskHasDataOnNode = new HashMap<>(); + @Getter + private final Set waitingTasks = new HashSet<>(); + private final Set runningTasks = new HashSet<>(); + private final Set finishedTasks = new HashSet<>(); + private final Set tasks = new HashSet<>(); + @Getter + private final Queue runningOrfinishedOnNodes = new PriorityQueue<>(); + private final Map nodeToShare = new HashMap<>(); + private final Map> taskHasDataOnNode = new HashMap<>(); /** * Get the number of tasks with this label From 034bc0184090ecb694702d8a9cf94635d5042891 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Mon, 8 Jul 2024 17:01:47 +0200 Subject: [PATCH 419/443] Refactor to hide excludedNodes variable Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/taskinputs/TaskInputs.java | 11 ++++++++++- .../scheduler/LocationAwareSchedulerV2.java | 4 ++-- .../scheduler/SchedulerWithDaemonSet.java | 9 +-------- .../scheduler/model/InputFileCollectorTest.java | 14 ++++++-------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java index f3861695..83937e3d 100644 --- a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java @@ -16,17 +16,26 @@ import java.util.stream.Collectors; @ToString -@Getter @Slf4j @RequiredArgsConstructor public class TaskInputs { + @Getter private final List symlinks; + @Getter private final List files; private final Set excludedNodes; private boolean sorted = false; + public boolean canRunOnLoc( Location loc ) { + return !excludedNodes.contains( loc ); + } + + public boolean hasExcludedNodes() { + return !excludedNodes.isEmpty(); + } + public boolean allFilesAreOnLocationAndNotOverwritten( Location loc, Set pathCurrentlyCopying ){ for (PathFileLocationTriple file : files) { if ( !file.locatedOnLocation(loc) || (pathCurrentlyCopying != null && pathCurrentlyCopying.contains(file.path.toString())) ) { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index dffa7238..2927edbc 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -139,7 +139,7 @@ public ScheduleObject getTaskNodeAlignment( .filter( node -> { final CurrentlyCopyingOnNode copyingFilesToNode = getCurrentlyCopying().get( node.getNodeLocation() ); //File version does not match and is in use - return !inputsOfTask.getExcludedNodes().contains( node.getNodeLocation() ) + return inputsOfTask.canRunOnLoc( node.getNodeLocation() ) //Affinities are correct and the node can run new pods && canSchedulePodOnNode( task.getPod(), node ) //All files are on the node and no copy task is overwriting them @@ -382,7 +382,7 @@ private TaskStat getDataOnNode( Task task, TaskInputs inputsOfTask, List matchingNodes, TaskInputs taskInputs ){ - final Iterator iterator = matchingNodes.iterator(); - final Set excludedNodes = taskInputs.getExcludedNodes(); - while ( iterator.hasNext() ){ - final NodeWithAlloc next = iterator.next(); - if( excludedNodes.contains( next.getNodeLocation() ) ){ - iterator.remove(); - } - } + matchingNodes.removeIf( next -> !taskInputs.canRunOnLoc( next.getNodeLocation() ) ); } public void taskHasFinishedCopyTask( String name ){ diff --git a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java index 7ecb3470..f89a3a69 100644 --- a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java @@ -10,7 +10,6 @@ import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -103,7 +102,7 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { assertEquals( 3, inputsOfTaskFiles.size() ); assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + assertFalse( inputsOfTask.hasExcludedNodes() ); //Not existing file final InputParam c = new InputParam<>("c", new FileHolder(null, root2 + "a.txt", null)); @@ -114,7 +113,7 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { assertEquals( 3, inputsOfTaskFiles.size() ); assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + assertFalse( inputsOfTask.hasExcludedNodes() ); //Symlink final InputParam d = new InputParam<>("d", new FileHolder(null, root2 + "b/c.txt", null)); @@ -126,7 +125,7 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { assertEquals( 4, inputsOfTaskFiles.size() ); assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); assertEquals( Set.of( symlink ), new HashSet<>(inputsOfTask.getSymlinks()) ); - assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + assertFalse( inputsOfTask.hasExcludedNodes() ); } @@ -177,9 +176,8 @@ public void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { assertEquals( 1, inputsOfTaskFiles.size() ); assertEquals( file1, inputsOfTaskFiles.get(0) ); assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertEquals( 1, inputsOfTask.getExcludedNodes().size() ); - Assert.assertEquals( NodeLocation.getLocation( "Node1" ), inputsOfTask.getExcludedNodes().iterator().next() ); - + assertTrue( inputsOfTask.hasExcludedNodes() ); + assertFalse( inputsOfTask.canRunOnLoc( NodeLocation.getLocation( "Node1" ) ) ); } @Test @@ -229,7 +227,7 @@ public void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundExceptio assertEquals( 1, inputsOfTaskFiles.size() ); assertEquals( file1, inputsOfTaskFiles.get(0) ); assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertTrue( inputsOfTask.getExcludedNodes().isEmpty() ); + assertFalse( inputsOfTask.hasExcludedNodes() ); } From 1b6734950fe83d8a31302e544144f43ca86c8dbf Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 18 Jul 2024 16:05:18 +0200 Subject: [PATCH 420/443] Create Copy Tasks for tasks that ran on a node that is not responsible for their labels. Signed-off-by: Lehmann_Fabian --- .../scheduler/model/InputFileCollector.java | 1 + .../java/cws/k8s/scheduler/model/Task.java | 15 +++ .../cws/k8s/scheduler/model/TaskConfig.java | 12 ++ .../k8s/scheduler/model/cluster/CopyTask.java | 60 +++++++++ .../scheduler/model/cluster/DataMissing.java | 18 +++ .../model/cluster/FakeTaskInputs.java | 28 ++++ .../model/cluster/FileSizeRankScoreGroup.java | 3 +- .../scheduler/model/cluster/GroupCluster.java | 126 ++++++++++++++++-- .../scheduler/model/cluster/LabelCount.java | 83 +++++++----- .../cluster/MostOutLabelsComparator.java | 4 +- .../scheduler/model/cluster/OutputFiles.java | 23 ++++ .../RankAndMinCopyingComparatorCopyTasks.java | 27 ++++ .../model/cluster/SimpleGroupCluster.java | 34 ++--- .../model/cluster/TasksOnNodeWrapper.java | 4 +- .../location/hierachy/RealHierarchyFile.java | 11 +- .../LocationAwareSchedulerGroups.java | 31 +++-- .../scheduler/LocationAwareSchedulerV2.java | 10 +- .../scheduler/SchedulerWithDaemonSet.java | 4 + ...yInAdvanceNodeWithMostDataIntelligent.java | 2 +- .../la2/copystrategy/LaListener.java | 4 + .../ready2run/OptimalReadyToRunToNode.java | 2 +- .../scheduler/util/score/CalculateScore.java | 4 +- .../util/score/FileSizeRankScore.java | 3 +- .../scheduler/util/score/FileSizeScore.java | 3 +- 24 files changed, 428 insertions(+), 84 deletions(-) create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java diff --git a/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java index 425de7a4..c6d4943b 100644 --- a/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java +++ b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java @@ -42,6 +42,7 @@ private void processNext( } } else { final RealHierarchyFile realFile = (RealHierarchyFile) file; + realFile.requestedByTask(); try { final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); if ( filesForTask.getExcludedNodes() != null ) { diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index 9ddbcb2f..49172fc4 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -2,8 +2,10 @@ import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.cluster.OutputFiles; import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; import cws.k8s.scheduler.model.tracing.TraceRecord; import cws.k8s.scheduler.util.Batch; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; @@ -70,6 +72,10 @@ public class Task { private final HierarchyWrapper hierarchyWrapper; + @Getter + @Setter + private OutputFiles outputFiles; + public Task( TaskConfig config, DAG dag ) { this( config, dag, null ); } @@ -80,6 +86,15 @@ public Task( TaskConfig config, DAG dag, HierarchyWrapper hierarchyWrapper ) { this.hierarchyWrapper = hierarchyWrapper; } + /** + * Constructor for inheritance + */ + protected Task( TaskConfig config, Process process ){ + this.config = config; + this.process = process; + this.hierarchyWrapper = null; + } + public int getCurrentCopyTaskId() { return copyTaskId.getAndIncrement(); } diff --git a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java index 3e162f4c..e681bac5 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java @@ -43,6 +43,18 @@ public TaskConfig(String task) { this.outLabel = null; } + public TaskConfig(String task, String name, String workDir, String runName ) { + this.task = task; + this.name = name; + this.schedulerParams = null; + this.inputs = new TaskInput( null, null, null, new LinkedList<>() ); + this.runName = runName; + this.cpus = 0; + this.memoryInBytes = 0; + this.workDir = workDir; + this.outLabel = null; + } + @Getter @ToString @NoArgsConstructor( access = AccessLevel.PRIVATE, force = true ) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java new file mode 100644 index 00000000..d01f2650 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java @@ -0,0 +1,60 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskConfig; +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; + +@Slf4j +public class CopyTask extends Task { + + private final static Process COPY_PROCESS = new Process( "Copy", Integer.MAX_VALUE ); + private final Task task; + private final NodeWithAlloc node; + private final LabelCount labelCount; + + protected CopyTask( Task task, NodeWithAlloc node, LabelCount labelCount ) { + super( new TaskConfig("Copy","Copy-" + task.getConfig().getName() + " -> " + node.getNodeLocation() + + " (" + labelCount.getLabel() + ")" , buildCopyTaskFolder(task), task.getConfig().getRunName() ), COPY_PROCESS ); + this.task = task; + this.node = node; + this.labelCount = labelCount; + } + + private static String buildCopyTaskFolder( Task task ) { + final Path path = Path.of( task.getConfig().getWorkDir() ); + Path p = Path.of( + path.getParent().getParent().toString(), + "copy", + path.getParent().getFileName().toString(), + path.getFileName().toString() + ); + return p.toString(); + } + + @Override + public Requirements getRequest() { + return new Requirements( 0, 0 ); + } + + @Override + public boolean equals( Object obj ) { + if ( obj instanceof CopyTask ) { + return this.task.equals( ((CopyTask) obj).task ); + } + else return false; + } + + @Override + public int hashCode() { + return task.hashCode() + 5; + } + + public void finished(){ + labelCount.taskIsNowOnNode( task, node ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java new file mode 100644 index 00000000..e64e5d0d --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java @@ -0,0 +1,18 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +public class DataMissing { + + private final Task task; + private final NodeWithAlloc node; + private final LabelCount labelCount; + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java b/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java new file mode 100644 index 00000000..96febd79 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java @@ -0,0 +1,28 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; + +import java.util.LinkedList; +import java.util.List; + +public class FakeTaskInputs extends TaskInputs { + + private final Location includedNode; + + public FakeTaskInputs( List files, Location includedNode ) { + super( new LinkedList<>(), files, null ); + this.includedNode = includedNode; + } + + @Override + public boolean canRunOnLoc( Location loc ) { + return includedNode == loc; + } + + @Override + public boolean hasExcludedNodes() { + return true; + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java index f0fc3315..96334a8d 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java @@ -1,5 +1,6 @@ package cws.k8s.scheduler.model.cluster; +import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.util.score.FileSizeRankScore; @@ -11,7 +12,7 @@ public class FileSizeRankScoreGroup extends FileSizeRankScore { private final GroupCluster groupCluster; @Override - public long getScore( Task task, NodeLocation location, long size ) { + public long getScore( Task task, NodeWithAlloc location, long size ) { final long score = super.getScore( task, location, size ); final long newScore = (long) (score * groupCluster.getScoreForTaskOnNode( task, location )); if ( newScore < 1 ) { diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java index 12347d32..97bca5bc 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java @@ -1,19 +1,41 @@ package cws.k8s.scheduler.model.cluster; +import cws.k8s.scheduler.client.KubernetesClient; +import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; +import cws.k8s.scheduler.model.location.hierachy.NoAlignmentFoundException; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import java.nio.file.Path; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; +@Slf4j +@RequiredArgsConstructor public abstract class GroupCluster { + private final double minScoreToCopy = 0.5; protected final Set allLabels = new HashSet<>(); protected final Map countPerLabel = new HashMap<>(); protected final LinkedList unscheduledTasks = new LinkedList<>(); protected final LinkedList scheduledTasks = new LinkedList<>(); protected final LinkedList finishedTasks = new LinkedList<>(); - protected final Map labelToNode = new HashMap<>(); - protected final Map> nodeToLabel = new HashMap<>(); + protected final Map labelToNode = new HashMap<>(); + protected final Map> nodeToLabel = new HashMap<>(); + @Getter + private final HierarchyWrapper hierarchyWrapper; + @Getter + private final KubernetesClient client; private void addAllOutLabels( List tasks ) { for ( Task task : tasks ) { @@ -34,10 +56,10 @@ public void tasksBecameAvailable( List tasks ) { } for ( String label : task.getOutLabel() ) { anyWithLabel = true; - countPerLabel.computeIfAbsent( label, k -> new LabelCount() ).addWaitingTask( task ); + countPerLabel.computeIfAbsent( label, k -> new LabelCount(k) ).addWaitingTask( task ); final LabelCount labelCount; if ( !countPerLabel.containsKey( label ) ) { - labelCount = new LabelCount(); + labelCount = new LabelCount( label ); countPerLabel.put( label, labelCount ); } else { labelCount = countPerLabel.get( label ); @@ -84,14 +106,14 @@ public void tasksHaveFinished( List tasks ) { abstract void recalculate(); - public double getScoreForTaskOnNode( Task task, NodeLocation nodeLocation ) { + public double getScoreForTaskOnNode( Task task, NodeWithAlloc node ) { if ( task.getOutLabel() == null || task.getOutLabel().isEmpty() ) { return 1; } //Jaccard similarity coefficient final Set outLabel = new HashSet<>( task.getOutLabel() ); int outLabelSize = outLabel.size(); - final Set nodeLabels = nodeToLabel.get( nodeLocation ); + final Set nodeLabels = nodeToLabel.get( node ); if ( nodeLabels == null ) { return 0.01; } @@ -102,8 +124,8 @@ public double getScoreForTaskOnNode( Task task, NodeLocation nodeLocation ) { return (double) outLabel.size() / outLabelSize + 0.01; // to avoid zero } - protected void addNodeToLabel( NodeLocation nodeLocation, String label ) { - final NodeLocation currentLocation = labelToNode.get( label ); + protected void addNodeToLabel( NodeWithAlloc nodeLocation, String label ) { + final NodeWithAlloc currentLocation = labelToNode.get( label ); if ( nodeLocation.equals( currentLocation ) ) { return; @@ -119,4 +141,90 @@ protected void addNodeToLabel( NodeLocation nodeLocation, String label ) { labelToNode.put( label, nodeLocation ); } + /** + * Return a list of all tasks for that a node exists where getScore is higher than minScoreToCopy and the task has at least one outLabel + * @return + */ + public List getTaskStatToCopy(){ + int maxCopiesPerNode = 1; + synchronized ( this ) { + //all tasks where score > minScoreToCopy and tasks have at least one outLabel + final Map taskForLocation = new HashMap<>(); + return getTasksThatNeedToBeCopied() + .sequential() + //Only create maxCopiesPerNode possible copy tasks + .filter( x -> shouldStillCreate( x, taskForLocation, maxCopiesPerNode ) ) + .map( this::getTaskStat ) + .filter( Objects::nonNull ) + .collect( Collectors.toList()); + } + } + + private static boolean shouldStillCreate( DataMissing x, Map taskForLocation, int maxCopiesPerNode ) { + if ( taskForLocation.containsKey( x.getNode() ) ) { + final Integer i = taskForLocation.get( x.getNode() ); + if ( i == maxCopiesPerNode ) { + return false; + } else { + taskForLocation.put( x.getNode(), i + 1 ); + return true; + } + } else { + taskForLocation.put( x.getNode(), 1 ); + return true; + } + } + + private TaskStat getTaskStat( DataMissing dataMissing ) { + final NodeWithAlloc node = dataMissing.getNode(); + final OutputFiles outputFilesWrapper = dataMissing.getTask().getOutputFiles(); + final Set outputFiles = outputFilesWrapper.getFiles(); + List files = outputFiles + .parallelStream() + .map( x -> convertToPathFileLocationTriple(x, dataMissing.getTask()) ) + .collect( Collectors.toList() ); + if ( dataMissing.getTask().getOutputFiles().isWasRequestedForRealTask() ) { + return null; + } + TaskInputs inputsOfTask = new FakeTaskInputs( files, node.getNodeLocation() ); + final CopyTask copyTask = new CopyTask( dataMissing.getTask(), node, dataMissing.getLabelCount() ); + final TaskStat taskStat = new TaskStat( copyTask, inputsOfTask ); + final TaskNodeStats taskNodeStats = new TaskNodeStats( inputsOfTask.calculateAvgSize(), 0, 0 ); + //client. + taskStat.add( dataMissing.getNode(), taskNodeStats ); + return taskStat; + } + + private PathFileLocationTriple convertToPathFileLocationTriple( PathLocationWrapperPair pair, Task task ){ + final Path path = pair.getPath(); + RealHierarchyFile file = (RealHierarchyFile) hierarchyWrapper.getFile( path ); + final RealHierarchyFile.MatchingLocationsPair filesForTask; + if ( file.wasRequestedByTask() ) { + task.getOutputFiles().wasRequestedForRealTask(); + return null; + } + try { + filesForTask = file.getFilesForTask( task ); + } catch ( NoAlignmentFoundException e ) { + throw new RuntimeException( e ); + } + return new PathFileLocationTriple( path, file, filesForTask.getMatchingLocations() ); + } + + private Stream getTasksThatNeedToBeCopied(){ + return countPerLabel.entrySet() + .parallelStream() + .unordered() + .flatMap( v -> { + final String label = v.getKey(); + final LabelCount value = v.getValue(); + final NodeWithAlloc node = labelToNode.get( label ); + return value.taskNotOnNode( node ); + } ) + .distinct() + .filter( d -> getScoreForTaskOnNode( d.getTask(), d.getNode() ) > minScoreToCopy ) + .sorted( (x, y) -> Double.compare( getScoreForTaskOnNode( y.getTask(), y.getNode() ), + getScoreForTaskOnNode( x.getTask(), x.getNode() ) ) ); + } + } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java index acafa56a..fa06e30f 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java @@ -2,16 +2,19 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; import lombok.AccessLevel; import lombok.Getter; -import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; import java.util.*; +import java.util.stream.Stream; -@NoArgsConstructor( access = AccessLevel.PACKAGE ) +@RequiredArgsConstructor( access = AccessLevel.PACKAGE ) public class LabelCount { + @Getter + private final String label; + @Getter private int countFinished = 0; @Getter @@ -27,7 +30,15 @@ public class LabelCount { @Getter private final Queue runningOrfinishedOnNodes = new PriorityQueue<>(); private final Map nodeToShare = new HashMap<>(); - private final Map> taskHasDataOnNode = new HashMap<>(); + private final Map> taskHasDataOnNode = new HashMap<>(); + + public Stream taskNotOnNode( NodeWithAlloc node ) { + return taskHasDataOnNode.entrySet() + .stream() + .filter( taskSetEntry -> !taskSetEntry.getValue().contains( node ) + && !taskSetEntry.getKey().getOutputFiles().isWasRequestedForRealTask() ) + .map( x -> new DataMissing( x.getKey(), node, this ) ); + } /** * Get the number of tasks with this label @@ -37,37 +48,41 @@ public int getCount(){ return countFinished + countStarted + countWaiting; } - public void addWaitingTask( Task task ){ - countWaiting++; - tasks.add(task); - waitingTasks.add(task); - } + public void addWaitingTask( Task task ){ + countWaiting++; + tasks.add(task); + waitingTasks.add(task); + } - public void makeTaskRunning( Task task ) { - final NodeWithAlloc node = task.getNode(); - final TasksOnNodeWrapper tasksOnNodeWrapper; - if ( nodeToShare.containsKey( node ) ) { - tasksOnNodeWrapper = nodeToShare.get( node ); - } else { - tasksOnNodeWrapper = new TasksOnNodeWrapper( node.getNodeLocation() ); - nodeToShare.put( node, tasksOnNodeWrapper ); - runningOrfinishedOnNodes.add( tasksOnNodeWrapper ); - } - tasksOnNodeWrapper.addRunningTask(); - countWaiting--; - countStarted++; - waitingTasks.remove( task ); - runningTasks.add( task ); + public void makeTaskRunning( Task task ) { + final NodeWithAlloc node = task.getNode(); + final TasksOnNodeWrapper tasksOnNodeWrapper; + if ( nodeToShare.containsKey( node ) ) { + tasksOnNodeWrapper = nodeToShare.get( node ); + } else { + tasksOnNodeWrapper = new TasksOnNodeWrapper( node ); + nodeToShare.put( node, tasksOnNodeWrapper ); + runningOrfinishedOnNodes.add( tasksOnNodeWrapper ); } + tasksOnNodeWrapper.addRunningTask(); + countWaiting--; + countStarted++; + waitingTasks.remove( task ); + runningTasks.add( task ); + } - public void makeTaskFinished( Task task ) { - countStarted--; - countFinished++; - runningTasks.remove( task ); - finishedTasks.add( task ); - final HashSet locations = new HashSet<>(); - locations.add( task.getNode().getNodeLocation() ); - taskHasDataOnNode.put( task, locations ); - } + public void makeTaskFinished( Task task ) { + countStarted--; + countFinished++; + runningTasks.remove( task ); + finishedTasks.add( task ); + final HashSet locations = new HashSet<>(); + locations.add( task.getNode() ); + taskHasDataOnNode.put( task, locations ); + } + + public void taskIsNowOnNode( Task task, NodeWithAlloc node ) { + taskHasDataOnNode.get( task ).add( node ); + } - } \ No newline at end of file +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java b/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java index aae7903f..42dd1d98 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java @@ -17,8 +17,8 @@ public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper final TaskNodeStats o2Stats = o2.getTaskNodeStats(); //same task does not necessarily have the same size if ( o1.getTask() == o2.getTask() || o1Stats.getTaskSize() == o2Stats.getTaskSize() ) { - double ratingO1 = groupCluster.getScoreForTaskOnNode( o1.getTask(), o1.getNode().getNodeLocation() ); - double ratingO2 = groupCluster.getScoreForTaskOnNode( o2.getTask(), o2.getNode().getNodeLocation() ); + double ratingO1 = groupCluster.getScoreForTaskOnNode( o1.getTask(), o1.getNode() ); + double ratingO2 = groupCluster.getScoreForTaskOnNode( o2.getTask(), o2.getNode() ); // Use the node with a higher rating if ( ratingO1 != ratingO2 ) { return Double.compare( ratingO2, ratingO1 ); diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java new file mode 100644 index 00000000..4a611a50 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java @@ -0,0 +1,23 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.Set; + +@Slf4j +@Getter +@RequiredArgsConstructor +public class OutputFiles { + + + private final Set files; + private boolean wasRequestedForRealTask = false; + + public void wasRequestedForRealTask(){ + wasRequestedForRealTask = true; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java b/src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java new file mode 100644 index 00000000..e057345a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java @@ -0,0 +1,27 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.scheduler.la2.RankAndMinCopyingComparator; +import cws.k8s.scheduler.scheduler.la2.TaskStat; + +import java.util.Comparator; + +public class RankAndMinCopyingComparatorCopyTasks extends RankAndMinCopyingComparator { + + public RankAndMinCopyingComparatorCopyTasks( Comparator comparator ) { + super( comparator ); + } + + @Override + public int compare( TaskStat o1, TaskStat o2 ) { + if ( o1.getTask() instanceof CopyTask ^ o2.getTask() instanceof CopyTask ) { + if ( o1.getTask() instanceof CopyTask ) { + return 1; + } else { + return -1; + } + } else { + return super.compare( o1, o2 ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java index 83c1e8f6..c277ae76 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java @@ -2,20 +2,20 @@ import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.model.NodeWithAlloc; -import cws.k8s.scheduler.model.location.NodeLocation; -import lombok.RequiredArgsConstructor; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import java.util.*; import java.util.stream.Collectors; -@RequiredArgsConstructor public class SimpleGroupCluster extends GroupCluster { - private final KubernetesClient client; + public SimpleGroupCluster( HierarchyWrapper hierarchyWrapper, KubernetesClient client ) { + super( hierarchyWrapper, client ); + } @Override void recalculate() { - final List allNodes = client.getAllNodes(); + final List allNodes = getClient().getAllNodes(); // Filter labels that have waiting tasks. final List> labelsWithWaitingTasks = countPerLabel @@ -30,7 +30,7 @@ void recalculate() { // store how many tasks would have been executed on every with the current alignment // this is an approximation since tasks can have multiple labels and would appear multiple times - Map tasksOnNode = new HashMap<>(); + Map tasksOnNode = new HashMap<>(); // For every label and node count how many tasks could run (affinities match), we ignore the current load completely for ( Map.Entry labelWithWaitingTasks : labelsWithWaitingTasks ) { @@ -43,15 +43,15 @@ void recalculate() { .filter( task -> node.affinitiesMatch( task.getPod() ) ) .count(); if ( count > 0 ) { - final TasksOnNodeWrapper tasksOnNodeWrapper = new TasksOnNodeWrapper( node.getNodeLocation(), (int) count ); + final TasksOnNodeWrapper tasksOnNodeWrapper = new TasksOnNodeWrapper( node, (int) count ); xTasksCanRunOnNode.add( tasksOnNodeWrapper ); } } if ( !xTasksCanRunOnNode.isEmpty() ) { - final NodeLocation nodeLocation = calculateBestFittingNode( labelWithWaitingTasks.getKey(), xTasksCanRunOnNode, labelWithWaitingTasks.getValue(), tasksOnNode ); - if ( nodeLocation != null ) { - addNodeToLabel( nodeLocation, labelWithWaitingTasks.getKey() ); - tasksOnNode.put( nodeLocation, tasksOnNode.getOrDefault( nodeLocation, 0 ) + labelWithWaitingTasks.getValue().getCountWaiting() ); + final NodeWithAlloc node = calculateBestFittingNode( labelWithWaitingTasks.getKey(), xTasksCanRunOnNode, labelWithWaitingTasks.getValue(), tasksOnNode ); + if ( node != null ) { + addNodeToLabel( node, labelWithWaitingTasks.getKey() ); + tasksOnNode.put( node, tasksOnNode.getOrDefault( node, 0 ) + labelWithWaitingTasks.getValue().getCountWaiting() ); } } } @@ -65,11 +65,11 @@ void recalculate() { * @param tasksOnNode * @return */ - private NodeLocation calculateBestFittingNode( String label, Queue xTasksCanRunOnNode, LabelCount labelCount, Map tasksOnNode ) { + private NodeWithAlloc calculateBestFittingNode( String label, Queue xTasksCanRunOnNode, LabelCount labelCount, Map tasksOnNode ) { if ( xTasksCanRunOnNode.isEmpty() ) { return null; } - final Set bestFittingNodes = new HashSet<>(); + final Set bestFittingNodes = new HashSet<>(); final TasksOnNodeWrapper bestFittingNode = xTasksCanRunOnNode.poll(); bestFittingNodes.add( bestFittingNode.getNode() ); while( !xTasksCanRunOnNode.isEmpty() && xTasksCanRunOnNode.peek().getShare() == bestFittingNode.getShare() ){ @@ -88,7 +88,7 @@ private NodeLocation calculateBestFittingNode( String label, Queue bestFittingNodes, Map tasksOnNode ) { - NodeLocation best = null; - for ( NodeLocation fittingNode : bestFittingNodes ) { + private NodeWithAlloc findBestFittingNode( final Set bestFittingNodes, Map tasksOnNode ) { + NodeWithAlloc best = null; + for ( NodeWithAlloc fittingNode : bestFittingNodes ) { if ( best == null || tasksOnNode.getOrDefault( fittingNode, 0 ) < tasksOnNode.getOrDefault( best, 0 ) ) { best = fittingNode; } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java index fd035a3c..8635b6d4 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java @@ -1,6 +1,6 @@ package cws.k8s.scheduler.model.cluster; -import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.NodeWithAlloc; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -13,7 +13,7 @@ @AllArgsConstructor public class TasksOnNodeWrapper implements Comparable { - private final NodeLocation node; + private final NodeWithAlloc node; private int share = 0; @Override diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java index 90811337..af0870d8 100644 --- a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -18,8 +18,9 @@ public class RealHierarchyFile extends AbstractHierarchyFile { @Getter private LocationWrapper[] locations; static final String LOCATION_IS_NULL = "location is null"; + private boolean wasRequestedByTask = false; - public RealHierarchyFile(LocationWrapper location ) { + public RealHierarchyFile( LocationWrapper location ) { if ( location == null ) { throw new IllegalArgumentException( LOCATION_IS_NULL ); } @@ -305,4 +306,12 @@ public LocationWrapper getLocationWrapper( Location location ){ throw new RuntimeException( "Not found: " + location.getIdentifier() ); } + public void requestedByTask(){ + wasRequestedByTask = true; + } + + public boolean wasRequestedByTask(){ + return wasRequestedByTask; + } + } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java index d3604b71..2bf9c1c6 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java @@ -3,18 +3,13 @@ import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.model.SchedulerConfig; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.cluster.FileSizeRankScoreGroup; -import cws.k8s.scheduler.model.cluster.GroupCluster; -import cws.k8s.scheduler.model.cluster.MostOutLabelsComparator; -import cws.k8s.scheduler.model.cluster.SimpleGroupCluster; +import cws.k8s.scheduler.model.cluster.*; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import cws.k8s.scheduler.scheduler.la2.MaxSizeComparator; -import cws.k8s.scheduler.scheduler.la2.MinCopyingComparator; -import cws.k8s.scheduler.scheduler.la2.MinSizeComparator; -import cws.k8s.scheduler.scheduler.la2.RankAndMinCopyingComparator; +import cws.k8s.scheduler.scheduler.la2.*; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; -import cws.k8s.scheduler.util.score.FileSizeRankScore; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import java.io.File; import java.util.List; public class LocationAwareSchedulerGroups extends LocationAwareSchedulerV2 { @@ -23,10 +18,10 @@ public class LocationAwareSchedulerGroups extends LocationAwareSchedulerV2 { public LocationAwareSchedulerGroups( String name, KubernetesClient client, String namespace, SchedulerConfig config, InputAlignment inputAlignment, ReadyToRunToNode readyToRunToNode ) { super( name, client, namespace, config, inputAlignment, readyToRunToNode, null ); - groupCluster = new SimpleGroupCluster( client ); + groupCluster = new SimpleGroupCluster( hierarchyWrapper, client ); readyToRunToNode.init( new FileSizeRankScoreGroup( groupCluster ) ); setPhaseTwoComparator( new MinCopyingComparator( MinSizeComparator.INSTANCE ) ); - setPhaseThreeComparator( new RankAndMinCopyingComparator( new MostOutLabelsComparator( groupCluster ) ) ); + setPhaseThreeComparator( new RankAndMinCopyingComparatorCopyTasks( new MostOutLabelsComparator( groupCluster ) ) ); } @Override @@ -47,4 +42,18 @@ int terminateTasks( List finishedTasks ) { groupCluster.tasksHaveFinished( finishedTasks ); return terminatedTasks; } + + @Override + List getAdditionalTaskStatPhaseThree(){ + return groupCluster.getTaskStatToCopy(); + } + + void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + if ( nodeTaskFilesAlignment.task instanceof CopyTask ) { + final String workingDir = nodeTaskFilesAlignment.task.getWorkingDir(); + new File( workingDir ).mkdirs(); + } + super.startCopyTask( nodeTaskFilesAlignment ); + } + } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 2927edbc..1a0dc4e1 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -200,6 +200,9 @@ void postScheduling( List unscheduledTasks, final Map unscheduledTasks, final Map getAdditionalTaskStatPhaseThree(){ + return new LinkedList<>(); + } + + + void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { nodeTaskFilesAlignment.task.getTraceRecord().copyTask(); final CopyTask copyTask = initializeCopyTask( nodeTaskFilesAlignment ); //Files that will be copied diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 83b80bd9..265cd4fd 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.model.cluster.OutputFiles; import cws.k8s.scheduler.model.location.hierachy.*; import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; @@ -143,12 +144,14 @@ int terminateTasks(List finishedTasks) { !finishedTask.wasSuccessfullyExecuted(), finishedTask ); + final Set outputFiles = new HashSet<>(); for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { hierarchyWrapper.addFile( newAndUpdatedFile.getPath(), ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() ); + outputFiles.add( (PathLocationWrapperPair) newAndUpdatedFile ); } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ hierarchyWrapper.addSymlink( newAndUpdatedFile.getPath(), @@ -156,6 +159,7 @@ int terminateTasks(List finishedTasks) { ); } } + finishedTask.setOutputFiles( new OutputFiles( outputFiles ) ); } } } catch ( Exception e ){ diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java index 1e197166..f2354098 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -192,7 +192,7 @@ private TaskWithScore getTaskWithScore( Task task, String error ) { .calculateDataOnNode( node.getNodeLocation() ); final long dataInSharedFS = task.getInputSize(); //The input should be similar to the first scheduling phase - return new TaskWithScore( task, calculateScore.getScore( task, node.getNodeLocation(), dataOnNode + dataInSharedFS ) ); + return new TaskWithScore( task, calculateScore.getScore( task, node, dataOnNode + dataInSharedFS ) ); } void addPlaned( Task task ) { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java index 497f4575..b4ddf8c1 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java @@ -80,6 +80,10 @@ public void onExit( int exitCode, Status reason ) { log.debug( name + " Exec Output: {} ", out ); log.debug( name + " Exec Error Output: {} ", error ); } + if ( copyTask.getTask() instanceof cws.k8s.scheduler.model.cluster.CopyTask ) { + cws.k8s.scheduler.model.cluster.CopyTask task = (cws.k8s.scheduler.model.cluster.CopyTask) copyTask.getTask(); + task.finished(); + } scheduler.copyTaskFinished( copyTask, exitCode == 0 ); close(); logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "finished(" + exitCode + ")" ); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index 28a57adf..c619202e 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -75,7 +75,7 @@ public List createAlignmentForTasksWithAllDataOnNod onlyOnOneNode.add( boolVar ); memUsed.get(node).addTerm( boolVar, ram ); cpuUsed.get(node).addTerm( boolVar, cpu ); - final long score = calculateScore.getScore( taskInputsNodes.getTask(), node.getNodeLocation(), taskInputsNodes.getTaskSize() ); + final long score = calculateScore.getScore( taskInputsNodes.getTask(), node, taskInputsNodes.getTaskSize() ); objective.addTerm( boolVar, score ); taskNodeBoolVars.add( new TaskNodeBoolVar( taskInputsNodes, node, boolVar ) ); } diff --git a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java index 00015d27..5b8eb523 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java @@ -1,7 +1,7 @@ package cws.k8s.scheduler.util.score; +import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; public interface CalculateScore { @@ -9,6 +9,6 @@ public interface CalculateScore { * Score must be higher than 0 * @return */ - long getScore( Task task, NodeLocation location, long inputSize ); + long getScore( Task task, NodeWithAlloc location, long inputSize ); } diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java index cfdd0d74..0e17f60e 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -1,12 +1,13 @@ package cws.k8s.scheduler.util.score; +import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; public class FileSizeRankScore extends FileSizeScore { @Override - public long getScore( Task task, NodeLocation location, long size ) { + public long getScore( Task task, NodeWithAlloc location, long size ) { //Add one to avoid becoming zero final int rank = task.getProcess().getRank() + 1; final long rankFactor = 100_000_000_000_000L * rank; // long would allow a rank of 92233 diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index 695c3b65..06802ddb 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -1,12 +1,13 @@ package cws.k8s.scheduler.util.score; +import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; public class FileSizeScore implements CalculateScore { @Override - public long getScore( Task task, NodeLocation location, long size ) { + public long getScore( Task task, NodeWithAlloc location, long size ) { //add one to prefer two tasks which sum up to the same score otherwise return size + 1; } From 544ff59e5ebda68919f8dd302ff6d02f30f4fa02 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:19:33 +0100 Subject: [PATCH 421/443] Add documentation for Scheduler.java Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index c40759d3..841a011c 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -168,6 +168,11 @@ abstract ScheduleObject getTaskNodeAlignment( final Map availableByNode ); + /** + * This method is called when tasks are finished + * @param finishedTasks the tasks that are finished + * @return the number of tasks that were not marked as finished successfully + */ int terminateTasks( final List finishedTasks ) { for (Task finishedTask : finishedTasks) { taskWasFinished( finishedTask ); @@ -176,7 +181,6 @@ int terminateTasks( final List finishedTasks ) { } /* Pod Event */ - void podEventReceived(Watcher.Action action, Pod pod){} void onPodTermination( PodWithAge pod ){ From e96c23c159c3e39607bf66d025a305185bb61861 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:20:40 +0100 Subject: [PATCH 422/443] Add a class comment to TaskOnNodeWrapper.java Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java index 8635b6d4..0d8066c6 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java @@ -7,6 +7,9 @@ import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; +/** + * This class is used to keep track of the number of tasks with a specific label on a node. + */ @Getter @EqualsAndHashCode @RequiredArgsConstructor From 4518b6e626fd13a66f8d22473dcc154e43eda068 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:21:17 +0100 Subject: [PATCH 423/443] Comment OutputFiles.java Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/cluster/OutputFiles.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java index 4a611a50..a19a2a27 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java @@ -14,6 +14,11 @@ public class OutputFiles { private final Set files; + + /** + * this is used to check if the output data was requested for a real task + * As soon as one file is requested for a real task, the output data is considered as requested for a real task + */ private boolean wasRequestedForRealTask = false; public void wasRequestedForRealTask(){ From 140eff3291ba29d69571e683fa2f223fabd8fbc4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:22:41 +0100 Subject: [PATCH 424/443] Comment DataMissing.java Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/cluster/DataMissing.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java index e64e5d0d..b35dd19c 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java @@ -6,6 +6,12 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; +import java.util.List; + +/** + * This class is used to create a triple of a task, a node and a label count. + * It is used to keep track which tasks are not on a node. + */ @Getter @EqualsAndHashCode @RequiredArgsConstructor From f76693d332ca9b604e14125561991ceec56bd489 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:23:00 +0100 Subject: [PATCH 425/443] Comment FakeTaskInputs.java Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java b/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java index 96febd79..2319f047 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java @@ -7,6 +7,10 @@ import java.util.LinkedList; import java.util.List; +/** + * Fake task inputs are used to create {@link CopyTask}s that are not real tasks but are used to copy data between nodes. + * This task can only be run on the node that is included in the constructor. + */ public class FakeTaskInputs extends TaskInputs { private final Location includedNode; From ae91bb5ef9d5e4361b73c7cfbe9ed44b6d3ae21f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:27:52 +0100 Subject: [PATCH 426/443] Optimize Imports Signed-off-by: Lehmann_Fabian --- .../java/cws/k8s/scheduler/model/Task.java | 1 - .../k8s/scheduler/model/TaskResultParser.java | 4 +++- .../model/cluster/FileSizeRankScoreGroup.java | 1 - .../scheduler/model/cluster/OutputFiles.java | 2 +- .../location/hierachy/LocationWrapper.java | 2 +- .../location/hierachy/RealHierarchyFile.java | 2 +- .../model/taskinputs/TaskInputs.java | 4 ++-- .../rest/SchedulerRestController.java | 14 +++++------ .../LocationAwareSchedulerGroups.java | 6 +++-- .../scheduler/LocationAwareSchedulerV2.java | 24 +++++++++++-------- .../scheduler/PrioritizeAssignScheduler.java | 3 ++- .../k8s/scheduler/scheduler/Scheduler.java | 2 +- .../scheduler/SchedulerWithDaemonSet.java | 18 ++++++++------ .../filealignment/GreedyAlignment.java | 2 +- .../filealignment/InputAlignmentClass.java | 2 +- .../k8s/scheduler/scheduler/la2/TaskStat.java | 3 ++- .../CapacityAvailableToNode.java | 4 ++-- .../CopyInAdvanceNodeWithMostData.java | 4 ++-- .../ready2run/OptimalReadyToRunToNode.java | 4 ++-- .../scheduler/nodeassign/NodeAssign.java | 2 +- .../nodeassign/RoundRobinAssign.java | 5 +++- .../java/cws/k8s/scheduler/util/CopyTask.java | 6 ++--- .../cws/k8s/scheduler/util/TaskStats.java | 6 +++-- .../util/score/FileSizeRankScore.java | 1 - .../scheduler/util/score/FileSizeScore.java | 1 - .../hierachy/HierarchyWrapperTest.java | 2 +- .../hierachy/RealHierarchyFileTest.java | 2 +- .../prioritize/RankPrioritizeTest.java | 4 ++-- .../k8s/scheduler/util/SortedListTest.java | 2 +- 29 files changed, 74 insertions(+), 59 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index 49172fc4..cd9dd4be 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -5,7 +5,6 @@ import cws.k8s.scheduler.model.cluster.OutputFiles; import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; -import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; import cws.k8s.scheduler.model.tracing.TraceRecord; import cws.k8s.scheduler.util.Batch; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; diff --git a/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java b/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java index 971b8e3b..8edd5fac 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java @@ -13,7 +13,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.HashSet; +import java.util.Scanner; +import java.util.Set; import java.util.stream.Stream; @Slf4j diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java index 96334a8d..68042667 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.util.score.FileSizeRankScore; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java index a19a2a27..4f3e67c3 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java @@ -14,7 +14,7 @@ public class OutputFiles { private final Set files; - + /** * this is used to check if the output data was requested for a real task * As soon as one file is requested for a real task, the output data is considered as requested for a real task diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java index 793c8583..ff3c3e5e 100644 --- a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java @@ -1,7 +1,7 @@ package cws.k8s.scheduler.model.location.hierachy; -import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; import lombok.Getter; import java.util.Objects; diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java index af0870d8..27921a61 100644 --- a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -1,9 +1,9 @@ package cws.k8s.scheduler.model.location.hierachy; import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.LocationType; -import cws.k8s.scheduler.model.Task; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java index 83937e3d..72121d4b 100644 --- a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java @@ -1,11 +1,11 @@ package cws.k8s.scheduler.model.taskinputs; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import cws.k8s.scheduler.util.TaskNodeStats; import cws.k8s.scheduler.util.Tuple; import cws.k8s.scheduler.util.copying.CopySource; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; -import cws.k8s.scheduler.model.location.Location; -import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 546ec2cf..aece5237 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -1,23 +1,23 @@ package cws.k8s.scheduler.rest; -import cws.k8s.scheduler.model.SchedulerConfig; -import cws.k8s.scheduler.model.TaskConfig; -import cws.k8s.scheduler.scheduler.*; -import cws.k8s.scheduler.scheduler.filealignment.GreedyAlignment; -import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; -import cws.k8s.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.InputEdge; import cws.k8s.scheduler.dag.Vertex; -import cws.k8s.scheduler.scheduler.prioritize.*; +import cws.k8s.scheduler.model.SchedulerConfig; +import cws.k8s.scheduler.model.TaskConfig; import cws.k8s.scheduler.rest.exceptions.NotARealFileException; import cws.k8s.scheduler.rest.response.getfile.FileResponse; +import cws.k8s.scheduler.scheduler.*; +import cws.k8s.scheduler.scheduler.filealignment.GreedyAlignment; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; import cws.k8s.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; import cws.k8s.scheduler.scheduler.nodeassign.FairAssign; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RandomNodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RoundRobinAssign; +import cws.k8s.scheduler.scheduler.prioritize.*; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java index 2bf9c1c6..6facb0a6 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java @@ -5,7 +5,9 @@ import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.cluster.*; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; -import cws.k8s.scheduler.scheduler.la2.*; +import cws.k8s.scheduler.scheduler.la2.MinCopyingComparator; +import cws.k8s.scheduler.scheduler.la2.MinSizeComparator; +import cws.k8s.scheduler.scheduler.la2.TaskStat; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; import cws.k8s.scheduler.util.NodeTaskFilesAlignment; @@ -45,7 +47,7 @@ int terminateTasks( List finishedTasks ) { @Override List getAdditionalTaskStatPhaseThree(){ - return groupCluster.getTaskStatToCopy(); + return groupCluster.getTaskStatToCopy(1); } void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index 1a0dc4e1..6b2fc45d 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -1,13 +1,7 @@ package cws.k8s.scheduler.scheduler; -import cws.k8s.scheduler.model.*; -import cws.k8s.scheduler.scheduler.la2.*; -import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; -import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; -import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; -import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; import cws.k8s.scheduler.client.KubernetesClient; -import cws.k8s.scheduler.util.*; +import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; @@ -16,18 +10,29 @@ import cws.k8s.scheduler.model.taskinputs.TaskInputs; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.*; import cws.k8s.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import cws.k8s.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; import cws.k8s.scheduler.scheduler.la2.copystrategy.CopyRunner; import cws.k8s.scheduler.scheduler.la2.copystrategy.ShellCopy; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; +import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.*; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import cws.k8s.scheduler.util.score.FileSizeRankScore; -import lombok.*; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import java.io.*; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -138,7 +143,6 @@ public ScheduleObject getTaskNodeAlignment( .stream() .filter( node -> { final CurrentlyCopyingOnNode copyingFilesToNode = getCurrentlyCopying().get( node.getNodeLocation() ); - //File version does not match and is in use return inputsOfTask.canRunOnLoc( node.getNodeLocation() ) //Affinities are correct and the node can run new pods && canSchedulePodOnNode( task.getPod(), node ) diff --git a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java index fb46bbf2..4cf2b1ef 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -8,7 +8,8 @@ import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; -import java.util.*; +import java.util.List; +import java.util.Map; @Slf4j public class PrioritizeAssignScheduler extends Scheduler { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index 841a011c..fd75bbcb 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -1,9 +1,9 @@ package cws.k8s.scheduler.scheduler; -import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.util.Batch; import cws.k8s.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Pod; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 265cd4fd..3fa98c6f 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -1,16 +1,13 @@ package cws.k8s.scheduler.scheduler; import com.fasterxml.jackson.databind.ObjectMapper; +import cws.k8s.scheduler.client.KubernetesClient; import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.model.cluster.OutputFiles; -import cws.k8s.scheduler.model.location.hierachy.*; -import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; -import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; -import cws.k8s.scheduler.client.KubernetesClient; -import cws.k8s.scheduler.util.*; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.LocationType; import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.*; import cws.k8s.scheduler.model.outfiles.OutputFile; import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; import cws.k8s.scheduler.model.outfiles.SymlinkOutput; @@ -20,16 +17,23 @@ import cws.k8s.scheduler.rest.response.getfile.FileResponse; import cws.k8s.scheduler.scheduler.copystrategy.CopyStrategy; import cws.k8s.scheduler.scheduler.copystrategy.FTPstrategy; +import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.*; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; -import io.fabric8.kubernetes.api.model.*; +import io.fabric8.kubernetes.api.model.ContainerStatus; +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.net.ftp.FTPClient; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java index a3c69bef..32b974c3 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -1,6 +1,5 @@ package cws.k8s.scheduler.scheduler.filealignment; -import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.Location; @@ -8,6 +7,7 @@ import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; import cws.k8s.scheduler.util.AlignmentWrapper; import cws.k8s.scheduler.util.FileAlignment; import cws.k8s.scheduler.util.FilePath; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java index 0f141268..ef9456ad 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -1,6 +1,5 @@ package cws.k8s.scheduler.scheduler.filealignment; -import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.Location; @@ -8,6 +7,7 @@ import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; import cws.k8s.scheduler.util.AlignmentWrapper; import cws.k8s.scheduler.util.FileAlignment; import cws.k8s.scheduler.util.FilePathWithTask; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java index e30c406f..68425f80 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java @@ -9,7 +9,8 @@ import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; -import java.util.*; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @RequiredArgsConstructor diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java index b0c8a7d6..2f7e80fc 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java @@ -1,13 +1,13 @@ package cws.k8s.scheduler.scheduler.la2.capacityavailable; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import cws.k8s.scheduler.util.TaskStats; import cws.k8s.scheduler.util.copying.CurrentlyCopying; -import cws.k8s.scheduler.model.NodeWithAlloc; -import cws.k8s.scheduler.model.Requirements; import java.util.List; import java.util.Map; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java index 8027ea04..7bcaa7c8 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -1,5 +1,7 @@ package cws.k8s.scheduler.scheduler.la2.copyinadvance; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; import cws.k8s.scheduler.scheduler.la2.TaskStat; @@ -7,8 +9,6 @@ import cws.k8s.scheduler.util.SortedList; import cws.k8s.scheduler.util.TaskStats; import cws.k8s.scheduler.util.copying.CurrentlyCopying; -import cws.k8s.scheduler.model.NodeWithAlloc; -import cws.k8s.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; import java.util.List; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java index c619202e..aeeeb4fd 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -1,13 +1,13 @@ package cws.k8s.scheduler.scheduler.la2.ready2run; +import com.google.ortools.Loader; +import com.google.ortools.sat.*; import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.util.LogCopyTask; import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; import cws.k8s.scheduler.util.score.CalculateScore; -import com.google.ortools.Loader; -import com.google.ortools.sat.*; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java index 1ef6f1de..8c58a8fd 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/NodeAssign.java @@ -1,9 +1,9 @@ package cws.k8s.scheduler.scheduler.nodeassign; -import cws.k8s.scheduler.scheduler.Scheduler; import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Requirements; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.scheduler.Scheduler; import cws.k8s.scheduler.util.NodeTaskAlignment; import java.util.List; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java index 0ba3528e..d56f4199 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/nodeassign/RoundRobinAssign.java @@ -8,7 +8,10 @@ import cws.k8s.scheduler.util.NodeTaskAlignment; import lombok.extern.slf4j.Slf4j; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; @Slf4j public class RoundRobinAssign extends NodeAssign implements Informable { diff --git a/src/main/java/cws/k8s/scheduler/util/CopyTask.java b/src/main/java/cws/k8s/scheduler/util/CopyTask.java index a0730e25..13b381aa 100644 --- a/src/main/java/cws/k8s/scheduler/util/CopyTask.java +++ b/src/main/java/cws/k8s/scheduler/util/CopyTask.java @@ -1,11 +1,11 @@ package cws.k8s.scheduler.util; -import cws.k8s.scheduler.model.TaskInputFileLocationWrapper; -import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; -import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskInputFileLocationWrapper; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/cws/k8s/scheduler/util/TaskStats.java b/src/main/java/cws/k8s/scheduler/util/TaskStats.java index 0db95707..8aaca2cd 100644 --- a/src/main/java/cws/k8s/scheduler/util/TaskStats.java +++ b/src/main/java/cws/k8s/scheduler/util/TaskStats.java @@ -1,10 +1,12 @@ package cws.k8s.scheduler.util; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.scheduler.la2.TaskStat; import cws.k8s.scheduler.scheduler.la2.TaskStatComparator; -import cws.k8s.scheduler.model.Task; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; public class TaskStats { diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java index 0e17f60e..79694528 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; public class FileSizeRankScore extends FileSizeScore { diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java index 06802ddb..52c5da88 100644 --- a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.model.location.NodeLocation; public class FileSizeScore implements CalculateScore { diff --git a/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java index 32abad3b..773597c9 100644 --- a/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -3,9 +3,9 @@ import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.Process; import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.TaskConfig; import cws.k8s.scheduler.model.location.NodeLocation; -import cws.k8s.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java index 98054da0..edee1741 100644 --- a/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -4,10 +4,10 @@ import cws.k8s.scheduler.dag.InputEdge; import cws.k8s.scheduler.dag.Process; import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.model.TaskConfig; import cws.k8s.scheduler.model.location.LocationType; import cws.k8s.scheduler.model.location.NodeLocation; -import cws.k8s.scheduler.model.Task; import org.apache.commons.collections4.iterators.PermutationIterator; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java index 7571ce91..ef211cb4 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankPrioritizeTest.java @@ -1,18 +1,18 @@ package cws.k8s.scheduler.scheduler.prioritize; -import cws.k8s.scheduler.model.TaskConfig; import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.InputEdge; import cws.k8s.scheduler.dag.Process; import cws.k8s.scheduler.dag.Vertex; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskConfig; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class RankPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/util/SortedListTest.java b/src/test/java/cws/k8s/scheduler/util/SortedListTest.java index 512d95c8..ac2768a3 100644 --- a/src/test/java/cws/k8s/scheduler/util/SortedListTest.java +++ b/src/test/java/cws/k8s/scheduler/util/SortedListTest.java @@ -7,7 +7,7 @@ import java.util.LinkedList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j class SortedListTest { From de5fda5934923200911fd2a8663a0e3353e1737a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 6 Nov 2024 16:35:58 +0100 Subject: [PATCH 427/443] Refactor and comment GroupCluster and LabelCount Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/model/cluster/CopyTask.java | 13 +- .../scheduler/model/cluster/DataMissing.java | 5 +- .../model/cluster/DataMissingIntern.java | 19 ++ .../scheduler/model/cluster/GroupCluster.java | 204 ++++++++++++------ .../scheduler/model/cluster/LabelCount.java | 129 ++++++++--- .../model/cluster/SimpleGroupCluster.java | 2 +- .../model/cluster/GroupClusterTest.java | 37 ++++ 7 files changed, 304 insertions(+), 105 deletions(-) create mode 100644 src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java create mode 100644 src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java index d01f2650..960af27e 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java @@ -8,6 +8,8 @@ import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; @Slf4j public class CopyTask extends Task { @@ -15,14 +17,15 @@ public class CopyTask extends Task { private final static Process COPY_PROCESS = new Process( "Copy", Integer.MAX_VALUE ); private final Task task; private final NodeWithAlloc node; - private final LabelCount labelCount; + private final List labelCounts; - protected CopyTask( Task task, NodeWithAlloc node, LabelCount labelCount ) { + protected CopyTask( Task task, NodeWithAlloc node, List labelCounts ) { super( new TaskConfig("Copy","Copy-" + task.getConfig().getName() + " -> " + node.getNodeLocation() - + " (" + labelCount.getLabel() + ")" , buildCopyTaskFolder(task), task.getConfig().getRunName() ), COPY_PROCESS ); + + " (" + labelCounts.stream().map( LabelCount::getLabel ).collect( Collectors.joining(",")) + ")" , + buildCopyTaskFolder(task), task.getConfig().getRunName() ), COPY_PROCESS ); this.task = task; this.node = node; - this.labelCount = labelCount; + this.labelCounts = labelCounts; } private static String buildCopyTaskFolder( Task task ) { @@ -55,6 +58,6 @@ public int hashCode() { } public void finished(){ - labelCount.taskIsNowOnNode( task, node ); + labelCounts.forEach( x -> x.taskIsNowOnNode( task, node )); } } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java index b35dd19c..50d597dc 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.Task; -import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -13,12 +12,12 @@ * It is used to keep track which tasks are not on a node. */ @Getter -@EqualsAndHashCode @RequiredArgsConstructor public class DataMissing { private final Task task; private final NodeWithAlloc node; - private final LabelCount labelCount; + private final List labelCounts; + private final double score; } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java new file mode 100644 index 00000000..0bf9def4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java @@ -0,0 +1,19 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * This is a temporary class to group a task, a node and a label count. + */ +@RequiredArgsConstructor +@Getter +public class DataMissingIntern { + + private final Task task; + private final NodeWithAlloc node; + private final LabelCount labelCount; + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java index 97bca5bc..979419b6 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java @@ -18,61 +18,58 @@ import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; @Slf4j @RequiredArgsConstructor public abstract class GroupCluster { private final double minScoreToCopy = 0.5; - protected final Set allLabels = new HashSet<>(); + // Group all tasks by label and deliver statistics protected final Map countPerLabel = new HashMap<>(); + // tasks that are not yet scheduled protected final LinkedList unscheduledTasks = new LinkedList<>(); + // tasks that are assigned to a task and running protected final LinkedList scheduledTasks = new LinkedList<>(); + // tasks that have finished protected final LinkedList finishedTasks = new LinkedList<>(); + // Map from label to node, which node is responsible for the label protected final Map labelToNode = new HashMap<>(); + // Map from node to label, which labels are assigned to the node protected final Map> nodeToLabel = new HashMap<>(); @Getter private final HierarchyWrapper hierarchyWrapper; @Getter private final KubernetesClient client; - private void addAllOutLabels( List tasks ) { - for ( Task task : tasks ) { - if ( task.getOutLabel() != null ) { - allLabels.addAll( task.getOutLabel() ); - } - } - } - + /** + * When new tasks become available and are ready to schedule, this method is called to register them. + * @param tasks the tasks that are ready to schedule + */ public void tasksBecameAvailable( List tasks ) { boolean anyWithLabel = false; synchronized ( this ) { unscheduledTasks.addAll( tasks ); - addAllOutLabels( tasks ); for ( Task task : tasks ) { if ( task.getOutLabel() == null || task.getOutLabel().isEmpty() ) { continue; } for ( String label : task.getOutLabel() ) { anyWithLabel = true; - countPerLabel.computeIfAbsent( label, k -> new LabelCount(k) ).addWaitingTask( task ); - final LabelCount labelCount; - if ( !countPerLabel.containsKey( label ) ) { - labelCount = new LabelCount( label ); - countPerLabel.put( label, labelCount ); - } else { - labelCount = countPerLabel.get( label ); - } + final LabelCount labelCount = countPerLabel.computeIfAbsent( label, LabelCount::new ); labelCount.addWaitingTask( task ); } } + // If we add at least one task with an outLabel we need to recalculate our alignment if ( anyWithLabel ) { recalculate(); } } } + /** + * This method is called when a task was scheduled to a node and can no longer be scheduled to another node. + * @param task the task that was scheduled + */ public void taskWasAssignedToNode( Task task ) { synchronized ( this ) { unscheduledTasks.remove( task ); @@ -86,6 +83,10 @@ public void taskWasAssignedToNode( Task task ) { } } + /** + * This method is called when a task has finished and output data can be considered. + * @param tasks the tasks that have finished + */ public void tasksHaveFinished( List tasks ) { synchronized ( this ) { scheduledTasks.removeAll( tasks ); @@ -95,35 +96,69 @@ public void tasksHaveFinished( List tasks ) { continue; } for ( String label : task.getOutLabel() ) { - if ( countPerLabel.get( label ) == null ) { - continue; - } countPerLabel.get( label ).makeTaskFinished( task ); } } } } + /** + * This method is called when the state changes. + * This is either the case when a new task becomes available, or a task is scheduled. + */ abstract void recalculate(); + /** + * Return the score for a task on a node. + * This calculates the score using jaccard similarity coefficient. + * Therefore, it calculates the shared labels between the task and the node and divides it by the total amount of labels of the task. + * @param task the task + * @param node the node + * @return the score in the range of 0 to 1, where 1 is the best score + */ public double getScoreForTaskOnNode( Task task, NodeWithAlloc node ) { - if ( task.getOutLabel() == null || task.getOutLabel().isEmpty() ) { + // If the task has no outLabel, it can run on any node + final Set outLabel = task.getOutLabel(); + if ( outLabel == null || outLabel.isEmpty() ) { return 1; } - //Jaccard similarity coefficient - final Set outLabel = new HashSet<>( task.getOutLabel() ); - int outLabelSize = outLabel.size(); + final Set nodeLabels = nodeToLabel.get( node ); - if ( nodeLabels == null ) { - return 0.01; + return calculateJaccardSimilarityCoefficient( outLabel, nodeLabels ); + } + + /** + * Calculate the Jaccard similarity coefficient between two sets of labels. + * If there is no overlap, the coefficient is 0.01. + * @param outLabel the first set of labels, this cannot be empty or null + * @param nodeLabels the second set of labels, this can be null or empty + * @return the Jaccard similarity coefficient in the range of 0.01 to 1 + */ + static double calculateJaccardSimilarityCoefficient( Set outLabel, Set nodeLabels ) { + if ( outLabel == null || outLabel.isEmpty() ) { + throw new IllegalArgumentException( "outLabel cannot be empty or null" ); + } + final double lowerBound = 0.01; + if ( nodeLabels == null || nodeLabels.isEmpty() ) { + return lowerBound; } + outLabel = new HashSet<>( outLabel ); + int outLabelSize = outLabel.size(); + //intersection of both sets. outLabel.retainAll( nodeLabels ); + // No common labels if ( outLabelSize == 0 ) { - return 1; + return lowerBound; } - return (double) outLabel.size() / outLabelSize + 0.01; // to avoid zero + final double result = (double) outLabel.size() / outLabelSize; + return Math.max( lowerBound, Math.min( 1, result )); // Clamp the result between 0.01 and 1 } + /** + * Assign a node to a label, such that tasks for this label can be prepared on the node. + * @param nodeLocation the node + * @param label the label + */ protected void addNodeToLabel( NodeWithAlloc nodeLocation, String label ) { final NodeWithAlloc currentLocation = labelToNode.get( label ); @@ -142,89 +177,122 @@ protected void addNodeToLabel( NodeWithAlloc nodeLocation, String label ) { } /** - * Return a list of all tasks for that a node exists where getScore is higher than minScoreToCopy and the task has at least one outLabel - * @return + * Create TaskStat for tasks that could be prepared on a node, return maximal {@code maxCopiesPerNode} TaskStat per node. + * Only tasks with a score higher than {@link #minScoreToCopy} are considered. + * @param maxCopiesPerNode the maximum amount of copies that will be created for a node + * @return a list of all TaskStats that should be copied */ - public List getTaskStatToCopy(){ - int maxCopiesPerNode = 1; + public List getTaskStatToCopy( final int maxCopiesPerNode ){ synchronized ( this ) { + final Map tasksForLocation = new HashMap<>(); //all tasks where score > minScoreToCopy and tasks have at least one outLabel - final Map taskForLocation = new HashMap<>(); - return getTasksThatNeedToBeCopied() - .sequential() - //Only create maxCopiesPerNode possible copy tasks - .filter( x -> shouldStillCreate( x, taskForLocation, maxCopiesPerNode ) ) - .map( this::getTaskStat ) - .filter( Objects::nonNull ) - .collect( Collectors.toList()); - } - } - - private static boolean shouldStillCreate( DataMissing x, Map taskForLocation, int maxCopiesPerNode ) { - if ( taskForLocation.containsKey( x.getNode() ) ) { - final Integer i = taskForLocation.get( x.getNode() ); - if ( i == maxCopiesPerNode ) { - return false; - } else { - taskForLocation.put( x.getNode(), i + 1 ); - return true; + final List tasksThatNeedToBeCopied = getTasksThatNeedToBeCopied(); + List taskStats = new ArrayList<>(); + for ( DataMissing dataMissing : tasksThatNeedToBeCopied ) { + Integer currentCopies = tasksForLocation.getOrDefault( dataMissing.getNode(), 0 ); + // Only create maxCopiesPerNode possible TaskStats per node + TaskStat taskStat = currentCopies < maxCopiesPerNode ? getTaskStat( dataMissing ) : null; + // TaskStat is null if the output data was requested for a real task already and should not be copied anymore + if ( taskStat != null ) { + taskStats.add( taskStat ); + tasksForLocation.put( dataMissing.getNode(), ++currentCopies ); + } } - } else { - taskForLocation.put( x.getNode(), 1 ); - return true; + return taskStats; } } + /** + * Create a TaskStat for a {@link DataMissing} object. + * @param dataMissing the DataMissing object + * @return the TaskStat or null if the output data was requested for a real task already + */ private TaskStat getTaskStat( DataMissing dataMissing ) { final NodeWithAlloc node = dataMissing.getNode(); final OutputFiles outputFilesWrapper = dataMissing.getTask().getOutputFiles(); final Set outputFiles = outputFilesWrapper.getFiles(); + // Do not check for OutputFiles#wasRequestedForRealTask() here, + // because it is checked when the DataMissing object is created in LabelCount.tasksNotOnNode List files = outputFiles .parallelStream() .map( x -> convertToPathFileLocationTriple(x, dataMissing.getTask()) ) .collect( Collectors.toList() ); + // If in the previous step at least one file was requested for a real task, + // all the output data is considered as requested for a real task if ( dataMissing.getTask().getOutputFiles().isWasRequestedForRealTask() ) { return null; } TaskInputs inputsOfTask = new FakeTaskInputs( files, node.getNodeLocation() ); - final CopyTask copyTask = new CopyTask( dataMissing.getTask(), node, dataMissing.getLabelCount() ); + final CopyTask copyTask = new CopyTask( dataMissing.getTask(), node, dataMissing.getLabelCounts() ); final TaskStat taskStat = new TaskStat( copyTask, inputsOfTask ); + // TaskNodeStats is created with the total size and 0 for the data that is on the node final TaskNodeStats taskNodeStats = new TaskNodeStats( inputsOfTask.calculateAvgSize(), 0, 0 ); - //client. taskStat.add( dataMissing.getNode(), taskNodeStats ); return taskStat; } + /** + * Convert a PathLocationWrapperPair to a PathFileLocationTriple. + * Check for every file if it was requested for a real task. + * @param pair the PathLocationWrapperPair containing the LocationWrapper to process + * @param task the task that created the output data + * @return the PathFileLocationTriple or null if the output data was requested for a real task + */ private PathFileLocationTriple convertToPathFileLocationTriple( PathLocationWrapperPair pair, Task task ){ final Path path = pair.getPath(); RealHierarchyFile file = (RealHierarchyFile) hierarchyWrapper.getFile( path ); - final RealHierarchyFile.MatchingLocationsPair filesForTask; + // if at least one file was requested for a real task, all the output data is considered as requested for a real task + // return null as we ignore this output data for proactively copying if ( file.wasRequestedByTask() ) { task.getOutputFiles().wasRequestedForRealTask(); return null; } try { - filesForTask = file.getFilesForTask( task ); + // task is the task that created the output data + final RealHierarchyFile.MatchingLocationsPair filesForTask = file.getFilesForTask( task ); + return new PathFileLocationTriple( path, file, filesForTask.getMatchingLocations() ); } catch ( NoAlignmentFoundException e ) { throw new RuntimeException( e ); } - return new PathFileLocationTriple( path, file, filesForTask.getMatchingLocations() ); } - private Stream getTasksThatNeedToBeCopied(){ - return countPerLabel.entrySet() + /** + * Get all MissingData for tasks. + * This method checks for each task if the output data is on the node that is responsible for the label. + * If the data is not on the node, a DataMissing object is created. But only if the score is higher than {@link #minScoreToCopy}. + * @return a stream of DataMissing objects + */ + private List getTasksThatNeedToBeCopied(){ + final Map, List> collect = countPerLabel.entrySet() .parallelStream() .unordered() .flatMap( v -> { final String label = v.getKey(); final LabelCount value = v.getValue(); + // which node is responsible for the label final NodeWithAlloc node = labelToNode.get( label ); - return value.taskNotOnNode( node ); + // get missing data for all tasks that are not on the node + return value.tasksNotOnNode( node ); } ) - .distinct() - .filter( d -> getScoreForTaskOnNode( d.getTask(), d.getNode() ) > minScoreToCopy ) - .sorted( (x, y) -> Double.compare( getScoreForTaskOnNode( y.getTask(), y.getNode() ), - getScoreForTaskOnNode( x.getTask(), x.getNode() ) ) ); + // Group all Labels for DataMissingIntern with the same task and node into a map + .collect( Collectors.groupingBy( + dmi -> Map.entry( dmi.getTask(), dmi.getNode() ), + Collectors.mapping( DataMissingIntern::getLabelCount, Collectors.toList() ) + ) ); + List result = new ArrayList<>( collect.size() ); + for ( Map.Entry, List> e : collect.entrySet() ) { + final Task task = e.getKey().getKey(); + final NodeWithAlloc node = e.getKey().getValue(); + final List labelCounts = e.getValue(); + final double score = getScoreForTaskOnNode( task, node ); + // Only return tasks that have a score higher than minScoreToCopy + if ( score > minScoreToCopy ) { + result.add( new DataMissing( task, node, labelCounts, score ) ); + } + } + // Sort the tasks by score in descending order: highest score first + result.sort( (x, y) -> Double.compare( y.getScore(), x.getScore() ) ); + return result; } } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java index fa06e30f..96bc6b6a 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java @@ -9,35 +9,90 @@ import java.util.*; import java.util.stream.Stream; +/** + * This class is used to keep track of the number of tasks with a specific label + * and on which nodes the tasks are running, waiting or there output is stored. + * This class is not thread safe! + */ @RequiredArgsConstructor( access = AccessLevel.PACKAGE ) public class LabelCount { + /** + * The label that all tasks have + */ @Getter private final String label; - @Getter - private int countFinished = 0; - @Getter - private int countStarted = 0; - @Getter - private int countWaiting = 0; - + /** + * The tasks that are not yet started + */ @Getter private final Set waitingTasks = new HashSet<>(); + + /** + * The tasks that are running + */ private final Set runningTasks = new HashSet<>(); + + /** + * The tasks that are finished + */ private final Set finishedTasks = new HashSet<>(); - private final Set tasks = new HashSet<>(); + + /** + * A Queue with a wrapper that contains the number of tasks running or finished on a node. + * The queue is sorted by the number of tasks running or finished on a node, such that the node with the most tasks is first. + */ @Getter private final Queue runningOrfinishedOnNodes = new PriorityQueue<>(); + + /** + * For each node store the number of tasks that are running or ran on it, + * this map is the index for the elements in {@link #runningOrfinishedOnNodes} + */ private final Map nodeToShare = new HashMap<>(); + + /** + * For each task with this label store the nodes where the output data of the task is stored + */ private final Map> taskHasDataOnNode = new HashMap<>(); - public Stream taskNotOnNode( NodeWithAlloc node ) { + /** + * Get output data for all tasks that are not on the node + * @param node the node to check + * @return a stream of DataMissing objects containing the tasks' output data that is missing on the node + */ + public Stream tasksNotOnNode( NodeWithAlloc node ) { return taskHasDataOnNode.entrySet() .stream() + // check if the task's output is not on the node and the output data was not requested for a real task .filter( taskSetEntry -> !taskSetEntry.getValue().contains( node ) && !taskSetEntry.getKey().getOutputFiles().isWasRequestedForRealTask() ) - .map( x -> new DataMissing( x.getKey(), node, this ) ); + .map( x -> new DataMissingIntern( x.getKey(), node, this ) ); + } + + /** + * Get the number of tasks with this label that are not yet started + * @return the number of tasks with this label that are not yet started + */ + public int getCountWaiting() { + return waitingTasks.size(); + } + + /** + * Get the number of tasks with this label that are running + * @return the number of tasks with this label that are running + */ + public int getCountRunning() { + return runningTasks.size(); + } + + /** + * Get the number of tasks with this label that are finished + * @return the number of tasks with this label that are finished + */ + public int getCountFinished() { + return finishedTasks.size(); } /** @@ -45,42 +100,60 @@ public Stream taskNotOnNode( NodeWithAlloc node ) { * @return the number of tasks with this label */ public int getCount(){ - return countFinished + countStarted + countWaiting; - } + return getCountWaiting() + getCountRunning() + getCountFinished(); + } + /** + * Add a task to the label count, the task is not yet started + * the task needs to have this label, however the label is not checked + * @param task the task to add with this label + */ public void addWaitingTask( Task task ){ - countWaiting++; - tasks.add(task); waitingTasks.add(task); } + /** + * Make a task running, the task needs to have this label, however the label is not checked + * Adds the task to {@link #runningOrfinishedOnNodes} + * @param task the task to make running + */ public void makeTaskRunning( Task task ) { final NodeWithAlloc node = task.getNode(); - final TasksOnNodeWrapper tasksOnNodeWrapper; - if ( nodeToShare.containsKey( node ) ) { - tasksOnNodeWrapper = nodeToShare.get( node ); - } else { - tasksOnNodeWrapper = new TasksOnNodeWrapper( node ); - nodeToShare.put( node, tasksOnNodeWrapper ); - runningOrfinishedOnNodes.add( tasksOnNodeWrapper ); - } + final TasksOnNodeWrapper tasksOnNodeWrapper = nodeToShare.computeIfAbsent(node, k -> { + TasksOnNodeWrapper newWrapper = new TasksOnNodeWrapper(k); + runningOrfinishedOnNodes.add(newWrapper); + return newWrapper; + }); tasksOnNodeWrapper.addRunningTask(); - countWaiting--; - countStarted++; - waitingTasks.remove( task ); + final boolean remove = waitingTasks.remove( task ); + if ( !remove ) { + throw new IllegalStateException( "Task " + task + " was not in waiting tasks" ); + } runningTasks.add( task ); } + /** + * Make a task finished, the task needs to have this label, however the label is not checked + * adds the task to {@link #taskHasDataOnNode}, to mark that the output data is on the node + * @param task the task to make finished + */ public void makeTaskFinished( Task task ) { - countStarted--; - countFinished++; - runningTasks.remove( task ); + final boolean remove = runningTasks.remove( task ); + if ( !remove ) { + throw new IllegalStateException( "Task " + task + " was not in running tasks" ); + } finishedTasks.add( task ); final HashSet locations = new HashSet<>(); locations.add( task.getNode() ); taskHasDataOnNode.put( task, locations ); } + /** + * Mark that the task's output data is on a node. + * This method is called after the task was copied to the node. + * @param task the task + * @param node the node the task's output was copied to + */ public void taskIsNowOnNode( Task task, NodeWithAlloc node ) { taskHasDataOnNode.get( task ).add( node ); } diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java index c277ae76..6189e149 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java @@ -78,7 +78,7 @@ private NodeWithAlloc calculateBestFittingNode( String label, Queue runningOrfinishedOnNodes = new ArrayList<>(labelCount.getRunningOrfinishedOnNodes()); if ( runningOrfinishedOnNodes.isEmpty() ) { - // if tass with this label have not been executed + // if tasks with this label have not been executed return findBestFittingNode( bestFittingNodes, tasksOnNode ); } diff --git a/src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java b/src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java new file mode 100644 index 00000000..24a51cdd --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java @@ -0,0 +1,37 @@ +package cws.k8s.scheduler.model.cluster; + +import org.apache.commons.collections4.map.HashedMap; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GroupClusterTest { + + @Test + void jaccard() { + Set a = Set.of("a", "b", "c"); + Set b = Set.of("a", "b", "c"); + assertEquals(1.0, GroupCluster.calculateJaccardSimilarityCoefficient(a, b)); + + a = Set.of("a", "b", "c"); + b = Set.of("a", "b", "d"); + assertEquals(0.6666666666666666, GroupCluster.calculateJaccardSimilarityCoefficient(a, b)); + + a = Set.of("a", "b", "c"); + b = Set.of("d", "e", "f"); + assertEquals(0.01, GroupCluster.calculateJaccardSimilarityCoefficient(a, b)); + } + + @Test + void testTasksNotOnNode() { + Map labels = new HashedMap<>(); + for (int i = 0; i < 10; i++) { + System.out.println( labels.merge( "a", 1, Integer::sum ) ); + } + + } + +} \ No newline at end of file From f1a506a42885afc3fe044098244f7afed3165320 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 14 Feb 2025 17:23:58 +0100 Subject: [PATCH 428/443] Fix problem that the node of the pod with the workflow engine was not transfered Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/client/KubernetesClient.java | 14 ++++++++++++++ .../scheduler/rest/SchedulerRestController.java | 7 ++++++- .../scheduler/SchedulerWithDaemonSet.java | 9 +++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java index 62964b10..d694d1b0 100644 --- a/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java +++ b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java @@ -33,6 +33,20 @@ public KubernetesClient(){ nodeSharedIndexInformer.start(); } + public Pod getPodByIp( String ip ) { + return this.pods() + .inAnyNamespace() + .list() + .getItems() + .parallelStream() + .filter( pod -> pod.getStatus().getPodIP().equals( ip ) ) + .findFirst() + .orElseGet( () -> { + log.warn("No Pod found for IP: {}", ip); + return null; + }); + } + public void addInformable( Informable informable ){ synchronized ( informables ){ informables.add( informable ); diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index aece5237..0ef8d576 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -33,6 +33,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.bind.annotation.*; +import javax.servlet.http.HttpServletRequest; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -96,7 +97,8 @@ private ResponseEntity noSchedulerFor( String execution ){ @PostMapping("/v1/scheduler/{execution}") ResponseEntity registerScheduler( @PathVariable String execution, - @RequestBody(required = false) SchedulerConfig config + @RequestBody(required = false) SchedulerConfig config, + HttpServletRequest request ) { final String namespace = config.namespace; @@ -170,6 +172,9 @@ ResponseEntity registerScheduler( } } } + if ( scheduler instanceof SchedulerWithDaemonSet ) { + ((SchedulerWithDaemonSet) scheduler).setWorkflowEngineNode( request.getRemoteAddr() ); + } schedulerHolder.put( execution, scheduler ); client.addInformable( scheduler ); diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 3fa98c6f..54d398e2 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -434,12 +434,13 @@ void taskWasScheduledSetState( Task task ){ task.getState().setState( State.SCHEDULED ); } + public void setWorkflowEngineNode( String ip ){ + this.workflowEngineNode = client.getPodByIp( ip ).getSpec().getNodeName(); + log.info( "WorkflowEngineNode was set to {}", workflowEngineNode ); + } + @Override void podEventReceived(Watcher.Action action, Pod pod){ - if ( pod.getMetadata().getName().equals( this.getExecution().replace('_', '-') ) ){ - this.workflowEngineNode = pod.getSpec().getNodeName(); - log.info( "WorkflowEngineNode was set to {}", workflowEngineNode ); - } //noinspection LoopConditionNotUpdatedInsideLoop while ( daemonHolder == null ){ //The Watcher can be started before the class is initialized From ef1cdaac63f8040110ac60d2cd658832a34765e6 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 14 Feb 2025 17:35:23 +0100 Subject: [PATCH 429/443] Fix not all pods have an ip Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/client/KubernetesClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java index d694d1b0..5da6aa37 100644 --- a/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java +++ b/src/main/java/cws/k8s/scheduler/client/KubernetesClient.java @@ -34,12 +34,15 @@ public KubernetesClient(){ } public Pod getPodByIp( String ip ) { + if ( ip == null ) { + throw new IllegalArgumentException("IP cannot be null"); + } return this.pods() .inAnyNamespace() .list() .getItems() .parallelStream() - .filter( pod -> pod.getStatus().getPodIP().equals( ip ) ) + .filter( pod -> ip.equals( pod.getStatus().getPodIP() ) ) .findFirst() .orElseGet( () -> { log.warn("No Pod found for IP: {}", ip); From f715e7aa77951921953853537fe6c6018b64a207 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Mar 2025 19:59:29 +0100 Subject: [PATCH 430/443] Update to Java 21 + improve Dockerfile Signed-off-by: Lehmann_Fabian --- Dockerfile | 45 +++++------ pom.xml | 80 +++++++------------ .../rest/SchedulerRestController.java | 2 +- src/main/resources/application.properties | 4 + 4 files changed, 58 insertions(+), 73 deletions(-) create mode 100644 src/main/resources/application.properties diff --git a/Dockerfile b/Dockerfile index 319924d9..e9be27cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,22 @@ -FROM maven:3-openjdk-18-slim AS build -WORKDIR /build -COPY pom.xml pom.xml -RUN mvn dependency:go-offline --no-transfer-progress -Dmaven.repo.local=/mvn/.m2nrepo/repository -COPY src/ src/ -#RUN mvn package -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository - -RUN mvn package --no-transfer-progress -f /build/pom.xml -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository - -ENTRYPOINT ["/bin/sh","-c", "mvn spring-boot:run -f pom.xml -Dmaven.repo.local=/mvn/.m2nrepo/repository"] - -# -# Package stage -# -#FROM openjdk:18-alpine -#WORKDIR /app -#RUN apk add --no-cache libstdc++ -#RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data -#COPY --from=build /build/target/k8s-scheduler*.jar k8s-scheduler.jar -#RUN chown -R javauser:javagroup /app -#USER javauser -#EXPOSE 8080 -#ENTRYPOINT ["java","-jar","/app/k8s-scheduler.jar"] \ No newline at end of file +FROM eclipse-temurin:21-jdk-jammy AS builder +WORKDIR /app + +RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/* + +COPY pom.xml . +RUN mvn dependency:go-offline --no-transfer-progress + +COPY src/ ./src/ +RUN mvn package --no-transfer-progress -DskipTests + +FROM eclipse-temurin:21-jre-jammy + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/target/cws-k8s-scheduler-*-SNAPSHOT.jar app.jar + +CMD ["java", "-jar", "app.jar"] diff --git a/pom.xml b/pom.xml index 8fe8cbb4..4d3ae7b7 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.springframework.boot spring-boot-starter-parent - 2.4.2 + 3.4.3 @@ -16,8 +16,8 @@ 1.2-SNAPSHOT - 11 - 11 + 21 + 21 cws.k8s.scheduler.Main @@ -26,7 +26,7 @@ io.fabric8 kubernetes-client - 6.9.0 + 7.1.0 @@ -38,7 +38,7 @@ org.projectlombok lombok - 1.18.26 + 1.18.36 provided @@ -58,71 +58,64 @@ org.springframework.boot spring-boot-starter-web - 3.1.4 + 3.4.3 org.springframework.boot spring-boot-starter-test - 3.1.4 + 3.4.3 test org.jetbrains annotations - 24.0.1 + 26.0.2 compile - - org.junit.vintage - junit-vintage-engine - 5.9.2 - test - - org.apache.commons commons-collections4 - 4.4 + 4.5.0-M3 test commons-net commons-net - 3.10.0 + 3.11.1 org.springdoc - springdoc-openapi-ui - 1.7.0 + springdoc-openapi-starter-webmvc-ui + 2.8.5 - + ch.qos.logback logback-core - 1.4.6 + 1.5.17 com.fasterxml.jackson.core jackson-databind - 2.14.2 + 2.18.3 com.fasterxml.jackson.core jackson-core - 2.15.0-rc2 + 2.18.3 com.fasterxml.jackson.core jackson-annotations - 2.15.0-rc2 + 2.18.3 @@ -132,6 +125,18 @@ 9.4.1874 + + org.springframework.boot + spring-boot-starter-validation + + + + org.junit.vintage + junit-vintage-engine + 5.8.2 + test + + @@ -160,12 +165,6 @@ implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> META-INF/spring.schemas - - - cws.k8s.scheduler.Main - @@ -174,7 +173,7 @@ org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.12 surefireArgLine @@ -198,11 +197,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M5 - - - ${surefireArgLine} --illegal-access=permit - + 3.5.2 default-test @@ -213,19 +208,6 @@ - - org.springframework.boot - spring-boot-maven-plugin - - - build-info - - build-info - - - - - \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 0ef8d576..a62feaa0 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -22,6 +22,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import jakarta.servlet.http.HttpServletRequest; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -33,7 +34,6 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.bind.annotation.*; -import javax.servlet.http.HttpServletRequest; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 00000000..bdca2cf6 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,4 @@ +logging.level.root=INFO +logging.level.fonda=INFO +server.port=8080 +server.servlet.context-path=/ \ No newline at end of file From e9156919f98f575a6d5edf9dad72674b72bcfa4b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Mar 2025 20:03:28 +0100 Subject: [PATCH 431/443] Update Java version to 21 Signed-off-by: Lehmann_Fabian --- .github/workflows/workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 5a26c528..4b7bb6b6 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - java_version: [ 11, 19 ] + java_version: [ 21, 23 ] steps: - name: Checkout uses: actions/checkout@v3 @@ -29,10 +29,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: 11 + java-version: 21 distribution: 'temurin' - name: Set outputs id: vars From 1d642b8d78cff5cbbec6d9b06189a7d8171a900a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 5 Mar 2025 20:06:39 +0100 Subject: [PATCH 432/443] Only test Java 21 Signed-off-by: Lehmann_Fabian --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 4b7bb6b6..14fc8375 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - java_version: [ 21, 23 ] + java_version: [ 21 ] steps: - name: Checkout uses: actions/checkout@v3 From 7eea10e526de4317800e28a241286f5b6798b08f Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 6 Mar 2025 15:11:14 +0100 Subject: [PATCH 433/443] Add warnings if execution cannot be registered Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/rest/SchedulerRestController.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index a62feaa0..80c542fc 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -116,13 +116,16 @@ ResponseEntity registerScheduler( if ( config.costFunction != null ) { switch (config.costFunction.toLowerCase()) { case "minsize": costFunction = new MinSizeCost(0); break; - default: return new ResponseEntity<>( "No cost function: " + config.costFunction, HttpStatus.NOT_FOUND ); + default: + log.warn( "Register execution: {} - No cost function for: {}", execution, config.costFunction ); + return new ResponseEntity<>( "No cost function: " + config.costFunction, HttpStatus.NOT_FOUND ); } } switch ( strategy.toLowerCase() ){ case "lav2" : if ( !config.locationAware ) { + log.warn( "Register execution: {} - LA scheduler only works if location aware", execution ); return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); } if ( costFunction == null ) { @@ -132,6 +135,7 @@ ResponseEntity registerScheduler( break; case "lagroup" : if ( !config.locationAware ) { + log.warn( "Register execution: {} - LA scheduler only works if location aware", execution ); return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); } if ( costFunction == null ) { @@ -153,6 +157,7 @@ ResponseEntity registerScheduler( case "max": prioritize = new MaxInputPrioritize(); break; case "min": prioritize = new MinInputPrioritize(); break; default: + log.warn( "Register execution: {} - No Prioritize for: {}", execution, split[0] ); return new ResponseEntity<>( "No Prioritize for: " + split[0], HttpStatus.NOT_FOUND ); } if ( split.length == 2 ) { @@ -161,6 +166,7 @@ ResponseEntity registerScheduler( case "roundrobin": case "rr": assign = new RoundRobinAssign(); break; case "fair": case "f": assign = new FairAssign(); break; default: + log.warn( "Register execution: {} - No Assign for: {}", execution, split[1] ); return new ResponseEntity<>( "No Assign for: " + split[1], HttpStatus.NOT_FOUND ); } } else { @@ -168,6 +174,7 @@ ResponseEntity registerScheduler( } scheduler = new PrioritizeAssignScheduler( execution, client, namespace, config, prioritize, assign ); } else { + log.warn( "Register execution: {} - No scheduler for strategy: {}", execution, strategy ); return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } } @@ -199,7 +206,7 @@ ResponseEntity registerScheduler( @PostMapping("/v1/scheduler/{execution}/task/{id}") ResponseEntity registerTask( @PathVariable String execution, @PathVariable int id, @RequestBody TaskConfig config ) { - log.trace( execution + " " + config.getTask() + " got: " + config ); + log.info( execution + " " + config.getTask() + " got: " + config ); final Scheduler scheduler = schedulerHolder.get( execution ); if ( scheduler == null ) { From 35750128b04be1d254ca2d2910d002b7a6792b45 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 6 Mar 2025 15:14:24 +0100 Subject: [PATCH 434/443] Reformat comments Signed-off-by: Lehmann_Fabian --- .../scheduler/model/cluster/GroupCluster.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java index 979419b6..d46a7e99 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java @@ -24,17 +24,29 @@ public abstract class GroupCluster { private final double minScoreToCopy = 0.5; - // Group all tasks by label and deliver statistics + /** + * Group all tasks by label and deliver statistics + */ protected final Map countPerLabel = new HashMap<>(); - // tasks that are not yet scheduled + /** + * tasks that are not yet scheduled + */ protected final LinkedList unscheduledTasks = new LinkedList<>(); - // tasks that are assigned to a task and running + /** + * tasks that are assigned to a task and running + */ protected final LinkedList scheduledTasks = new LinkedList<>(); - // tasks that have finished + /** + * tasks that have finished + */ protected final LinkedList finishedTasks = new LinkedList<>(); - // Map from label to node, which node is responsible for the label + /** + * Map from label to node, which node is responsible for the label + */ protected final Map labelToNode = new HashMap<>(); - // Map from node to label, which labels are assigned to the node + /** + * Map from node to label, which labels are assigned to the node + */ protected final Map> nodeToLabel = new HashMap<>(); @Getter private final HierarchyWrapper hierarchyWrapper; From 2d013c11a33f6dea6d13d719d7fe781857925cab Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 6 Mar 2025 15:14:40 +0100 Subject: [PATCH 435/443] improve comment Signed-off-by: Lehmann_Fabian --- .../cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java index 6189e149..28294be7 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java @@ -28,7 +28,7 @@ void recalculate() { .sorted( Comparator.comparingInt( kv -> - kv.getValue().getCount() ) ) .collect( Collectors.toList() ); - // store how many tasks would have been executed on every with the current alignment + // store how many tasks would have been executed on every node with the current alignment // this is an approximation since tasks can have multiple labels and would appear multiple times Map tasksOnNode = new HashMap<>(); From 8a755d4a7cfdc3b8d0f75158e047d094920051a2 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 12 Mar 2025 18:51:00 +0100 Subject: [PATCH 436/443] Convert Tests to Junit 5 Signed-off-by: Lehmann_Fabian --- pom.xml | 19 -- .../java/cws/k8s/scheduler/dag/DAGTest.java | 81 +++---- .../scheduler/dag/VertexDeserializerTest.java | 15 +- .../model/InputFileCollectorTest.java | 93 ++++---- .../scheduler/model/TaskResultParserTest.java | 37 ++-- .../hierachy/HierarchyWrapperTest.java | 100 +++++---- .../hierachy/RealHierarchyFileTest.java | 199 +++++++++--------- .../prediction/MemoryScalerTest.java | 2 +- .../LinearPredictorSquaredLossTest.java | 26 ++- .../predictor/PolynomialPredictorTest.java | 46 ++-- 10 files changed, 293 insertions(+), 325 deletions(-) diff --git a/pom.xml b/pom.xml index 5a9fcfb9..55e60408 100644 --- a/pom.xml +++ b/pom.xml @@ -61,11 +61,6 @@ provided - - junit - junit - - org.mockito mockito-inline @@ -93,12 +88,6 @@ compile - - org.junit.vintage - junit-vintage-engine - test - - org.apache.commons commons-collections4 @@ -148,7 +137,6 @@ 3.6.1 - com.google.ortools ortools-java @@ -160,13 +148,6 @@ spring-boot-starter-validation - - org.junit.vintage - junit-vintage-engine - 5.12.0 - test - - diff --git a/src/test/java/cws/k8s/scheduler/dag/DAGTest.java b/src/test/java/cws/k8s/scheduler/dag/DAGTest.java index 4b2777c1..ed8a1575 100644 --- a/src/test/java/cws/k8s/scheduler/dag/DAGTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/DAGTest.java @@ -1,31 +1,32 @@ package cws.k8s.scheduler.dag; import lombok.extern.slf4j.Slf4j; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.*; import java.util.stream.Collectors; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertThrows; @Slf4j -public class DAGTest { +class DAGTest { private void compare(int uid, List vertices, int[] ancestorIds, int[] descendantsIds ){ //noinspection OptionalGetWithoutIsPresent final Vertex vertex = vertices.stream().filter(v -> v.getUid() == uid).findFirst().get(); if( vertex.getAncestors() == null ){ - assertNull( ancestorIds ); + Assertions.assertNull( ancestorIds ); } else { final int[] a = vertex.getAncestors().stream().mapToInt( Vertex::getUid ).sorted().toArray(); - assertArrayEquals("Compare Ancestors for uid: " + uid, ancestorIds, a); + Assertions.assertArrayEquals( ancestorIds, a, "Compare Ancestors for uid: " + uid ); } if( vertex.getDescendants() == null ){ - assertNull( descendantsIds ); + Assertions.assertNull( descendantsIds ); } else { final int[] d = vertex.getDescendants().stream().mapToInt(Vertex::getUid).sorted().toArray(); - assertArrayEquals("Compare Descendants for uid: " + uid, descendantsIds, d); + Assertions.assertArrayEquals( descendantsIds, d, "Compare Descendants for uid: " + uid ); } } @@ -53,16 +54,16 @@ private HashSet setTo(DAG dag, int idA, int[]... otherIds){ private void isSameEdge( HashSet expectedIn, HashSet expectedOut, Vertex vertex ){ final String[] in = setToArrayEdges(expectedIn); final String[] out = setToArrayEdges(expectedOut); - assertArrayEquals( "In of " + vertex.getLabel(), in, setToArrayEdges(vertex.getIn()) ); - assertArrayEquals( "Out of " + vertex.getLabel(), out, setToArrayEdges(vertex.getOut()) ); + Assertions.assertArrayEquals( in, setToArrayEdges(vertex.getIn()), "In of " + vertex.getLabel() ); + Assertions.assertArrayEquals( out, setToArrayEdges(vertex.getOut()), "Out of " + vertex.getLabel() ); } private void isSameProc( HashSet expectedAncestors, HashSet expectedDescendants, Vertex vertex, int rank ){ final String[] descendants = setToArrayProcesses(expectedDescendants); final String[] ancestors = setToArrayProcesses(expectedAncestors); - assertArrayEquals( "Descendants of " + vertex.getLabel(), descendants, setToArrayProcesses(vertex.getDescendants()) ); - assertArrayEquals( "Ancestors of " + vertex.getLabel(), ancestors, setToArrayProcesses(vertex.getAncestors()) ); - assertEquals( rank, vertex.getRank() ); + Assertions.assertArrayEquals( descendants, setToArrayProcesses(vertex.getDescendants()), "Descendants of " + vertex.getLabel() ); + Assertions.assertArrayEquals( ancestors, setToArrayProcesses(vertex.getAncestors()), "Ancestors of " + vertex.getLabel() ); + Assertions.assertEquals( rank, vertex.getRank() ); } private String[] setToArrayProcesses(Set set) { @@ -132,7 +133,7 @@ public List genEdgeList(){ } @Test - public void testRelations() { + void testRelations() { for (int q = 0; q < 500 ; q++) { final DAG dag = new DAG(); List vertexList = genVertexList(); @@ -144,7 +145,7 @@ public void testRelations() { } @Test - public void testRelations2() { + void testRelations2() { for (int q = 0; q < 500 ; q++) { final DAG dag = new DAG(); List vertexList = genVertexList(); @@ -164,7 +165,7 @@ public void testRelations2() { } @Test - public void smallTest(){ + void smallTest(){ final DAG dag = new DAG(); @@ -188,26 +189,26 @@ public void smallTest(){ dag.registerVertices( vertexList ); dag.registerEdges( inputEdges ); - assertEquals( new HashSet<>(), a.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), a.getAncestors() ); final HashSet descA = new HashSet<>(); descA.add( b ); - assertEquals( descA, a.getDescendants() ); + Assertions.assertEquals( descA, a.getDescendants() ); final HashSet ancB = new HashSet<>(); ancB.add( a ); - assertEquals( new HashSet<>(), b.getDescendants() ); - assertEquals( ancB, b.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), b.getDescendants() ); + Assertions.assertEquals( ancB, b.getAncestors() ); - assertEquals( ancB, filter.getAncestors() ); - assertEquals( descA, filter.getDescendants() ); + Assertions.assertEquals( ancB, filter.getAncestors() ); + Assertions.assertEquals( descA, filter.getDescendants() ); - assertEquals( new HashSet<>(),o.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), o.getAncestors() ); final HashSet descO = new HashSet<>(); descO.add( a ); descO.add( b ); - assertEquals( descO, o.getDescendants() ); + Assertions.assertEquals( descO, o.getDescendants() ); } @@ -223,7 +224,7 @@ public void smallTest(){ * */ @Test - public void deleteTest(){ + void deleteTest(){ final DAG dag = new DAG(); @@ -258,37 +259,37 @@ public void deleteTest(){ dag.removeVertices( filter2.getUid() ); } - assertEquals( new HashSet<>(), a.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), a.getAncestors() ); if ( i == 0 ) { - assertEquals( new HashSet<>( Arrays.asList( new Edge(2,a,filter), new Edge(3,a,filter2) ) ), a.getOut() ); + Assertions.assertEquals( new HashSet<>( Arrays.asList( new Edge(2,a,filter), new Edge(3,a,filter2) ) ), a.getOut() ); } else { - assertEquals( new HashSet<>(List.of(new Edge(2, a, filter))), a.getOut() ); + Assertions.assertEquals( new HashSet<>( List.of(new Edge(2, a, filter))), a.getOut() ); } final HashSet descA = new HashSet<>(); descA.add( b ); - assertEquals( descA, a.getDescendants() ); + Assertions.assertEquals( descA, a.getDescendants() ); final HashSet ancB = new HashSet<>(); ancB.add( a ); - assertEquals( new HashSet<>(), b.getDescendants() ); - assertEquals( ancB, b.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), b.getDescendants() ); + Assertions.assertEquals( ancB, b.getAncestors() ); if ( i == 0 ) { - assertEquals( new HashSet<>( Arrays.asList( new Edge(4,filter,b), new Edge(5,filter2,b) ) ), b.getIn() ); + Assertions.assertEquals( new HashSet<>( Arrays.asList( new Edge(4,filter,b), new Edge(5,filter2,b) ) ), b.getIn() ); } else { - assertEquals( new HashSet<>(List.of(new Edge(4, filter, b))), b.getIn() ); + Assertions.assertEquals( new HashSet<>( List.of(new Edge(4, filter, b))), b.getIn() ); } - assertEquals( ancB, filter.getAncestors() ); - assertEquals( descA, filter.getDescendants() ); + Assertions.assertEquals( ancB, filter.getAncestors() ); + Assertions.assertEquals( descA, filter.getDescendants() ); - assertEquals( new HashSet<>(),o.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), o.getAncestors() ); final HashSet descO = new HashSet<>(); descO.add( a ); descO.add( b ); - assertEquals( descO, o.getDescendants() ); + Assertions.assertEquals( descO, o.getDescendants() ); } } @@ -297,7 +298,7 @@ public void deleteTest(){ @Test - public void deleteTest2() { + void deleteTest2() { final DAG dag = createDag(); dag.removeEdges( 3 ); @@ -325,7 +326,7 @@ public void deleteTest2() { } @Test - public void deleteTest3() { + void deleteTest3() { final DAG dag = createDag(); dag.removeVertices( dag.getByProcess("c").getUid(), dag.getByProcess("e").getUid() ); @@ -360,7 +361,7 @@ public void deleteTest3() { * e */ @Test - public void deleteTest4() { + void deleteTest4() { final DAG dag = createDag(); final Process a = new Process("a", 1); @@ -415,7 +416,7 @@ public void deleteTest4() { * e */ @Test - public void deleteTest5() { + void deleteTest5() { final DAG dag = new DAG(); final Process a = new Process("a", 1); diff --git a/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java index 7b2ab3f4..56cd13aa 100644 --- a/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java @@ -2,28 +2,27 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.springframework.boot.test.autoconfigure.json.JsonTest; import java.io.IOException; -import static org.junit.Assert.assertEquals; - @JsonTest -public class VertexDeserializerTest { +class VertexDeserializerTest { @Test - public void testDeserialize() throws IOException { + void testDeserialize() throws IOException { String json = "{\"label\":\"a\", \"type\":\"PROCESS\", \"uid\":0}"; final ObjectMapper objectMapper = new ObjectMapper(); SimpleModule module = new SimpleModule(); module.addDeserializer(Vertex.class, new VertexDeserializer()); objectMapper.registerModule(module); final Vertex process = objectMapper.readValue(json, Vertex.class); - assertEquals( "a", process.getLabel() ); - assertEquals( Type.PROCESS, process.getType() ); - assertEquals( 0, process.getUid() ); + Assertions.assertEquals( "a", process.getLabel() ); + Assertions.assertEquals( Type.PROCESS, process.getType() ); + Assertions.assertEquals( 0, process.getUid() ); System.out.println( process ); } diff --git a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java index f89a3a69..0497831e 100644 --- a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java +++ b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java @@ -10,8 +10,9 @@ import cws.k8s.scheduler.model.taskinputs.SymlinkInput; import cws.k8s.scheduler.model.taskinputs.TaskInputs; import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.nio.file.Path; import java.nio.file.Paths; @@ -20,10 +21,8 @@ import java.util.List; import java.util.Set; -import static org.junit.Assert.*; - @Slf4j -public class InputFileCollectorTest { +class InputFileCollectorTest { private LocationWrapper location11; private LocationWrapper location12; @@ -34,7 +33,7 @@ public class InputFileCollectorTest { private LocationWrapper location44; private LocationWrapper location45; - @Before + @BeforeEach public void init(){ location11 = new LocationWrapper( NodeLocation.getLocation("Node1"), 1, 100); location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 1, 101); @@ -47,7 +46,7 @@ public void init(){ } @Test - public void getInputsOfTaskTest() throws NoAlignmentFoundException { + void getInputsOfTaskTest() throws NoAlignmentFoundException { final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; final String root2 = "/workdir/01/db62d739d658b839f07a1a77d877d1/"; @@ -56,27 +55,27 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); final Path path1 = Paths.get(root + "a.txt"); - assertNotNull( hierarchyWrapper.addFile(path1, location11) ); - assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location12) ); final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) hierarchyWrapper.getFile(path1), List.of(location11, location12)); final Path path2 = Paths.get(root + "a/b.txt"); - assertNotNull( hierarchyWrapper.addFile(path2, location21) ); - assertNotNull( hierarchyWrapper.addFile(path2, location23) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path2, location21) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path2, location23) ); final PathFileLocationTriple file2 = new PathFileLocationTriple(path2, (RealHierarchyFile) hierarchyWrapper.getFile(path2), List.of(location21, location23)); final Path path3 = Paths.get(root2 + "a/b.txt"); - assertNotNull( hierarchyWrapper.addFile(path3, location34) ); - assertNotNull( hierarchyWrapper.addFile(path3, location35) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path3, location34) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path3, location35) ); final PathFileLocationTriple file3 = new PathFileLocationTriple(path3, (RealHierarchyFile) hierarchyWrapper.getFile(path3), List.of(location34, location35)); final Path path4 = Paths.get(root3 + "b/c.txt"); - assertNotNull( hierarchyWrapper.addFile(path4, location44) ); - assertNotNull( hierarchyWrapper.addFile(path4, location45) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path4, location44) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path4, location45) ); final PathFileLocationTriple file4 = new PathFileLocationTriple(path4, (RealHierarchyFile) hierarchyWrapper.getFile(path4), List.of(location44, location45)); final Path path5 = Paths.get(root2 + "b/c.txt"); - assertTrue(hierarchyWrapper.addSymlink( path5, path4 ) ); + Assertions.assertTrue( hierarchyWrapper.addSymlink( path5, path4 ) ); final SymlinkInput symlink = new SymlinkInput( path5, path4 ); @@ -99,10 +98,10 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); - assertEquals( 3, inputsOfTaskFiles.size() ); - assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); - assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertFalse( inputsOfTask.hasExcludedNodes() ); + Assertions.assertEquals( 3, inputsOfTaskFiles.size() ); + Assertions.assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); //Not existing file final InputParam c = new InputParam<>("c", new FileHolder(null, root2 + "a.txt", null)); @@ -110,10 +109,10 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); inputsOfTaskFiles = inputsOfTask.getFiles(); - assertEquals( 3, inputsOfTaskFiles.size() ); - assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); - assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertFalse( inputsOfTask.hasExcludedNodes() ); + Assertions.assertEquals( 3, inputsOfTaskFiles.size() ); + Assertions.assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); //Symlink final InputParam d = new InputParam<>("d", new FileHolder(null, root2 + "b/c.txt", null)); @@ -122,16 +121,16 @@ public void getInputsOfTaskTest() throws NoAlignmentFoundException { inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); inputsOfTaskFiles = inputsOfTask.getFiles(); - assertEquals( 4, inputsOfTaskFiles.size() ); - assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); - assertEquals( Set.of( symlink ), new HashSet<>(inputsOfTask.getSymlinks()) ); - assertFalse( inputsOfTask.hasExcludedNodes() ); + Assertions.assertEquals( 4, inputsOfTaskFiles.size() ); + Assertions.assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + Assertions.assertEquals( Set.of( symlink ), new HashSet<>(inputsOfTask.getSymlinks()) ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); } @Test - public void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { + void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; @@ -164,24 +163,24 @@ public void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { location11.use(); final Path path1 = Paths.get(root + "a.txt"); - assertNotNull( hierarchyWrapper.addFile(path1, location11) ); - assertNotNull( hierarchyWrapper.addFile(path1, location12) ); - assertNotNull( hierarchyWrapper.addFile(path1, location13) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location13) ); final HierarchyFile file = hierarchyWrapper.getFile(path1); - assertNotNull( file ); + Assertions.assertNotNull( file ); final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); - assertEquals( 1, inputsOfTaskFiles.size() ); - assertEquals( file1, inputsOfTaskFiles.get(0) ); - assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertTrue( inputsOfTask.hasExcludedNodes() ); - assertFalse( inputsOfTask.canRunOnLoc( NodeLocation.getLocation( "Node1" ) ) ); + Assertions.assertEquals( 1, inputsOfTaskFiles.size() ); + Assertions.assertEquals( file1, inputsOfTaskFiles.get(0) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertTrue( inputsOfTask.hasExcludedNodes() ); + Assertions.assertFalse( inputsOfTask.canRunOnLoc( NodeLocation.getLocation( "Node1" ) ) ); } @Test - public void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundException { + void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundException { final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; @@ -214,20 +213,20 @@ public void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundExceptio location13.use(); final Path path1 = Paths.get(root + "a.txt"); - assertNotNull( hierarchyWrapper.addFile(path1, location11) ); - assertNotNull( hierarchyWrapper.addFile(path1, location12) ); - assertNotNull( hierarchyWrapper.addFile(path1, location13) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location13) ); final HierarchyFile file = hierarchyWrapper.getFile(path1); - assertNotNull( file ); + Assertions.assertNotNull( file ); final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); log.info(file1.toString()); TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC, Integer.MAX_VALUE); List inputsOfTaskFiles = inputsOfTask.getFiles(); - assertEquals( 1, inputsOfTaskFiles.size() ); - assertEquals( file1, inputsOfTaskFiles.get(0) ); - assertTrue( inputsOfTask.getSymlinks().isEmpty() ); - assertFalse( inputsOfTask.hasExcludedNodes() ); + Assertions.assertEquals( 1, inputsOfTaskFiles.size() ); + Assertions.assertEquals( file1, inputsOfTaskFiles.get(0) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); } diff --git a/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java b/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java index a7cc37dc..5dbdf32a 100644 --- a/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java +++ b/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java @@ -9,8 +9,9 @@ import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; import cws.k8s.scheduler.model.outfiles.SymlinkOutput; import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.BufferedWriter; import java.io.FileWriter; @@ -19,10 +20,8 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.*; -import static org.junit.Assert.assertEquals; - @Slf4j -public class TaskResultParserTest { +class TaskResultParserTest { private final String NL = "\n"; @@ -52,7 +51,7 @@ private Path storeData( String[] inputdata, String[] outputdata ){ private DAG dag; - @Before + @BeforeEach public void before(){ dag = new DAG(); List vertexList = new LinkedList<>(); @@ -63,7 +62,7 @@ public void before(){ @Test - public void test1(){ + void test1(){ String[] infiles = { "1636549091222254911", @@ -93,12 +92,12 @@ public void test1(){ expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt", "/pvcdata/testfile.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt", "/pvcdata/testfile.txt") ); - assertEquals( expected, newAndUpdatedFiles ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); } @Test - public void test2(){ + void test2(){ String[] infiles = { "1636720962171455407", @@ -132,12 +131,12 @@ public void test2(){ expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); - assertEquals( expected, newAndUpdatedFiles ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); } @Test - public void test3(){ + void test3(){ final TaskResultParser taskResultParser = new TaskResultParser(); @@ -177,12 +176,12 @@ public void test3(){ expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt") ); expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt") ); - assertEquals( expected, newAndUpdatedFiles ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); } @Test - public void test4(){ + void test4(){ String[] infiles = { "---", @@ -199,7 +198,7 @@ public void test4(){ } @Test - public void test5(){ + void test5(){ final TaskResultParser taskResultParser = new TaskResultParser(); @@ -235,12 +234,12 @@ public void test5(){ expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); - assertEquals( expected, newAndUpdatedFiles ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); } @Test - public void test6(){ + void test6(){ final TaskResultParser taskResultParser = new TaskResultParser(); @@ -277,12 +276,12 @@ public void test6(){ expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/a"), new LocationWrapper( node1, 1658328569683L, 249228, task )) ); expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/b"), new LocationWrapper( node1, 1658328569659L, 260036, task )) ); expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/c"), new LocationWrapper( node1, 1658328569707L, 265581, task )) ); - assertEquals( expected, newAndUpdatedFiles ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); } @Test - public void test7(){ + void test7(){ long a = System.currentTimeMillis(); long b = (long) (1658328570467617203l / 1.0E6); log.info("{} {}", a - b, b); @@ -290,7 +289,7 @@ public void test7(){ @Test - public void fileWalker() throws IOException { + void fileWalker() throws IOException { FileVisitor visitor = new FileVisitor() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { diff --git a/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java index 773597c9..5a429d63 100644 --- a/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -7,18 +7,17 @@ import cws.k8s.scheduler.model.TaskConfig; import cws.k8s.scheduler.model.location.NodeLocation; import lombok.extern.slf4j.Slf4j; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; -import static org.junit.Assert.*; - @Slf4j -public class HierarchyWrapperTest { +class HierarchyWrapperTest { final String workdir = "/folder/localworkdir/"; HierarchyWrapper hw; @@ -32,7 +31,7 @@ private LocationWrapper getLocationWrapper( String location ){ return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, new Task( new TaskConfig("processA"), dag) ); } - @Before + @BeforeEach public void init() { dag = new DAG(); List vertexList = new LinkedList<>(); @@ -54,22 +53,19 @@ public void init() { files.add( Paths.get(temporaryDir + "b/c/test.abc" )); files.add( Paths.get(temporaryDir + "bc/file.abc" )); - files.parallelStream().forEach( x -> assertNotNull(hw.addFile(x, node1))); + files.parallelStream().forEach( x -> Assertions.assertNotNull( hw.addFile(x, node1) ) ); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } private void compare( List a, Collection b){ - assertEquals( - new HashSet<>(a.stream().map(Path::normalize).collect(Collectors.toList())), - new HashSet<>(b) - ); - assertEquals(a.size(),b.size()); + Assertions.assertEquals( new HashSet<>(a.stream().map( Path::normalize).collect( Collectors.toList())), new HashSet<>(b) ); + Assertions.assertEquals( a.size(), b.size() ); } @Test - public void getAllFilesInDir() { + void getAllFilesInDir() { log.info("{}", this.result); Collection result; @@ -85,27 +81,27 @@ public void getAllFilesInDir() { } @Test - public void getAllFilesInFile() { - assertNull(hw.getAllFilesInDir( Paths.get(temporaryDir + "test/" ) )); - assertNull(hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/" ) )); - assertNull(hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/c/" ) )); + void getAllFilesInFile() { + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(temporaryDir + "test/" ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/" ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/c/" ) ) ); } @Test - public void getAllFilesOutOfScrope() { - assertNull(hw.getAllFilesInDir( Paths.get("/somewhere/" ) )); - assertNull(hw.getAllFilesInDir( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/" ) )); + void getAllFilesOutOfScrope() { + Assertions.assertNull( hw.getAllFilesInDir( Paths.get("/somewhere/" ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/" ) ) ); } @Test - public void getAllFilesinAllWorkdirs() { - assertNull(hw.getAllFilesInDir( Paths.get(workdir ) )); - assertNull(hw.getAllFilesInDir( Paths.get(workdir + "/ab/" ) )); + void getAllFilesinAllWorkdirs() { + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(workdir ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(workdir + "/ab/" ) ) ); } @Test - public void createFileinFile() { - assertNotNull(hw.addFile( Paths.get(temporaryDir + "test/b.txt"), node1)); + void createFileinFile() { + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "test/b.txt"), node1) ); files.remove( Paths.get(temporaryDir + "test" )); files.add( Paths.get(temporaryDir + "test/b.txt") ); @@ -115,33 +111,33 @@ public void createFileinFile() { } @Test - public void createFileinWorkdir() { - assertNull(hw.addFile(Paths.get(workdir + "ab/b.txt"), node1)); + void createFileinWorkdir() { + Assertions.assertNull( hw.addFile( Paths.get(workdir + "ab/b.txt"), node1) ); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @Test - public void createFileOutOfScope() { - assertNull(hw.addFile(Paths.get("/somewhere/test.txt"), node1)); - assertNull(hw.addFile(Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/test.txt"), node1)); + void createFileOutOfScope() { + Assertions.assertNull( hw.addFile( Paths.get("/somewhere/test.txt"), node1) ); + Assertions.assertNull( hw.addFile( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/test.txt"), node1) ); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @Test - public void createFileTwice() { - assertNotNull(hw.addFile(Paths.get(temporaryDir + "bc/file.abc"), node1)); + void createFileTwice() { + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "bc/file.abc"), node1) ); result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); compare( files, result); } @Test - public void createFileButWasFolder() { - assertNotNull(hw.addFile(Paths.get(temporaryDir + "bc"),node1)); + void createFileButWasFolder() { + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "bc"),node1) ); files.remove( Paths.get(temporaryDir + "bc/file.abc") ); files.add( Paths.get(temporaryDir + "bc") ); @@ -152,7 +148,7 @@ public void createFileButWasFolder() { } @Test - public void testParallelAdd() { + void testParallelAdd() { System.gc(); long intialMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); @@ -179,7 +175,7 @@ public void testParallelAdd() { intialMem = finalMem; - files.parallelStream().forEach( x -> assertNotNull(hw.addFile(x, getLocationWrapper("Node1")))); + files.parallelStream().forEach( x -> Assertions.assertNotNull( hw.addFile(x, getLocationWrapper("Node1")) ) ); finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); @@ -198,7 +194,7 @@ public void testParallelAdd() { } @Test - public void testGetFile() { + void testGetFile() { hw = new HierarchyWrapper(workdir); files = new LinkedList<>(); @@ -215,7 +211,7 @@ public void testGetFile() { LocationWrapper[] lw = new LocationWrapper[ files.size() ]; for ( Path file : files) { lw[index] = getLocationWrapper( "Node" + index ); - assertNotNull(hw.addFile(file,lw[index++])); + Assertions.assertNotNull( hw.addFile(file,lw[index++]) ); } result = hw.getAllFilesInDir( Paths.get(temporaryDir) ).keySet(); @@ -226,34 +222,34 @@ public void testGetFile() { RealHierarchyFile realFile = (RealHierarchyFile) hw.getFile(file); final LocationWrapper[] locations = realFile.getLocations(); - assertEquals(lw[index++], locations[0]); - assertEquals(1,locations.length); + Assertions.assertEquals( lw[index++], locations[0] ); + Assertions.assertEquals( 1, locations.length ); } } @Test - public void testGetFileOutOfScope() { - assertNull(hw.getFile( Paths.get("/file.txt" ) )); - assertNull(hw.getFile( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/file.txt" ) )); + void testGetFileOutOfScope() { + Assertions.assertNull( hw.getFile( Paths.get("/file.txt" ) ) ); + Assertions.assertNull( hw.getFile( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/file.txt" ) ) ); } @Test - public void testGetFileWorkdir() { - assertNull(hw.getFile( Paths.get(workdir + "ab/test.txt" ))); + void testGetFileWorkdir() { + Assertions.assertNull( hw.getFile( Paths.get(workdir + "ab/test.txt" )) ); } @Test - public void testGetFileButIsDir() { + void testGetFileButIsDir() { final HierarchyFile file = hw.getFile(Paths.get(temporaryDir + "d")); - assertNotNull( file ); - assertTrue( file.isDirectory() ); + Assertions.assertNotNull( file ); + Assertions.assertTrue( file.isDirectory() ); } @Test - public void testFileIsNowDir(){ - assertNotNull(hw.addFile( Paths.get(temporaryDir + "d"), getLocationWrapper("nodeXY") )); + void testFileIsNowDir(){ + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "d"), getLocationWrapper("nodeXY") ) ); result = hw.getAllFilesInDir( Paths.get(temporaryDir ) ).keySet(); - assertNotNull( hw.getFile( Paths.get(temporaryDir + "d" )) ); - assertNull( hw.getFile( Paths.get(temporaryDir + "d/e/file.txt" )) ); + Assertions.assertNotNull( hw.getFile( Paths.get(temporaryDir + "d" )) ); + Assertions.assertNull( hw.getFile( Paths.get(temporaryDir + "d/e/file.txt" )) ); } } \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java index edee1741..098cd273 100644 --- a/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -9,17 +9,17 @@ import cws.k8s.scheduler.model.location.LocationType; import cws.k8s.scheduler.model.location.NodeLocation; import org.apache.commons.collections4.iterators.PermutationIterator; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; import static org.junit.jupiter.api.Assertions.assertThrows; -public class RealHierarchyFileTest { +class RealHierarchyFileTest { private Task processA; private Task processB; @@ -27,7 +27,7 @@ public class RealHierarchyFileTest { private Task processD; private Task processE; - @Before + @BeforeEach public void before(){ DAG dag = new DAG(); List vertexList = new LinkedList<>(); @@ -56,7 +56,7 @@ private LocationWrapper getLocationWrapper( String location ){ @Test - public void addLocation() { + void addLocation() { List locations = new LinkedList<>(); @@ -64,33 +64,33 @@ public void addLocation() { final LocationWrapper node1 = getLocationWrapper("Node1"); locations.add(node1); final RealHierarchyFile realFile = new RealHierarchyFile(node1); - assertArrayEquals( locations.toArray(), realFile.getLocations() ); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); final LocationWrapper node2 = getLocationWrapper("Node2"); locations.add(node2); realFile.addOrUpdateLocation( false, node2); - assertArrayEquals( locations.toArray(), realFile.getLocations() ); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); final LocationWrapper node3 = getLocationWrapper("Node3"); locations.add(node3); realFile.addOrUpdateLocation( false, node3); - assertArrayEquals( locations.toArray(), realFile.getLocations() ); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); final LocationWrapper node1New = getLocationWrapper("Node1"); realFile.addOrUpdateLocation( false, node1New); - assertArrayEquals( locations.toArray(), realFile.getLocations()); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); } @Test - public void addEmptyLocation() { + void addEmptyLocation() { final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("node1") ); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( true, null )); } @Test - public void addInParallel() { + void addInParallel() { final LocationWrapper node0 = getLocationWrapper("Node0"); final RealHierarchyFile realFile = new RealHierarchyFile(node0); @@ -105,15 +105,12 @@ public void addInParallel() { locations.parallelStream().forEach( r -> realFile.addOrUpdateLocation( false, r ) ); locations.add( node0 ); - assertEquals( - new HashSet<>(locations), - new HashSet<>(Arrays.asList(realFile.getLocations())) - ); + Assertions.assertEquals( new HashSet<>(locations), new HashSet<>( Arrays.asList(realFile.getLocations())) ); } @Test - public void changeFile() { + void changeFile() { final LocationWrapper node0 = getLocationWrapper("Node0"); final RealHierarchyFile realFile = new RealHierarchyFile(node0); @@ -127,72 +124,72 @@ public void changeFile() { final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, processB ); realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; - assertArrayEquals( expected, realFile.getLocations() ); + Assertions.assertArrayEquals( expected, realFile.getLocations() ); } @Test - public void overwriteFile() { + void overwriteFile() { final LinkedList results = new LinkedList<>(); final LocationWrapper node0 = getLocationWrapper("Node0"); results.add( node0 ); - assertTrue(node0.isActive()); + Assertions.assertTrue( node0.isActive() ); final RealHierarchyFile realFile = new RealHierarchyFile(node0); - assertArrayEquals( results.toArray(), realFile.getLocations() ); - assertTrue(node0.isActive()); + Assertions.assertArrayEquals( results.toArray(), realFile.getLocations() ); + Assertions.assertTrue( node0.isActive() ); final LocationWrapper node1 = getLocationWrapper("Node1"); results.add( node1 ); realFile.addOrUpdateLocation( true, node1); - assertArrayEquals( results.toArray(), realFile.getLocations() ); - assertFalse(node0.isActive()); - assertTrue(node1.isActive()); + Assertions.assertArrayEquals( results.toArray(), realFile.getLocations() ); + Assertions.assertFalse( node0.isActive() ); + Assertions.assertTrue( node1.isActive() ); final LocationWrapper node2 = getLocationWrapper("Node2"); - assertArrayEquals( results.toArray(), realFile.getLocations() ); + Assertions.assertArrayEquals( results.toArray(), realFile.getLocations() ); results.add( node2 ); realFile.addOrUpdateLocation( true, node2); - assertFalse(node0.isActive()); - assertFalse(node1.isActive()); - assertTrue(node2.isActive()); + Assertions.assertFalse( node0.isActive() ); + Assertions.assertFalse( node1.isActive() ); + Assertions.assertTrue( node2.isActive() ); } @Test - public void changeFileOnExistingLocation() { + void changeFileOnExistingLocation() { final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("Node0") ); final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, processA ); realFile.addOrUpdateLocation( false, nodeNew ); LocationWrapper[] expected = { nodeNew }; - assertArrayEquals( expected, realFile.getLocations() ); + Assertions.assertArrayEquals( expected, realFile.getLocations() ); final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, processB ); realFile.addOrUpdateLocation( false, nodeNew2 ); LocationWrapper[] expected2 = { nodeNew2 }; - assertArrayEquals( expected2, realFile.getLocations() ); + Assertions.assertArrayEquals( expected2, realFile.getLocations() ); } @Test - public void isFile() { + void isFile() { final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("Node0") ); - assertFalse( realFile.isDirectory() ); - assertFalse( realFile.isSymlink() ); + Assertions.assertFalse( realFile.isDirectory() ); + Assertions.assertFalse( realFile.isSymlink() ); } @Test - public void getFilesForTaskTest() throws InterruptedException, NoAlignmentFoundException { + void getFilesForTaskTest() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); List locationWrapperList = List.of(loc1, loc2, loc3, loc4); @@ -205,11 +202,11 @@ public void getFilesForTaskTest() throws InterruptedException, NoAlignmentFoundE realFile.addOrUpdateLocation(false, next.get(2)); realFile.addOrUpdateLocation(false, next.get(3)); - assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); } locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); @@ -223,27 +220,27 @@ public void getFilesForTaskTest() throws InterruptedException, NoAlignmentFoundE realFile.addOrUpdateLocation(false, next.get(3)); realFile.addOrUpdateLocation(false, next.get(4)); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); - assertEquals(new HashSet<>(List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); } } @Test - public void getFilesForTaskTestInitFiles() throws InterruptedException, NoAlignmentFoundException { + void getFilesForTaskTestInitFiles() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); @@ -257,28 +254,28 @@ public void getFilesForTaskTestInitFiles() throws InterruptedException, NoAlignm realFile.addOrUpdateLocation(false, next.get(3)); realFile.addOrUpdateLocation(false, next.get(4)); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); } } @Test - public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException, NoAlignmentFoundException { + void getFilesForTaskTestMultipleInitFiles() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); @@ -293,32 +290,32 @@ public void getFilesForTaskTestMultipleInitFiles() throws InterruptedException, realFile.addOrUpdateLocation(false, next.get(3)); realFile.addOrUpdateLocation(false, next.get(4)); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations())); - assertEquals(new HashSet<>(Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations())); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); } } @Test - public void getFilesForTaskTestDifferentAncestors() throws InterruptedException, NoAlignmentFoundException { + void getFilesForTaskTestDifferentAncestors() throws InterruptedException, NoAlignmentFoundException { final CountDownLatch waiter = new CountDownLatch(1); final NodeLocation location1 = NodeLocation.getLocation("Node1"); LocationWrapper loc1 = new LocationWrapper( location1, System.currentTimeMillis() - 2, 2, null ); loc1.use(); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); final NodeLocation location3 = NodeLocation.getLocation("Node3"); LocationWrapper loc3 = new LocationWrapper( location3, System.currentTimeMillis() - 5, 2, null ); loc3.use(); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); final NodeLocation location4 = NodeLocation.getLocation("Node4"); LocationWrapper loc4 = new LocationWrapper( location4, System.currentTimeMillis() - 2, 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); final NodeLocation location5 = NodeLocation.getLocation("Node5"); LocationWrapper loc5 = new LocationWrapper( location5, System.currentTimeMillis() - 5, 2, null ); loc5.use(); @@ -336,8 +333,8 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException, for (Task task : List.of(processA, processB, processC, processD, processE)) { final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask( task ); - assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); - assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); + Assertions.assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); + Assertions.assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); } } @@ -361,8 +358,8 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException, for (Task task : List.of(processA, processB, processC, processD, processE)) { final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask( task ); - assertEquals( new HashSet<>( List.of( loc4, loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); - assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); + Assertions.assertEquals( new HashSet<>( List.of( loc4, loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); + Assertions.assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); } } @@ -370,62 +367,62 @@ public void getFilesForTaskTestDifferentAncestors() throws InterruptedException, } @Test - public void getLastLocationTest() throws InterruptedException { + void getLastLocationTest() throws InterruptedException { final CountDownLatch waiter = new CountDownLatch(1); LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); RealHierarchyFile realFile = new RealHierarchyFile( loc1 ); - assertEquals( loc1, realFile.getLastUpdate( LocationType.NODE )); + Assertions.assertEquals( loc1, realFile.getLastUpdate( LocationType.NODE ) ); realFile.addOrUpdateLocation(false, loc4 ); - assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); + Assertions.assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE ) ); realFile.addOrUpdateLocation(false, loc2 ); - assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); + Assertions.assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE ) ); realFile.addOrUpdateLocation(false, loc3 ); - assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE )); + Assertions.assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE ) ); realFile.addOrUpdateLocation(false, loc5 ); - assertEquals( loc5, realFile.getLastUpdate( LocationType.NODE )); + Assertions.assertEquals( loc5, realFile.getLastUpdate( LocationType.NODE ) ); realFile.addOrUpdateLocation(true, loc2 ); - assertEquals( loc2, realFile.getLastUpdate( LocationType.NODE )); + Assertions.assertEquals( loc2, realFile.getLastUpdate( LocationType.NODE ) ); } @Test - public void getLocationWrapperForNodeTest() throws InterruptedException { + void getLocationWrapperForNodeTest() throws InterruptedException { final CountDownLatch waiter = new CountDownLatch(1); LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); - assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); RealHierarchyFile realFile = new RealHierarchyFile(loc1); - assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); + Assertions.assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); realFile.addOrUpdateLocation(false, loc4 ); - assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + Assertions.assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); realFile.addOrUpdateLocation(false, loc3 ); - assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + Assertions.assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); realFile.addOrUpdateLocation(false, loc2 ); - assertEquals( loc2, realFile.getLocationWrapper( NodeLocation.getLocation("Node2") ) ); + Assertions.assertEquals( loc2, realFile.getLocationWrapper( NodeLocation.getLocation("Node2") ) ); realFile.addOrUpdateLocation(false, loc5 ); - assertEquals( loc5, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); - assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); + Assertions.assertEquals( loc5, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + Assertions.assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); final NodeLocation node99 = NodeLocation.getLocation("Node99"); assertThrows( RuntimeException.class, () -> realFile.getLocationWrapper( node99 ) ); diff --git a/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java b/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java index 47b93ee8..25567e41 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java @@ -5,7 +5,7 @@ import cws.k8s.scheduler.model.SchedulerConfig; import cws.k8s.scheduler.model.TaskMetrics; import cws.k8s.scheduler.prediction.predictor.TestTask; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; diff --git a/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java b/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java index 81295ac7..2cdde0f4 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java @@ -1,42 +1,40 @@ package cws.k8s.scheduler.prediction.predictor; import org.jetbrains.annotations.NotNull; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class LinearPredictorSquaredLossTest { +class LinearPredictorSquaredLossTest { @Test - public void testOneTask() { + void testOneTask() { LinearPredictorSquaredLoss lp = getLinearPredictor(); lp.addTask( new TestTask( 1d, 1d ) ); - assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testTwoTasks() { + void testTwoTasks() { LinearPredictorSquaredLoss lp = getLinearPredictor(); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); - assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testPredict() { + void testPredict() { LinearPredictorSquaredLoss lp = getLinearPredictor(); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); lp.addTask( new TestTask( 3d, 3d ) ); - assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); - assertEquals( (Double) 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ) ); + Assertions.assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertEquals( (Double) 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ) ); } @Test - public void noData() { + void noData() { LinearPredictorSquaredLoss lp = getLinearPredictor(); - assertNull(lp.queryPrediction( new TestTask( 4d, 4d ) )); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @NotNull diff --git a/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java b/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java index 7b4a8c05..4e8b07bd 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java @@ -3,58 +3,56 @@ import cws.k8s.scheduler.prediction.Predictor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; @Slf4j -public class PolynomialPredictorTest { +class PolynomialPredictorTest { @Test - public void testOneTask() { + void testOneTask() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); - assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testTwoTasks() { + void testTwoTasks() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); - assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testThreeTasksLinear() { + void testThreeTasksLinear() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); lp.addTask( new TestTask( 3d, 3d ) ); - assertEquals( 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ), 0.0001 ); - assertEquals( 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ), 0.0001 ); + Assertions.assertEquals( 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ), 0.0001 ); } @Test - public void testThreeTasksPoly() { + void testThreeTasksPoly() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); lp.addTask( new TestTask( 3d, 9d ) ); - assertEquals( 1d, lp.queryPrediction( new TestTask( 1d, 0d ) ), 0.0001 ); - assertEquals( 2d, lp.queryPrediction( new TestTask( 2d, 0d ) ), 0.0001 ); - assertEquals( 9d, lp.queryPrediction( new TestTask( 3d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 1d, lp.queryPrediction( new TestTask( 1d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 2d, lp.queryPrediction( new TestTask( 2d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 9d, lp.queryPrediction( new TestTask( 3d, 0d ) ), 0.0001 ); } @Test - public void noData() { + void noData() { Predictor lp = getPolyPredictor(2); - assertNull(lp.queryPrediction( new TestTask( 4d, 4d ) )); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void measure(){ + void measure(){ int iterations = 1000; Predictor lp1 = getPolyPredictor(2); TestTask[] tasks = new TestTask[iterations]; @@ -69,7 +67,7 @@ public void measure(){ } @Test - public void compareLinear(){ + void compareLinear(){ int iterations = 100; Predictor lp = getPolyPredictor(2); TestTask[] tasks = new TestTask[iterations]; @@ -83,12 +81,12 @@ public void compareLinear(){ if ( prediction1 == null ) { continue; } - assertEquals( task.y, lp.queryPrediction( task ), 1 ); + Assertions.assertEquals( task.y, lp.queryPrediction( task ), 1 ); } } @Test - public void comparePoly(){ + void comparePoly(){ int iterations = 1000; Predictor lp = getPolyPredictor(2); TestTask[] tasks = new TestTask[iterations]; @@ -103,11 +101,11 @@ public void comparePoly(){ if ( prediction1 == null ) { continue; } - assertEquals( task.y, lp.queryPrediction( task ), 1 ); + Assertions.assertEquals( task.y, lp.queryPrediction( task ), 1 ); } for ( int x = 0; x < 1000; x++ ) { final TestTask task = new TestTask( x, 0d ); - assertEquals( getY( x ), lp.queryPrediction( task ), 1 ); + Assertions.assertEquals( getY( x ), lp.queryPrediction( task ), 1 ); } } From 21d7c6ca2be95738b9c9aa5320bab7bb02fd282b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 12 Mar 2025 18:56:00 +0100 Subject: [PATCH 437/443] Fix VertexDeserializerTest Signed-off-by: Lehmann_Fabian --- src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java index 56cd13aa..c33f9ab3 100644 --- a/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java @@ -4,11 +4,9 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.springframework.boot.test.autoconfigure.json.JsonTest; import java.io.IOException; -@JsonTest class VertexDeserializerTest { From 784df499f5f706637ff1f638ac8ef74b17e6e8b9 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 12 Mar 2025 19:13:36 +0100 Subject: [PATCH 438/443] Read spring-boot-maven-plugin Signed-off-by: Lehmann_Fabian --- pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pom.xml b/pom.xml index 55e60408..2014fa7b 100644 --- a/pom.xml +++ b/pom.xml @@ -219,6 +219,18 @@ + + org.springframework.boot + spring-boot-maven-plugin + + + build-info + + build-info + + + + From 7cafa0e3b4188fba59763636530ff84b97c551d5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 12 Mar 2025 19:27:58 +0100 Subject: [PATCH 439/443] update OR-Tools to 9.12.4544 Signed-off-by: Lehmann_Fabian --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2014fa7b..95aeeea9 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ com.google.ortools ortools-java - 9.4.1874 + 9.12.4544 From ce39b75252da73990253b91b90b0fbd90103b9e3 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 13 Mar 2025 11:11:49 +0100 Subject: [PATCH 440/443] Change naming to WOW Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/rest/SchedulerRestController.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 0003ddde..0eb6ba33 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -11,22 +11,16 @@ import cws.k8s.scheduler.scheduler.filealignment.GreedyAlignment; import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; import cws.k8s.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; -import cws.k8s.scheduler.dag.DAG; -import cws.k8s.scheduler.dag.InputEdge; -import cws.k8s.scheduler.dag.Vertex; import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.model.TaskMetrics; import cws.k8s.scheduler.scheduler.PrioritizeAssignScheduler; import cws.k8s.scheduler.scheduler.Scheduler; import cws.k8s.scheduler.scheduler.prioritize.*; -import cws.k8s.scheduler.rest.exceptions.NotARealFileException; -import cws.k8s.scheduler.rest.response.getfile.FileResponse; import cws.k8s.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; import cws.k8s.scheduler.scheduler.nodeassign.FairAssign; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RandomNodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RoundRobinAssign; -import cws.k8s.scheduler.scheduler.prioritize.*; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -132,7 +126,7 @@ ResponseEntity registerScheduler( } switch ( strategy.toLowerCase() ){ - case "lav2" : + case "wow" : if ( !config.locationAware ) { log.warn( "Register execution: {} - LA scheduler only works if location aware", execution ); return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); @@ -142,7 +136,7 @@ ResponseEntity registerScheduler( } scheduler = new LocationAwareSchedulerV2( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ), new OptimalReadyToRunToNode() ); break; - case "lagroup" : + case "wowgroup" : if ( !config.locationAware ) { log.warn( "Register execution: {} - LA scheduler only works if location aware", execution ); return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); From 073f9718b2aebaa6e48ea1de6df3d20c1ca6f31c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 13 Mar 2025 11:14:48 +0100 Subject: [PATCH 441/443] Optimize Imports Signed-off-by: Lehmann_Fabian --- .../k8s/scheduler/client/CWSKubernetesClient.java | 11 ++--------- .../java/cws/k8s/scheduler/model/DateParser.java | 1 - .../cws/k8s/scheduler/model/NodeWithAlloc.java | 2 +- .../java/cws/k8s/scheduler/model/PodWithAge.java | 3 ++- .../prediction/offset/VarianceOffset.java | 2 -- .../prediction/predictor/LinearPredictor.java | 1 - .../predictor/LinearPredictorCustomLoss.java | 2 -- .../predictor/LinearPredictorSquaredLoss.java | 1 - .../prediction/predictor/MeanPredictor.java | 1 - .../scheduler/rest/SchedulerRestController.java | 8 +++----- .../scheduler/LocationAwareSchedulerV2.java | 14 +++++++------- .../scheduler/PrioritizeAssignScheduler.java | 2 +- .../cws/k8s/scheduler/scheduler/Scheduler.java | 3 +-- .../scheduler/SchedulerWithDaemonSet.java | 6 ++++-- .../scheduler/copystrategy/CopyStrategy.java | 1 - .../scheduler/prioritize/RankMaxPrioritize.java | 1 - .../prediction/offset/VarianceOffsetTest.java | 2 +- .../scheduler/prioritize/FifoPrioritizeTest.java | 3 ++- .../LeastFinishedFirstPrioritizeTest.java | 2 +- .../prioritize/MaxInputPrioritizeTest.java | 3 ++- .../prioritize/RankMaxPrioritizeTest.java | 2 +- 21 files changed, 28 insertions(+), 43 deletions(-) diff --git a/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java b/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java index 50db1463..1ea0f75f 100644 --- a/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java +++ b/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java @@ -5,16 +5,9 @@ import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.util.MyExecListner; import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.dsl.ExecWatch; -import io.fabric8.kubernetes.client.WatcherException; -import io.fabric8.kubernetes.client.*; import io.fabric8.kubernetes.client.Config; -import io.fabric8.kubernetes.client.dsl.MixedOperation; -import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; -import io.fabric8.kubernetes.client.dsl.PodResource; -import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.*; +import io.fabric8.kubernetes.client.dsl.*; import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; diff --git a/src/main/java/cws/k8s/scheduler/model/DateParser.java b/src/main/java/cws/k8s/scheduler/model/DateParser.java index 53879509..845bfda8 100644 --- a/src/main/java/cws/k8s/scheduler/model/DateParser.java +++ b/src/main/java/cws/k8s/scheduler/model/DateParser.java @@ -3,7 +3,6 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; -import java.nio.file.attribute.FileTime; import java.text.SimpleDateFormat; @NoArgsConstructor(access = AccessLevel.PRIVATE) diff --git a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java index a8412e88..c8b01f56 100644 --- a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java @@ -1,7 +1,7 @@ package cws.k8s.scheduler.model; -import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; diff --git a/src/main/java/cws/k8s/scheduler/model/PodWithAge.java b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java index f3a6ad65..e111d92a 100644 --- a/src/main/java/cws/k8s/scheduler/model/PodWithAge.java +++ b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java @@ -1,7 +1,8 @@ package cws.k8s.scheduler.model; import cws.k8s.scheduler.util.PodPhase; -import io.fabric8.kubernetes.api.model.*; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Quantity; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java b/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java index 908d0597..6f368251 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java +++ b/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java @@ -2,8 +2,6 @@ import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.prediction.Predictor; -import org.apache.commons.math3.stat.descriptive.moment.Variance; -import org.apache.commons.math3.stat.descriptive.rank.Percentile; import java.util.List; diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java index bf044daa..64e0412e 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java @@ -1,6 +1,5 @@ package cws.k8s.scheduler.prediction.predictor; -import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.prediction.Predictor; public interface LinearPredictor extends Predictor { diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java index 0f74a4fe..27c93ee8 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java @@ -1,10 +1,8 @@ package cws.k8s.scheduler.prediction.predictor; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.prediction.Predictor; import cws.k8s.scheduler.prediction.extractor.VariableExtractor; import cws.k8s.scheduler.prediction.predictor.loss.UnequalLossFunction; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.math3.optim.InitialGuess; import org.apache.commons.math3.optim.MaxEval; diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java index e427b680..459c4143 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java @@ -1,7 +1,6 @@ package cws.k8s.scheduler.prediction.predictor; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.prediction.Predictor; import cws.k8s.scheduler.prediction.extractor.VariableExtractor; import lombok.RequiredArgsConstructor; import org.apache.commons.math3.stat.regression.SimpleRegression; diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java index 5243a05a..710fbb08 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java @@ -4,7 +4,6 @@ import cws.k8s.scheduler.prediction.Predictor; import cws.k8s.scheduler.prediction.extractor.VariableExtractor; import lombok.RequiredArgsConstructor; -import org.apache.commons.math3.stat.regression.SimpleRegression; import java.util.concurrent.atomic.AtomicLong; diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 0eb6ba33..24cfdf22 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -1,26 +1,24 @@ package cws.k8s.scheduler.rest; +import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.InputEdge; import cws.k8s.scheduler.dag.Vertex; import cws.k8s.scheduler.model.SchedulerConfig; import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.model.TaskMetrics; import cws.k8s.scheduler.rest.exceptions.NotARealFileException; import cws.k8s.scheduler.rest.response.getfile.FileResponse; import cws.k8s.scheduler.scheduler.*; import cws.k8s.scheduler.scheduler.filealignment.GreedyAlignment; import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; import cws.k8s.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; -import cws.k8s.scheduler.client.CWSKubernetesClient; -import cws.k8s.scheduler.model.TaskMetrics; -import cws.k8s.scheduler.scheduler.PrioritizeAssignScheduler; -import cws.k8s.scheduler.scheduler.Scheduler; -import cws.k8s.scheduler.scheduler.prioritize.*; import cws.k8s.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; import cws.k8s.scheduler.scheduler.nodeassign.FairAssign; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RandomNodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RoundRobinAssign; +import cws.k8s.scheduler.scheduler.prioritize.*; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java index d9199be1..94b3a09e 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -1,13 +1,7 @@ package cws.k8s.scheduler.scheduler; -import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.client.CWSKubernetesClient; -import cws.k8s.scheduler.scheduler.la2.*; -import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; -import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; -import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; -import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; -import cws.k8s.scheduler.util.*; +import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.model.location.Location; import cws.k8s.scheduler.model.location.NodeLocation; import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; @@ -16,11 +10,17 @@ import cws.k8s.scheduler.model.taskinputs.TaskInputs; import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.*; import cws.k8s.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; import cws.k8s.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; import cws.k8s.scheduler.scheduler.la2.copystrategy.CopyRunner; import cws.k8s.scheduler.scheduler.la2.copystrategy.ShellCopy; import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; +import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.*; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import cws.k8s.scheduler.util.score.FileSizeRankScore; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java index 2329a694..0682ec83 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -1,8 +1,8 @@ package cws.k8s.scheduler.scheduler; +import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.model.*; -import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.prioritize.Prioritize; import cws.k8s.scheduler.util.NodeTaskAlignment; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index 0efe5fef..fdd92173 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -1,14 +1,13 @@ package cws.k8s.scheduler.scheduler; +import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.client.CannotPatchException; import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.model.*; -import cws.k8s.scheduler.model.*; import cws.k8s.scheduler.prediction.MemoryScaler; import cws.k8s.scheduler.prediction.TaskScaler; import cws.k8s.scheduler.util.Batch; -import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java index 614f68bc..e3a760d8 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -2,11 +2,10 @@ import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.model.*; -import cws.k8s.scheduler.model.location.hierachy.*; -import cws.k8s.scheduler.util.*; import cws.k8s.scheduler.model.cluster.OutputFiles; import cws.k8s.scheduler.model.location.LocationType; import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.*; import cws.k8s.scheduler.model.outfiles.OutputFile; import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; import cws.k8s.scheduler.model.outfiles.SymlinkOutput; @@ -16,6 +15,9 @@ import cws.k8s.scheduler.rest.response.getfile.FileResponse; import cws.k8s.scheduler.scheduler.copystrategy.CopyStrategy; import cws.k8s.scheduler.scheduler.copystrategy.FTPstrategy; +import cws.k8s.scheduler.util.DaemonHolder; +import cws.k8s.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; import cws.k8s.scheduler.util.copying.CurrentlyCopying; import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import io.fabric8.kubernetes.api.model.ContainerStatus; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java index a46c62c9..65479eb7 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -1,6 +1,5 @@ package cws.k8s.scheduler.scheduler.copystrategy; -import cws.k8s.scheduler.model.Task; import lombok.extern.slf4j.Slf4j; import java.io.*; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java index c8417132..55a5a093 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.Task; -import java.util.Comparator; import java.util.List; public class RankMaxPrioritize implements Prioritize { diff --git a/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java b/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java index 7f4d0c1c..e60165c2 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java @@ -2,7 +2,7 @@ import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class VarianceOffsetTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java index 9617757c..e16f13dc 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java @@ -6,7 +6,8 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; class FifoPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java index 463414fc..699798ab 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java @@ -6,7 +6,7 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class LeastFinishedFirstPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java index 9067985c..bc036806 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java @@ -6,7 +6,8 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; class MaxInputPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java index a4e20e0b..a4c539d6 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java @@ -6,7 +6,7 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class RankMaxPrioritizeTest { From e547e012f9542672b108295a8bbf218f8fced88a Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 13 Mar 2025 11:17:34 +0100 Subject: [PATCH 442/443] Reorder attributes Signed-off-by: Lehmann_Fabian --- src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java index 4e2b97cc..967433ef 100644 --- a/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java +++ b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java @@ -14,7 +14,7 @@ @Slf4j public class CopyTask extends Task { - private final static Process COPY_PROCESS = new Process( "Copy", Integer.MAX_VALUE ); + private static final Process COPY_PROCESS = new Process( "Copy", Integer.MAX_VALUE ); private final Task task; private final NodeWithAlloc node; private final List labelCounts; From a02b84ba220c94a9e6e34741bd73232050e9ac0c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 13 Mar 2025 11:24:15 +0100 Subject: [PATCH 443/443] Add WOW to Readme Signed-off-by: Lehmann_Fabian --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c3e9162c..b11996dd 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,7 @@ The following strategies are available: | random | Randomly prioritize tasks. | | max | Prioritize tasks with larger input size. | | min | Prioritize tasks with smaller input size. | +| wow | WOW scheduler for data location awareness. This is scheduling + node assignment. Details are provided in our paper [tbd](tbd). | | Node Assignment Strategy | Behaviour | |--------------------------|-----------------------------------------------------------------------------------------|