diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 5f182801..688fc93f 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - java_version: [ 17, 21 ] + java_version: [ 21 ] steps: - name: Checkout uses: actions/checkout@v3 @@ -32,10 +32,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: 17 + java-version: 21 distribution: 'temurin' - name: Set outputs id: vars diff --git a/Dockerfile b/Dockerfile index 280e032c..e9be27cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,22 @@ -FROM maven:3-openjdk-17-slim AS build -WORKDIR /build -COPY pom.xml pom.xml -RUN mvn dependency:go-offline --no-transfer-progress -Dmaven.repo.local=/mvn/.m2nrepo/repository -COPY src/ src/ -RUN mvn package --no-transfer-progress -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository - -# -# Package stage -# -FROM openjdk:17-alpine +FROM eclipse-temurin:21-jdk-jammy AS builder WORKDIR /app -RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data -COPY --from=build /build/target/cws-k8s-scheduler*.jar cws-k8s-scheduler.jar -RUN chown -R javauser:javagroup /app -USER javauser -EXPOSE 8080 -ENTRYPOINT ["java","-jar","/app/cws-k8s-scheduler.jar"] + +RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/* + +COPY pom.xml . +RUN mvn dependency:go-offline --no-transfer-progress + +COPY src/ ./src/ +RUN mvn package --no-transfer-progress -DskipTests + +FROM eclipse-temurin:21-jre-jammy + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/target/cws-k8s-scheduler-*-SNAPSHOT.jar app.jar + +CMD ["java", "-jar", "app.jar"] diff --git a/Dockerfile-development b/Dockerfile-development index 445f1b5a..ecf73f63 100644 --- a/Dockerfile-development +++ b/Dockerfile-development @@ -1,4 +1,4 @@ -FROM maven:3-openjdk-17-slim AS build +FROM maven:3-openjdk-18-slim AS build WORKDIR /build COPY pom.xml pom.xml RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..2b77db2c --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,136 @@ +pipeline { + agent { + kubernetes { + yamlFile 'jenkins-pod.yaml' + } + } + environment { + // creates DOCKERHUB_USR and DOCKERHUB_PSW env variables + DOCKERHUB = credentials('fondahub-dockerhub') + } + + stages { + stage('Build') { + steps { + container('maven') { + // run a clean build without tests to see if the project compiles + sh 'mvn clean test-compile -DskipTests=true -Dmaven.javadoc.skip=true -B -V' + } + } + } + + stage('Test') { + steps { + container('maven') { + // run JUnit tests + sh 'mvn test -B -V' + } + } + post { + // collect test results + always { + junit 'target/surefire-reports/TEST-*.xml' + jacoco classPattern: 'target/classes,target/test-classes', execPattern: 'target/coverage-reports/*.exec', inclusionPattern: '**/*.class', sourcePattern: 'src/main/java,src/test/java' + archiveArtifacts 'target/surefire-reports/TEST-*.xml' + archiveArtifacts 'target/*.exec' + } + } + } + + stage('Package') { + steps { + container('maven') { + sh 'mvn package -DskipTests=true -Dmaven.javadoc.skip=true -B -V' + } + } + post { + success { + archiveArtifacts 'target/*.jar' + } + } + } + + stage('Static Code Analysis') { + steps { + container('maven') { + withSonarQubeEnv('fonda-sonarqube') { + sh ''' + mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \ + -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \ + -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \ + -Dsonar.junit.reportPaths=target/surefire-reports + ''' + } + } + } + } + + stage('Build and push Docker') { + // only push images from the master branch + when { + branch "master" + } + // agents are specified per stage to enable real parallel execution + parallel { + stage('workflow-k8s-scheduler') { + agent { + kubernetes { + yamlFile 'jenkins-pod.yaml' + } + } + steps { + container('hadolint') { + sh "hadolint --format json Dockerfile | tee -a hadolint_scheduler.json" + } + // build and push image to fondahub/workflow-k8s-scheduler + container('docker') { + sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin" + sh "docker build . -t fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}" + sh "docker tag fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]} fondahub/workflow-k8s-scheduler:latest" + sh "docker push fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}" + sh "docker push fondahub/workflow-k8s-scheduler:latest" + } + } + post { + always { + archiveArtifacts "hadolint_scheduler.json" + recordIssues( + aggregatingResults: true, + tools: [hadoLint(pattern: "hadolint_scheduler.json", id: "scheduler")] + ) + } + } + } + stage('vsftpd') { + agent { + kubernetes { + yamlFile 'jenkins-pod.yaml' + } + } + steps { + container('hadolint') { + sh "hadolint --format json daemons/ftp/Dockerfile | tee -a hadolint_vsftpd.json" + } + // build and push image to fondahub/vsftpd + container('docker') { + sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin" + sh "docker build daemons/ftp/ -t fondahub/vsftpd:${GIT_COMMIT[0..7]}" + sh "docker tag fondahub/vsftpd:${GIT_COMMIT[0..7]} fondahub/vsftpd:latest" + sh "docker push fondahub/vsftpd:${GIT_COMMIT[0..7]}" + sh "docker push fondahub/vsftpd:latest" + } + } + post { + always { + archiveArtifacts "hadolint_vsftpd.json" + recordIssues( + aggregatingResults: true, + tools: [hadoLint(pattern: "hadolint_vsftpd.json", id: "vsfptd")] + ) + } + } + } + } + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index 14740999..b11996dd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ -# Common Workflow Scheduler for Kubernetes -[![DOI](https://zenodo.org/badge/596122315.svg)](https://zenodo.org/badge/latestdoi/596122315) +# Kubernetes Workflow Scheduler -In this repository, you will find the Common Workflow Scheduler for Kubernetes proposed in the paper "**How Workflow Engines Should Talk to Resource Managers: A Proposal for a Common Workflow Scheduling Interface**." +SWAGGER: http://localhost:8080/swagger-ui.html + +API-DOCS: http://localhost:8080/v3/api-docs/ --- #### Build @@ -188,6 +189,7 @@ The following strategies are available: | random | Randomly prioritize tasks. | | max | Prioritize tasks with larger input size. | | min | Prioritize tasks with smaller input size. | +| wow | WOW scheduler for data location awareness. This is scheduling + node assignment. Details are provided in our paper [tbd](tbd). | | Node Assignment Strategy | Behaviour | |--------------------------|-----------------------------------------------------------------------------------------| @@ -215,4 +217,4 @@ Lehmann Fabian, Jonathan Bader, Friedrich Tschirpke, Lauritz Thamsen, and Ulf Le ``` --- #### Acknowledgement: -This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis." \ No newline at end of file +This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis." diff --git a/daemons/ftp/Dockerfile b/daemons/ftp/Dockerfile new file mode 100644 index 00000000..57099808 --- /dev/null +++ b/daemons/ftp/Dockerfile @@ -0,0 +1,17 @@ +FROM python:slim +RUN apt update && apt install -y \ + vsftpd \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /var/run/vsftpd/empty + +COPY vsftpd.conf /etc/vsftpd.conf + +USER root +RUN echo 'root:password' | chpasswd + +COPY ftp.py /code/ftp.py + +WORKDIR /code + +ENTRYPOINT ["sh","-c","/usr/sbin/vsftpd /etc/vsftpd.conf"] \ No newline at end of file diff --git a/daemons/ftp/ftp.py b/daemons/ftp/ftp.py new file mode 100644 index 00000000..4a779dde --- /dev/null +++ b/daemons/ftp/ftp.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +import ftplib +import json +import logging as log +import os +import shutil +import signal +import sys +import time +import urllib.request +import urllib.parse +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from time import sleep + +######################################################################################## +# Call this class with three arguments: trace enabled, name to store logs, config json # +######################################################################################## + +exitIfFileWasNotFound = True +CLOSE = False +UNEXPECTED_ERROR = "Unexpected error" +EXIT = 0 +log.basicConfig( + format='%(levelname)s: %(message)s', + level=log.DEBUG, + handlers=[ + log.FileHandler(".command.init." + sys.argv[2] + ".log"), + log.StreamHandler() + ] +) +trace = {} +traceFilePath = ".command.scheduler.trace" +errors = 0 + + +def myExit(code): + global EXIT + EXIT = code + global CLOSE + CLOSE = True + writeTrace(trace) + exit(EXIT) + + +def close(signalnum, syncFile): + log.info("Killed: %s", str(signalnum)) + closeWithWarning(50, syncFile) + + +def closeWithWarning(errorCode, syncFile): + syncFile.write('##FAILURE##\n') + syncFile.flush() + syncFile.close() + myExit(errorCode) + + +def getIP(node, dns, execution): + ip = urllib.request.urlopen(dns + "daemon/" + execution + "/" + node).read() + return str(ip.decode("utf-8")) + + +# True if the file was deleted or did not exist +def clearLocation(path, dst=None): + if os.path.exists(path): + log.debug("Delete %s", path) + if os.path.islink(path): + if dst is not None and os.readlink(path) == dst: + return False + else: + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + return True + + +def getFTP(node, currentIP, dns, execution, syncFile): + global errors + connectionProblem = 0 + while connectionProblem < 8: + try: + if currentIP is None: + log.info("Request ip for node: %s", node) + ip = getIP(node, dns, execution) + else: + ip = currentIP + log.info("Try to connect to %s", ip) + ftp = ftplib.FTP(ip, timeout=10) + ftp.login("root", "password") + ftp.set_pasv(True) + ftp.encoding = 'utf-8' + log.info("Connection established") + return ftp + except ConnectionRefusedError: + errors += 1 + log.warning("Connection refused! Try again...") + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + connectionProblem += 1 + time.sleep(2 ** connectionProblem) + closeWithWarning(8, syncFile) + + +def closeFTP(ftp): + global errors + if ftp is None: + return + try: + ftp.quit() + ftp.close() + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + + +def downloadFile(ftp, filename, size, index, node, syncFile, speed): + global errors + log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) + try: + syncFile.write("S-" + filename + '\n') + clearLocation(filename) + Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) + start = time.time() + with open(filename, 'wb') as file: + if speed == 100: + ftp.retrbinary('RETR %s' % filename, file.write, 102400) + else: + timer = {"t": time.time_ns()} + + def callback(data): + now = time.time_ns() + diff = now - timer["t"] + file.write(data) + timeToSleep = (diff * (100 / speed) - diff) / 1_000_000_000 + # sleep at least 10ms + if timeToSleep > 0.01: + time.sleep(timeToSleep) + timer["t"] = time.time_ns() + + ftp.retrbinary('RETR %s' % filename, callback, 102400) + end = time.time() + sizeInMB = os.path.getsize(filename) / 1048576 + delta = (end - start) + log.info("Speed: %.3f Mb/s", sizeInMB / delta) + return sizeInMB, delta + except ftplib.error_perm as err: + errors += 1 + if str(err) == "550 Failed to open file.": + log.warning("File not found node: %s file: %s", node, filename) + if exitIfFileWasNotFound: + closeWithWarning(40, syncFile) + except FileNotFoundError: + errors += 1 + log.warning("File not found node: %s file: %s", node, filename) + if exitIfFileWasNotFound: + closeWithWarning(41, syncFile) + except EOFError: + errors += 1 + log.warning("It seems the connection was lost! Try again...") + return None + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + return None + return 0, 0 + + +def download(node, currentIP, files, dns, execution, syncFile, speed): + ftp = None + size = len(files) + global CLOSE + sizeInMB = 0 + downloadTime = 0 + while not CLOSE and len(files) > 0: + if ftp is None: + ftp = getFTP(node, currentIP, dns, execution, syncFile) + currentIP = None + filename = files[0] + index = size - len(files) + 1 + result = downloadFile(ftp, filename, size, index, node, syncFile, speed) + if result is None: + ftp = None + continue + sizeInMB += result[0] + downloadTime += result[1] + files.pop(0) + syncFile.write("F-" + filename + '\n') + closeFTP(ftp) + return node, sizeInMB / downloadTime + + +def waitForFiles(syncFilePath, files, startTime): + # wait max. 60 seconds + while True: + if startTime + 60 < time.time(): + return False + if os.path.isfile(syncFilePath): + break + log.debug("Wait for file creation") + time.sleep(0.1) + + # Read file live + with open(syncFilePath, 'r') as syncFileTask: + current = [] + while len(files) > 0: + data = syncFileTask.read() + if not data: + time.sleep(0.3) + else: + for d in data: + if d != "\n": + current.append(d) + else: + text = ''.join(current) + current = [] + if text.startswith("S-"): + continue + if text == "##FAILURE##": + log.debug("Read FAILURE in %s", syncFilePath) + myExit(51) + if text == "##FINISHED##": + log.debug("Read FINISHED in " + syncFilePath + " before all files were found") + myExit(52) + log.debug("Look for " + text[:2] + " with " + text[2:] + " in " + str(files)) + if text[:2] == "F-" and text[2:] in files: + files.remove(text[2:]) + if len(files) == 0: + return True + return len(files) == 0 + + +def loadConfig(): + log.info("Load config") + with open(sys.argv[3]) as jsonFile: + config = json.load(jsonFile) + os.makedirs(config["syncDir"], exist_ok=True) + return config + + +def registerSignal(syncFile): + signal.signal(signal.SIGINT, lambda signalnum, handler: close(signalnum, syncFile)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: close(signalnum, syncFile)) + + +def registerSignal2(): + signal.signal(signal.SIGINT, lambda signalnum, handler: myExit(1)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: myExit(1)) + + +def generateSymlinks(symlinks): + for s in symlinks: + src = s["src"] + dst = s["dst"] + if clearLocation(src, dst): + Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) + try: + os.symlink(dst, src) + except FileExistsError: + log.warning("File exists: %s -> %s", src, dst) + + +def downloadAllData(data, dns, execution, syncFile, speed): + global trace + throughput = [] + with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: + futures = [] + for d in data: + files = d["files"] + node = d["node"] + currentIP = d["currentIP"] + futures.append(executor.submit(download, node, currentIP, files, dns, execution, syncFile, speed)) + lastNum = -1 + while len(futures) > 0: + if lastNum != len(futures): + log.info("Wait for %d threads to finish", len(futures)) + lastNum = len(futures) + for f in futures[:]: + if f.done(): + throughput.append(f.result()) + futures.remove(f) + sleep(0.1) + trace["scheduler_init_throughput"] = "\"" + ",".join("{}:{:.3f}".format(*x) for x in throughput) + "\"" + + +def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): + # Now check for files of other tasks + for waitForTask in waitForFilesOfTask: + waitForFilesSet = set(waitForFilesOfTask[waitForTask]) + if not waitForFiles(syncDir + waitForTask, waitForFilesSet, startTime): + log.error(syncDir + waitForTask + " was not successful") + myExit(200) + + +def writeTrace(dataMap): + if sys.argv[1] == 'true': + global errors + if len(dataMap) == 0 or errors > 0: + return + with open(traceFilePath, "a") as traceFile: + for d in dataMap: + traceFile.write(d + "=" + str(dataMap[d]) + "\n") + traceFile.write("scheduler_init_errors=" + str(errors) + "\n") + + +def finishedDownload(dns, execution, taskname): + try: + dns = dns + "downloadtask/" + execution + log.info("Request: %s", dns) + urllib.request.urlopen(dns, taskname.encode("utf-8")) + except BaseException as err: + log.exception(err) + myExit(100) + + +def run(): + global trace + startTime = time.time() + log.info("Start to setup the environment") + config = loadConfig() + + dns = config["dns"] + execution = config["execution"] + data = config["data"] + symlinks = config["symlinks"] + taskname = config["hash"] + + with open(config["syncDir"] + config["hash"], 'w') as syncFile: + registerSignal(syncFile) + syncFile.write('##STARTED##\n') + syncFile.flush() + startTimeSymlinks = time.time() + generateSymlinks(symlinks) + trace["scheduler_init_symlinks_runtime"] = int((time.time() - startTimeSymlinks) * 1000) + syncFile.write('##SYMLINKS##\n') + syncFile.flush() + startTimeDownload = time.time() + downloadAllData(data, dns, execution, syncFile, config["speed"]) + trace["scheduler_init_download_runtime"] = int((time.time() - startTimeDownload) * 1000) + if CLOSE: + log.debug("Closed with code %s", str(EXIT)) + exit(EXIT) + log.info("Finished Download") + syncFile.write('##FINISHED##\n') + registerSignal2() + + # finishedDownload(dns, execution, taskname) + + # startTimeDependingTasks = time.time() + # waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) + # trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) + # log.info("Waited for all tasks") + + # runtime = int((time.time() - startTime) * 1000) + # trace["scheduler_init_runtime"] = runtime + # writeTrace(trace) + + +if __name__ == '__main__': + run() diff --git a/daemons/ftp/vsftpd.conf b/daemons/ftp/vsftpd.conf new file mode 100644 index 00000000..d14bf2df --- /dev/null +++ b/daemons/ftp/vsftpd.conf @@ -0,0 +1,164 @@ +# Example config file /etc/vsftpd.conf +# +# The default compiled in settings are fairly paranoid. This sample file +# loosens things up a bit, to make the ftp daemon more usable. +# Please see vsftpd.conf.5 for all compiled in defaults. +# +# READ THIS: This example file is NOT an exhaustive list of vsftpd options. +# Please read the vsftpd.conf.5 manual page to get a full idea of vsftpd's +# capabilities. +# +# +# Run standalone? vsftpd can run either from an inetd or as a standalone +# daemon started from an initscript. +listen=YES +# +# This directive enables listening on IPv6 sockets. By default, listening +# on the IPv6 "any" address (::) will accept connections from both IPv6 +# and IPv4 clients. It is not necessary to listen on *both* IPv4 and IPv6 +# sockets. If you want that (perhaps because you want to listen on specific +# addresses) then you must run two copies of vsftpd with two configuration +# files. +listen_ipv6=NO +# +# Allow anonymous FTP? (Disabled by default). +anonymous_enable=YES +# +# Uncomment this to allow local users to log in. +local_enable=YES +# +# Uncomment this to enable any form of FTP write command. +write_enable=YES +# +# Default umask for local users is 077. You may wish to change this to 022, +# if your users expect that (022 is used by most other ftpd's) +#local_umask=022 +# +# Uncomment this to allow the anonymous FTP user to upload files. This only +# has an effect if the above global write enable is activated. Also, you will +# obviously need to create a directory writable by the FTP user. +anon_upload_enable=NO +# +# Uncomment this if you want the anonymous FTP user to be able to create +# new directories. +anon_mkdir_write_enable=NO +# +anon_other_write_enable=NO +# +# Activate directory messages - messages given to remote users when they +# go into a certain directory. +dirmessage_enable=YES +# +# If enabled, vsftpd will display directory listings with the time +# in your local time zone. The default is to display GMT. The +# times returned by the MDTM FTP command are also affected by this +# option. +use_localtime=YES +# +# Activate logging of uploads/downloads. +xferlog_enable=YES +# +# Make sure PORT transfer connections originate from port 20 (ftp-data). +connect_from_port_20=NO +# +# If you want, you can arrange for uploaded anonymous files to be owned by +# a different user. Note! Using "root" for uploaded files is not +# recommended! +#chown_uploads=YES +#chown_username=whoever +# +# You may override where the log file goes if you like. The default is shown +# below. +#xferlog_file=/var/log/vsftpd.log +# +# If you want, you can have your log file in standard ftpd xferlog format. +# Note that the default log file location is /var/log/xferlog in this case. +#xferlog_std_format=YES +# +# You may change the default value for timing out an idle session. +#idle_session_timeout=600 +# +# You may change the default value for timing out a data connection. +#data_connection_timeout=120 +# +# It is recommended that you define on your system a unique user which the +# ftp server can use as a totally isolated and unprivileged user. +#nopriv_user=ftpsecure +# +# Enable this and the server will recognise asynchronous ABOR requests. Not +# recommended for security (the code is non-trivial). Not enabling it, +# however, may confuse older FTP clients. +#async_abor_enable=YES +# +# By default the server will pretend to allow ASCII mode but in fact ignore +# the request. Turn on the below options to have the server actually do ASCII +# mangling on files when in ASCII mode. +# Beware that on some FTP servers, ASCII support allows a denial of service +# attack (DoS) via the command "SIZE /big/file" in ASCII mode. vsftpd +# predicted this attack and has always been safe, reporting the size of the +# raw file. +# ASCII mangling is a horrible feature of the protocol. +#ascii_upload_enable=YES +#ascii_download_enable=YES +# +# You may fully customise the login banner string: +#ftpd_banner=Welcome to blah FTP service. +# +# You may specify a file of disallowed anonymous e-mail addresses. Apparently +# useful for combatting certain DoS attacks. +#deny_email_enable=YES +# (default follows) +#banned_email_file=/etc/vsftpd.banned_emails +# +# You may restrict local users to their home directories. See the FAQ for +# the possible risks in this before using chroot_local_user or +# chroot_list_enable below. +#chroot_local_user=YES +# +# You may specify an explicit list of local users to chroot() to their home +# directory. If chroot_local_user is YES, then this list becomes a list of +# users to NOT chroot(). +# (Warning! chroot'ing can be very dangerous. If using chroot, make sure that +# the user does not have write access to the top level directory within the +# chroot) +#chroot_local_user=YES +#chroot_list_enable=YES +# (default follows) +#chroot_list_file=/etc/vsftpd.chroot_list +# +# You may activate the "-R" option to the builtin ls. This is disabled by +# default to avoid remote users being able to cause excessive I/O on large +# sites. However, some broken FTP clients such as "ncftp" and "mirror" assume +# the presence of the "-R" option, so there is a strong case for enabling it. +#ls_recurse_enable=YES +# +# Customization +# +# Some of vsftpd's settings don't fit the filesystem layout by +# default. +# +# This option should be the name of a directory which is empty. Also, the +# directory should not be writable by the ftp user. This directory is used +# as a secure chroot() jail at times vsftpd does not require filesystem +# access. +secure_chroot_dir=/var/run/vsftpd/empty +# +# This string is the name of the PAM service vsftpd will use. +pam_service_name=ftp +# +# This option specifies the location of the RSA certificate to use for SSL +# encrypted connections. +rsa_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +rsa_private_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +ssl_enable=NO + +anonymous_enable=yes +anon_root=/ + +pasv_enable=Yes +pasv_max_port=10090 +pasv_min_port=11090 + +# +# Uncomment this to indicate that vsftpd use a utf8 filesystem. +#utf8_filesystem=YES \ No newline at end of file diff --git a/jenkins-pod.yaml b/jenkins-pod.yaml new file mode 100644 index 00000000..eb40c2b4 --- /dev/null +++ b/jenkins-pod.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: Pod +spec: + containers: + - name: maven + image: maven:3.8.3-jdk-11-slim + command: ['sleep', '99d'] + imagePullPolicy: Always + tty: true + + - name: hadolint + image: hadolint/hadolint:latest-debian + imagePullPolicy: Always + command: + - cat + tty: true + + - name: docker + image: docker:20.10.12 + command: ['sleep', '99d'] + env: + - name: DOCKER_HOST + value: tcp://localhost:2375 + + - name: docker-daemon + image: docker:20.10.12-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" diff --git a/pom.xml b/pom.xml index ed662853..95aeeea9 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.springframework.boot spring-boot-starter-parent - 3.1.5 + 3.4.3 @@ -36,7 +36,8 @@ - 17 + 21 + 21 cws.k8s.scheduler.Main @@ -45,18 +46,19 @@ io.fabric8 kubernetes-client - 6.13.1 + 7.1.0 - org.projectlombok - lombok - provided + org.javatuples + javatuples + 1.2 - junit - junit + org.projectlombok + lombok + provided @@ -69,66 +71,64 @@ org.springframework.boot spring-boot-starter-web - 3.3.1 + 3.4.3 org.springframework.boot spring-boot-starter-test + 3.4.3 test - 3.3.1 org.jetbrains annotations - 24.0.1 + 26.0.2 compile - - org.junit.vintage - junit-vintage-engine - test - - org.apache.commons commons-collections4 - 4.4 + 4.5.0-M3 test commons-net commons-net - 3.10.0 + 3.11.1 - - org.springdoc - springdoc-openapi-starter-webmvc-ui - 2.2.0 - + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.8.5 + ch.qos.logback logback-core - 1.5.3 + 1.5.17 + com.fasterxml.jackson.core jackson-databind + 2.18.3 com.fasterxml.jackson.core jackson-core + 2.18.3 com.fasterxml.jackson.core jackson-annotations + 3.0-rc1 @@ -137,11 +137,21 @@ 3.6.1 + + com.google.ortools + ortools-java + 9.12.4544 + + + + org.springframework.boot + spring-boot-starter-validation + + - org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.12 surefireArgLine @@ -204,10 +208,7 @@ org.apache.maven.plugins maven-surefire-plugin - - - ${surefireArgLine} --illegal-access=permit - + 3.5.2 default-test @@ -232,5 +233,4 @@ - diff --git a/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java b/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java index 89f1e2e9..1ea0f75f 100644 --- a/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java +++ b/src/main/java/cws/k8s/scheduler/client/CWSKubernetesClient.java @@ -3,18 +3,14 @@ import cws.k8s.scheduler.model.NodeWithAlloc; import cws.k8s.scheduler.model.PodWithAge; import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.util.MyExecListner; import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.WatcherException; -import io.fabric8.kubernetes.client.*; import io.fabric8.kubernetes.client.Config; -import io.fabric8.kubernetes.client.dsl.MixedOperation; -import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; -import io.fabric8.kubernetes.client.dsl.PodResource; -import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.*; +import io.fabric8.kubernetes.client.dsl.*; import lombok.extern.slf4j.Slf4j; +import java.io.ByteArrayOutputStream; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.*; @@ -24,7 +20,7 @@ public class CWSKubernetesClient { private final KubernetesClient client; - private final Map nodeHolder = new HashMap<>(); + private final Map nodeHolder= new HashMap<>(); private final List informables = new LinkedList<>(); public CWSKubernetesClient() { @@ -37,6 +33,23 @@ public CWSKubernetesClient() { this.nodes().watch( new NodeWatcher( this ) ); } + public Pod getPodByIp( String ip ) { + if ( ip == null ) { + throw new IllegalArgumentException("IP cannot be null"); + } + return this.pods() + .inAnyNamespace() + .list() + .getItems() + .parallelStream() + .filter( pod -> ip.equals( pod.getStatus().getPodIP() ) ) + .findFirst() + .orElseGet( () -> { + log.warn("No Pod found for IP: {}", ip); + return null; + }); + } + public NonNamespaceOperation> nodes() { return client.nodes(); } @@ -137,6 +150,49 @@ public BigDecimal getMemoryOfNode(NodeWithAlloc node ){ return Quantity.getAmountInBytes(memory); } + private void forceDeletePod( Pod pod ) { + this.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .withName(pod.getMetadata().getName()) + .withGracePeriod(0) + .withPropagationPolicy( DeletionPropagation.BACKGROUND ) + .delete(); + } + + private void createPod( Pod pod ) { + this.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .resource( pod ) + .create(); + } + + public void assignPodToNodeAndRemoveInit( PodWithAge pod, String node ) { + if ( pod.getSpec().getInitContainers().size() > 0 ) { + pod.getSpec().getInitContainers().remove( 0 ); + } + pod.getMetadata().setResourceVersion( null ); + pod.getMetadata().setManagedFields( null ); + pod.getSpec().setNodeName( node ); + + forceDeletePod( pod ); + createPod( pod ); + } + + public void execCommand( String podName, String namespace, String[] command, MyExecListner listener ){ + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream error = new ByteArrayOutputStream(); + final ExecWatch exec = this.pods() + .inNamespace( namespace ) + .withName( podName ) + .writingOutput( out ) + .writingError( error ) + .usingListener( listener ) + .exec( command ); + listener.setExec( exec ); + listener.setError( error ); + listener.setOut( out ); + } + static class NodeWatcher implements Watcher{ private final CWSKubernetesClient kubernetesClient; @@ -146,7 +202,7 @@ public NodeWatcher(CWSKubernetesClient kubernetesClient) { } @Override - public void eventReceived(Action action, Node node) { + public void eventReceived( Watcher.Action action, Node node) { boolean change = false; NodeWithAlloc processedNode = null; switch (action) { @@ -269,7 +325,7 @@ public boolean featureGateActive( String featureGate ){ * It will create a patch for the memory limits and request values and submit it * to the cluster. * Moreover, it updates the task with the new pod. - * + * * @param t the task to be patched * @return false if patching failed because of InPlacePodVerticalScaling */ diff --git a/src/main/java/cws/k8s/scheduler/dag/Origin.java b/src/main/java/cws/k8s/scheduler/dag/Origin.java index b9b64051..3536b2d1 100644 --- a/src/main/java/cws/k8s/scheduler/dag/Origin.java +++ b/src/main/java/cws/k8s/scheduler/dag/Origin.java @@ -19,7 +19,7 @@ public Type getType() { @Override public void addInbound(Edge e) { - throw new IllegalStateException("Cannot add an inbound to an Origin"); + throw new IllegalStateException("Cannot add an Edge(uid: " + e.getUid() + "; " + e.getFrom().getUid() + " -> " + e.getTo().getUid() + ") inbound to an Origin (uid: " + getUid() + ")"); } public Set getAncestors() { diff --git a/src/main/java/cws/k8s/scheduler/dag/Process.java b/src/main/java/cws/k8s/scheduler/dag/Process.java index 3e03817b..18da2788 100644 --- a/src/main/java/cws/k8s/scheduler/dag/Process.java +++ b/src/main/java/cws/k8s/scheduler/dag/Process.java @@ -19,10 +19,6 @@ public int getSuccessfullyFinished() { return successfullyFinished.get(); } - public int getFailed() { - return failed.get(); - } - public void incrementSuccessfullyFinished() { successfullyFinished.incrementAndGet(); } diff --git a/src/main/java/cws/k8s/scheduler/model/DateParser.java b/src/main/java/cws/k8s/scheduler/model/DateParser.java new file mode 100644 index 00000000..845bfda8 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/DateParser.java @@ -0,0 +1,32 @@ +package cws.k8s.scheduler.model; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.text.SimpleDateFormat; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class DateParser { + + public static Long millisFromString( String date ) { + if( date == null || date.isEmpty() || date.equals("-") || date.equals("w") ) { + return null; + } + try { + if (Character.isLetter(date.charAt(0))) { + // if ls was used, date has the format "Nov 2 08:49:30 2021" + return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime(); + } else { + // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" + String[] parts = date.split(" "); + parts[1] = parts[1].substring(0, 12); + // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" + String shortenedDate = String.join(" ", parts); + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); + } + } catch ( Exception e ){ + return null; + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/FileHolder.java b/src/main/java/cws/k8s/scheduler/model/FileHolder.java index e7d45a5a..6b7ff6f9 100644 --- a/src/main/java/cws/k8s/scheduler/model/FileHolder.java +++ b/src/main/java/cws/k8s/scheduler/model/FileHolder.java @@ -6,7 +6,7 @@ import lombok.ToString; @ToString( exclude = {"stageName", "storePath"}) -@NoArgsConstructor(access = AccessLevel.NONE) +@NoArgsConstructor(access = AccessLevel.NONE, force = true) @RequiredArgsConstructor public class FileHolder { diff --git a/src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java b/src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java new file mode 100644 index 00000000..540187db --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/ImmutableRequirements.java @@ -0,0 +1,42 @@ +package cws.k8s.scheduler.model; + +import java.math.BigDecimal; + +public class ImmutableRequirements extends Requirements { + + public static final Requirements ZERO = new ImmutableRequirements(); + + private static final String ERROR_MESSAGE = "ImmutableRequirements cannot be modified"; + + public ImmutableRequirements( BigDecimal cpu, BigDecimal ram ) { + super( cpu, ram ); + } + + public ImmutableRequirements(){ + super(); + } + + public ImmutableRequirements( Requirements requirements ){ + super( requirements.getCpu(), requirements.getRam() ); + } + + @Override + public Requirements addCPUtoThis( BigDecimal cpu ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } + + @Override + public Requirements addRAMtoThis( BigDecimal ram ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } + + @Override + public Requirements addToThis( Requirements requirements ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } + + @Override + public Requirements subFromThis( Requirements requirements ) { + throw new IllegalStateException( ERROR_MESSAGE ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java new file mode 100644 index 00000000..c6d4943b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/InputFileCollector.java @@ -0,0 +1,91 @@ +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.*; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.Tuple; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.util.*; + +@Slf4j +@RequiredArgsConstructor +public class InputFileCollector { + + private final HierarchyWrapper hierarchyWrapper; + + private void processNext( + final LinkedList> toProcess, + final List symlinks, + final List files, + final Set excludedLocations, + final Task task + ) throws NoAlignmentFoundException { + final Tuple tuple = toProcess.removeLast(); + final HierarchyFile file = tuple.getA(); + if( file == null ) { + return; + } + final Path path = tuple.getB(); + if ( file.isSymlink() ){ + final Path linkTo = ((LinkHierarchyFile) file).getDst(); + symlinks.add( new SymlinkInput( path, linkTo ) ); + toProcess.push( new Tuple<>( hierarchyWrapper.getFile( linkTo ), linkTo ) ); + } else if( file.isDirectory() ){ + final Map allChildren = ((Folder) file).getAllChildren(path); + for (Map.Entry pathAbstractFileEntry : allChildren.entrySet()) { + toProcess.push( new Tuple<>( pathAbstractFileEntry.getValue(), pathAbstractFileEntry.getKey() ) ); + } + } else { + final RealHierarchyFile realFile = (RealHierarchyFile) file; + realFile.requestedByTask(); + try { + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask(task); + if ( filesForTask.getExcludedNodes() != null ) { + excludedLocations.addAll(filesForTask.getExcludedNodes()); + } + files.add( new PathFileLocationTriple( path, realFile, filesForTask.getMatchingLocations()) ); + } catch ( NoAlignmentFoundException e ){ + log.error( "No alignment for task: " + task.getConfig().getName() + " path: " + path, e ); + throw e; + } + } + } + + public TaskInputs getInputsOfTask( Task task, int numberNode ) throws NoAlignmentFoundException { + + final List> fileInputs = task.getConfig().getInputs().fileInputs; + final LinkedList> toProcess = filterFilesToProcess( fileInputs ); + + final List symlinks = new ArrayList<>( fileInputs.size() ); + final List files = new ArrayList<>( fileInputs.size() ); + final Set excludedLocations = new HashSet<>(); + + while ( !toProcess.isEmpty() && excludedLocations.size() < numberNode ){ + processNext( toProcess, symlinks, files, excludedLocations, task ); + } + + if( excludedLocations.size() == numberNode ) { + return null; + } + + return new TaskInputs( symlinks, files, excludedLocations ); + + } + + private LinkedList> filterFilesToProcess(List> fileInputs ){ + final LinkedList> toProcess = new LinkedList<>(); + for ( InputParam fileInput : fileInputs) { + final Path path = Path.of(fileInput.value.sourceObj); + if ( this.hierarchyWrapper.isInScope( path ) ){ + toProcess.add( new Tuple<>(hierarchyWrapper.getFile( path ), path) ); + } + } + return toProcess; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java index 734d808e..c8b01f56 100644 --- a/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java +++ b/src/main/java/cws/k8s/scheduler/model/NodeWithAlloc.java @@ -1,6 +1,7 @@ package cws.k8s.scheduler.model; import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.location.NodeLocation; import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; @@ -10,8 +11,7 @@ import lombok.extern.slf4j.Slf4j; import java.math.BigDecimal; -import java.util.HashMap; -import java.util.Map; +import java.util.*; @Getter @Slf4j @@ -25,10 +25,16 @@ public class NodeWithAlloc extends Node implements Comparable { private final Map assignedPods; + private final List startingTaskCopyingData = new LinkedList<>(); + + @Getter + private final NodeLocation nodeLocation; + public NodeWithAlloc( String name ) { this.kubernetesClient = null; this.maxResources = null; this.assignedPods = null; + this.nodeLocation = null; this.setMetadata( new ObjectMeta() ); this.getMetadata().setName( name ); } @@ -41,6 +47,8 @@ public NodeWithAlloc( Node node, CWSKubernetesClient kubernetesClient ) { assignedPods = new HashMap<>(); + this.nodeLocation = NodeLocation.getLocation( node ); + } @@ -98,20 +106,46 @@ private void setNodeData( Node node, boolean isCreate ) { public void addPod( PodWithAge pod ) { Requirements request = pod.getRequest(); synchronized (assignedPods) { - assignedPods.put( pod.getMetadata().getUid(), request ); + assignedPods.put( getPodInternalId( pod ) , request ); + } + } + + private String getPodInternalId( Pod pod ) { + return pod.getMetadata().getNamespace() + "||" + pod.getMetadata().getName(); + } + + private void removeStartingTaskCopyingDataByUid( String uid ) { + synchronized ( startingTaskCopyingData ) { + final Iterator iterator = startingTaskCopyingData.iterator(); + while ( iterator.hasNext() ) { + final PodWithAge podWithAge = iterator.next(); + if ( podWithAge.getMetadata().getUid().equals( uid ) ) { + iterator.remove(); + break; + } + } } } public boolean removePod( Pod pod ){ synchronized (assignedPods) { - return assignedPods.remove( pod.getMetadata().getUid() ) != null; + return assignedPods.remove( getPodInternalId( pod ) ) != null; } } + public void startingTaskCopyingDataFinished( Task task ) { + final String uid = task.getPod().getMetadata().getUid(); + removeStartingTaskCopyingDataByUid( uid ); + } + public int getRunningPods(){ return assignedPods.size(); } + public int getStartingPods(){ + return startingTaskCopyingData.size(); + } + /** * @return max(Requested by all and currently used ) */ @@ -173,4 +207,25 @@ public int hashCode() { public boolean isReady(){ return Readiness.isNodeReady(this); } + + public boolean affinitiesMatch( PodWithAge pod ){ + + final boolean nodeCouldRunThisPod = this.getMaxResources().higherOrEquals( pod.getRequest() ); + if ( !nodeCouldRunThisPod ){ + return false; + } + + final Map podsNodeSelector = pod.getSpec().getNodeSelector(); + final Map nodesLabels = this.getMetadata().getLabels(); + if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) { + return true; + } + //cannot be fulfilled if podsNodeSelector is not empty + if ( nodesLabels == null || nodesLabels.isEmpty() ) { + return false; + } + + return nodesLabels.entrySet().containsAll( podsNodeSelector.entrySet() ); + } + } diff --git a/src/main/java/cws/k8s/scheduler/model/PodWithAge.java b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java index fb989052..e111d92a 100644 --- a/src/main/java/cws/k8s/scheduler/model/PodWithAge.java +++ b/src/main/java/cws/k8s/scheduler/model/PodWithAge.java @@ -1,7 +1,8 @@ package cws.k8s.scheduler.model; import cws.k8s.scheduler.util.PodPhase; -import io.fabric8.kubernetes.api.model.*; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Quantity; import lombok.Getter; import lombok.Setter; @@ -13,21 +14,13 @@ public class PodWithAge extends Pod { @Setter private BigDecimal age; - public PodWithAge(ObjectMeta metadata, PodSpec spec, PodStatus status) { - super("v1", "Pod", metadata, spec, status); - this.age = BigDecimal.ZERO; - } - public PodWithAge() { - super("v1", "Pod", null, null, null); - this.age = BigDecimal.ZERO; - } public PodWithAge(Pod pod) { super(pod.getApiVersion(), pod.getKind(), pod.getMetadata(), pod.getSpec(), pod.getStatus()); this.age = BigDecimal.ZERO; } public Requirements getRequest(){ - return this + return new ImmutableRequirements( this .getSpec().getContainers().stream() .filter( x -> x.getResources() != null && x.getResources().getRequests() != null ) @@ -36,7 +29,7 @@ public Requirements getRequest(){ x.getResources().getRequests().get("cpu") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("cpu")), x.getResources().getRequests().get("memory") == null ? null : Quantity.getAmountInBytes(x.getResources().getRequests().get("memory")) ) - ).reduce( new Requirements(), Requirements::addToThis ); + ).reduce( new Requirements(), Requirements::addToThis ) ); } public String getName(){ diff --git a/src/main/java/cws/k8s/scheduler/model/Requirements.java b/src/main/java/cws/k8s/scheduler/model/Requirements.java index 2ba931b4..1fcb57cc 100644 --- a/src/main/java/cws/k8s/scheduler/model/Requirements.java +++ b/src/main/java/cws/k8s/scheduler/model/Requirements.java @@ -7,7 +7,7 @@ import static cws.k8s.scheduler.util.Formater.formatBytes; -public class Requirements implements Serializable { +public class Requirements implements Serializable, Cloneable { private static final long serialVersionUID = 1L; @@ -23,6 +23,16 @@ public Requirements( BigDecimal cpu, BigDecimal ram ) { this.ram = ram == null ? BigDecimal.ZERO : ram; } + /** + * Basically used for testing + * @param cpu + * @param ram + */ + public Requirements( int cpu, int ram ) { + this.cpu = BigDecimal.valueOf(cpu); + this.ram = BigDecimal.valueOf(ram); + } + public Requirements(){ this( BigDecimal.ZERO, BigDecimal.ZERO ); } @@ -33,6 +43,13 @@ public Requirements addToThis( Requirements requirements ){ return this; } + public Requirements add( Requirements requirements ){ + return new Requirements( + this.cpu.add(requirements.cpu), + this.ram.add(requirements.ram) + ); + } + public Requirements addRAMtoThis( BigDecimal ram ){ this.ram = this.ram.add( ram ); return this; @@ -56,6 +73,19 @@ public Requirements sub( Requirements requirements ){ ); } + public Requirements multiply( BigDecimal factor ){ + return new Requirements( + this.cpu.multiply(factor), + this.ram.multiply(factor) + ); + } + + public Requirements multiplyToThis( BigDecimal factor ){ + this.cpu = this.cpu.multiply(factor); + this.ram = this.ram.multiply(factor); + return this; + } + public boolean higherOrEquals( Requirements requirements ){ return this.cpu.compareTo( requirements.cpu ) >= 0 && this.ram.compareTo( requirements.ram ) >= 0; @@ -84,4 +114,29 @@ public int hashCode() { result = 31 * result + (getRam() != null ? getRam().hashCode() : 0); return result; } + + /** + * Always returns a mutable copy of this object + * @return + */ + @Override + public Requirements clone() { + return new Requirements( this.cpu, this.ram ); + } + + public boolean smaller( Requirements request ) { + return this.cpu.compareTo( request.cpu ) < 0 + && this.ram.compareTo( request.ram ) < 0; + } + + public boolean smallerEquals( Requirements request ) { + return this.cpu.compareTo( request.cpu ) <= 0 + && this.ram.compareTo( request.ram ) <= 0; + } + + public boolean atLeastOneBigger( Requirements request ) { + return this.cpu.compareTo( request.cpu ) > 0 + || this.ram.compareTo( request.ram ) > 0; + } + } diff --git a/src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java b/src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java index d55233f7..743f0965 100644 --- a/src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/SchedulerConfig.java @@ -12,18 +12,40 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE,force = true) public class SchedulerConfig { + public final List localClaims; public final List volumeClaims; public final String workDir; public final String dns; + public final String copyStrategy; + public final boolean locationAware; public final boolean traceEnabled; public final String namespace; public final String costFunction; public final String strategy; + + public final Integer maxCopyTasksPerNode; + + public final Integer maxWaitingCopyTasksPerNode; + public final Integer maxHeldCopyTaskReady; + public final Integer prioPhaseThree; + public final Map additional; public final String memoryPredictor; public final Long maxMemory; public final Long minMemory; + @ToString + public static class LocalClaim { + public final String mountPath; + public final String hostPath; + + private LocalClaim(){ + this.mountPath = null; + this.hostPath = null; + } + + } + @ToString @NoArgsConstructor(access = AccessLevel.PRIVATE,force = true) public static class VolumeClaim { diff --git a/src/main/java/cws/k8s/scheduler/model/Task.java b/src/main/java/cws/k8s/scheduler/model/Task.java index 0b934c5c..b3a832ba 100644 --- a/src/main/java/cws/k8s/scheduler/model/Task.java +++ b/src/main/java/cws/k8s/scheduler/model/Task.java @@ -2,15 +2,23 @@ import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.cluster.OutputFiles; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; import cws.k8s.scheduler.model.tracing.TraceRecord; import cws.k8s.scheduler.util.Batch; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.math.BigDecimal; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; @Slf4j public class Task { @@ -28,6 +36,14 @@ public class Task { @Getter private final Process process; + @Getter + @Setter + private List inputFiles; + + @Getter + @Setter + private List< TaskInputFileLocationWrapper > copiedFiles; + @Getter private PodWithAge pod = null; @@ -39,6 +55,10 @@ public class Task { @Setter private Batch batch; + @Getter + @Setter + private CurrentlyCopyingOnNode copyingToNode; + @Getter private final TraceRecord traceRecord = new TraceRecord(); @@ -49,7 +69,6 @@ public class Task { private final Requirements oldRequirements; - @Getter private Requirements planedRequirements; @Getter @@ -58,11 +77,39 @@ public class Task { @Getter private long cpuPredictionVersion = -1; + private final AtomicInteger copyTaskId = new AtomicInteger(0); + + private final HierarchyWrapper hierarchyWrapper; + + @Getter + @Setter + private OutputFiles outputFiles; + public Task( TaskConfig config, DAG dag ) { + this( config, dag, null ); + } + + public Task( TaskConfig config, DAG dag, HierarchyWrapper hierarchyWrapper ) { this.config = config; oldRequirements = new Requirements( BigDecimal.valueOf(config.getCpus()), BigDecimal.valueOf(config.getMemoryInBytes()) ); planedRequirements = new Requirements( BigDecimal.valueOf(config.getCpus()), BigDecimal.valueOf(config.getMemoryInBytes()) ); this.process = dag.getByProcess( config.getTask() ); + this.hierarchyWrapper = hierarchyWrapper; + } + + /** + * Constructor for inheritance + */ + protected Task( TaskConfig config, Process process ){ + this.config = config; + oldRequirements = new Requirements( BigDecimal.valueOf(config.getCpus()), BigDecimal.valueOf(config.getMemoryInBytes()) ); + planedRequirements = new Requirements( BigDecimal.valueOf(config.getCpus()), BigDecimal.valueOf(config.getMemoryInBytes()) ); + this.process = process; + this.hierarchyWrapper = null; + } + + public int getCurrentCopyTaskId() { + return copyTaskId.getAndIncrement(); } public synchronized void setTaskMetrics( TaskMetrics taskMetrics ){ @@ -104,8 +151,16 @@ public void submitted(){ traceRecord.setSchedulerTimeInQueue( System.currentTimeMillis() - timeAddedToQueue ); } + public Set getOutLabel(){ + return config.getOutLabel(); + } + private long inputSize = -1; + /** + * Calculates the size of all input files in bytes in the shared filesystem. + * @return The sum of all input files in bytes. + */ public long getInputSize(){ if ( config.getInputSize() != null ) { return config.getInputSize(); @@ -113,10 +168,18 @@ public long getInputSize(){ synchronized ( this ) { if ( inputSize == -1 ) { //calculate - inputSize = getConfig() + Stream> inputParamStream = getConfig() .getInputs() .fileInputs - .parallelStream() + .parallelStream(); + //If LA Scheduling, filter out files that are not in sharedFS + if ( hierarchyWrapper != null ) { + inputParamStream = inputParamStream.filter( x -> { + final Path path = Path.of( x.value.sourceObj ); + return !hierarchyWrapper.isInScope( path ); + } ); + } + inputSize = inputParamStream .mapToLong( input -> new File(input.value.sourceObj).length() ) .sum(); } @@ -130,10 +193,15 @@ public String toString() { return "Task{" + "state=" + state + ", pod=" + (pod == null ? "--" : pod.getMetadata().getName()) + + ", node='" + (node != null ? node.getNodeLocation().getIdentifier() : "--") + '\'' + ", workDir='" + getWorkingDir() + '\'' + '}'; } + public Requirements getPlanedRequirements() { + return planedRequirements.clone(); + } + public long getNewMemoryRequest(){ return getPlanedRequirements().getRam().longValue(); } diff --git a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java index f3e1ec44..9619663f 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskConfig.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskConfig.java @@ -8,6 +8,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; @Getter @ToString @@ -20,6 +21,7 @@ public class TaskConfig { private final String runName; private final float cpus; private final long memoryInBytes; + private final Set outLabel; private final String workDir; private final int repetition; private final Long inputSize; @@ -40,6 +42,21 @@ public TaskConfig(String task) { this.cpus = 0; this.memoryInBytes = 0; this.workDir = null; + this.outLabel = null; + this.repetition = 0; + this.inputSize = null; + } + + public TaskConfig(String task, String name, String workDir, String runName ) { + this.task = task; + this.name = name; + this.schedulerParams = null; + this.inputs = new TaskInput( null, null, null, new LinkedList<>() ); + this.runName = runName; + this.cpus = 0; + this.memoryInBytes = 0; + this.workDir = workDir; + this.outLabel = null; this.repetition = 0; this.inputSize = null; } diff --git a/src/main/java/cws/k8s/scheduler/model/TaskInput.java b/src/main/java/cws/k8s/scheduler/model/TaskInput.java index d0b8b3ab..7aac8b53 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskInput.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskInput.java @@ -27,4 +27,5 @@ public String toString() { ", fileInputs=#" + (fileInputs != null ? fileInputs.size() : 0) + '}'; } + } diff --git a/src/main/java/cws/k8s/scheduler/model/TaskInputFileLocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/TaskInputFileLocationWrapper.java new file mode 100644 index 00000000..bbbb1811 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/TaskInputFileLocationWrapper.java @@ -0,0 +1,24 @@ +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class TaskInputFileLocationWrapper { + + private final String path; + private final RealHierarchyFile file; + private final LocationWrapper wrapper; + + public void success(){ + file.addOrUpdateLocation( false, wrapper ); + } + + public void failure(){ + file.removeLocation( wrapper ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java b/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java new file mode 100644 index 00000000..8edd5fac --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/TaskResultParser.java @@ -0,0 +1,177 @@ +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.outfiles.OutputFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.outfiles.SymlinkOutput; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Scanner; +import java.util.Set; +import java.util.stream.Stream; + +@Slf4j +public class TaskResultParser { + + static final int VIRTUAL_PATH = 0; + static final int FILE_EXISTS = 1; + static final int REAL_PATH = 2; + static final int SIZE = 3; + static final int FILE_TYPE = 4; + static final int CREATION_DATE = 5; + static final int ACCESS_DATE = 6; + static final int MODIFICATION_DATE = 7; + + public String getIndex( File file, int index ){ + try ( Scanner sc = new Scanner( file ) ) { + int i = 0; + while( sc.hasNext() && i++ < index ) { + sc.nextLine(); + } + if ( sc.hasNext() ) { + return sc.nextLine(); + } + } catch (FileNotFoundException e) { + log.error( "Cannot read " + file, e); + } + return null; + } + + private String getRootDir( File file, int index ){ + String data = getIndex( file, index ); + if ( data == null ) { + return null; + } + return data.split(";")[0]; + } + + private Long getDateDir(File file ){ + String data = getIndex( file, 0 ); + if ( data == null ) { + return null; + } + return Long.parseLong( data ); + } + + public void processInput( Stream in, final Set inputdata ){ + in.skip( 2 ) + .forEach( line -> { + String[] data = line.split(";"); + if( data[ FILE_EXISTS ].equals("0") ) { + return; + } + if ( data[ 3 ].equals("directory") ) { + return; + } + String path = data[ REAL_PATH ].equals("") ? data[ VIRTUAL_PATH ] : data[ REAL_PATH ]; + inputdata.add( path ); + }); + } + + private Set processOutput( + final Stream out, + final Set inputdata, + final Location location, + final boolean onlyUpdated, + final Task finishedTask, + final String outputRootDir, + final long initailDate + ){ + final Set newOrUpdated = new HashSet<>(); + out.skip( 1 ) + .forEach( line -> { + String[] data = line.split(";"); + if( data[ FILE_EXISTS ].equals("0") && data.length != 8 ) { + return; + } + boolean isSymlink = !data[ REAL_PATH ].equals(""); + String path = isSymlink ? data[ REAL_PATH ] : data[ VIRTUAL_PATH ]; + String modificationDate = data[ MODIFICATION_DATE ]; + if ( "directory".equals( data[ FILE_TYPE ] ) ) { + return; + } + String lockupPath = isSymlink ? path : path.substring( outputRootDir.length() ); + long modificationDateNano = Long.parseLong( modificationDate ); + if ( ( !inputdata.contains(lockupPath) && !onlyUpdated ) + || + modificationDateNano > initailDate ) + { + final LocationWrapper locationWrapper = new LocationWrapper( + location, + modificationDateNano / (int) 1.0E6, + Long.parseLong(data[ SIZE ]), + finishedTask + ); + newOrUpdated.add( new PathLocationWrapperPair( Paths.get(path), locationWrapper ) ); + } + if( isSymlink ){ + newOrUpdated.add( new SymlinkOutput( data[ VIRTUAL_PATH ], path )); + } + }); + return newOrUpdated; + } + + /** + * + * @param workdir + * @param location + * @param onlyUpdated + * @param finishedTask + * @return A list of all new or updated files + */ + public Set getNewAndUpdatedFiles( + final Path workdir, + final Location location, + final boolean onlyUpdated, + Task finishedTask + ){ + + final Path infile = workdir.resolve(".command.infiles"); + final Path outfile = workdir.resolve(".command.outfiles"); + if ( !outfile.toFile().exists() ) { + log.error( "Cannot find outfile " + infile ); + return new HashSet<>(); + } + + final String taskRootDir = getRootDir( infile.toFile(), 1 ); + if( taskRootDir == null + && (finishedTask.getInputFiles() == null || finishedTask.getInputFiles().isEmpty()) ) { + throw new IllegalStateException("taskRootDir is null"); + } + + + final String outputRootDir = getRootDir( outfile.toFile(), 0 ); + //No outputs defined / found + if( outputRootDir == null ) { + return new HashSet<>(); + } + + final Set inputdata = new HashSet<>(); + + + try ( + Stream in = Files.lines(infile); + Stream out = Files.lines(outfile) + ) { + + processInput( in, inputdata ); + log.trace( "{}", inputdata ); + final Long initialDate = getDateDir( infile.toFile() ); + return processOutput( out, inputdata, location, onlyUpdated, finishedTask, outputRootDir, initialDate ); + + } catch (IOException e) { + log.error( "Cannot read in/outfile in workdir: " + workdir, e); + } + return new HashSet<>(); + + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/TaskState.java b/src/main/java/cws/k8s/scheduler/model/TaskState.java index 04d7b6d7..05f8278a 100644 --- a/src/main/java/cws/k8s/scheduler/model/TaskState.java +++ b/src/main/java/cws/k8s/scheduler/model/TaskState.java @@ -16,5 +16,6 @@ public void error (String error){ this.state = State.ERROR; this.error = error; } + } diff --git a/src/main/java/cws/k8s/scheduler/model/WriteConfigResult.java b/src/main/java/cws/k8s/scheduler/model/WriteConfigResult.java new file mode 100644 index 00000000..6f4246c4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/WriteConfigResult.java @@ -0,0 +1,20 @@ +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.List; +import java.util.Map; + +@Getter +@AllArgsConstructor +public class WriteConfigResult { + + private final List inputFiles; + private final Map waitForTask; + private final CurrentlyCopyingOnNode copyingToNode; + private final boolean wroteConfig; + private boolean copyDataToNode; + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java new file mode 100644 index 00000000..967433ef --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/CopyTask.java @@ -0,0 +1,63 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskConfig; +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class CopyTask extends Task { + + private static final Process COPY_PROCESS = new Process( "Copy", Integer.MAX_VALUE ); + private final Task task; + private final NodeWithAlloc node; + private final List labelCounts; + + protected CopyTask( Task task, NodeWithAlloc node, List labelCounts ) { + super( new TaskConfig("Copy","Copy-" + task.getConfig().getName() + " -> " + node.getNodeLocation() + + " (" + labelCounts.stream().map( LabelCount::getLabel ).collect( Collectors.joining(",")) + ")" , + buildCopyTaskFolder(task), task.getConfig().getRunName() ), COPY_PROCESS ); + this.task = task; + this.node = node; + this.labelCounts = labelCounts; + } + + private static String buildCopyTaskFolder( Task task ) { + final Path path = Path.of( task.getConfig().getWorkDir() ); + Path p = Path.of( + path.getParent().getParent().toString(), + "copy", + path.getParent().getFileName().toString(), + path.getFileName().toString() + ); + return p.toString(); + } + + @Override + public Requirements getPlanedRequirements() { + return new Requirements( 0, 0 ); + } + + @Override + public boolean equals( Object obj ) { + if ( obj instanceof CopyTask ) { + return this.task.equals( ((CopyTask) obj).task ); + } + else return false; + } + + @Override + public int hashCode() { + return task.hashCode() + 5; + } + + public void finished(){ + labelCounts.forEach( x -> x.taskIsNowOnNode( task, node )); + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java new file mode 100644 index 00000000..50d597dc --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissing.java @@ -0,0 +1,23 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +/** + * This class is used to create a triple of a task, a node and a label count. + * It is used to keep track which tasks are not on a node. + */ +@Getter +@RequiredArgsConstructor +public class DataMissing { + + private final Task task; + private final NodeWithAlloc node; + private final List labelCounts; + private final double score; + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java new file mode 100644 index 00000000..0bf9def4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/DataMissingIntern.java @@ -0,0 +1,19 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * This is a temporary class to group a task, a node and a label count. + */ +@RequiredArgsConstructor +@Getter +public class DataMissingIntern { + + private final Task task; + private final NodeWithAlloc node; + private final LabelCount labelCount; + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java b/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java new file mode 100644 index 00000000..2319f047 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FakeTaskInputs.java @@ -0,0 +1,32 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; + +import java.util.LinkedList; +import java.util.List; + +/** + * Fake task inputs are used to create {@link CopyTask}s that are not real tasks but are used to copy data between nodes. + * This task can only be run on the node that is included in the constructor. + */ +public class FakeTaskInputs extends TaskInputs { + + private final Location includedNode; + + public FakeTaskInputs( List files, Location includedNode ) { + super( new LinkedList<>(), files, null ); + this.includedNode = includedNode; + } + + @Override + public boolean canRunOnLoc( Location loc ) { + return includedNode == loc; + } + + @Override + public boolean hasExcludedNodes() { + return true; + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java new file mode 100644 index 00000000..68042667 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/FileSizeRankScoreGroup.java @@ -0,0 +1,23 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.util.score.FileSizeRankScore; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class FileSizeRankScoreGroup extends FileSizeRankScore { + + private final GroupCluster groupCluster; + + @Override + public long getScore( Task task, NodeWithAlloc location, long size ) { + final long score = super.getScore( task, location, size ); + final long newScore = (long) (score * groupCluster.getScoreForTaskOnNode( task, location )); + if ( newScore < 1 ) { + return 1; + } + return newScore; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java new file mode 100644 index 00000000..f105ebb1 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/GroupCluster.java @@ -0,0 +1,310 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; +import cws.k8s.scheduler.model.location.hierachy.NoAlignmentFoundException; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +public abstract class GroupCluster { + + private final double minScoreToCopy = 0.5; + /** + * Group all tasks by label and deliver statistics + */ + protected final Map countPerLabel = new HashMap<>(); + /** + * tasks that are not yet scheduled + */ + protected final LinkedList unscheduledTasks = new LinkedList<>(); + /** + * tasks that are assigned to a task and running + */ + protected final LinkedList scheduledTasks = new LinkedList<>(); + /** + * tasks that have finished + */ + protected final LinkedList finishedTasks = new LinkedList<>(); + /** + * Map from label to node, which node is responsible for the label + */ + protected final Map labelToNode = new HashMap<>(); + /** + * Map from node to label, which labels are assigned to the node + */ + protected final Map> nodeToLabel = new HashMap<>(); + @Getter + private final HierarchyWrapper hierarchyWrapper; + @Getter + private final CWSKubernetesClient client; + + /** + * When new tasks become available and are ready to schedule, this method is called to register them. + * @param tasks the tasks that are ready to schedule + */ + public void tasksBecameAvailable( List tasks ) { + boolean anyWithLabel = false; + synchronized ( this ) { + unscheduledTasks.addAll( tasks ); + for ( Task task : tasks ) { + if ( task.getOutLabel() == null || task.getOutLabel().isEmpty() ) { + continue; + } + for ( String label : task.getOutLabel() ) { + anyWithLabel = true; + final LabelCount labelCount = countPerLabel.computeIfAbsent( label, LabelCount::new ); + labelCount.addWaitingTask( task ); + } + } + // If we add at least one task with an outLabel we need to recalculate our alignment + if ( anyWithLabel ) { + recalculate(); + } + } + } + + /** + * This method is called when a task was scheduled to a node and can no longer be scheduled to another node. + * @param task the task that was scheduled + */ + public void taskWasAssignedToNode( Task task ) { + synchronized ( this ) { + unscheduledTasks.remove( task ); + scheduledTasks.add( task ); + if ( task.getOutLabel() != null ) { + for ( String label : task.getOutLabel() ) { + countPerLabel.get( label ).makeTaskRunning( task ); + } + recalculate(); + } + } + } + + /** + * This method is called when a task has finished and output data can be considered. + * @param tasks the tasks that have finished + */ + public void tasksHaveFinished( List tasks ) { + synchronized ( this ) { + scheduledTasks.removeAll( tasks ); + finishedTasks.addAll( tasks ); + for ( Task task : tasks ) { + if ( task.getOutLabel() == null ) { + continue; + } + for ( String label : task.getOutLabel() ) { + countPerLabel.get( label ).makeTaskFinished( task ); + } + } + } + } + + /** + * This method is called when the state changes. + * This is either the case when a new task becomes available, or a task is scheduled. + */ + abstract void recalculate(); + + /** + * Return the score for a task on a node. + * This calculates the score using jaccard similarity coefficient. + * Therefore, it calculates the shared labels between the task and the node and divides it by the total amount of labels of the task. + * @param task the task + * @param node the node + * @return the score in the range of 0 to 1, where 1 is the best score + */ + public double getScoreForTaskOnNode( Task task, NodeWithAlloc node ) { + // If the task has no outLabel, it can run on any node + final Set outLabel = task.getOutLabel(); + if ( outLabel == null || outLabel.isEmpty() ) { + return 1; + } + + final Set nodeLabels = nodeToLabel.get( node ); + return calculateJaccardSimilarityCoefficient( outLabel, nodeLabels ); + } + + /** + * Calculate the Jaccard similarity coefficient between two sets of labels. + * If there is no overlap, the coefficient is 0.01. + * @param outLabel the first set of labels, this cannot be empty or null + * @param nodeLabels the second set of labels, this can be null or empty + * @return the Jaccard similarity coefficient in the range of 0.01 to 1 + */ + static double calculateJaccardSimilarityCoefficient( Set outLabel, Set nodeLabels ) { + if ( outLabel == null || outLabel.isEmpty() ) { + throw new IllegalArgumentException( "outLabel cannot be empty or null" ); + } + final double lowerBound = 0.01; + if ( nodeLabels == null || nodeLabels.isEmpty() ) { + return lowerBound; + } + outLabel = new HashSet<>( outLabel ); + int outLabelSize = outLabel.size(); + //intersection of both sets. + outLabel.retainAll( nodeLabels ); + // No common labels + if ( outLabelSize == 0 ) { + return lowerBound; + } + final double result = (double) outLabel.size() / outLabelSize; + return Math.max( lowerBound, Math.min( 1, result )); // Clamp the result between 0.01 and 1 + } + + /** + * Assign a node to a label, such that tasks for this label can be prepared on the node. + * @param nodeLocation the node + * @param label the label + */ + protected void addNodeToLabel( NodeWithAlloc nodeLocation, String label ) { + final NodeWithAlloc currentLocation = labelToNode.get( label ); + + if ( nodeLocation.equals( currentLocation ) ) { + return; + } + + Set labels = nodeToLabel.computeIfAbsent( nodeLocation, k -> new HashSet<>() ); + labels.add( label ); + + if ( currentLocation != null ) { + labels = nodeToLabel.get( currentLocation ); + labels.remove( label ); + } + labelToNode.put( label, nodeLocation ); + } + + /** + * Create TaskStat for tasks that could be prepared on a node, return maximal {@code maxCopiesPerNode} TaskStat per node. + * Only tasks with a score higher than {@link #minScoreToCopy} are considered. + * @param maxCopiesPerNode the maximum amount of copies that will be created for a node + * @return a list of all TaskStats that should be copied + */ + public List getTaskStatToCopy( final int maxCopiesPerNode ){ + synchronized ( this ) { + final Map tasksForLocation = new HashMap<>(); + //all tasks where score > minScoreToCopy and tasks have at least one outLabel + final List tasksThatNeedToBeCopied = getTasksThatNeedToBeCopied(); + List taskStats = new ArrayList<>(); + for ( DataMissing dataMissing : tasksThatNeedToBeCopied ) { + Integer currentCopies = tasksForLocation.getOrDefault( dataMissing.getNode(), 0 ); + // Only create maxCopiesPerNode possible TaskStats per node + TaskStat taskStat = currentCopies < maxCopiesPerNode ? getTaskStat( dataMissing ) : null; + // TaskStat is null if the output data was requested for a real task already and should not be copied anymore + if ( taskStat != null ) { + taskStats.add( taskStat ); + tasksForLocation.put( dataMissing.getNode(), ++currentCopies ); + } + } + return taskStats; + } + } + + /** + * Create a TaskStat for a {@link DataMissing} object. + * @param dataMissing the DataMissing object + * @return the TaskStat or null if the output data was requested for a real task already + */ + private TaskStat getTaskStat( DataMissing dataMissing ) { + final NodeWithAlloc node = dataMissing.getNode(); + final OutputFiles outputFilesWrapper = dataMissing.getTask().getOutputFiles(); + final Set outputFiles = outputFilesWrapper.getFiles(); + // Do not check for OutputFiles#wasRequestedForRealTask() here, + // because it is checked when the DataMissing object is created in LabelCount.tasksNotOnNode + List files = outputFiles + .parallelStream() + .map( x -> convertToPathFileLocationTriple(x, dataMissing.getTask()) ) + .collect( Collectors.toList() ); + // If in the previous step at least one file was requested for a real task, + // all the output data is considered as requested for a real task + if ( dataMissing.getTask().getOutputFiles().isWasRequestedForRealTask() ) { + return null; + } + TaskInputs inputsOfTask = new FakeTaskInputs( files, node.getNodeLocation() ); + final CopyTask copyTask = new CopyTask( dataMissing.getTask(), node, dataMissing.getLabelCounts() ); + final TaskStat taskStat = new TaskStat( copyTask, inputsOfTask ); + // TaskNodeStats is created with the total size and 0 for the data that is on the node + final TaskNodeStats taskNodeStats = new TaskNodeStats( inputsOfTask.calculateAvgSize(), 0, 0 ); + taskStat.add( dataMissing.getNode(), taskNodeStats ); + return taskStat; + } + + /** + * Convert a PathLocationWrapperPair to a PathFileLocationTriple. + * Check for every file if it was requested for a real task. + * @param pair the PathLocationWrapperPair containing the LocationWrapper to process + * @param task the task that created the output data + * @return the PathFileLocationTriple or null if the output data was requested for a real task + */ + private PathFileLocationTriple convertToPathFileLocationTriple( PathLocationWrapperPair pair, Task task ){ + final Path path = pair.getPath(); + RealHierarchyFile file = (RealHierarchyFile) hierarchyWrapper.getFile( path ); + // if at least one file was requested for a real task, all the output data is considered as requested for a real task + // return null as we ignore this output data for proactively copying + if ( file.wasRequestedByTask() ) { + task.getOutputFiles().wasRequestedForRealTask(); + return null; + } + try { + // task is the task that created the output data + final RealHierarchyFile.MatchingLocationsPair filesForTask = file.getFilesForTask( task ); + return new PathFileLocationTriple( path, file, filesForTask.getMatchingLocations() ); + } catch ( NoAlignmentFoundException e ) { + throw new RuntimeException( e ); + } + } + + /** + * Get all MissingData for tasks. + * This method checks for each task if the output data is on the node that is responsible for the label. + * If the data is not on the node, a DataMissing object is created. But only if the score is higher than {@link #minScoreToCopy}. + * @return a stream of DataMissing objects + */ + private List getTasksThatNeedToBeCopied(){ + final Map, List> collect = countPerLabel.entrySet() + .parallelStream() + .unordered() + .flatMap( v -> { + final String label = v.getKey(); + final LabelCount value = v.getValue(); + // which node is responsible for the label + final NodeWithAlloc node = labelToNode.get( label ); + // get missing data for all tasks that are not on the node + return value.tasksNotOnNode( node ); + } ) + // Group all Labels for DataMissingIntern with the same task and node into a map + .collect( Collectors.groupingBy( + dmi -> Map.entry( dmi.getTask(), dmi.getNode() ), + Collectors.mapping( DataMissingIntern::getLabelCount, Collectors.toList() ) + ) ); + List result = new ArrayList<>( collect.size() ); + for ( Map.Entry, List> e : collect.entrySet() ) { + final Task task = e.getKey().getKey(); + final NodeWithAlloc node = e.getKey().getValue(); + final List labelCounts = e.getValue(); + final double score = getScoreForTaskOnNode( task, node ); + // Only return tasks that have a score higher than minScoreToCopy + if ( score > minScoreToCopy ) { + result.add( new DataMissing( task, node, labelCounts, score ) ); + } + } + // Sort the tasks by score in descending order: highest score first + result.sort( (x, y) -> Double.compare( y.getScore(), x.getScore() ) ); + return result; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java new file mode 100644 index 00000000..96bc6b6a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/LabelCount.java @@ -0,0 +1,161 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.*; +import java.util.stream.Stream; + +/** + * This class is used to keep track of the number of tasks with a specific label + * and on which nodes the tasks are running, waiting or there output is stored. + * This class is not thread safe! + */ +@RequiredArgsConstructor( access = AccessLevel.PACKAGE ) +public class LabelCount { + + /** + * The label that all tasks have + */ + @Getter + private final String label; + + /** + * The tasks that are not yet started + */ + @Getter + private final Set waitingTasks = new HashSet<>(); + + /** + * The tasks that are running + */ + private final Set runningTasks = new HashSet<>(); + + /** + * The tasks that are finished + */ + private final Set finishedTasks = new HashSet<>(); + + /** + * A Queue with a wrapper that contains the number of tasks running or finished on a node. + * The queue is sorted by the number of tasks running or finished on a node, such that the node with the most tasks is first. + */ + @Getter + private final Queue runningOrfinishedOnNodes = new PriorityQueue<>(); + + /** + * For each node store the number of tasks that are running or ran on it, + * this map is the index for the elements in {@link #runningOrfinishedOnNodes} + */ + private final Map nodeToShare = new HashMap<>(); + + /** + * For each task with this label store the nodes where the output data of the task is stored + */ + private final Map> taskHasDataOnNode = new HashMap<>(); + + /** + * Get output data for all tasks that are not on the node + * @param node the node to check + * @return a stream of DataMissing objects containing the tasks' output data that is missing on the node + */ + public Stream tasksNotOnNode( NodeWithAlloc node ) { + return taskHasDataOnNode.entrySet() + .stream() + // check if the task's output is not on the node and the output data was not requested for a real task + .filter( taskSetEntry -> !taskSetEntry.getValue().contains( node ) + && !taskSetEntry.getKey().getOutputFiles().isWasRequestedForRealTask() ) + .map( x -> new DataMissingIntern( x.getKey(), node, this ) ); + } + + /** + * Get the number of tasks with this label that are not yet started + * @return the number of tasks with this label that are not yet started + */ + public int getCountWaiting() { + return waitingTasks.size(); + } + + /** + * Get the number of tasks with this label that are running + * @return the number of tasks with this label that are running + */ + public int getCountRunning() { + return runningTasks.size(); + } + + /** + * Get the number of tasks with this label that are finished + * @return the number of tasks with this label that are finished + */ + public int getCountFinished() { + return finishedTasks.size(); + } + + /** + * Get the number of tasks with this label + * @return the number of tasks with this label + */ + public int getCount(){ + return getCountWaiting() + getCountRunning() + getCountFinished(); + } + + /** + * Add a task to the label count, the task is not yet started + * the task needs to have this label, however the label is not checked + * @param task the task to add with this label + */ + public void addWaitingTask( Task task ){ + waitingTasks.add(task); + } + + /** + * Make a task running, the task needs to have this label, however the label is not checked + * Adds the task to {@link #runningOrfinishedOnNodes} + * @param task the task to make running + */ + public void makeTaskRunning( Task task ) { + final NodeWithAlloc node = task.getNode(); + final TasksOnNodeWrapper tasksOnNodeWrapper = nodeToShare.computeIfAbsent(node, k -> { + TasksOnNodeWrapper newWrapper = new TasksOnNodeWrapper(k); + runningOrfinishedOnNodes.add(newWrapper); + return newWrapper; + }); + tasksOnNodeWrapper.addRunningTask(); + final boolean remove = waitingTasks.remove( task ); + if ( !remove ) { + throw new IllegalStateException( "Task " + task + " was not in waiting tasks" ); + } + runningTasks.add( task ); + } + + /** + * Make a task finished, the task needs to have this label, however the label is not checked + * adds the task to {@link #taskHasDataOnNode}, to mark that the output data is on the node + * @param task the task to make finished + */ + public void makeTaskFinished( Task task ) { + final boolean remove = runningTasks.remove( task ); + if ( !remove ) { + throw new IllegalStateException( "Task " + task + " was not in running tasks" ); + } + finishedTasks.add( task ); + final HashSet locations = new HashSet<>(); + locations.add( task.getNode() ); + taskHasDataOnNode.put( task, locations ); + } + + /** + * Mark that the task's output data is on a node. + * This method is called after the task was copied to the node. + * @param task the task + * @param node the node the task's output was copied to + */ + public void taskIsNowOnNode( Task task, NodeWithAlloc node ) { + taskHasDataOnNode.get( task ).add( node ); + } + +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java b/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java new file mode 100644 index 00000000..42dd1d98 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/MostOutLabelsComparator.java @@ -0,0 +1,34 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.RequiredArgsConstructor; + +import java.util.Comparator; + +@RequiredArgsConstructor +public class MostOutLabelsComparator implements Comparator { + + private final GroupCluster groupCluster; + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + //same task does not necessarily have the same size + if ( o1.getTask() == o2.getTask() || o1Stats.getTaskSize() == o2Stats.getTaskSize() ) { + double ratingO1 = groupCluster.getScoreForTaskOnNode( o1.getTask(), o1.getNode() ); + double ratingO2 = groupCluster.getScoreForTaskOnNode( o2.getTask(), o2.getNode() ); + // Use the node with a higher rating + if ( ratingO1 != ratingO2 ) { + return Double.compare( ratingO2, ratingO1 ); + } + //Prefer the one with fewer remaining data + return Long.compare( o1Stats.getSizeRemaining(), o2Stats.getSizeRemaining() ); + } else { + //Prefer the one with larger task size + return Long.compare( o2Stats.getTaskSize(), o1Stats.getTaskSize() ); + } + } + +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java new file mode 100644 index 00000000..4f3e67c3 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/OutputFiles.java @@ -0,0 +1,28 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.Set; + +@Slf4j +@Getter +@RequiredArgsConstructor +public class OutputFiles { + + + private final Set files; + + /** + * this is used to check if the output data was requested for a real task + * As soon as one file is requested for a real task, the output data is considered as requested for a real task + */ + private boolean wasRequestedForRealTask = false; + + public void wasRequestedForRealTask(){ + wasRequestedForRealTask = true; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java b/src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java new file mode 100644 index 00000000..e057345a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/RankAndMinCopyingComparatorCopyTasks.java @@ -0,0 +1,27 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.scheduler.la2.RankAndMinCopyingComparator; +import cws.k8s.scheduler.scheduler.la2.TaskStat; + +import java.util.Comparator; + +public class RankAndMinCopyingComparatorCopyTasks extends RankAndMinCopyingComparator { + + public RankAndMinCopyingComparatorCopyTasks( Comparator comparator ) { + super( comparator ); + } + + @Override + public int compare( TaskStat o1, TaskStat o2 ) { + if ( o1.getTask() instanceof CopyTask ^ o2.getTask() instanceof CopyTask ) { + if ( o1.getTask() instanceof CopyTask ) { + return 1; + } else { + return -1; + } + } else { + return super.compare( o1, o2 ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java new file mode 100644 index 00000000..2297019b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/SimpleGroupCluster.java @@ -0,0 +1,116 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; + +import java.util.*; +import java.util.stream.Collectors; + +public class SimpleGroupCluster extends GroupCluster { + + public SimpleGroupCluster( HierarchyWrapper hierarchyWrapper, CWSKubernetesClient client ) { + super( hierarchyWrapper, client ); + } + + @Override + void recalculate() { + final List allNodes = getClient().getAllNodes(); + + // Filter labels that have waiting tasks. + final List> labelsWithWaitingTasks = countPerLabel + .entrySet() + .stream() + // If nothing waiting, we do not need a suitable node, so we do not reassign it + // If there is only one task with a label it can run where ever resources are available. Do not force a location. + .filter( kv -> kv.getValue().getCountWaiting() > 0 && kv.getValue().getCount() > 1 ) + // Sort reverse by count + .sorted( Comparator.comparingInt( kv -> - kv.getValue().getCount() ) ) + .collect( Collectors.toList() ); + + // store how many tasks would have been executed on every node with the current alignment + // this is an approximation since tasks can have multiple labels and would appear multiple times + Map tasksOnNode = new HashMap<>(); + + // For every label and node count how many tasks could run (affinities match), we ignore the current load completely + for ( Map.Entry labelWithWaitingTasks : labelsWithWaitingTasks ) { + Queue xTasksCanRunOnNode = new PriorityQueue<>(); + for ( NodeWithAlloc node : allNodes ) { + final long count = labelWithWaitingTasks + .getValue() + .getWaitingTasks() + .stream() + .filter( task -> node.affinitiesMatch( task.getPod() ) ) + .count(); + if ( count > 0 ) { + final TasksOnNodeWrapper tasksOnNodeWrapper = new TasksOnNodeWrapper( node, (int) count ); + xTasksCanRunOnNode.add( tasksOnNodeWrapper ); + } + } + if ( !xTasksCanRunOnNode.isEmpty() ) { + final NodeWithAlloc node = calculateBestFittingNode( labelWithWaitingTasks.getKey(), xTasksCanRunOnNode, labelWithWaitingTasks.getValue(), tasksOnNode ); + if ( node != null ) { + addNodeToLabel( node, labelWithWaitingTasks.getKey() ); + tasksOnNode.put( node, tasksOnNode.getOrDefault( node, 0 ) + labelWithWaitingTasks.getValue().getCountWaiting() ); + } + } + } + } + + /** + * This method first looks which nodes ran the most tasks with the current label already and then selects the node where the most tasks can run. + * @param label + * @param xTasksCanRunOnNode + * @param labelCount + * @param tasksOnNode + * @return + */ + private NodeWithAlloc calculateBestFittingNode( String label, Queue xTasksCanRunOnNode, LabelCount labelCount, Map tasksOnNode ) { + if ( xTasksCanRunOnNode.isEmpty() ) { + return null; + } + final Set bestFittingNodes = new HashSet<>(); + final TasksOnNodeWrapper bestFittingNode = xTasksCanRunOnNode.poll(); + bestFittingNodes.add( bestFittingNode.getNode() ); + while( !xTasksCanRunOnNode.isEmpty() && xTasksCanRunOnNode.peek().getShare() == bestFittingNode.getShare() ){ + bestFittingNodes.add( xTasksCanRunOnNode.poll().getNode() ); + } + + final List runningOrfinishedOnNodes = new ArrayList<>(labelCount.getRunningOrfinishedOnNodes()); + if ( runningOrfinishedOnNodes.isEmpty() ) { + // if tasks with this label have not been executed + return findBestFittingNode( bestFittingNodes, tasksOnNode ); + } + + for ( TasksOnNodeWrapper tasksOnNodeWrapper : runningOrfinishedOnNodes ) { + if ( bestFittingNodes.contains( tasksOnNodeWrapper.getNode() ) ) { + return tasksOnNodeWrapper.getNode(); + } + } + + final NodeWithAlloc nodeLocation = calculateBestFittingNode( label, xTasksCanRunOnNode, labelCount, tasksOnNode ); + if ( nodeLocation != null ) { + return nodeLocation; + } + // If no node is found, return a random one + return findBestFittingNode( bestFittingNodes, tasksOnNode ); + + } + + /** + * Assign to the node that has the least tasks to process yet + * @param bestFittingNodes list of potential nodes + * @param tasksOnNode map of tasks on each node + * @return + */ + private NodeWithAlloc findBestFittingNode( final Set bestFittingNodes, Map tasksOnNode ) { + NodeWithAlloc best = null; + for ( NodeWithAlloc fittingNode : bestFittingNodes ) { + if ( best == null || tasksOnNode.getOrDefault( fittingNode, 0 ) < tasksOnNode.getOrDefault( best, 0 ) ) { + best = fittingNode; + } + } + return best; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java new file mode 100644 index 00000000..0d8066c6 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/cluster/TasksOnNodeWrapper.java @@ -0,0 +1,30 @@ +package cws.k8s.scheduler.model.cluster; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; + +/** + * This class is used to keep track of the number of tasks with a specific label on a node. + */ +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@AllArgsConstructor +public class TasksOnNodeWrapper implements Comparable { + + private final NodeWithAlloc node; + private int share = 0; + + @Override + public int compareTo( @NotNull TasksOnNodeWrapper o ) { + return Integer.compare( o.share, share ); + } + + public void addRunningTask() { + share++; + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/Location.java b/src/main/java/cws/k8s/scheduler/model/location/Location.java new file mode 100644 index 00000000..ad4cc207 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/Location.java @@ -0,0 +1,30 @@ +package cws.k8s.scheduler.model.location; + +import java.io.Serializable; +import java.util.Objects; + +public abstract class Location implements Serializable { + + public abstract String getIdentifier(); + + public abstract LocationType getType(); + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Location)) { + return false; + } + Location that = (Location) o; + return ( getType() == that.getType() ) && ( getIdentifier().equals( that.getIdentifier() )); + } + + @Override + public int hashCode() { + return Objects.hash(getIdentifier(),getType()); + } + + +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/LocationType.java b/src/main/java/cws/k8s/scheduler/model/location/LocationType.java new file mode 100644 index 00000000..ed623a2f --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/LocationType.java @@ -0,0 +1,7 @@ +package cws.k8s.scheduler.model.location; + +public enum LocationType { + + NODE + +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/NodeLocation.java b/src/main/java/cws/k8s/scheduler/model/location/NodeLocation.java new file mode 100644 index 00000000..c97b585e --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/NodeLocation.java @@ -0,0 +1,70 @@ +package cws.k8s.scheduler.model.location; + +import io.fabric8.kubernetes.api.model.Node; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@RequiredArgsConstructor( access = AccessLevel.PRIVATE ) +public class NodeLocation extends Location { + + private static final long serialVersionUID = 1L; + + private static final ConcurrentMap< String, NodeLocation > locationHolder = new ConcurrentHashMap<>(); + + @Getter + private final String identifier; + + public static NodeLocation getLocation( Node node ){ + return getLocation( node.getMetadata().getName() ); + } + + public static NodeLocation getLocation( String node ){ + if ( node == null ) { + throw new IllegalArgumentException("Node cannot be null"); + } + final NodeLocation nodeLocation = locationHolder.get(node); + if ( nodeLocation == null ){ + locationHolder.putIfAbsent( node, new NodeLocation( node ) ); + return locationHolder.get( node ); + } + return nodeLocation; + } + + @Override + public LocationType getType() { + return LocationType.NODE; + } + + @Override + public String toString() { + return "Node(" + identifier + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof NodeLocation)) { + return false; + } + if (!super.equals(o)) { + return false; + } + + NodeLocation that = (NodeLocation) o; + + return getIdentifier() != null ? getIdentifier().equals(that.getIdentifier()) : that.getIdentifier() == null; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getIdentifier() != null ? getIdentifier().hashCode() : 0); + return result; + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/AbstractHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/AbstractHierarchyFile.java new file mode 100644 index 00000000..3a431269 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/AbstractHierarchyFile.java @@ -0,0 +1,5 @@ +package cws.k8s.scheduler.model.location.hierachy; + +public abstract class AbstractHierarchyFile extends HierarchyFile { + +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/Folder.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/Folder.java new file mode 100644 index 00000000..ef4c0ae4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/Folder.java @@ -0,0 +1,84 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PACKAGE) +public class Folder extends HierarchyFile { + + private final ConcurrentMap children = new ConcurrentHashMap<>(); + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isSymlink() { + return false; + } + + public HierarchyFile get(String name ){ + return children.get( name ); + } + + /** + * Creates folder if not existing + * @param name name of the folder to create + * @return the file with the name, or the new created folder + */ + public Folder getOrCreateFolder(String name ){ + final HierarchyFile file = children.get(name); + if( file == null || !file.isDirectory() ){ + return (Folder) children.compute( name, (key,value) -> (value == null || !value.isDirectory()) ? new Folder() : value ); + } + return (Folder) file; + } + + public Map getAllChildren(Path currentPath ){ + Map result = new TreeMap<>(); + getAllChildren( result, currentPath ); + return result; + } + + private void getAllChildren(final Map result, Path currentPath ){ + for (Map.Entry entry : children.entrySet()) { + Path resolve = currentPath.resolve(entry.getKey()); + if ( !entry.getValue().isSymlink() && entry.getValue().isDirectory() ){ + final Folder value = (Folder) entry.getValue(); + value.getAllChildren( result, resolve ); + } else { + result.put( resolve, (AbstractHierarchyFile) entry.getValue()); + } + } + } + + public LocationWrapper addOrUpdateFile(final String name, boolean overwrite, final LocationWrapper location ) { + final RealHierarchyFile file = (RealHierarchyFile) children.compute( name, (k, v) -> { + if (v == null || v.isDirectory() || v.isSymlink() ) { + return new RealHierarchyFile( location ); + } + return v; + } ); + return file.addOrUpdateLocation( overwrite, location ); + } + + public boolean addSymlink( final String name, final Path dst ){ + children.compute( name, (k,v) -> { + if ( v == null || !v.isSymlink() || !((LinkHierarchyFile) v).getDst().equals(dst) ) { + return new LinkHierarchyFile( dst ); + } + return v; + } ); + return true; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyFile.java new file mode 100644 index 00000000..a32b6226 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyFile.java @@ -0,0 +1,9 @@ +package cws.k8s.scheduler.model.location.hierachy; + +public abstract class HierarchyFile { + + public abstract boolean isDirectory(); + + public abstract boolean isSymlink(); + +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapper.java new file mode 100644 index 00000000..0d2069a4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapper.java @@ -0,0 +1,169 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Slf4j +public class HierarchyWrapper { + + private final Path workdir; + + private final ConcurrentMap workDirs = new ConcurrentHashMap<>(2); + + public HierarchyWrapper( String workdir ) { + if ( workdir == null ) { + throw new IllegalArgumentException( "Workdir is not defined" ); + } + this.workdir = Paths.get( workdir ).normalize(); + } + + private Path relativize( Path path ){ + return workdir.relativize( path ).normalize(); + } + + private Folder getWorkdir( Iterator iterator, boolean create ){ + if(!iterator.hasNext()) { + return null; + } + final String hash1 = iterator.next().toString(); + if(!iterator.hasNext()) { + return null; + } + final String hash2 = iterator.next().toString(); + final String key = hash1 + hash2; + final Folder folder = workDirs.get( key ); + if( create && folder == null ){ + workDirs.putIfAbsent( key, new Folder()); + return workDirs.get( key ); + } + return folder; + } + + /** + * + * @param path get all files recursively in this folder (absolute path) + * @return Null if folder is empty, or not found + */ + public Map getAllFilesInDir(final Path path ){ + final Path relativePath = relativize( path ); + Iterator iterator = relativePath.iterator(); + HierarchyFile current = getWorkdir( iterator, false ); + if( current == null ) { + return null; + } + while(iterator.hasNext()){ + Path p = iterator.next(); + if ( current != null && current.isDirectory() ){ + current = ((Folder) current).get( p.toString() ); + } else { + return null; + } + } + if( current.isDirectory() ) { + return ((Folder) current).getAllChildren( path.normalize() ); + } else { + return null; + } + } + + /** + * + * @param path file to add (absolute path) + * @param location location where the file is located + * @return null if file can not be created + */ + public LocationWrapper addFile(final Path path, final LocationWrapper location ){ + return addFile( path, false, location ); + } + + public LocationWrapper addFile(final Path path, boolean overwrite, final LocationWrapper location ){ + + final Folder folderToInsert = findFolderToInsert(path); + + if( folderToInsert == null ) { + return null; + } else { + return folderToInsert.addOrUpdateFile( path.getFileName().toString(), overwrite, location ); + } + + } + + public boolean addSymlink( final Path src, final Path dst ){ + + final Folder folderToInsert = findFolderToInsert( src ); + + if( folderToInsert == null ) { + return false; + } else { + return folderToInsert.addSymlink( src.getFileName().toString(), dst ); + } + + } + + private Folder findFolderToInsert( final Path path ){ + final Path relativePath = relativize( path ); + if (relativePath.startsWith("..")){ + return null; + } + Iterator iterator = relativePath.iterator(); + Folder current = getWorkdir( iterator, true ); + if( current == null ) { + return null; + } + while(iterator.hasNext()) { + Path p = iterator.next(); + if( iterator.hasNext() ){ + //folder + current = current.getOrCreateFolder( p.toString() ); + } else { + //file + return current; + } + } + //This would add a file in working hierarchy + return null; + } + + /** + * + * @param path file to get (absolute path) + * @return File or null if file does not exist + */ + public HierarchyFile getFile(Path path ){ + final Path relativePath = relativize( path ); + if (relativePath.startsWith("..")){ + return null; + } + Iterator iterator = relativePath.iterator(); + Folder current = getWorkdir( iterator, false ); + if( current == null ) { + return null; + } + if( !iterator.hasNext() ) { + return current; + } + while( iterator.hasNext() ) { + Path p = iterator.next(); + final HierarchyFile file = current.get( p.toString() ); + if( iterator.hasNext() && file.isDirectory() ){ + //folder + current = (Folder) file; + } else if ( !iterator.hasNext() ) { + return file; + } else { + break; + } + } + return null; + } + + public boolean isInScope( Path path ){ + return path.startsWith( workdir ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LinkHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LinkHierarchyFile.java new file mode 100644 index 00000000..7b36b85c --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LinkHierarchyFile.java @@ -0,0 +1,23 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.nio.file.Path; + +@RequiredArgsConstructor +public class LinkHierarchyFile extends AbstractHierarchyFile { + + @Getter + private final Path dst; + + @Override + public boolean isDirectory() { + throw new IllegalStateException("Call on link"); + } + + @Override + public boolean isSymlink() { + return true; + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java new file mode 100644 index 00000000..ff3c3e5e --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/LocationWrapper.java @@ -0,0 +1,142 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import lombok.Getter; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +@Getter +public class LocationWrapper { + + private static final AtomicLong nextID = new AtomicLong(0); + + private final long id = nextID.getAndIncrement(); + private final Location location; + private long timestamp; + private long sizeInBytes; + private long createTime = System.currentTimeMillis(); + private Task createdByTask; + private LocationWrapper copyOf; + //Deactivated if file was maybe not copied completely or if one file was changed by the workflow engine. + private boolean active = true; + private int inUse = 0; + + public LocationWrapper(Location location, long timestamp, long sizeInBytes) { + this( location, timestamp, sizeInBytes ,null); + } + + public LocationWrapper(Location location, long timestamp, long sizeInBytes, Task task) { + this( location, timestamp, sizeInBytes ,task, null ); + } + + private LocationWrapper(Location location, long timestamp, long sizeInBytes, Task createdByTask, LocationWrapper copyOf) { + this.location = location; + this.timestamp = timestamp; + this.sizeInBytes = sizeInBytes; + this.createdByTask = createdByTask; + this.copyOf = copyOf; + } + + public void update( LocationWrapper update ){ + if (location != update.location) { + throw new IllegalArgumentException( "Can only update LocationWrapper with the same location." ); + } + synchronized ( this ) { + this.timestamp = update.timestamp; + this.sizeInBytes = update.sizeInBytes; + this.createTime = update.createTime; + this.createdByTask = update.createdByTask; + this.copyOf = update.copyOf; + this.active = update.active; + } + } + + public void deactivate(){ + this.active = false; + } + + /** + * use the file, if you copy it to a node, or a task uses it as input + */ + public void use(){ + synchronized ( this ) { + inUse++; + } + } + + /** + * free the file, if you finished copy it to a node, or a task the task finished that used it as an input + */ + public void free(){ + synchronized ( this ) { + inUse--; + } + } + + /** + * Any task currently reading or writing to this file + */ + public boolean isInUse(){ + return inUse > 0; + } + + public LocationWrapper getCopyOf( Location location ) { + synchronized ( this ) { + return new LocationWrapper(location, timestamp, sizeInBytes, createdByTask, copyOf == null ? this : copyOf); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof LocationWrapper)) { + return false; + } + + LocationWrapper that = (LocationWrapper) o; + + synchronized ( this ) { + if (getTimestamp() != that.getTimestamp()) { + return false; + } + if (getSizeInBytes() != that.getSizeInBytes()) { + return false; + } + if (!getLocation().equals(that.getLocation())) { + return false; + } + if (getCreatedByTask() != null ? !getCreatedByTask().equals(that.getCreatedByTask()) : that.getCreatedByTask() != null) { + return false; + } + return getCopyOf() != null ? getCopyOf().equals(that.getCopyOf()) : that.getCopyOf() == null; + } + } + + @Override + public int hashCode() { + synchronized ( this ) { + return Objects.hash(getLocation(), getTimestamp(), getSizeInBytes(), getCreatedByTask()); + } + } + + @Override + public String toString() { + synchronized ( this ) { + return "LocationWrapper{" + + "id=" + id + + ", active=" + active + + ", location=" + location.getIdentifier() + + ", timestamp=" + timestamp + + ", inUse=" + inUse + "x" + + ", sizeInBytes=" + sizeInBytes + + ", createTime=" + createTime + + ", createdByTask=" + createdByTask + + ", copyOf=" + copyOf + + '}'; + } + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/NoAlignmentFoundException.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/NoAlignmentFoundException.java new file mode 100644 index 00000000..e73de13a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/NoAlignmentFoundException.java @@ -0,0 +1,4 @@ +package cws.k8s.scheduler.model.location.hierachy; + +public class NoAlignmentFoundException extends Exception { +} diff --git a/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java new file mode 100644 index 00000000..27921a61 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFile.java @@ -0,0 +1,317 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.LocationType; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class RealHierarchyFile extends AbstractHierarchyFile { + + /** + * This field contains the newest LocationWrapper of one file for each node. + */ + @Getter + private LocationWrapper[] locations; + static final String LOCATION_IS_NULL = "location is null"; + private boolean wasRequestedByTask = false; + + public RealHierarchyFile( LocationWrapper location ) { + if ( location == null ) { + throw new IllegalArgumentException( LOCATION_IS_NULL ); + } + this.locations = new LocationWrapper[]{ location }; + } + + @Override + public boolean isDirectory(){ + return false; + } + + @Override + public boolean isSymlink() { + return false; + } + + public void removeLocation( LocationWrapper location ){ + if ( location == null ) { + throw new IllegalArgumentException( LOCATION_IS_NULL ); + } + synchronized ( this ){ + for ( LocationWrapper locationWrapper : locations ) { + if ( location.getLocation().equals( locationWrapper.getLocation() ) ) { + locationWrapper.deactivate(); + } + } + } + } + + public LocationWrapper addOrUpdateLocation( boolean overwrite, LocationWrapper location ){ + if ( location == null ) { + throw new IllegalArgumentException( LOCATION_IS_NULL ); + } + synchronized ( this ){ + LocationWrapper locationWrapperToUpdate = null; + for (LocationWrapper locationWrapper : locations) { + if ( location.getLocation().equals( locationWrapper.getLocation() ) ) { + locationWrapperToUpdate = locationWrapper; + if ( overwrite || location.getTimestamp() > locationWrapper.getTimestamp() ) { + locationWrapperToUpdate.update( location ); + } + if ( !overwrite ) { + return locationWrapperToUpdate; + } + } else if ( overwrite ){ + locationWrapper.deactivate(); + } + } + if ( overwrite && locationWrapperToUpdate != null ) { + return locationWrapperToUpdate; + } + final LocationWrapper[] newLocation = Arrays.copyOf(locations, locations.length + 1); + newLocation[ locations.length ] = location; + locations = newLocation; + } + return location; + } + + private List combineResultsWithInitial ( + LinkedList current, + LinkedList ancestors, + LinkedList descendants, + LinkedList unrelated, + LinkedList initial + ) { + LinkedList result; + long time = 0; + //Only keep last update + if ( initial.size() > 1 ) { + result = new LinkedList<>(); + for (LocationWrapper locationWrapper : initial) { + if ( locationWrapper.getCreateTime() > time ) { + result.clear(); + result.add( locationWrapper ); + time = locationWrapper.getCreateTime(); + } else if ( locationWrapper.getCreateTime() == time ) { + result.add( locationWrapper ); + } + } + } else { + time = initial.get(0).getCreateTime(); + result = initial; + } + addAllLaterLocationsToResult( current, result, time ); + if( current == null ) { + addAllLaterLocationsToResult( ancestors, result, time ); + } + addAllLaterLocationsToResult( unrelated, result, time ); + addAllLaterLocationsToResult( descendants, result, time ); + return result; + } + + private List combineResultsEmptyInitial ( + LinkedList current, + LinkedList ancestors, + LinkedList descendants, + LinkedList unrelated + ) { + LinkedList result = null; + if ( current != null ) { + result = current; + } + if ( ancestors != null ) { + if (current == null ) { + result = ancestors; + } else { + result.addAll( ancestors ); + } + } + if ( unrelated != null ) { + if ( result == null ) { + result = unrelated; + } else { + result.addAll( unrelated ); + } + } + if ( descendants != null ) { + if ( result == null ) { + result = descendants; + } else { + result.addAll( descendants ); + } + } + return result; + } + + private void addToAncestors(LinkedList ancestors, LocationWrapper location, Process locationProcess) { + //Add location to list if it could be the last version + final Iterator iterator = ancestors.iterator(); + Set locationAncestors = null; + while (iterator.hasNext()) { + final LocationWrapper next = iterator.next(); + final Process currentProcess = next.getCreatedByTask().getProcess(); + if (locationProcess == currentProcess) { + break; + } else { + if( locationAncestors == null ) { + locationAncestors = locationProcess.getAncestors(); + } + if ( locationAncestors.contains(currentProcess) ) { + iterator.remove(); + } else if (locationProcess.getDescendants().contains(currentProcess)) { + return; + } + } + } + ancestors.add(location); + } + + private LinkedList addAndCreateList(LinkedList list, LocationWrapper toAdd ){ + if ( list == null ) { + list = new LinkedList<>(); + } + list.add( toAdd ); + return list; + } + + /** + * This method is used to find all possible LocationWrappers of a file for a specific task. + * @return a list of all LocationWrapper of this file that could be used and a list of all Locations that are in use and are not in a version that this task could use. + */ + public MatchingLocationsPair getFilesForTask( Task task ) throws NoAlignmentFoundException { + LocationWrapper[] locationsRef = this.locations; + + LinkedList current = null; + LinkedList ancestors = null; + LinkedList descendants = null; + LinkedList unrelated = null; + LinkedList initial = null; + + final Process taskProcess = task.getProcess(); + final Set taskAncestors = taskProcess.getAncestors(); + final Set taskDescendants = taskProcess.getDescendants(); + + Set inUse = null; + + for ( LocationWrapper location : locationsRef ) { + + if( location.isInUse() ) { + if ( inUse == null ) { + inUse = new HashSet<>(); + } + inUse.add(location.getLocation()); + } + + if ( !location.isActive() ) { + continue; + } + + //File was modified by an operator (no relation known) + if ( location.getCreatedByTask() == null ) { + initial = addAndCreateList( initial, location ); + continue; + } + + final Process locationProcess = location.getCreatedByTask().getProcess(); + if ( locationProcess == taskProcess) + //Location was created by the same process == does definitely fit. + { + current = addAndCreateList( current, location ); + } else if ( taskAncestors.contains(locationProcess) ) { + // location is a direct ancestor + if ( ancestors == null ) { + ancestors = new LinkedList<>(); + ancestors.add( location ); + } else { + addToAncestors( ancestors, location, locationProcess ); + } + } + else if ( taskDescendants.contains(locationProcess) ) + // location is a direct descendant + { + descendants = addAndCreateList( descendants, location ); + } else + // location was possibly generated in parallel + { + unrelated = addAndCreateList( unrelated, location ); + } + } + + final List matchingLocations = ( initial == null ) + ? combineResultsEmptyInitial( current, ancestors, descendants, unrelated ) + : combineResultsWithInitial( current, ancestors, descendants, unrelated, initial ); + + if ( matchingLocations == null ) { + throw new NoAlignmentFoundException(); + } + removeMatchingLocations( matchingLocations, inUse ); + + return new MatchingLocationsPair( matchingLocations, inUse ); + } + + private void removeMatchingLocations( List matchingLocations, Set locations ){ + if ( locations == null || matchingLocations == null ) { + return; + } + for ( LocationWrapper matchingLocation : matchingLocations ) { + if( matchingLocation.isInUse() ) { + locations.remove(matchingLocation.getLocation()); + } + } + } + + @Getter + public class MatchingLocationsPair { + + private final List matchingLocations; + private final Set excludedNodes; + + private MatchingLocationsPair(List matchingLocations, Set excludedNodes) { + this.matchingLocations = matchingLocations; + this.excludedNodes = excludedNodes; + } + + } + + private void addAllLaterLocationsToResult( List list, List result, long time ){ + if( list != null ) { + for ( LocationWrapper l : list ) { + if( l.getCreateTime() >= time ) { + result.add( l ); + } + } + } + } + + public LocationWrapper getLastUpdate( LocationType type ){ + LocationWrapper lastLocation = null; + for (LocationWrapper location : locations) { + if( location.isActive() && location.getLocation().getType() == type && (lastLocation == null || lastLocation.getCreateTime() < location.getCreateTime() )){ + lastLocation = location; + } + } + return lastLocation; + } + + public LocationWrapper getLocationWrapper( Location location ){ + for (LocationWrapper locationWrapper : locations) { + if ( locationWrapper.isActive() && locationWrapper.getLocation() == location ) { + return locationWrapper; + } + } + throw new RuntimeException( "Not found: " + location.getIdentifier() ); + } + + public void requestedByTask(){ + wasRequestedByTask = true; + } + + public boolean wasRequestedByTask(){ + return wasRequestedByTask; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/outfiles/OutputFile.java b/src/main/java/cws/k8s/scheduler/model/outfiles/OutputFile.java new file mode 100644 index 00000000..2e43afc4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/outfiles/OutputFile.java @@ -0,0 +1,31 @@ +package cws.k8s.scheduler.model.outfiles; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.nio.file.Path; +import java.util.Objects; + +@Getter +@RequiredArgsConstructor +public class OutputFile { + + private final Path path; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OutputFile)) { + return false; + } + OutputFile that = (OutputFile) o; + return Objects.equals(getPath(), that.getPath()); + } + + @Override + public int hashCode() { + return Objects.hash(getPath()); + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/outfiles/PathLocationWrapperPair.java b/src/main/java/cws/k8s/scheduler/model/outfiles/PathLocationWrapperPair.java new file mode 100644 index 00000000..1465f079 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/outfiles/PathLocationWrapperPair.java @@ -0,0 +1,47 @@ +package cws.k8s.scheduler.model.outfiles; + +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import lombok.Getter; + +import java.nio.file.Path; +import java.util.Objects; + +@Getter +public class PathLocationWrapperPair extends OutputFile { + + private final LocationWrapper locationWrapper; + + public PathLocationWrapperPair(Path path, LocationWrapper locationWrapper) { + super( path ); + this.locationWrapper = locationWrapper; + } + + @Override + public String toString() { + return "PathLocationWrapperPair{" + + "path=" + getPath() + + ", locationWrapper=" + locationWrapper + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PathLocationWrapperPair)) { + return false; + } + if (!super.equals(o)) { + return false; + } + PathLocationWrapperPair that = (PathLocationWrapperPair) o; + return Objects.equals(getLocationWrapper(), that.getLocationWrapper()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getLocationWrapper()); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/outfiles/SymlinkOutput.java b/src/main/java/cws/k8s/scheduler/model/outfiles/SymlinkOutput.java new file mode 100644 index 00000000..d3345f73 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/outfiles/SymlinkOutput.java @@ -0,0 +1,43 @@ +package cws.k8s.scheduler.model.outfiles; + +import lombok.Getter; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +@Getter +public class SymlinkOutput extends OutputFile { + + private final Path dst; + + public SymlinkOutput( String path, String dst ) { + super( Paths.get(path) ); + this.dst = Paths.get(dst); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SymlinkOutput)) { + return false; + } + if (!super.equals(o)) { + return false; + } + SymlinkOutput that = (SymlinkOutput) o; + return Objects.equals(dst, that.dst); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dst); + } + + @Override + public String toString() { + return "SymlinkOutput{" + super.getPath() + " -> " + getDst() + "}"; + } +} diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java new file mode 100644 index 00000000..a1b71c29 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/Input.java @@ -0,0 +1,4 @@ +package cws.k8s.scheduler.model.taskinputs; + +public interface Input { +} diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/PathFileLocationTriple.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/PathFileLocationTriple.java new file mode 100644 index 00000000..aab7cdcf --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/PathFileLocationTriple.java @@ -0,0 +1,66 @@ +package cws.k8s.scheduler.model.taskinputs; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +import java.nio.file.Path; +import java.util.List; + +@ToString( exclude = "file" ) +@EqualsAndHashCode +@RequiredArgsConstructor +public class PathFileLocationTriple implements Input { + + public final Path path; + public final RealHierarchyFile file; + public final List locations; + private long size = -1; + + public long getSizeInBytes() { + if ( this.size != -1 ) { + return this.size; + } + long currentSize = 0; + for (LocationWrapper location : locations) { + currentSize += location.getSizeInBytes(); + } + this.size = currentSize / locations.size(); + return this.size; + } + + public long getMinSizeInBytes() { + if ( locations.isEmpty() ) { + throw new IllegalStateException("No locations for file " + path); + } + long minSize = Long.MAX_VALUE; + for (LocationWrapper location : locations) { + if ( location.getSizeInBytes() < minSize ) { + minSize = location.getSizeInBytes(); + } + } + return minSize; + } + + public LocationWrapper locationWrapperOnLocation(Location loc){ + for (LocationWrapper location : locations) { + if ( location.getLocation().equals(loc) ) { + return location; + } + } + throw new IllegalStateException("LocationWrapper not found for location " + loc); + } + + public boolean locatedOnLocation(Location loc){ + for (LocationWrapper location : locations) { + if ( location.getLocation() == loc ) { + return true; + } + } + return false; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/SymlinkInput.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/SymlinkInput.java new file mode 100644 index 00000000..cc08e6dd --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/SymlinkInput.java @@ -0,0 +1,20 @@ +package cws.k8s.scheduler.model.taskinputs; + +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.nio.file.Path; + +@Getter +@EqualsAndHashCode +public class SymlinkInput implements Input { + + private final String src; + private final String dst; + + public SymlinkInput(Path src, Path dst) { + this.src = src.toAbsolutePath().toString(); + this.dst = dst.toAbsolutePath().toString(); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java new file mode 100644 index 00000000..72121d4b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/model/taskinputs/TaskInputs.java @@ -0,0 +1,125 @@ +package cws.k8s.scheduler.model.taskinputs; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.util.TaskNodeStats; +import cws.k8s.scheduler.util.Tuple; +import cws.k8s.scheduler.util.copying.CopySource; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@ToString +@Slf4j +@RequiredArgsConstructor +public class TaskInputs { + + @Getter + private final List symlinks; + @Getter + private final List files; + private final Set excludedNodes; + + private boolean sorted = false; + + public boolean canRunOnLoc( Location loc ) { + return !excludedNodes.contains( loc ); + } + + public boolean hasExcludedNodes() { + return !excludedNodes.isEmpty(); + } + + public boolean allFilesAreOnLocationAndNotOverwritten( Location loc, Set pathCurrentlyCopying ){ + for (PathFileLocationTriple file : files) { + if ( !file.locatedOnLocation(loc) || (pathCurrentlyCopying != null && pathCurrentlyCopying.contains(file.path.toString())) ) { + return false; + } + } + return true; + } + + public List allLocationWrapperOnLocation( Location loc){ + return files + .parallelStream() + .map( file -> file.locationWrapperOnLocation( loc ) ) + .collect( Collectors.toList() ); + } + + public long calculateDataOnNode( Location loc ) { + return calculateDataOnNodeAdditionalInfo( loc ).getB(); + } + + /** + * Calculates the data on a node and returns whether all data is on the location + * @return boolean: true if all files are on location, Long: data on location + */ + public Tuple calculateDataOnNodeAdditionalInfo( Location loc ) { + long size = 0; + boolean allOnNode = true; + for ( PathFileLocationTriple fileLocation : files ) { + if (fileLocation.locatedOnLocation(loc)) { + size += fileLocation.getSizeInBytes(); + } else { + allOnNode = false; + } + } + return new Tuple<>( allOnNode, size ); + } + + /** + * Calculates the data on a node and returns whether all data is on the location + * @return the size remaining and the amount of data currently copying. Null if the task cannot run on this node. + */ + public TaskNodeStats calculateMissingData( Location loc, CurrentlyCopyingOnNode currentlyCopying ) { + long sizeRemaining = 0; + long sizeCurrentlyCopying = 0; + long sizeOnNode = 0; + for ( PathFileLocationTriple fileLocation : files ) { + final long minSizeInBytes = fileLocation.getMinSizeInBytes(); + //Is the file already on the node? + if ( fileLocation.locatedOnLocation(loc) ) { + sizeOnNode += minSizeInBytes; + } else { + //is the file currently copying? + final CopySource copySource = currentlyCopying.getCopySource( fileLocation.path.toString() ); + if ( copySource != null ) { + //Is this file compatible with the task? + if ( fileLocation.locatedOnLocation( copySource.getLocation() ) ) { + sizeCurrentlyCopying += minSizeInBytes; + } else { + //currently copying file is incompatible with this task + return null; + } + } else { + sizeRemaining += minSizeInBytes; + } + } + } + return new TaskNodeStats( sizeRemaining, sizeCurrentlyCopying, sizeOnNode ); + } + + public long calculateAvgSize() { + long size = 0; + for ( PathFileLocationTriple file : files ) { + size += file.getSizeInBytes(); + } + return size; + } + + public void sort(){ + synchronized ( files ) { + if (!sorted) { + files.sort((x, y) -> Long.compare(y.getSizeInBytes(), x.getSizeInBytes())); + sorted = true; + } + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java index c44d7988..9e331397 100644 --- a/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java +++ b/src/main/java/cws/k8s/scheduler/model/tracing/TraceRecord.java @@ -6,12 +6,40 @@ import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; public class TraceRecord { + @Getter + @Setter + /*Filesize required for task*/ + private Long schedulerFilesBytes = null; + + @Getter + @Setter + /*Filesize required for task and already on node*/ + private Long schedulerFilesNodeBytes = null; + + @Getter + @Setter + /*Filesize required for task and already copied by other task*/ + private Long schedulerFilesNodeOtherTaskBytes = null; + + @Getter + @Setter + private Integer schedulerFiles = null; + + @Getter + @Setter + private Integer schedulerFilesNode = null; + + @Getter + @Setter + private Integer schedulerFilesNodeOtherTask = null; + @Getter @Setter private Integer schedulerDependingTask = null; @@ -24,6 +52,10 @@ public class TraceRecord { @Setter private Integer schedulerPlaceInQueue = null; + @Getter + @Setter + private Integer schedulerLocationCount = null; + @Getter @Setter private Integer schedulerNodesTried = null; @@ -42,10 +74,18 @@ public class TraceRecord { private int schedulerTriedToSchedule = 0; + @Getter + @Setter + private Integer schedulerNodesToCopyFrom = null; + @Getter @Setter private Integer schedulerTimeToSchedule = null; + @Getter + @Setter + private Integer schedulerNoAlignmentFound = null; + private Integer schedulerDeltaScheduleSubmitted = null; private Integer schedulerDeltaScheduleAlignment = null; @@ -75,19 +115,40 @@ public class TraceRecord { /*Time delta between a task was submitted and the batch became schedulable*/ private Integer schedulerDeltaSubmittedBatchEnd = null; + @Getter + private List schedulerTimeDeltaPhaseThree = null; + + private int schedulerCopyTasks = 0; + + public void addSchedulerTimeDeltaPhaseThree( Integer schedulerTimeDeltaPhaseThree ) { + if ( this.schedulerTimeDeltaPhaseThree == null ) { + this.schedulerTimeDeltaPhaseThree = new ArrayList<>(); + } + this.schedulerTimeDeltaPhaseThree.add( schedulerTimeDeltaPhaseThree ); + } + public void writeRecord( String tracePath ) throws IOException { try ( BufferedWriter bw = new BufferedWriter( new FileWriter( tracePath ) ) ) { bw.write("nextflow.scheduler.trace/v1\n"); + writeValue("scheduler_files_bytes", schedulerFilesBytes, bw); + writeValue("scheduler_files_node_bytes", schedulerFilesNodeBytes, bw); + writeValue("scheduler_files_node_other_task_bytes", schedulerFilesNodeOtherTaskBytes, bw); + writeValue("scheduler_files", schedulerFiles, bw); + writeValue("scheduler_files_node", schedulerFilesNode, bw); + writeValue("scheduler_files_node_other_task", schedulerFilesNodeOtherTask, bw); writeValue("scheduler_depending_task", schedulerDependingTask, bw); writeValue("scheduler_time_in_queue", schedulerTimeInQueue, bw); writeValue("scheduler_place_in_queue", schedulerPlaceInQueue, bw); + writeValue("scheduler_location_count", schedulerLocationCount, bw); writeValue("scheduler_nodes_tried", schedulerNodesTried, bw); writeValue("scheduler_nodes_cost", schedulerNodesCost, bw); writeValue("scheduler_could_stop_fetching", schedulerCouldStopFetching, bw); writeValue("scheduler_best_cost", schedulerBestCost, bw); writeValue("scheduler_tried_to_schedule", schedulerTriedToSchedule, bw); + writeValue("scheduler_nodes_to_copy_from", schedulerNodesToCopyFrom, bw); writeValue("scheduler_time_to_schedule", schedulerTimeToSchedule, bw); + writeValue("scheduler_no_alignment_found", schedulerNoAlignmentFound, bw); writeValue("scheduler_delta_schedule_submitted", schedulerDeltaScheduleSubmitted, bw); writeValue("scheduler_delta_schedule_alignment", schedulerDeltaScheduleAlignment, bw); writeValue("scheduler_batch_id", schedulerBatchId, bw); @@ -95,7 +156,8 @@ public void writeRecord( String tracePath ) throws IOException { writeValue("scheduler_delta_batch_start_received", schedulerDeltaBatchStartReceived, bw); writeValue("scheduler_delta_batch_closed_batch_end", schedulerDeltaBatchClosedBatchEnd, bw); writeValue("scheduler_delta_submitted_batch_end", schedulerDeltaSubmittedBatchEnd, bw); - + writeValue("scheduler_time_delta_phase_three", schedulerTimeDeltaPhaseThree, bw); + writeValue("scheduler_copy_tasks", schedulerCopyTasks, bw); } } @@ -106,7 +168,7 @@ private void writeValue( String name, T value, BufferedWriter } } - private void writeValue( String name, List value, BufferedWriter bw ) throws IOException { + private void writeValue( String name, List value, BufferedWriter bw ) throws IOException { if ( value != null ) { final String collect = value.stream() .map( x -> x==null ? "null" : x.toString() ) @@ -136,5 +198,9 @@ public void tryToSchedule( long startSchedule ){ schedulerTriedToSchedule++; } + public void copyTask(){ + schedulerCopyTasks++; + } + } diff --git a/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java b/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java index 908d0597..6f368251 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java +++ b/src/main/java/cws/k8s/scheduler/prediction/offset/VarianceOffset.java @@ -2,8 +2,6 @@ import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.prediction.Predictor; -import org.apache.commons.math3.stat.descriptive.moment.Variance; -import org.apache.commons.math3.stat.descriptive.rank.Percentile; import java.util.List; diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java index bf044daa..64e0412e 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictor.java @@ -1,6 +1,5 @@ package cws.k8s.scheduler.prediction.predictor; -import cws.k8s.scheduler.model.Task; import cws.k8s.scheduler.prediction.Predictor; public interface LinearPredictor extends Predictor { diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java index 0f74a4fe..27c93ee8 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorCustomLoss.java @@ -1,10 +1,8 @@ package cws.k8s.scheduler.prediction.predictor; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.prediction.Predictor; import cws.k8s.scheduler.prediction.extractor.VariableExtractor; import cws.k8s.scheduler.prediction.predictor.loss.UnequalLossFunction; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.math3.optim.InitialGuess; import org.apache.commons.math3.optim.MaxEval; diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java index e427b680..459c4143 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLoss.java @@ -1,7 +1,6 @@ package cws.k8s.scheduler.prediction.predictor; import cws.k8s.scheduler.model.Task; -import cws.k8s.scheduler.prediction.Predictor; import cws.k8s.scheduler.prediction.extractor.VariableExtractor; import lombok.RequiredArgsConstructor; import org.apache.commons.math3.stat.regression.SimpleRegression; diff --git a/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java b/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java index 5243a05a..710fbb08 100644 --- a/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java +++ b/src/main/java/cws/k8s/scheduler/prediction/predictor/MeanPredictor.java @@ -4,7 +4,6 @@ import cws.k8s.scheduler.prediction.Predictor; import cws.k8s.scheduler.prediction.extractor.VariableExtractor; import lombok.RequiredArgsConstructor; -import org.apache.commons.math3.stat.regression.SimpleRegression; import java.util.concurrent.atomic.AtomicLong; diff --git a/src/main/java/cws/k8s/scheduler/rest/PathAttributes.java b/src/main/java/cws/k8s/scheduler/rest/PathAttributes.java new file mode 100644 index 00000000..fafc03b7 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/rest/PathAttributes.java @@ -0,0 +1,18 @@ +package cws.k8s.scheduler.rest; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@Getter +@ToString +@NoArgsConstructor( access = AccessLevel.PRIVATE, force = true ) +public class PathAttributes { + + private final String path; + private final long size; + private final long timestamp; + private final long locationWrapperID; + +} diff --git a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java index 2012e7d7..24cfdf22 100644 --- a/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java +++ b/src/main/java/cws/k8s/scheduler/rest/SchedulerRestController.java @@ -1,14 +1,19 @@ package cws.k8s.scheduler.rest; +import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.dag.DAG; import cws.k8s.scheduler.dag.InputEdge; -import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.dag.Vertex; import cws.k8s.scheduler.model.SchedulerConfig; import cws.k8s.scheduler.model.TaskConfig; import cws.k8s.scheduler.model.TaskMetrics; -import cws.k8s.scheduler.scheduler.PrioritizeAssignScheduler; -import cws.k8s.scheduler.scheduler.Scheduler; +import cws.k8s.scheduler.rest.exceptions.NotARealFileException; +import cws.k8s.scheduler.rest.response.getfile.FileResponse; +import cws.k8s.scheduler.scheduler.*; +import cws.k8s.scheduler.scheduler.filealignment.GreedyAlignment; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.MinSizeCost; +import cws.k8s.scheduler.scheduler.la2.ready2run.OptimalReadyToRunToNode; import cws.k8s.scheduler.scheduler.nodeassign.FairAssign; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.nodeassign.RandomNodeAssign; @@ -18,6 +23,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import jakarta.servlet.http.HttpServletRequest; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -29,6 +35,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.bind.annotation.*; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -91,7 +98,8 @@ private ResponseEntity noSchedulerFor( String execution ){ @PostMapping("/v1/scheduler/{execution}") ResponseEntity registerScheduler( @PathVariable String execution, - @RequestBody(required = false) SchedulerConfig config + @RequestBody(required = false) SchedulerConfig config, + HttpServletRequest request ) { final String namespace = config.namespace; @@ -105,7 +113,37 @@ ResponseEntity registerScheduler( return noSchedulerFor( execution ); } + CostFunction costFunction = null; + if ( config.costFunction != null ) { + switch (config.costFunction.toLowerCase()) { + case "minsize": costFunction = new MinSizeCost(0); break; + default: + log.warn( "Register execution: {} - No cost function for: {}", execution, config.costFunction ); + return new ResponseEntity<>( "No cost function: " + config.costFunction, HttpStatus.NOT_FOUND ); + } + } + switch ( strategy.toLowerCase() ){ + case "wow" : + if ( !config.locationAware ) { + log.warn( "Register execution: {} - LA scheduler only works if location aware", execution ); + return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); + } + if ( costFunction == null ) { + costFunction = new MinSizeCost( 0 ); + } + scheduler = new LocationAwareSchedulerV2( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ), new OptimalReadyToRunToNode() ); + break; + case "wowgroup" : + if ( !config.locationAware ) { + log.warn( "Register execution: {} - LA scheduler only works if location aware", execution ); + return new ResponseEntity<>( "LA scheduler only works if location aware", HttpStatus.BAD_REQUEST ); + } + if ( costFunction == null ) { + costFunction = new MinSizeCost( 0 ); + } + scheduler = new LocationAwareSchedulerGroups( execution, client, namespace, config, new GreedyAlignment( 0.5, costFunction ), new OptimalReadyToRunToNode() ); + break; default: { final String[] split = strategy.split( "-" ); Prioritize prioritize; @@ -124,6 +162,7 @@ ResponseEntity registerScheduler( case "max": prioritize = new MaxInputPrioritize(); break; case "min": prioritize = new MinInputPrioritize(); break; default: + log.warn( "Register execution: {} - No Prioritize for: {}", execution, split[0] ); return new ResponseEntity<>( "No Prioritize for: " + split[0], HttpStatus.NOT_FOUND ); } if ( split.length == 2 ) { @@ -132,6 +171,7 @@ ResponseEntity registerScheduler( case "roundrobin": case "rr": assign = new RoundRobinAssign(); break; case "fair": case "f": assign = new FairAssign(); break; default: + log.warn( "Register execution: {} - No Assign for: {}", execution, split[1] ); return new ResponseEntity<>( "No Assign for: " + split[1], HttpStatus.NOT_FOUND ); } } else { @@ -139,10 +179,14 @@ ResponseEntity registerScheduler( } scheduler = new PrioritizeAssignScheduler( execution, client, namespace, config, prioritize, assign ); } else { + log.warn( "Register execution: {} - No scheduler for strategy: {}", execution, strategy ); return new ResponseEntity<>( "No scheduler for strategy: " + strategy, HttpStatus.NOT_FOUND ); } } } + if ( scheduler instanceof SchedulerWithDaemonSet ) { + ((SchedulerWithDaemonSet) scheduler).setWorkflowEngineNode( request.getRemoteAddr() ); + } schedulerHolder.put( execution, scheduler ); client.addInformable( scheduler ); @@ -167,7 +211,7 @@ ResponseEntity registerScheduler( @PostMapping("/v1/scheduler/{execution}/task/{id}") ResponseEntity registerTask( @PathVariable String execution, @PathVariable int id, @RequestBody TaskConfig config ) { - log.trace( execution + " " + config.getTask() + " got: " + config ); + log.info( execution + " " + config.getTask() + " got: " + config ); final Scheduler scheduler = schedulerHolder.get( execution ); if ( scheduler == null ) { @@ -322,6 +366,98 @@ ResponseEntity delete( @PathVariable String execution ) { return new ResponseEntity<>( HttpStatus.OK ); } + @GetMapping("/v1/daemon/{execution}/{node}") + ResponseEntity getDaemonName( @PathVariable String execution, @PathVariable String node ) { + + log.info( "Asking for Daemon exec: {} node: {}", execution, node ); + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { + return noSchedulerFor( execution ); + } + + String daemon = ((SchedulerWithDaemonSet) scheduler).getDaemonIpOnNode( node ); + + if ( daemon == null ){ + return new ResponseEntity<>( "No daemon for node found: " + node , HttpStatus.NOT_FOUND ); + } + + return new ResponseEntity<>( daemon, HttpStatus.OK ); + + } + + @PostMapping("/v1/downloadtask/{execution}") + ResponseEntity finishDownload( @PathVariable String execution, @RequestBody byte[] task ) { + + String name = new String( task, StandardCharsets.UTF_8 ); + if ( name.endsWith( "=" ) ) { + name = name.substring( 0, name.length() - 1 ); + } + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { + return noSchedulerFor( execution ); + } + ((SchedulerWithDaemonSet) scheduler).taskHasFinishedCopyTask( name ); + return new ResponseEntity<>( HttpStatus.OK ); + + } + + @GetMapping("/v1/file/{execution}") + ResponseEntity getNodeForFile( @PathVariable String execution, @RequestParam String path ) { + + log.info( "Get file location request: {} {}", execution, path ); + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { + return noSchedulerFor( execution ); + } + + FileResponse fileResponse; + try { + fileResponse = ((SchedulerWithDaemonSet) scheduler).nodeOfLastFileVersion( path ); + log.info(fileResponse.toString()); + } catch (NotARealFileException e) { + return new ResponseEntity<>( "Requested path is not a real file: " + path , HttpStatus.BAD_REQUEST ); + } + + if ( fileResponse == null ){ + return new ResponseEntity<>( "No node for file found: " + path , HttpStatus.NOT_FOUND ); + } + + return new ResponseEntity<>( fileResponse, HttpStatus.OK ); + + } + + @PostMapping("/v1/file/{execution}/location/{method}") + ResponseEntity changeLocationForFile( @PathVariable String method, @PathVariable String execution, @RequestBody PathAttributes pa ) { + return changeLocationForFile( method, execution, null, pa ); + } + + @PostMapping("/v1/file/{execution}/location/{method}/{node}") + ResponseEntity changeLocationForFile( @PathVariable String method, @PathVariable String execution, @PathVariable String node, @RequestBody PathAttributes pa ) { + + log.info( "Change file location request: {} {} {}", method, execution, pa ); + + final Scheduler scheduler = schedulerHolder.get( execution ); + if ( !(scheduler instanceof SchedulerWithDaemonSet) ) { + log.info( "No scheduler for: " + execution ); + return noSchedulerFor( execution ); + } + + if ( !method.equals("add") && !method.equals("overwrite") ) { + log.info("Method not found: " + method); + return new ResponseEntity<>( "Method not found: " + method , HttpStatus.NOT_FOUND ); + } + + boolean overwrite = method.equals("overwrite"); + + ((SchedulerWithDaemonSet) scheduler).addFile( pa.getPath(), pa.getSize(), pa.getTimestamp(), pa.getLocationWrapperID(), overwrite, node ); + + return new ResponseEntity<>( HttpStatus.OK ); + + } + @GetMapping ("/health") ResponseEntity checkHealth() { return new ResponseEntity<>( HttpStatus.OK ); diff --git a/src/main/java/cws/k8s/scheduler/rest/exceptions/NotARealFileException.java b/src/main/java/cws/k8s/scheduler/rest/exceptions/NotARealFileException.java new file mode 100644 index 00000000..98bb4cf8 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/rest/exceptions/NotARealFileException.java @@ -0,0 +1,8 @@ +package cws.k8s.scheduler.rest.exceptions; + +public class NotARealFileException extends Exception { + + public NotARealFileException() { + super( "Not a real file" ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/rest/response/getfile/FileResponse.java b/src/main/java/cws/k8s/scheduler/rest/response/getfile/FileResponse.java new file mode 100644 index 00000000..b7bc9656 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/rest/response/getfile/FileResponse.java @@ -0,0 +1,39 @@ +package cws.k8s.scheduler.rest.response.getfile; + +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import lombok.ToString; + +import java.util.List; + +@ToString( exclude = "locationWrapperID" ) +public class FileResponse { + + public final String path; + public final boolean sameAsEngine; + public final String node; + public final String daemon; + public final List symlinks; + public final boolean notInContext; + public final long locationWrapperID; + + public FileResponse( String path, String node, String daemon, boolean sameAsEngine, List symlinks, long locationWrapperID) { + this.path = path; + this.sameAsEngine = sameAsEngine; + this.node = node; + this.daemon = daemon; + this.symlinks = symlinks; + notInContext = false; + this.locationWrapperID = locationWrapperID; + } + + public FileResponse( String path, List symlinks) { + this.path = path; + this.sameAsEngine = true; + this.node = null; + this.daemon = null; + this.symlinks = symlinks; + notInContext = true; + locationWrapperID = -1; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java new file mode 100644 index 00000000..adbd0709 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerGroups.java @@ -0,0 +1,61 @@ +package cws.k8s.scheduler.scheduler; + +import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.SchedulerConfig; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.cluster.*; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.MinCopyingComparator; +import cws.k8s.scheduler.scheduler.la2.MinSizeComparator; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; + +import java.io.File; +import java.util.List; + +public class LocationAwareSchedulerGroups extends LocationAwareSchedulerV2 { + + GroupCluster groupCluster; + + public LocationAwareSchedulerGroups( String name, CWSKubernetesClient client, String namespace, SchedulerConfig config, InputAlignment inputAlignment, ReadyToRunToNode readyToRunToNode ) { + super( name, client, namespace, config, inputAlignment, readyToRunToNode, null ); + groupCluster = new SimpleGroupCluster( hierarchyWrapper, client ); + readyToRunToNode.init( new FileSizeRankScoreGroup( groupCluster ) ); + setPhaseTwoComparator( new MinCopyingComparator( MinSizeComparator.INSTANCE ) ); + setPhaseThreeComparator( new RankAndMinCopyingComparatorCopyTasks( new MostOutLabelsComparator( groupCluster ) ) ); + } + + @Override + protected void tasksWhereAddedToQueue( List newTasks ){ + super.tasksWhereAddedToQueue( newTasks ); + groupCluster.tasksBecameAvailable( newTasks ); + } + + @Override + void taskWasScheduled(Task task ) { + super.taskWasScheduled( task ); + groupCluster.taskWasAssignedToNode( task ); + } + + @Override + int terminateTasks( List finishedTasks ) { + final int terminatedTasks = super.terminateTasks( finishedTasks ); + groupCluster.tasksHaveFinished( finishedTasks ); + return terminatedTasks; + } + + @Override + List getAdditionalTaskStatPhaseThree(){ + return groupCluster.getTaskStatToCopy(1); + } + + void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + if ( nodeTaskFilesAlignment.task instanceof CopyTask ) { + final String workingDir = nodeTaskFilesAlignment.task.getWorkingDir(); + new File( workingDir ).mkdirs(); + } + super.startCopyTask( nodeTaskFilesAlignment ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java new file mode 100644 index 00000000..94b3a09e --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/LocationAwareSchedulerV2.java @@ -0,0 +1,457 @@ +package cws.k8s.scheduler.scheduler; + +import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.NoAlignmentFoundException; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.*; +import cws.k8s.scheduler.scheduler.la2.capacityavailable.CapacityAvailableToNode; +import cws.k8s.scheduler.scheduler.la2.capacityavailable.SimpleCapacityAvailableToNode; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvance; +import cws.k8s.scheduler.scheduler.la2.copyinadvance.CopyInAdvanceNodeWithMostData; +import cws.k8s.scheduler.scheduler.la2.copystrategy.CopyRunner; +import cws.k8s.scheduler.scheduler.la2.copystrategy.ShellCopy; +import cws.k8s.scheduler.scheduler.la2.ready2run.ReadyToRunToNode; +import cws.k8s.scheduler.scheduler.schedulingstrategy.InputEntry; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.*; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import cws.k8s.scheduler.util.score.FileSizeRankScore; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Slf4j +public class LocationAwareSchedulerV2 extends SchedulerWithDaemonSet { + + @Getter(AccessLevel.PACKAGE) + private final InputAlignment inputAlignment; + @Getter(AccessLevel.PACKAGE) + private final int maxCopyTasksPerNode; + @Getter(AccessLevel.PACKAGE) + private final int maxWaitingCopyTasksPerNode; + + private final LogCopyTask logCopyTask = new LogCopyTask(); + + private final ReadyToRunToNode readyToRunToNode; + + private final CopyRunner copyRunner; + + private final CapacityAvailableToNode capacityAvailableToNode; + + private final CopyInAdvance copyInAdvance; + + @Setter(AccessLevel.PACKAGE) + private TaskStatComparator phaseTwoComparator; + @Setter(AccessLevel.PACKAGE) + private TaskStatComparator phaseThreeComparator; + + private final int copySameTaskInParallel; + private final int maxHeldCopyTaskReady; + + /** + * This value must be between 1 and 100. + * 100 means that the data will be copied with full speed. + */ + private final int prioPhaseThree; + + /** + * This lock is used to synchronize the creation of the copy tasks and the finishing. + * Otherwise, it could happen that: + * 1. Copy tasks checks for data already on the node + * 2. Copy tasks finished, data is removed from currently copying and added to onNode + * 3. Task checks if data is currently copied -> no -> create a new task + */ + private final Object copyLock = new Object(); + + public LocationAwareSchedulerV2( + String name, + CWSKubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment, + ReadyToRunToNode readyToRunToNode + ) { + this( + name, + client, + namespace, + config, + inputAlignment, + readyToRunToNode, + new FileSizeRankScore() + ); + setPhaseTwoComparator( new MinCopyingComparator( MinSizeComparator.INSTANCE ) ); + setPhaseThreeComparator( new RankAndMinCopyingComparator( MaxSizeComparator.INSTANCE ) ); + } + + LocationAwareSchedulerV2( + String name, + CWSKubernetesClient client, + String namespace, + SchedulerConfig config, + InputAlignment inputAlignment, + ReadyToRunToNode readyToRunToNode, + FileSizeRankScore calculateScore + ) { + super( name, client, namespace, config ); + this.inputAlignment = inputAlignment; + this.maxCopyTasksPerNode = config.maxCopyTasksPerNode == null ? 1 : config.maxCopyTasksPerNode; + this.maxWaitingCopyTasksPerNode = config.maxWaitingCopyTasksPerNode == null ? 1 : config.maxWaitingCopyTasksPerNode; + this.readyToRunToNode = readyToRunToNode; + if ( calculateScore != null ) { + this.readyToRunToNode.init( calculateScore ); + } + readyToRunToNode.setLogger( logCopyTask ); + this.copyRunner = new ShellCopy( client, this, logCopyTask ); + this.copySameTaskInParallel = 2; + this.capacityAvailableToNode = new SimpleCapacityAvailableToNode( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + this.copyInAdvance = new CopyInAdvanceNodeWithMostData( getCurrentlyCopying(), inputAlignment, this.copySameTaskInParallel ); + this.maxHeldCopyTaskReady = config.maxHeldCopyTaskReady == null ? 3 : config.maxHeldCopyTaskReady; + this.prioPhaseThree = config.prioPhaseThree == null ? 70 : config.prioPhaseThree; + } + + @Override + public ScheduleObject getTaskNodeAlignment( + final List unscheduledTasks, + final Map availableByNode + ){ + final List tasksAndData = unscheduledTasks + .parallelStream() + .map( task -> { + final TaskInputs inputsOfTask = extractInputsOfData( task ); + if ( inputsOfTask == null ) return null; + //all nodes that contain all files + final List nodesWithAllData = availableByNode + .keySet() + .stream() + .filter( node -> { + final CurrentlyCopyingOnNode copyingFilesToNode = getCurrentlyCopying().get( node.getNodeLocation() ); + return inputsOfTask.canRunOnLoc( node.getNodeLocation() ) + //Affinities are correct and the node can run new pods + && canSchedulePodOnNode( task, node ) + //All files are on the node and no copy task is overwriting them + && inputsOfTask.allFilesAreOnLocationAndNotOverwritten( node.getNodeLocation(), copyingFilesToNode.getAllFilesCurrentlyCopying() ); + } ) + .collect( Collectors.toList() ); + return new TaskInputsNodes( task, nodesWithAllData, inputsOfTask ); + } ) + .filter( Objects::nonNull ) + .collect( Collectors.toList() ); + + final List taskWithAllData = tasksAndData + .stream() + .filter( td -> !td.getNodesWithAllData().isEmpty() ) + .collect( Collectors.toList() ); + final List alignment = readyToRunToNode.createAlignmentForTasksWithAllDataOnNode( taskWithAllData, availableByNode ); + final ScheduleObject scheduleObject = new ScheduleObject( (List) alignment ); + scheduleObject.setCheckStillPossible( true ); + scheduleObject.setStopSubmitIfOneFails( true ); + return scheduleObject; + } + + @Override + void postScheduling( List unscheduledTasks, final Map availableByNode ) { + long start = System.currentTimeMillis(); + final Map currentlyCopyingTasksOnNode = getCurrentlyCopying().getCurrentlyCopyingTasksOnNode(); + final List allNodes = client.getAllNodes(); + allNodes.removeIf( x -> !x.isReady() ); + final List nodeTaskFilesAlignments; + Map< NodeWithAlloc, List > readyTasksPerNode = new ConcurrentHashMap<>(); + synchronized ( copyLock ) { + final TaskStats taskStats = new TaskStats(); + //Calculate the stats of available data for each task and node. + unscheduledTasks + .parallelStream() + .map( task -> { + final TaskInputs inputsOfTask = extractInputsOfData( task ); + if ( inputsOfTask == null ) return null; + return getDataOnNode( task, inputsOfTask, allNodes, readyTasksPerNode ); + } ) + .filter( Objects::nonNull ) + .filter( TaskStat::canStartSomewhere ) + .sequential() + .forEach( taskStats::add ); + + taskStats.setComparator( phaseTwoComparator ); + + + final CurrentlyCopying planedToCopy = new CurrentlyCopying(); + //Fill the currently available resources as fast as possible: start the tasks with the least data missing on a node. + nodeTaskFilesAlignments = capacityAvailableToNode + .createAlignmentForTasksWithEnoughCapacity( + taskStats, + planedToCopy, + availableByNode, + allNodes, + getMaxCopyTasksPerNode(), + currentlyCopyingTasksOnNode, + 100 + ); + for ( TaskStat taskStat : getAdditionalTaskStatPhaseThree() ) { + taskStats.add( taskStat ); + } + taskStats.setComparator( phaseThreeComparator ); + //Generate copy tasks for tasks that cannot yet run. + copyInAdvance.createAlignmentForTasksWithEnoughCapacity( + nodeTaskFilesAlignments, + taskStats, + planedToCopy, + allNodes, + getMaxCopyTasksPerNode(), + maxHeldCopyTaskReady, + currentlyCopyingTasksOnNode, + prioPhaseThree, + readyTasksPerNode + ); + } + log.info( "Post Scheduling: Created {} copy tasks in {} ms", nodeTaskFilesAlignments.size(), System.currentTimeMillis() - start ); + nodeTaskFilesAlignments.parallelStream().forEach( this::startCopyTask ); + } + + List getAdditionalTaskStatPhaseThree(){ + return new LinkedList<>(); + } + + + void startCopyTask( final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + nodeTaskFilesAlignment.task.getTraceRecord().copyTask(); + final CopyTask copyTask = initializeCopyTask( nodeTaskFilesAlignment ); + //Files that will be copied + reserveCopyTask( copyTask ); + try { + copyRunner.startCopyTasks( copyTask, nodeTaskFilesAlignment ); + } catch ( Exception e ) { + log.error( "Could not start copy task", e ); + undoReserveCopyTask( copyTask ); + } + + } + + + /** + * Creates config + * + * @param nodeTaskFilesAlignment + * @return + */ + CopyTask initializeCopyTask( NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + final CopyTask copyTask = createCopyTask(nodeTaskFilesAlignment); + copyTask.setNodeLocation( nodeTaskFilesAlignment.node.getNodeLocation() ); + final List allLocationWrappers = nodeTaskFilesAlignment.fileAlignment.getAllLocationWrappers(); + copyTask.setAllLocationWrapper( allLocationWrappers ); + log.info( "addToCopyingToNode task: {}, node: {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); + return copyTask; + } + + private void reserveCopyTask( CopyTask copyTask ) { + //Store files to copy + addToCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + useLocations( copyTask.getAllLocationWrapper() ); + } + + private void undoReserveCopyTask( CopyTask copyTask ) { + //Store files to copy + removeFromCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + freeLocations( copyTask.getAllLocationWrapper() ); + } + + public void copyTaskFinished( CopyTask copyTask, boolean success ) { + synchronized ( copyLock ) { + freeLocations( copyTask.getAllLocationWrapper() ); + if( success ){ + copyTask.getInputFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); + removeFromCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + } else { + removeFromCopyingToNode( copyTask.getTask(), copyTask.getNodeLocation(), copyTask.getFilesForCurrentNode() ); + handleProblematicCopy( copyTask ); + } + } + } + + private void handleProblematicCopy( CopyTask copyTask ){ + String file = this.localWorkDir + "/sync/" + copyTask.getInputs().execution; + try { + Map wrapperByPath = new HashMap<>(); + copyTask.getInputFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); + log.info( "Get daemon on node {}; daemons: {}", copyTask.getNodeLocation().getIdentifier(), daemonHolder ); + final InputStream inputStream = getConnection( getDaemonIpOnNode(copyTask.getNodeLocation().getIdentifier())).retrieveFileStream(file); + if (inputStream == null) { + //Init has not even started + return; + } + Scanner scanner = new Scanner(inputStream); + Set openedFiles = new HashSet<>(); + while( scanner.hasNext() ){ + String line = scanner.nextLine(); + if ( line.startsWith( "S-" ) ){ + openedFiles.add( line.substring( 2 ) ); + } else if ( line.startsWith( "F-" ) ){ + openedFiles.remove( line.substring( 2 ) ); + wrapperByPath.get( line.substring( 2 ) ).success(); + log.info("task {}, file: {} success", copyTask.getInputs().execution, line); + } + } + for ( String openedFile : openedFiles ) { + wrapperByPath.get( openedFile ).failure(); + log.info("task {}, file: {} deactivated on node {}", copyTask.getInputs().execution, openedFile, wrapperByPath.get( openedFile ).getWrapper().getLocation()); + } + } catch ( Exception e ){ + log.error( "Can't handle failed init from pod " + copyTask.getInputs().execution, e); + } + } + + + /** + * @param alignment + * @return null if the task cannot be scheduled + */ + CopyTask createCopyTask( NodeTaskFilesAlignment alignment ) { + LinkedList< TaskInputFileLocationWrapper > inputFiles = new LinkedList<>(); + final CurrentlyCopyingOnNode filesForCurrentNode = new CurrentlyCopyingOnNode(); + final NodeLocation currentNode = alignment.node.getNodeLocation(); + final Inputs inputs = new Inputs( + this.getDns(), + getExecution(), + this.localWorkDir + "/sync/", + alignment.task.getConfig().getRunName(), + alignment.prio + ); + + for (Map.Entry entry : alignment.fileAlignment.getNodeFileAlignment().entrySet()) { + if( entry.getKey() == currentNode ) { + continue; + } + + final NodeLocation location = (NodeLocation) entry.getKey(); + final AlignmentWrapper alignmentWrapper = entry.getValue(); + + final List collect = new LinkedList<>(); + for (FilePath filePath : alignmentWrapper.getFilesToCopy()) { + final LocationWrapper locationWrapper = filePath.getFile().getLocationWrapper(location); + inputFiles.add( + new TaskInputFileLocationWrapper( + filePath.getPath(), + filePath.getFile(), + locationWrapper.getCopyOf( currentNode ) + ) + ); + collect.add(filePath.getPath()); + filesForCurrentNode.add( filePath.getPath(), alignment.task, location ); + } + if( !collect.isEmpty() ) { + inputs.data.add(new InputEntry( getDaemonIpOnNode(entry.getKey().getIdentifier()), entry.getKey().getIdentifier(), collect, alignmentWrapper.getToCopySize())); + } + } + + inputs.sortData(); + return new CopyTask( inputs, inputFiles, filesForCurrentNode, alignment.task ); + } + + + private TaskInputs extractInputsOfData( Task task ) { + if ( task == null ) return null; + final TaskInputs inputsOfTask; + try { + inputsOfTask = getInputsOfTask( task ); + } catch ( NoAlignmentFoundException e ) { + return null; + } + if ( inputsOfTask == null ) { + log.info( "No node where the pod can start, pod: {}", task.getConfig().getRunName() ); + return null; + } + return inputsOfTask; + } + + + /** + * Calculate the remaining data on each node. + * + * @param task + * @param inputsOfTask + * @param allNodes + * @param readyTasksPerNode + * @return A wrapper containing the remaining data on each node, the nodes where all data is available, the inputs and the task. + */ + private TaskStat getDataOnNode( Task task, TaskInputs inputsOfTask, List allNodes, Map> readyTasksPerNode ) { + TaskStat taskStats = new TaskStat( task, inputsOfTask ); + final CurrentlyCopying currentlyCopying = getCurrentlyCopying(); + for ( NodeWithAlloc node : allNodes ) { + if ( inputsOfTask.canRunOnLoc( node.getNodeLocation() ) && node.affinitiesMatch( task.getPod() ) ) { + final CurrentlyCopyingOnNode currentlyCopyingOnNode = currentlyCopying.get( node.getNodeLocation() ); + final TaskNodeStats taskNodeStats = inputsOfTask.calculateMissingData( node.getNodeLocation(), currentlyCopyingOnNode ); + if ( taskNodeStats != null ) { + taskStats.add( node, taskNodeStats ); + if ( taskNodeStats.allOnNodeOrCopying() ) { + readyTasksPerNode.compute(node, (key, value) -> { + final List tasks = value == null ? new LinkedList<>() : value; + tasks.add(task); + return tasks; + }); + } + } + } + } + return taskStats; + } + + @Override + boolean assignTaskToNode( NodeTaskAlignment alignment ) { + final NodeTaskLocalFilesAlignment nodeTaskFilesAlignment = (NodeTaskLocalFilesAlignment) alignment; + + if ( !nodeTaskFilesAlignment.symlinks.isEmpty() ) { + try ( BufferedWriter writer = new BufferedWriter( new FileWriter( alignment.task.getWorkingDir() + '/' + ".command.symlinks" ) ) ) { + writer.write( "create_symlink() {" ); + writer.newLine(); + writer.write( " rm -rf \"$2\" || true" ); + writer.newLine(); + writer.write( " mkdir -p \"$(dirname \"$2\")\" || true" ); + writer.newLine(); + writer.write( " ln -s \"$1\" \"$2\" || true" ); + writer.newLine(); + writer.write( "}" ); + writer.newLine(); + for ( SymlinkInput symlink : nodeTaskFilesAlignment.symlinks ) { + writer.write( "create_symlink \"" + symlink.getDst().replace( "\"", "\\\"" ) + "\" \"" + symlink.getSrc().replace( "\"", "\\\"" ) + "\"" ); + writer.newLine(); + } + } catch ( IOException ex ) { + ex.printStackTrace(); + } + } + + alignment.task.setInputFiles( nodeTaskFilesAlignment.locationWrappers ); + useLocations( nodeTaskFilesAlignment.locationWrappers ); + + return super.assignTaskToNode( alignment ); + } + + @Override + public void close() { + logCopyTask.close(); + super.close(); + } + + @Override + Task createTask( TaskConfig conf ){ + return new Task( conf, getDag(), hierarchyWrapper ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/MatchingFilesAndNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/MatchingFilesAndNodes.java new file mode 100644 index 00000000..012872a9 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/MatchingFilesAndNodes.java @@ -0,0 +1,17 @@ +package cws.k8s.scheduler.scheduler; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Set; + +@Getter +@RequiredArgsConstructor +public class MatchingFilesAndNodes { + + private final Set nodes; + private final TaskInputs inputsOfTask; + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java index 2329a694..0682ec83 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/PrioritizeAssignScheduler.java @@ -1,8 +1,8 @@ package cws.k8s.scheduler.scheduler; +import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.model.*; -import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.scheduler.nodeassign.NodeAssign; import cws.k8s.scheduler.scheduler.prioritize.Prioritize; import cws.k8s.scheduler.util.NodeTaskAlignment; diff --git a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java index d693de3a..fdd92173 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/Scheduler.java @@ -1,5 +1,6 @@ package cws.k8s.scheduler.scheduler; +import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.client.CannotPatchException; import cws.k8s.scheduler.client.Informable; import cws.k8s.scheduler.dag.DAG; @@ -7,12 +8,11 @@ import cws.k8s.scheduler.prediction.MemoryScaler; import cws.k8s.scheduler.prediction.TaskScaler; import cws.k8s.scheduler.util.Batch; -import cws.k8s.scheduler.client.CWSKubernetesClient; import cws.k8s.scheduler.util.NodeTaskAlignment; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.informers.ResourceEventHandler; +import io.fabric8.kubernetes.client.informers.SharedIndexInformer; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.util.StringUtils; @@ -39,18 +39,16 @@ public abstract class Scheduler implements Informable { private boolean close; @Getter private final DAG dag; - private final Object batchHelper = new Object(); private int currentBatch = 0; private Batch currentBatchInstance = null; - final CWSKubernetesClient client; private final Set upcomingTasks = new HashSet<>(); private final List unscheduledTasks = new ArrayList<>( 100 ); private final List unfinishedTasks = new ArrayList<>( 100 ); final Map tasksByPodName = new HashMap<>(); final Map tasksById = new HashMap<>(); - private final Watch watcher; + private final SharedIndexInformer podHandler; private final TaskprocessingThread schedulingThread; private final TaskprocessingThread finishThread; @@ -58,7 +56,7 @@ public abstract class Scheduler implements Informable { // TaskScaler will observe tasks and modify their memory assignments final List taskScaler = new LinkedList<>(); - + Scheduler(String execution, CWSKubernetesClient client, String namespace, SchedulerConfig config){ this.execution = execution; this.name = System.getenv( "SCHEDULER_NAME" ) + "-" + execution; @@ -69,7 +67,7 @@ public abstract class Scheduler implements Informable { this.dag = new DAG(); this.traceEnabled = config.traceEnabled; - PodWatcher podWatcher = new PodWatcher(this); + PodHandler handler = new PodHandler(this ); schedulingThread = new TaskprocessingThread( unscheduledTasks, this::schedule ); schedulingThread.start(); @@ -77,10 +75,13 @@ public abstract class Scheduler implements Informable { finishThread = new TaskprocessingThread(unfinishedTasks, this::terminateTasks ); finishThread.start(); - log.info("Start watching"); - watcher = client.pods().inNamespace( this.namespace ).watch(podWatcher); + log.info("Start watching: {}", this.namespace ); + + this.podHandler = client.pods().inNamespace( this.namespace ).inform( handler ); + this.podHandler.start(); + log.info("Watching"); - + if ( StringUtils.hasText(config.memoryPredictor) ) { if ( client.inPlacePodVerticalScalingActive() ) { // create a new TaskScaler for each Scheduler instance @@ -97,17 +98,18 @@ public abstract class Scheduler implements Informable { * @return the number of unscheduled Tasks */ public int schedule( final List unscheduledTasks ) { + final LinkedList unscheduledTasksCopy = new LinkedList<>( unscheduledTasks ); long startSchedule = System.currentTimeMillis(); if( traceEnabled ) { unscheduledTasks.forEach( x -> x.getTraceRecord().tryToSchedule( startSchedule ) ); } - + if ( !taskScaler.isEmpty() ) { // change resource requests and limits here taskScaler.parallelStream().forEach( x -> x.beforeTasksScheduled( unscheduledTasks ) ); } - - final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode()); + + final ScheduleObject scheduleObject = getTaskNodeAlignment(unscheduledTasks, getAvailableByNode( true )); final List taskNodeAlignment = scheduleObject.getTaskAlignments(); //check if still possible... @@ -115,6 +117,7 @@ public int schedule( final List unscheduledTasks ) { boolean possible = validSchedulePlan ( taskNodeAlignment ); if (!possible) { log.info("The whole scheduling plan is not possible anymore."); + informOtherResourceChange(); return taskNodeAlignment.size(); } } @@ -144,11 +147,20 @@ public int schedule( final List unscheduledTasks ) { continue; } taskWasScheduled(nodeTaskAlignment.task); + unscheduledTasksCopy.remove( nodeTaskAlignment.task ); scheduled++; } + //Use instance object that does not contain yet scheduled tasks + postScheduling( unscheduledTasksCopy, getAvailableByNode( false ) ); return unscheduledTasks.size() - taskNodeAlignment.size() + failure; } + /** + * This method is called when a SchedulePlan was successfully executed. + * @param unscheduledTasks + */ + void postScheduling( final List unscheduledTasks, Map availableByNode ) {} + /** * Call this method in case of any scheduling problems */ @@ -156,16 +168,18 @@ void undoTaskScheduling( Task task ){} public boolean validSchedulePlan( List taskNodeAlignment ){ - Map< NodeWithAlloc, Requirements> availableByNode = getAvailableByNode(); + Map< NodeWithAlloc, Requirements> availableByNode = getAvailableByNode( false ); for ( NodeTaskAlignment nodeTaskAlignment : taskNodeAlignment ) { final Requirements requirements = availableByNode.get(nodeTaskAlignment.node); if ( requirements == null ) { + log.info( "Node {} is not available anymore", nodeTaskAlignment.node.getMetadata().getName() ); return false; } requirements.subFromThis(nodeTaskAlignment.task.getPlanedRequirements()); } for ( Map.Entry e : availableByNode.entrySet() ) { - if ( ! e.getValue().higherOrEquals( Requirements.ZERO ) ) { + if ( ! e.getValue().higherOrEquals( ImmutableRequirements.ZERO ) ) { + log.info( "Node {} has not enough resources. Available: {}, After assignment it would be: {}", e.getKey().getMetadata().getName(), e.getKey().getAvailableResources(), e.getValue() ); return false; } } @@ -177,6 +191,11 @@ abstract ScheduleObject getTaskNodeAlignment( final Map availableByNode ); + /** + * This method is called when tasks are finished + * @param finishedTasks the tasks that are finished + * @return the number of tasks that were not marked as finished successfully + */ int terminateTasks( final List finishedTasks ) { for (Task finishedTask : finishedTasks) { taskWasFinished( finishedTask ); @@ -185,7 +204,6 @@ int terminateTasks( final List finishedTasks ) { } /* Pod Event */ - void podEventReceived(Watcher.Action action, Pod pod){} void onPodTermination( PodWithAge pod ){ @@ -241,6 +259,7 @@ public void schedulePod(PodWithAge pod ) { if ( task.getBatch() == null ){ synchronized (unscheduledTasks){ unscheduledTasks.add( task ); + tasksWhereAddedToQueue( List.of(task) ); unscheduledTasks.notifyAll(); synchronized ( upcomingTasks ){ upcomingTasks.remove( task ); @@ -263,6 +282,7 @@ private void tryToScheduleBatch( Batch batch ){ synchronized (unscheduledTasks){ final List tasksToScheduleAndDestroy = batch.getTasksToScheduleAndDestroy(); unscheduledTasks.addAll(tasksToScheduleAndDestroy); + tasksWhereAddedToQueue( tasksToScheduleAndDestroy ); unscheduledTasks.notifyAll(); synchronized ( upcomingTasks ){ tasksToScheduleAndDestroy.forEach(upcomingTasks::remove); @@ -271,6 +291,18 @@ private void tryToScheduleBatch( Batch batch ){ } } + /** + * This methode is called whenever new task(s) are available to be scheduled. + * E.g. when a batch is full. + * The methode is called, before the scheduling is performed. + * @param newTasks the tasks that are now ready. + */ + protected void tasksWhereAddedToQueue( List newTasks ){} + + /** + * This method is called whenever a task was scheduled and the assignment is final. + * @param task the task that was scheduled + */ void taskWasScheduled(Task task ) { synchronized (unscheduledTasks){ unscheduledTasks.remove( task ); @@ -287,10 +319,13 @@ public void markPodAsDeleted( PodWithAge pod ) { task.setPod( pod ); } - /* External access to Tasks */ + Task createTask( TaskConfig conf ){ + return new Task( conf, dag ); + } + /* External access to Tasks */ public void addTask( int id, TaskConfig conf ) { - final Task task = new Task( conf, dag ); + final Task task = createTask( conf ); synchronized ( tasksByPodName ) { if ( !tasksByPodName.containsKey( conf.getRunName() ) ) { tasksByPodName.put( conf.getRunName(), task ); @@ -353,12 +388,20 @@ public boolean canScheduleTaskOnNode( Requirements availableByNode, Task task, N if ( availableByNode == null ) { return false; } - return node.canScheduleNewPod() - && availableByNode.higherOrEquals( task.getPlanedRequirements() ) - && affinitiesMatch( task.getPod(), node ); + return canSchedulePodOnNode( task, node ) && availableByNode.higherOrEquals( task.getPlanedRequirements() ); + } + + public boolean canSchedulePodOnNode(Task task, NodeWithAlloc node ) { + return node.canScheduleNewPod() && affinitiesMatch( task.getPod(), node ); } boolean affinitiesMatch( PodWithAge pod, NodeWithAlloc node ){ + + final boolean nodeCouldRunThisPod = node.getMaxResources().higherOrEquals( pod.getRequest() ); + if ( !nodeCouldRunThisPod ){ + return false; + } + final Map podsNodeSelector = pod.getSpec().getNodeSelector(); final Map nodesLabels = node.getMetadata().getLabels(); if ( podsNodeSelector == null || podsNodeSelector.isEmpty() ) { @@ -389,6 +432,10 @@ boolean canPodBeScheduled( Requirements requests, NodeWithAlloc node ){ return node.canSchedule( requests ); } + void assignPodToNode( PodWithAge pod, NodeTaskAlignment alignment ){ + client.assignPodToNode( pod, alignment.node.getMetadata().getName() ); + } + boolean assignTaskToNode( NodeTaskAlignment alignment ){ final File nodeFile = new File(alignment.task.getWorkingDir() + '/' + ".command.node"); @@ -397,7 +444,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ printWriter.write( alignment.node.getName() ); printWriter.write( '\n' ); } catch (IOException e) { - log.error( "Cannot read " + nodeFile, e); + log.error( "Cannot write " + nodeFile, e); } if ( alignment.task.requirementsChanged() ){ @@ -412,7 +459,7 @@ boolean assignTaskToNode( NodeTaskAlignment alignment ){ log.info ( "Assign pod: " + pod.getMetadata().getName() + " to node: " + alignment.node.getMetadata().getName() ); - client.assignPodToNode( pod, alignment.node.getMetadata().getName() ); + assignPodToNode( pod, alignment ); pod.getSpec().setNodeName( alignment.node.getMetadata().getName() ); log.info ( "Assigned pod to:" + pod.getSpec().getNodeName()); @@ -467,11 +514,19 @@ Task changeStateOfTask(Pod pod, State state) { * starts the scheduling routine */ public void informResourceChange() { + informOtherResourceChange(); synchronized (unscheduledTasks){ unscheduledTasks.notifyAll(); } } + /** + * Call if something was changed withing the scheduling loop. In all other cases, call informResourceChange() + */ + private void informOtherResourceChange() { + schedulingThread.otherResourceChange(); + } + Task getTaskByPod( Pod pod ) { Task t = null; synchronized ( tasksByPodName ) { @@ -487,20 +542,29 @@ Task getTaskByPod( Pod pod ) { return t; } - Map getAvailableByNode(){ + Map getAvailableByNode( boolean logging ){ Map availableByNode = new HashMap<>(); - List logInfo = new LinkedList<>(); - logInfo.add("------------------------------------"); + final List logInfo; + if ( logging ){ + logInfo = new LinkedList<>(); + logInfo.add("------------------------------------"); + } else { + logInfo = null; + } for (NodeWithAlloc item : getNodeList()) { if ( !item.isReady() ) { continue; } final Requirements availableResources = item.getAvailableResources(); availableByNode.put(item, availableResources); - logInfo.add("Node: " + item.getName() + " " + availableResources); + if ( logging ){ + logInfo.add("Node: " + item.getName() + " " + availableResources); + } + } + if ( logging ) { + logInfo.add("------------------------------------"); + log.info(String.join("\n", logInfo)); } - logInfo.add("------------------------------------"); - log.info(String.join("\n", logInfo)); return availableByNode; } @@ -525,22 +589,21 @@ LinkedList getUpcomingTasksCopy() { * Close used resources */ public void close(){ - watcher.close(); + podHandler.close(); schedulingThread.interrupt(); finishThread.interrupt(); this.close = true; } - static class PodWatcher implements Watcher { + static class PodHandler implements ResourceEventHandler { private final Scheduler scheduler; - public PodWatcher(Scheduler scheduler) { + public PodHandler( Scheduler scheduler ) { this.scheduler = scheduler; } - @Override - public void eventReceived(Action action, Pod pod) { + public void eventReceived( Watcher.Action action, Pod pod) { scheduler.podEventReceived(action, pod); @@ -570,7 +633,10 @@ public void eventReceived(Action action, Pod pod) { } break; case MODIFIED: - if (!pod.getStatus().getContainerStatuses().isEmpty() && pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null) { + if ( "DeadlineExceeded".equals( pod.getStatus().getReason() ) || //Task ran out of time + ( !pod.getStatus().getContainerStatuses().isEmpty() && + pod.getStatus().getContainerStatuses().get(0).getState().getTerminated() != null ) + ) { scheduler.onPodTermination(pwa); } else { final Task task = scheduler.getTaskByPod(pwa); @@ -585,12 +651,20 @@ public void eventReceived(Action action, Pod pod) { } + @Override + public void onAdd( Pod pod ) { + eventReceived( Watcher.Action.ADDED, pod ); + } @Override - public void onClose(WatcherException cause) { - log.info( "Watcher was closed" ); + public void onUpdate( Pod oldPod, Pod newPod ) { + eventReceived( Watcher.Action.MODIFIED, newPod ); } + @Override + public void onDelete( Pod pod, boolean deletedFinalStateUnknown ) { + eventReceived( Watcher.Action.DELETED, pod ); + } } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java new file mode 100644 index 00000000..e3a760d8 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/SchedulerWithDaemonSet.java @@ -0,0 +1,413 @@ +package cws.k8s.scheduler.scheduler; + +import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.model.*; +import cws.k8s.scheduler.model.cluster.OutputFiles; +import cws.k8s.scheduler.model.location.LocationType; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.*; +import cws.k8s.scheduler.model.outfiles.OutputFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.outfiles.SymlinkOutput; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.rest.exceptions.NotARealFileException; +import cws.k8s.scheduler.rest.response.getfile.FileResponse; +import cws.k8s.scheduler.scheduler.copystrategy.CopyStrategy; +import cws.k8s.scheduler.scheduler.copystrategy.FTPstrategy; +import cws.k8s.scheduler.util.DaemonHolder; +import cws.k8s.scheduler.util.NodeTaskAlignment; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import io.fabric8.kubernetes.api.model.ContainerStatus; +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.Watcher; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.net.ftp.FTPClient; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +public abstract class SchedulerWithDaemonSet extends Scheduler { + + @Getter(AccessLevel.PROTECTED) + final DaemonHolder daemonHolder = new DaemonHolder(); + @Getter + private String workflowEngineNode = null; + @Getter + private final CopyStrategy copyStrategy; + final HierarchyWrapper hierarchyWrapper; + private final InputFileCollector inputFileCollector; + private final ConcurrentHashMap requestedLocations = new ConcurrentHashMap<>(); + final String localWorkDir; + + /** + * Which node is currently copying files from which node + */ + @Getter(AccessLevel.PACKAGE) + private final CurrentlyCopying currentlyCopying = new CurrentlyCopying(); + + SchedulerWithDaemonSet( String execution, CWSKubernetesClient client, String namespace, SchedulerConfig config) { + super(execution, client, namespace, config); + this.hierarchyWrapper = new HierarchyWrapper( config.workDir ); + this.inputFileCollector = new InputFileCollector( hierarchyWrapper ); + if ( config.copyStrategy == null ) { + throw new IllegalArgumentException( "Copy strategy is null" ); + } + switch ( config.copyStrategy ){ + case "ftp": + case "copy": + copyStrategy = new FTPstrategy(); + break; + default: + throw new IllegalArgumentException( "Copy strategy is unknown " + config.copyStrategy ); + } + this.localWorkDir = config.workDir; + } + + public String getDaemonIpOnNode( String node ){ + return daemonHolder.getDaemonIp( node ); + } + + public String getDaemonNameOnNode( String node ){ + return daemonHolder.getDaemonName( node ); + } + + String getDaemonIpOnNode( Node node ){ + return getDaemonIpOnNode( node.getMetadata().getName() ); + } + + /** + * Mark all locationWrappers as used + */ + void useLocations( List locationWrappers ){ + locationWrappers.parallelStream().forEach( LocationWrapper::use ); + } + + /** + * Mark all locationWrappers as unused + */ + void freeLocations( List locationWrappers ){ + locationWrappers.parallelStream().forEach( LocationWrapper::free ); + } + + @Override + void assignPodToNode( PodWithAge pod, NodeTaskAlignment alignment ) { + if ( !pod.getSpec().getInitContainers().isEmpty() && ((NodeTaskFilesAlignment) alignment).isRemoveInit() ) { + log.info( "Removing init container from pod {}", pod.getMetadata().getName() ); + client.assignPodToNodeAndRemoveInit( pod, alignment.node.getName() ); + } else { + super.assignPodToNode( pod, alignment ); + } + } + + @Override + void undoTaskScheduling( Task task ){ + if ( task.getInputFiles() != null ) { + freeLocations( task.getInputFiles() ); + task.setInputFiles( null ); + } + if ( task.getCopyingToNode() != null ) { + removeFromCopyingToNode( task, task.getNode().getNodeLocation(), task.getCopyingToNode()); + task.setCopyingToNode( null ); + } + task.setCopiedFiles( null ); + task.setNode( null ); + } + + @Override + int terminateTasks(List finishedTasks) { + final TaskResultParser taskResultParser = new TaskResultParser(); + finishedTasks.parallelStream().forEach( finishedTask -> { + try{ + freeLocations( finishedTask.getInputFiles() ); + if ( !"DeadlineExceeded".equals( finishedTask.getPod().getStatus().getReason() ) ) { //If Deadline exceeded, task cannot write out files and containerStatuses.terminated is not available + final Integer exitCode = finishedTask.getPod().getStatus().getContainerStatuses().get(0).getState().getTerminated().getExitCode(); + log.info( "Pod finished with exitCode: {}", exitCode ); + //Init failure + final Path workdir = Paths.get(finishedTask.getWorkingDir()); + if ( exitCode == 123 && Files.exists( workdir.resolve(".command.init.failure") ) ) { + log.info( "Task " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ") had an init failure: won't parse the in- and out files" ); + } else { + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles( + workdir, + finishedTask.getNode().getNodeLocation(), + !finishedTask.wasSuccessfullyExecuted(), + finishedTask + ); + final Set outputFiles = new HashSet<>(); + for (OutputFile newAndUpdatedFile : newAndUpdatedFiles) { + if( newAndUpdatedFile instanceof PathLocationWrapperPair ) { + hierarchyWrapper.addFile( + newAndUpdatedFile.getPath(), + ((PathLocationWrapperPair) newAndUpdatedFile).getLocationWrapper() + ); + outputFiles.add( (PathLocationWrapperPair) newAndUpdatedFile ); + } else if ( newAndUpdatedFile instanceof SymlinkOutput ){ + hierarchyWrapper.addSymlink( + newAndUpdatedFile.getPath(), + ((SymlinkOutput) newAndUpdatedFile).getDst() + ); + } + } + finishedTask.setOutputFiles( new OutputFiles( outputFiles ) ); + } + } + } catch ( Exception e ){ + log.info( "Problem while finishing task: " + finishedTask.getConfig().getRunName() + " (" + finishedTask.getConfig().getName() + ")", e ); + } + super.taskWasFinished( finishedTask ); + }); + return 0; + } + + /** + * Register that file is copied to node + */ + void addToCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode toAdd ){ + if ( nodeLocation == null ) { + throw new IllegalArgumentException( "NodeLocation cannot be null" ); + } + currentlyCopying.add( task, nodeLocation, toAdd ); + } + + /** + * Remove that file is copied to node + */ + void removeFromCopyingToNode( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode toRemove ) { + if (nodeLocation == null) { + throw new IllegalArgumentException("NodeLocation cannot be null"); + } + currentlyCopying.remove( task, nodeLocation, toRemove ); + } + + TaskInputs getInputsOfTask(Task task ) throws NoAlignmentFoundException { + return inputFileCollector.getInputsOfTask( task, client.getNumberOfNodes() ); + } + + + public FileResponse nodeOfLastFileVersion( String path ) throws NotARealFileException { + LinkedList symlinks = new LinkedList<>(); + Path currentPath = Paths.get(path); + HierarchyFile currentFile = hierarchyWrapper.getFile( currentPath ); + while ( currentFile instanceof LinkHierarchyFile){ + final LinkHierarchyFile linkFile = (LinkHierarchyFile) currentFile; + symlinks.add( new SymlinkInput( currentPath, linkFile.getDst() ) ); + currentPath = linkFile.getDst(); + currentFile = hierarchyWrapper.getFile( currentPath ); + } + Collections.reverse( symlinks ); + //File is maybe out of scope + if ( currentFile == null ) { + return new FileResponse( currentPath.toString(), symlinks ); + } + if ( ! (currentFile instanceof RealHierarchyFile) ){ + log.info( "File was: {}", currentFile ); + throw new NotARealFileException(); + } + final RealHierarchyFile file = (RealHierarchyFile) currentFile; + final LocationWrapper lastUpdate = file.getLastUpdate(LocationType.NODE); + if( lastUpdate == null ) { + return null; + } + requestedLocations.put( lastUpdate.getId(), lastUpdate ); + String node = lastUpdate.getLocation().getIdentifier(); + return new FileResponse( currentPath.toString(), node, getDaemonIpOnNode(node), node.equals(workflowEngineNode), symlinks, lastUpdate.getId() ); + } + + MatchingFilesAndNodes getMatchingFilesAndNodes( final Task task, final Map availableByNode ){ + final Set matchingNodesForTask = getMatchingNodesForTask(availableByNode, task); + if( matchingNodesForTask.isEmpty() ) { + log.trace( "No node with enough resources for {}", task.getConfig().getRunName() ); + return null; + } + + final TaskInputs inputsOfTask; + try { + inputsOfTask = getInputsOfTask(task); + } catch (NoAlignmentFoundException e) { + return null; + } + if( inputsOfTask == null ) { + log.info( "No node where the pod can start, pod: {}", task.getConfig().getRunName() ); + return null; + } + + filterNotMatchingNodesForTask( matchingNodesForTask, inputsOfTask ); + if( matchingNodesForTask.isEmpty() ) { + log.info( "No node which fulfills all requirements {}", task.getConfig().getRunName() ); + return null; + } + + return new MatchingFilesAndNodes( matchingNodesForTask, inputsOfTask ); + } + + /** + * Register a new local file + */ + public void addFile( String path, long size, long timestamp, long locationWrapperID, boolean overwrite, String node ){ + final NodeLocation location = NodeLocation.getLocation( node == null ? workflowEngineNode : node ); + + LocationWrapper locationWrapper; + if( !overwrite && locationWrapperID != -1 ){ + locationWrapper = requestedLocations.get( locationWrapperID ).getCopyOf( location ); + } else { + locationWrapper = new LocationWrapper( location, timestamp, size ); + } + + hierarchyWrapper.addFile( Paths.get( path ), overwrite, locationWrapper ); + } + + private void handleProblematicInit( Task task ){ + String file = this.localWorkDir + "/sync/" + task.getConfig().getRunName(); + try { + Map wrapperByPath = new HashMap<>(); + task.getCopiedFiles().forEach( x -> wrapperByPath.put( x.getPath(), x )); + log.info( "Get daemon on node {}; daemons: {}", task.getNode().getNodeLocation().getIdentifier(), daemonHolder ); + final InputStream inputStream = getConnection( getDaemonIpOnNode(task.getNode().getNodeLocation().getIdentifier())).retrieveFileStream(file); + if (inputStream == null) { + //Init has not even started + return; + } + Scanner scanner = new Scanner(inputStream); + Set openedFiles = new HashSet<>(); + while( scanner.hasNext() ){ + String line = scanner.nextLine(); + if ( line.startsWith( "S-" ) ){ + openedFiles.add( line.substring( 2 ) ); + } else if ( line.startsWith( "F-" ) ){ + openedFiles.remove( line.substring( 2 ) ); + wrapperByPath.get( line.substring( 2 ) ).success(); + log.info("task {}, file: {} success", task.getConfig().getName(), line); + } + } + for ( String openedFile : openedFiles ) { + wrapperByPath.get( openedFile ).failure(); + log.info("task {}, file: {} deactivated on node {}", task.getConfig().getName(), openedFile, wrapperByPath.get( openedFile ).getWrapper().getLocation()); + } + } catch ( Exception e ){ + log.error( "Can't handle failed init from pod " + task.getPod().getName(), e); + } + } + + FTPClient getConnection( String daemon ){ + int trial = 0; + while ( true ) { + try { + FTPClient f = new FTPClient(); + f.connect(daemon); + f.login("ftp", "nextflowClient"); + f.enterLocalPassiveMode(); + return f; + } catch ( IOException e ) { + if ( trial > 5 ) { + throw new RuntimeException(e); + } + log.error("Cannot create FTP client: {}", daemon); + try { + Thread.sleep((long) Math.pow(2, trial++)); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + log.error( "Interrupted while waiting for retry to connect to FTP client", e); + } + } + } + } + + private void podWasInitialized( Pod pod ){ + final Integer exitCode = pod.getStatus().getInitContainerStatuses().get(0).getState().getTerminated().getExitCode(); + final Task task = changeStateOfTask(pod, exitCode == 0 ? State.PREPARED : State.INIT_WITH_ERRORS); + task.setPod( new PodWithAge( pod ) ); + log.info( "Pod {}, Init Code: {}", pod.getMetadata().getName(), exitCode); + removeFromCopyingToNode( task, task.getNode().getNodeLocation(), task.getCopyingToNode() ); + if( exitCode == 0 ){ + task.getCopiedFiles().parallelStream().forEach( TaskInputFileLocationWrapper::success ); + } else { + handleProblematicInit( task ); + task.setInputFiles( null ); + } + task.setCopiedFiles( null ); + task.setCopyingToNode( null ); + } + + /** + * Remove all Nodes with a location contained in taskInputs.excludedNodes + */ + void filterNotMatchingNodesForTask(Set matchingNodes, TaskInputs taskInputs ){ + matchingNodes.removeIf( next -> !taskInputs.canRunOnLoc( next.getNodeLocation() ) ); + } + + public void taskHasFinishedCopyTask( String name ){ + final Task task = tasksByPodName.get( name ); + task.getNode().startingTaskCopyingDataFinished( task ); + informResourceChange(); + } + + /** + * Since task was not yet initialized: set scheduled + * @param task task that was scheduled + */ + @Override + void taskWasScheduledSetState( Task task ){ + task.getState().setState( State.SCHEDULED ); + } + + public void setWorkflowEngineNode( String ip ){ + this.workflowEngineNode = client.getPodByIp( ip ).getSpec().getNodeName(); + log.info( "WorkflowEngineNode was set to {}", workflowEngineNode ); + } + + @Override + void podEventReceived(Watcher.Action action, Pod pod){ + //noinspection LoopConditionNotUpdatedInsideLoop + while ( daemonHolder == null ){ + //The Watcher can be started before the class is initialized + try { + Thread.sleep(20); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + if( ( "mount-" + this.getExecution().replace('_', '-') + "-" ).equals(pod.getMetadata().getGenerateName()) ){ + final String nodeName = pod.getSpec().getNodeName(); + if ( nodeName != null ){ + synchronized ( daemonHolder ) { + final String podName = pod.getMetadata().getName(); + final boolean podIsCurrentDaemon = pod.getStatus().getPodIP() != null && pod.getStatus().getPodIP().equals(daemonHolder.getDaemonIp(nodeName)); + if ( action == Watcher.Action.DELETED ) { + if (podIsCurrentDaemon) { + daemonHolder.removeDaemon(nodeName); + } + } else if ( pod.getStatus().getPhase().equals("Running") ) { + daemonHolder.addDaemon( nodeName, podName, pod.getStatus().getPodIP() ); + informResourceChange(); + } else if ( podIsCurrentDaemon ) { + daemonHolder.removeDaemon(nodeName); + if( !pod.getStatus().getPhase().equals("Failed") ){ + log.info( "Unexpected phase {} for daemon: {}", pod.getStatus().getPhase(), podName ); + } + } + } + } + } else if ( this.getName().equals(pod.getSpec().getSchedulerName()) + && action == Watcher.Action.MODIFIED + && getTaskByPod( pod ).getState().getState() == State.SCHEDULED ) + { + final List initContainerStatuses = pod.getStatus().getInitContainerStatuses(); + if ( ! initContainerStatuses.isEmpty() && initContainerStatuses.get(0).getState().getTerminated() != null ) { + podWasInitialized( pod ); + } + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java b/src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java index fbac232c..270bcb9d 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/TaskprocessingThread.java @@ -15,6 +15,12 @@ public class TaskprocessingThread extends Thread { private final List unprocessedTasks; private final Function, Integer> function; + private boolean otherResourceChange = false; + + public void otherResourceChange() { + otherResourceChange = true; + } + @Override public void run() { int unscheduled = 0; @@ -23,12 +29,13 @@ public void run() { LinkedList tasks; synchronized (unprocessedTasks) { do { - if (unscheduled == unprocessedTasks.size()) { + if ( !otherResourceChange && unscheduled == unprocessedTasks.size()) { unprocessedTasks.wait( 10000 ); } if( Thread.interrupted() ) { return; } + otherResourceChange = false; } while ( unprocessedTasks.isEmpty() ); tasks = new LinkedList<>(unprocessedTasks); } diff --git a/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java new file mode 100644 index 00000000..65479eb7 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/CopyStrategy.java @@ -0,0 +1,38 @@ +package cws.k8s.scheduler.scheduler.copystrategy; + +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +@Slf4j +public abstract class CopyStrategy { + + private void write( BufferedWriter pw, File file, String resource ) { + ClassLoader classLoader = getClass().getClassLoader(); + try (InputStream inputStream = classLoader.getResourceAsStream( resource )) { + assert inputStream != null; + try (InputStreamReader streamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(streamReader)) { + + String line; + while ((line = reader.readLine()) != null) { + pw.write( line ); + pw.write( '\n' ); + } + + Set executable = PosixFilePermissions.fromString("rwxrwxrwx"); + Files.setPosixFilePermissions( file.toPath(), executable ); + } + } catch (IOException e) { + log.error( "Cannot write " + file, e); + } + } + + abstract String getResource(); + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/FTPstrategy.java b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/FTPstrategy.java new file mode 100644 index 00000000..4369efb8 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/copystrategy/FTPstrategy.java @@ -0,0 +1,10 @@ +package cws.k8s.scheduler.scheduler.copystrategy; + +public class FTPstrategy extends CopyStrategy { + + @Override + String getResource() { + return "copystrategies/ftp.py"; + } + +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java new file mode 100644 index 00000000..1b0115af --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/data/TaskInputsNodes.java @@ -0,0 +1,25 @@ +package cws.k8s.scheduler.scheduler.data; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +/** + * This class is used to store the data of a task, the task, and the nodes that contain all data for this task. + */ +@Getter +@RequiredArgsConstructor +public class TaskInputsNodes { + private final Task task; + private final List nodesWithAllData; + private final TaskInputs inputsOfTask; + + public long getTaskSize() { + final long filesInSharedFS = task.getInputSize(); + return filesInSharedFS + inputsOfTask.calculateAvgSize(); + } +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java new file mode 100644 index 00000000..32b974c3 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/GreedyAlignment.java @@ -0,0 +1,79 @@ +package cws.k8s.scheduler.scheduler.filealignment; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.CostFunction; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.FilePath; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; + +@Slf4j +public class GreedyAlignment extends InputAlignmentClass { + + private final CostFunction cf; + + public GreedyAlignment( double weightForSingleSource, CostFunction cf ) { + super( weightForSingleSource ); + this.cf = cf; + } + + Costs findAlignmentForFile ( + PathFileLocationTriple pathFileLocationTriple, + NodeLocation scheduledNode, + HashMap map, + final Costs costs + ){ + double minCost = Double.MAX_VALUE; + double currentMaxCostForIndividualNode = costs.getMaxCostForIndividualNode(); + double currentSumOfCosts = costs.getSumOfCosts(); + Location bestLoc = null; + LocationWrapper bestLocationWrapper = null; + for (LocationWrapper locationWrapper : pathFileLocationTriple.locations) { + final Location currentLoc = locationWrapper.getLocation(); + final double maxCostOnIndividualNode; + final double tmpCurrentSumOfCosts; + if ( currentLoc != scheduledNode) { + final AlignmentWrapper alignmentWrapper = map.get(currentLoc); + tmpCurrentSumOfCosts = costs.getSumOfCosts() + cf.getCost( locationWrapper ); + final double costOnIndividualNode = cf.calculateCost(alignmentWrapper, locationWrapper); + maxCostOnIndividualNode = Math.max( costOnIndividualNode, costs.getMaxCostForIndividualNode() ); + } else { + tmpCurrentSumOfCosts = costs.getSumOfCosts(); + maxCostOnIndividualNode = costs.getMaxCostForIndividualNode(); + } + final double calculatedCost = calculateCost( maxCostOnIndividualNode, tmpCurrentSumOfCosts ); + if ( calculatedCost < minCost ) { + bestLoc = currentLoc; + minCost = calculatedCost; + bestLocationWrapper = locationWrapper; + currentMaxCostForIndividualNode = maxCostOnIndividualNode; + currentSumOfCosts = tmpCurrentSumOfCosts; + } + } + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(bestLoc, k -> new AlignmentWrapper() ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, bestLocationWrapper ), minCost, bestLocationWrapper.getSizeInBytes() ); + return new Costs( currentMaxCostForIndividualNode, currentSumOfCosts, minCost ); + } + + @Override + public FileAlignment getInputAlignment(@NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy, + double maxCost) { + inputsOfTask.sort(); + return super.getInputAlignment( task, inputsOfTask, node, currentlyCopying, currentlyPlanedToCopy, maxCost ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignment.java new file mode 100644 index 00000000..5fbbeb36 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignment.java @@ -0,0 +1,43 @@ +package cws.k8s.scheduler.scheduler.filealignment; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import org.jetbrains.annotations.NotNull; + +public interface InputAlignment { + + /** + * Calculate a alignment for the input data. Stop if costs are higher than maxCost + * @param task + * @param inputsOfTask + * @param node + * @param maxCost + * @return null if no or no better than maxCost alignment is found + */ + FileAlignment getInputAlignment( @NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy, + double maxCost ); + + /** + * Calculate a alignment for the input data + * @param task + * @param inputsOfTask + * @param node + * @return null if no alignment is found + */ + default FileAlignment getInputAlignment( @NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy + ){ + return getInputAlignment( task, inputsOfTask, node, currentlyCopying, currentlyPlanedToCopy, Double.MAX_VALUE ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java new file mode 100644 index 00000000..ef9456ad --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/InputAlignmentClass.java @@ -0,0 +1,127 @@ +package cws.k8s.scheduler.scheduler.filealignment; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.FilePathWithTask; +import cws.k8s.scheduler.util.Tuple; +import cws.k8s.scheduler.util.copying.CopySource; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +abstract class InputAlignmentClass implements InputAlignment { + + /** + * This weight weights the cost of copying files from one node. 1 - this value weights the overall data copied. + * How much should the individual costs of one node be weighted compared to the overall data copied. + */ + private final double weightForIndividualNode; + + public InputAlignmentClass( double weightForIndividualNode ) { + if ( weightForIndividualNode < 0 || weightForIndividualNode > 1 ) { + throw new IllegalArgumentException( "Weight for single source must be between 0 and 1" ); + } + this.weightForIndividualNode = weightForIndividualNode; + } + + /** + * Check if another scheduled task is already copying a required file + * + */ + private Tuple alreadyCopying( CurrentlyCopyingOnNode currentlyCopying, String path, List locations, String debug ){ + if ( currentlyCopying != null && currentlyCopying.isCurrentlyCopying( path ) ){ + final CopySource copySource = currentlyCopying.getCopySource( path ); + final Location copyFrom = copySource.getLocation(); + for ( LocationWrapper locationWrapper : locations ) { + if ( locationWrapper.getLocation() == copyFrom ) { + return new Tuple<>(locationWrapper,copySource.getTask()); + } + } + throw new NoAligmentPossibleException( "Node is a already copying file: " + path + " but in an incompatible version." ); + } + return null; + } + + private boolean canUseFileFromOtherTask ( + CurrentlyCopyingOnNode currentlyCopying, + PathFileLocationTriple pathFileLocationTriple, + Map map, + String debug + ) { + final Tuple copying = alreadyCopying( + currentlyCopying, + pathFileLocationTriple.path.toString(), + pathFileLocationTriple.locations, + debug + ); + if ( copying != null ) { + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(copying.getA().getLocation(), k -> new AlignmentWrapper()); + alignmentWrapper.addAlignmentToWaitFor(new FilePathWithTask(pathFileLocationTriple, copying.getA(), copying.getB()), copying.getA().getSizeInBytes()); + return true; + } + return false; + } + + @Override + public FileAlignment getInputAlignment(@NotNull Task task, + @NotNull TaskInputs inputsOfTask, + @NotNull NodeWithAlloc node, + CurrentlyCopyingOnNode currentlyCopying, + CurrentlyCopyingOnNode currentlyPlanedToCopy, + double maxCost) { + final HashMap map = new HashMap<>(); + Costs costs = new Costs(); + for (PathFileLocationTriple pathFileLocationTriple : inputsOfTask.getFiles()) { + if ( !canUseFileFromOtherTask( currentlyCopying, pathFileLocationTriple, map, "copying" ) + && + !canUseFileFromOtherTask( currentlyPlanedToCopy, pathFileLocationTriple, map, "currentSchedule" ) + ) { + costs = findAlignmentForFile( pathFileLocationTriple, node.getNodeLocation(), map, costs ); + if ( costs.calculatedCost > maxCost ) { + return null; + } + } + } + return new FileAlignment( map, inputsOfTask.getSymlinks(), costs.calculatedCost); + } + + protected double calculateCost( double maxIndividual, double sumOfCost ) { + return maxIndividual * weightForIndividualNode + sumOfCost * ( 1 - weightForIndividualNode ); + } + + abstract Costs findAlignmentForFile( + PathFileLocationTriple pathFileLocationTriple, + NodeLocation scheduledNode, + HashMap map, + final Costs costs + ); + + @Getter + @RequiredArgsConstructor + class Costs { + + private final double maxCostForIndividualNode; + private final double sumOfCosts; + private final double calculatedCost; + + public Costs() { + this( 0, 0, 0 ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/RandomAlignment.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/RandomAlignment.java new file mode 100644 index 00000000..e8d919d0 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/RandomAlignment.java @@ -0,0 +1,44 @@ +package cws.k8s.scheduler.scheduler.filealignment; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FilePath; + +import java.util.HashMap; +import java.util.Optional; +import java.util.Random; + +public class RandomAlignment extends InputAlignmentClass { + + private final Random random = new Random(); + + public RandomAlignment() { + super( 0 ); + } + + @Override + Costs findAlignmentForFile ( + PathFileLocationTriple pathFileLocationTriple, + NodeLocation scheduledNode, + HashMap map, + final Costs costs + ){ + final Optional first = pathFileLocationTriple + .locations + .stream() + .filter(x -> x.getLocation() == scheduledNode) + .findFirst(); + final LocationWrapper locationWrapper = first.orElseGet(() -> pathFileLocationTriple.locations.get( + random.nextInt(pathFileLocationTriple.locations.size()) + )); + final Location location = locationWrapper.getLocation(); + final AlignmentWrapper alignmentWrapper = map.computeIfAbsent(location, k -> new AlignmentWrapper() ); + alignmentWrapper.addAlignmentToCopy( new FilePath( pathFileLocationTriple, locationWrapper ), 0, locationWrapper.getSizeInBytes() ); + final double calculatedCost = costs.getSumOfCosts() + locationWrapper.getSizeInBytes(); + return new Costs( 0, calculatedCost, calculatedCost ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/CostFunction.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/CostFunction.java new file mode 100644 index 00000000..d7b8a40b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/CostFunction.java @@ -0,0 +1,17 @@ +package cws.k8s.scheduler.scheduler.filealignment.costfunctions; + +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.util.AlignmentWrapper; + +public interface CostFunction { + + double getInitCost(); + double getCost( LocationWrapper fileToTest ); + + default double calculateCost(AlignmentWrapper alignment, LocationWrapper fileToTest ) { + double currentCost = alignment == null || alignment.empty() ? getInitCost() : alignment.getCost(); + return currentCost + getCost( fileToTest ); + } + + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java new file mode 100644 index 00000000..d20a709c --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/MinSizeCost.java @@ -0,0 +1,21 @@ +package cws.k8s.scheduler.scheduler.filealignment.costfunctions; + +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class MinSizeCost implements CostFunction { + + private final double initCost; + + @Override + public double getInitCost() { + return initCost; + } + + @Override + public double getCost(LocationWrapper fileToTest) { + return fileToTest.getSizeInBytes(); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java new file mode 100644 index 00000000..812ff26c --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/filealignment/costfunctions/NoAligmentPossibleException.java @@ -0,0 +1,8 @@ +package cws.k8s.scheduler.scheduler.filealignment.costfunctions; + +public class NoAligmentPossibleException extends RuntimeException { + + public NoAligmentPossibleException(String message) { + super(message); + } +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/CreateCopyTasks.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/CreateCopyTasks.java new file mode 100644 index 00000000..458c4b8b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/CreateCopyTasks.java @@ -0,0 +1,77 @@ +package cws.k8s.scheduler.scheduler.la2; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.filealignment.costfunctions.NoAligmentPossibleException; +import cws.k8s.scheduler.util.FileAlignment; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.RequiredArgsConstructor; + +import java.util.List; +import java.util.Map; + +@RequiredArgsConstructor +public abstract class CreateCopyTasks { + + protected final CurrentlyCopying currentlyCopying; + protected final InputAlignment inputAlignment; + protected final int copySameTaskInParallel; + + protected FileAlignment getFileAlignmentForTaskAndNode( + final NodeWithAlloc node, + final Task task, + final TaskInputs inputsOfTask, + final CurrentlyCopying planedToCopy + ) { + final CurrentlyCopyingOnNode currentlyCopyingOnNode = this.currentlyCopying.get(node.getNodeLocation()); + final CurrentlyCopyingOnNode currentlyPlanedToCopy = planedToCopy.get(node.getNodeLocation()); + try { + return inputAlignment.getInputAlignment( + task, + inputsOfTask, + node, + currentlyCopyingOnNode, + currentlyPlanedToCopy, + Double.MAX_VALUE + ); + } catch ( NoAligmentPossibleException e ){ + return null; + } + } + + /** + * + * @return the success of this operation + */ + protected boolean createFileAlignment( + CurrentlyCopying planedToCopy, + List nodeTaskAlignments, + Map currentlyCopyingTasksOnNode, + TaskStat poll, + Task task, + NodeWithAlloc node, + int prio + ) { + final FileAlignment fileAlignmentForTaskAndNode = getFileAlignmentForTaskAndNode( node, task, poll.getInputsOfTask(), planedToCopy ); + if ( fileAlignmentForTaskAndNode != null && fileAlignmentForTaskAndNode.copyFromSomewhere( node.getNodeLocation() ) ) { + planedToCopy.addAlignment( fileAlignmentForTaskAndNode.getNodeFileAlignment(), task, node ); + nodeTaskAlignments.add( new NodeTaskFilesAlignment( node, task, fileAlignmentForTaskAndNode, prio ) ); + currentlyCopyingTasksOnNode.compute( node.getNodeLocation(), ( nodeLocation, value ) -> value == null ? 1 : value + 1 ); + poll.copyToNodeWithAvailableResources(); + return true; + } else { + return false; + } + } + + protected void removeTasksThatAreCopiedMoreThanXTimeCurrently( SortedList taskStats, int maxParallelTasks ) { + taskStats.removeIf( elem -> currentlyCopying.getNumberOfNodesForTask( elem.getTask() ) >= maxParallelTasks ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java new file mode 100644 index 00000000..fbc6c891 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MaxSizeComparator.java @@ -0,0 +1,32 @@ +package cws.k8s.scheduler.scheduler.la2; + +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.Comparator; + +/** + * This comparator prioritizes tasks with a larger size. Exception is when two tasks have the same size, then the + * option where fewer data is copied is preferred. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MaxSizeComparator implements Comparator { + + public static final MaxSizeComparator INSTANCE = new MaxSizeComparator(); + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + //same task does not necessarily have the same size + if ( o1.getTask() == o2.getTask() || o1Stats.getTaskSize() == o2Stats.getTaskSize() ) { + //Prefer the one with fewer remaining data + return Long.compare( o1Stats.getSizeRemaining(), o2Stats.getSizeRemaining() ); + } else { + //Prefer the one with larger task size + return Long.compare( o2Stats.getTaskSize(), o1Stats.getTaskSize() ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java new file mode 100644 index 00000000..5c7ac5a0 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinCopyingComparator.java @@ -0,0 +1,28 @@ +package cws.k8s.scheduler.scheduler.la2; + +import java.util.Comparator; + +/** + * This comparator first prioritizes tasks that are on fewer nodes yet. + * If two tasks are on the same number of nodes, the task that is currently copying to fewer nodes is preferred. + * If two tasks are on the same number of nodes and are copying to the same number of nodes, the comparator + * that is passed to the constructor is used. + */ +public class MinCopyingComparator extends TaskStatComparator { + + public MinCopyingComparator( Comparator comparator ) { + super( comparator ); + } + + @Override + public int compare( TaskStat o1, TaskStat o2 ) { + if ( o1.getCompleteOnNodes() != o2.getCompleteOnNodes() ) { + return Long.compare( o1.getCompleteOnNodes(), o2.getCompleteOnNodes() ); + } + if ( o1.getCopyingToNodes() != o2.getCopyingToNodes() ) { + return Long.compare( o1.getCopyingToNodes(), o2.getCopyingToNodes() ); + } + return getComparator().compare( o1.getBestStats(), o2.getBestStats() ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/MinSizeComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinSizeComparator.java new file mode 100644 index 00000000..443bb012 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/MinSizeComparator.java @@ -0,0 +1,26 @@ +package cws.k8s.scheduler.scheduler.la2; + +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.Comparator; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MinSizeComparator implements Comparator { + + public static final MinSizeComparator INSTANCE = new MinSizeComparator(); + + @Override + public int compare( TaskStat.NodeAndStatWrapper o1, TaskStat.NodeAndStatWrapper o2 ) { + final TaskNodeStats o1Stats = o1.getTaskNodeStats(); + final TaskNodeStats o2Stats = o2.getTaskNodeStats(); + if ( o1Stats.getSizeRemaining() == o2Stats.getSizeRemaining() ) { + //Prefer the one that is copying fewer data --> expected to finish faster + return Long.compare( o1Stats.getSizeCurrentlyCopying(), o2Stats.getSizeCurrentlyCopying() ); + } else { + return Long.compare( o1Stats.getSizeRemaining(), o2Stats.getSizeRemaining() ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java new file mode 100644 index 00000000..16f4303e --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/RankAndMinCopyingComparator.java @@ -0,0 +1,27 @@ +package cws.k8s.scheduler.scheduler.la2; + +import java.util.Comparator; + +/** + * This comparator first prioritizes tasks with a higher rank. + * If two tasks have the same rank, the comparator that is passed to the constructor is used. + */ +public class RankAndMinCopyingComparator extends MinCopyingComparator { + + public RankAndMinCopyingComparator( Comparator comparator ) { + super( comparator ); + } + + @Override + public int compare( TaskStat o1, TaskStat o2 ) { + final int rankO1 = o1.getTask().getProcess().getRank(); + final int rankO2 = o2.getTask().getProcess().getRank(); + if ( rankO1 == rankO2 ) { + return super.compare( o1, o2 ); + } else { + //Prefer larger rank + return Integer.compare( rankO2, rankO1 ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java new file mode 100644 index 00000000..68425f80 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStat.java @@ -0,0 +1,142 @@ +package cws.k8s.scheduler.scheduler.la2; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import cws.k8s.scheduler.util.TaskNodeStats; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +@RequiredArgsConstructor +public class TaskStat implements Comparable { + + @Getter + private final Task task; + @Getter + private final TaskInputs inputsOfTask; + private TaskStatComparator comparator; + private final List taskStats = new ArrayList<>(); + @Getter + private int completeOnNodes = 0; + @Getter + private int copyingToNodes = 0; + private boolean finished = false; + private int indexToCompare = 0; + + @Getter + private boolean copyToNodeWithAvailableResources = false; + + public boolean canStartSomewhere() { + return !taskStats.isEmpty(); + } + + /** + * @return number is the number of nodes, that are better for this task depending on the comparator. + */ + public int getCurrentIndex() { + return indexToCompare; + } + + /** + * @return number of nodes that have already all data or are copying remaining data to this node. + */ + public int dataOnNodes() { + return completeOnNodes + copyingToNodes; + } + + public void setComparator( TaskStatComparator comparator ) { + this.comparator = comparator; + this.indexToCompare = 0; + finish(); + } + + public void copyToNodeWithAvailableResources() { + this.copyToNodeWithAvailableResources = true; + } + + public boolean missingDataOnAnyNode() { + return !taskStats.isEmpty(); + } + + /** + * Call this to sort the taskStats list + */ + private void finish() { + finished = true; + taskStats.sort( comparator.getComparator() ); + } + + /** + * Increases the index to compare. + * False if no other opportunity exists + * @return + */ + public boolean increaseIndexToCompare() { + indexToCompare++; + return indexToCompare < taskStats.size(); + } + + private final AtomicInteger canStartAfterCopying = new AtomicInteger( 0 ); + + + /** + * Will only be added to the list if there is missing data on the node. + * @param node + * @param taskNodeStats + */ + public void add( NodeWithAlloc node, TaskNodeStats taskNodeStats ) { + assert !finished; + if ( taskNodeStats.allOnNode() ) { + this.completeOnNodes++; + } else if ( taskNodeStats.allOnNodeOrCopying() ) { + this.copyingToNodes++; + } else { + //Only add in the case that not all data is already on the node + this.taskStats.add( new NodeAndStatWrapper( node, taskNodeStats, task ) ); + } + } + + /** + * Mark this task as it is currently copying to a node with enough available resources. + */ + public void canStartAfterCopying() { + this.canStartAfterCopying.incrementAndGet(); + } + + public int getCanStartAfterCopying() { + return this.canStartAfterCopying.get(); + } + + public NodeAndStatWrapper getBestStats() { + assert finished; + assert indexToCompare < taskStats.size(); + return taskStats.get( indexToCompare ); + } + + @Override + public int compareTo( @NotNull TaskStat o ) { + if ( indexToCompare >= taskStats.size() ) { + throw new IllegalStateException( "Cannot compare task " + task.getConfig().getName() + " size = " + taskStats.size() + " indexToCompare = " + indexToCompare); + } + if ( o.indexToCompare >= o.taskStats.size() ) { + throw new IllegalStateException( "Cannot compare task " + o.task.getConfig().getName() + " size = " + o.taskStats.size() + " indexToCompare = " + o.indexToCompare ); + } + return comparator.compare( this, o ); + } + + @Getter + @RequiredArgsConstructor(access = AccessLevel.PRIVATE) + public static class NodeAndStatWrapper { + private final NodeWithAlloc node; + private final TaskNodeStats taskNodeStats; + private final Task task; + } + + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStatComparator.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStatComparator.java new file mode 100644 index 00000000..5af7adc8 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/TaskStatComparator.java @@ -0,0 +1,15 @@ +package cws.k8s.scheduler.scheduler.la2; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Comparator; + +@RequiredArgsConstructor +public abstract class TaskStatComparator implements Comparator { + + @Getter + private final Comparator comparator; + + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java new file mode 100644 index 00000000..2f7e80fc --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/CapacityAvailableToNode.java @@ -0,0 +1,34 @@ +package cws.k8s.scheduler.scheduler.la2.capacityavailable; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; + +import java.util.List; +import java.util.Map; + +public abstract class CapacityAvailableToNode extends CreateCopyTasks { + + public CapacityAvailableToNode( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + public abstract List createAlignmentForTasksWithEnoughCapacity( + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final Map availableByNodes, + final List allNodes, + final int maxCopyingTaskPerNode, + final Map currentlyCopyingTasksOnNode, + int prio + ); + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java new file mode 100644 index 00000000..dca8d995 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/capacityavailable/SimpleCapacityAvailableToNode.java @@ -0,0 +1,102 @@ +package cws.k8s.scheduler.scheduler.la2.capacityavailable; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import lombok.extern.slf4j.Slf4j; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class SimpleCapacityAvailableToNode extends CapacityAvailableToNode { + + public SimpleCapacityAvailableToNode( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + public List createAlignmentForTasksWithEnoughCapacity( + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final Map availableByNodes, + final List allNodes, + final int maxCopyingTaskPerNode, + final Map currentlyCopyingTasksOnNode, + final int prio + ) { + + + List nodeTaskAlignments = new LinkedList<>(); + + //Remove available resources if a copy task is already running: this logic may not be optimal for more than 2 parallel copy tasks (unclear which task starts first) + removeAvailableResources( taskStats, availableByNodes, allNodes ); + //Sort tasks by missing data: prefer tasks where the least data is missing on the node + final SortedList stats = new SortedList<>( taskStats.getTaskStats().stream().filter( TaskStat::missingDataOnAnyNode ).collect( Collectors.toList() ) ); + removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); + + final boolean cannotAdd; + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode + && + availableByNodes.get( node ).higherOrEquals( task.getPlanedRequirements() ) ) { + if ( createFileAlignment( planedToCopy, nodeTaskAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { + cannotAdd = false; + availableByNodes.get( node ).subFromThis( task.getPlanedRequirements() ); + } else { + cannotAdd = true; + } + } else { + cannotAdd = true; + } + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( cannotAdd && poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } + } + return nodeTaskAlignments; + } + + + + private void removeAvailableResources( TaskStats taskStats, Map availableByNodes, List allNodes ) { + allNodes.parallelStream().forEach( node -> { + final Requirements availableOnNode = availableByNodes.get( node ); + final Requirements clone = availableOnNode.clone(); + for ( Task task : currentlyCopying.getTasksOnNode( node.getNodeLocation() ) ) { + final Requirements request = task.getPlanedRequirements(); + if ( clone.higherOrEquals( request ) ) { + if ( availableOnNode.higherOrEquals( request ) ) { + //Here we remove the first tasks in the list. However, the ordering of copy tasks finished might be different. + availableOnNode.subFromThis( request ); + } + final TaskStat taskStat = taskStats.get( task ); + //We might still copy data for tasks that have already been started. + if ( taskStat != null ) { + taskStat.canStartAfterCopying(); + } + } + } + } ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java new file mode 100644 index 00000000..cd8b75e6 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvance.java @@ -0,0 +1,32 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.CreateCopyTasks; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; + +import java.util.List; +import java.util.Map; + +public abstract class CopyInAdvance extends CreateCopyTasks { + + public CopyInAdvance( CurrentlyCopying currentlyCopying, InputAlignment inputAlignment, int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + public abstract void createAlignmentForTasksWithEnoughCapacity( + final List nodeTaskFilesAlignments, + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final List allNodes, + final int maxCopyingTaskPerNode, + final int maxHeldCopyTaskReady, + final Map currentlyCopyingTasksOnNode, + int prio, + Map> readyTasksPerNode ); + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java new file mode 100644 index 00000000..7bcaa7c8 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostData.java @@ -0,0 +1,77 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; + +@Slf4j +public class CopyInAdvanceNodeWithMostData extends CopyInAdvance { + + public CopyInAdvanceNodeWithMostData( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + } + + + /** + * Do not filter maxHeldCopyTaskReady, that could lead to starving if another node has resources. + */ + public void createAlignmentForTasksWithEnoughCapacity( + final List nodeTaskFilesAlignments, + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final List allNodes, + final int maxCopyingTaskPerNode, + final int maxHeldCopyTaskReady, + final Map currentlyCopyingTasksOnNode, + int prio, + Map> readyTasksPerNode ) { + final SortedList stats = new SortedList<>( taskStats.getTaskStats() ); + removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); + if ( !poll.missingDataOnAnyNode() || poll.isCopyToNodeWithAvailableResources() ) { + continue; + } + long start = System.currentTimeMillis(); + + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); + + final boolean cannotAdd; + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode ) { + if ( createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) { + cannotAdd = false; + log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); + } else { + cannotAdd = true; + } + } else { + cannotAdd = true; + } + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( cannotAdd && poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } + task.getTraceRecord().addSchedulerTimeDeltaPhaseThree( (int) (System.currentTimeMillis() - start) ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java new file mode 100644 index 00000000..a2d42980 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/CopyInAdvanceNodeWithMostDataIntelligent.java @@ -0,0 +1,211 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.HierarchyWrapper; +import cws.k8s.scheduler.scheduler.filealignment.InputAlignment; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import cws.k8s.scheduler.util.SortedList; +import cws.k8s.scheduler.util.TaskStats; +import cws.k8s.scheduler.util.copying.CurrentlyCopying; +import cws.k8s.scheduler.util.score.CalculateScore; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This strategy does not perform better than the normal CopyInAdvanceNodeWithMostData. + */ +@Slf4j +public class CopyInAdvanceNodeWithMostDataIntelligent extends CopyInAdvance { + + /** + * Try to distribute the tasks evenly on the nodes. More important tasks can be on x more nodes. (x = TASKS_READY_FACTOR) + */ + private static final int TASKS_READY_FACTOR = 2; + + CalculateScore calculateScore; + + HierarchyWrapper hierarchyWrapper; + + public CopyInAdvanceNodeWithMostDataIntelligent( + CurrentlyCopying currentlyCopying, + InputAlignment inputAlignment, + int copySameTaskInParallel , + CalculateScore calculateScore, + HierarchyWrapper hierarchyWrapper + ) { + super( currentlyCopying, inputAlignment, copySameTaskInParallel ); + this.calculateScore = calculateScore; + this.hierarchyWrapper = hierarchyWrapper; + } + + /** + * Do not filter maxHeldCopyTaskReady, that could lead to starving if another node has resources. + */ + public void createAlignmentForTasksWithEnoughCapacity( + final List nodeTaskFilesAlignments, + final TaskStats taskStats, + final CurrentlyCopying planedToCopy, + final List allNodes, + final int maxCopyingTaskPerNode, + final int maxHeldCopyTaskReady, + final Map currentlyCopyingTasksOnNode, + int prio, + Map> readyTasksPerNode + ) { + Map cache = new HashMap<>(); + long startMethod = System.currentTimeMillis(); + final SortedList stats = new SortedList<>( taskStats.getTaskStats().stream().filter( x -> x.missingDataOnAnyNode() && !x.isCopyToNodeWithAvailableResources() ).collect( Collectors.toList()) ); + removeTasksThatAreCopiedMoreThanXTimeCurrently( stats, copySameTaskInParallel ); + + int readyOnNodes = 0; + //Outer loop to only process tasks that are not yet ready on enough nodes + while( !stats.isEmpty() ) { + LinkedList tasksReadyOnMoreNodes = new LinkedList<>(); + while( !stats.isEmpty() ) { + final TaskStat poll = stats.poll(); + if ( poll.dataOnNodes() > readyOnNodes + TASKS_READY_FACTOR ) { + tasksReadyOnMoreNodes.add( poll ); + continue; + } + + long start = System.currentTimeMillis(); + final TaskStat.NodeAndStatWrapper bestStats = poll.getBestStats(); + final Task task = poll.getTask(); + final NodeWithAlloc node = bestStats.getNode(); + final NodeCache nodeCache = cache.computeIfAbsent( node, + n -> new NodeCache( + //Only consider workflow tasks, other will not finish soon + n.getAssignedPods().entrySet().stream().filter( x -> x.getKey().contains( "||nf-" ) ).map( Map.Entry::getValue ).collect( Collectors.toList() ), + readyTasksPerNode.get( n ), + planedToCopy.getTasksOnNode( n.getNodeLocation() ), + taskStats, + node + ) + ); + + //Check if the node has still enough resources to run the task + if ( currentlyCopyingTasksOnNode.getOrDefault( node.getNodeLocation(), 0 ) < maxCopyingTaskPerNode + && reasonableToCopyData( node, task, nodeCache ) + && createFileAlignment( planedToCopy, nodeTaskFilesAlignments, currentlyCopyingTasksOnNode, poll, task, node, prio ) ) + { + log.info( "Start copy task with {} missing bytes", poll.getBestStats().getTaskNodeStats().getSizeRemaining() ); + nodeCache.addPlaned( task ); + } else { + //if not enough resources or too many tasks are running, mark next node as to compare and add again into the list + if ( poll.increaseIndexToCompare() ) { + //Only re-add if still other opportunities exist + stats.add( poll ); + } + } + task.getTraceRecord().addSchedulerTimeDeltaPhaseThree( (int) (System.currentTimeMillis() - start) ); + } + stats.addAll( tasksReadyOnMoreNodes ); + readyOnNodes++; + } + log.info( "Time to create alignment for tasks with enough capacity: {}", System.currentTimeMillis() - startMethod ); + } + + /** + * Check if a single task or two tasks could replace one running task + * @param node + * @param task + * @param cache + * @return + */ + private boolean reasonableToCopyData( NodeWithAlloc node, Task task, NodeCache cache ) { + final Requirements availableResources = node.getAvailableResources(); + final ShouldCopyChecker shouldCopyChecker = new ShouldCopyChecker( + cache.getTaskWithScore( task, "Problem in reasonableToCopyData" ).score, + cache.waiting, + task.getPlanedRequirements() + ); + return cache.running + .parallelStream() + .anyMatch( r -> shouldCopyChecker.couldBeStarted( r.add( availableResources ) ) ); + } + + private class NodeCache { + + /** + * A sorted list of tasks that are currently running on the node. + */ + private final Set running; + /** + * A sorted list of tasks that are currently waiting on the node. + */ + private final List waiting; + + private final NodeWithAlloc node; + + private final TaskStats taskStats; + + NodeCache( final Collection running, final List tasksAlreadyReady, final List tasksPlanedToCopy, TaskStats taskStats, NodeWithAlloc node ) { + this.node = node; + this.taskStats = taskStats; + /** + * remove tasks with similar requirements, as replacement would be the same. + */ + this.running = new HashSet<>(running); + + Stream waitingTemp = null; + + // A list of tasks that are ready to run, or data is copied so that they can start soon. + if ( tasksAlreadyReady != null ) { + waitingTemp = tasksAlreadyReady + .stream() + .map( x -> getTaskWithScore( x, "Problem with tasksAlreadyReady" ) ); + } + if ( tasksPlanedToCopy != null && !tasksPlanedToCopy.isEmpty() ) { + final Stream planned = tasksPlanedToCopy.stream() + .map( x -> getTaskWithScore( x, "Problem with tasksPlanedToCopy" ) ); + if ( waitingTemp == null ) { + waitingTemp = planned; + } else { + waitingTemp = Stream.concat( waitingTemp, planned ); + } + } else if ( waitingTemp == null ) { + waitingTemp = Stream.empty(); + } + + waiting = waitingTemp + .collect( Collectors.toList() ); + + } + + @NotNull + private TaskWithScore getTaskWithScore( Task task, String error ) { + if ( taskStats.get( task ) == null ) { + throw new RuntimeException( error ); + } + final long dataOnNode = taskStats + .get( task ) + .getInputsOfTask() + .calculateDataOnNode( node.getNodeLocation() ); + final long dataInSharedFS = task.getInputSize(); + //The input should be similar to the first scheduling phase + return new TaskWithScore( task, calculateScore.getScore( task, node, dataOnNode + dataInSharedFS ) ); + } + + void addPlaned( Task task ) { + waiting.add( getTaskWithScore( task, "Problem in addPlanned" ) ); + } + + } + + @RequiredArgsConstructor + static class TaskWithScore { + final Task task; + final long score; + } + +} + diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java new file mode 100644 index 00000000..9654a783 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copyinadvance/ShouldCopyChecker.java @@ -0,0 +1,56 @@ +package cws.k8s.scheduler.scheduler.la2.copyinadvance; + +import cws.k8s.scheduler.model.Requirements; +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; + +import java.util.List; +import java.util.Optional; + +@RequiredArgsConstructor +public class ShouldCopyChecker { + + private final long taskScore; + private final List waiting; + private final Requirements taskRequest; + + boolean couldBeStarted( Requirements becomesAvailable ) { + //True if the task to check would be started in the current case + Result results = new Result( true, taskScore ); + final Optional reduce = waiting.parallelStream().map( waitingTask -> { + final Result result = new Result( true, taskScore ); + final Requirements waitingRequest = waitingTask.task.getPlanedRequirements(); + //In all cases waitingRequest is used and accordingly the request must always be smaller than the available resources + if ( waitingRequest.smallerEquals( becomesAvailable ) ) { + checkSingle( becomesAvailable, result, waitingRequest.add( taskRequest ), waitingTask.score ); + } + return result; + } ).reduce( ( a, b ) -> a.bestScore > b.bestScore ? a : b ); + return reduce.map( result -> result.couldBeStarted ).orElseGet( () -> results.couldBeStarted ); + } + + /** + * + * @param becomesAvailable resources available + * @param results wrapper for the result + * @param requestAndTaskRequest sum of task to check and waiting task + * @param score score of the waiting task + */ + private void checkSingle( Requirements becomesAvailable, Result results, Requirements requestAndTaskRequest, long score ) { + if ( results.bestScore < score ) { + results.bestScore = score; + results.couldBeStarted = false; + } + if ( results.bestScore < score + taskScore && requestAndTaskRequest.smallerEquals( becomesAvailable ) ) { + results.bestScore = score + taskScore; + results.couldBeStarted = true; + } + } + + @AllArgsConstructor + private static class Result { + boolean couldBeStarted; + long bestScore; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java new file mode 100644 index 00000000..28503c12 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/CopyRunner.java @@ -0,0 +1,8 @@ +package cws.k8s.scheduler.scheduler.la2.copystrategy; + +import cws.k8s.scheduler.util.CopyTask; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; + +public interface CopyRunner { + void startCopyTasks( CopyTask copyTask, NodeTaskFilesAlignment nodeTaskFilesAlignment ); +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java new file mode 100644 index 00000000..b4ddf8c1 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/LaListener.java @@ -0,0 +1,91 @@ +package cws.k8s.scheduler.scheduler.la2.copystrategy; + +import cws.k8s.scheduler.scheduler.LocationAwareSchedulerV2; +import cws.k8s.scheduler.util.CopyTask; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.MyExecListner; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import io.fabric8.kubernetes.api.model.Status; +import io.fabric8.kubernetes.client.dsl.ExecWatch; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; + +@Slf4j +@RequiredArgsConstructor +public class LaListener implements MyExecListner { + + @Setter + private ExecWatch exec; + private final CopyTask copyTask; + private final String name; + @Setter + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + @Setter + private ByteArrayOutputStream error = new ByteArrayOutputStream(); + private boolean finished = false; + + private final NodeTaskFilesAlignment nodeTaskFilesAlignment; + + private final LocationAwareSchedulerV2 scheduler; + + private final LogCopyTask logCopyTask; + + private void close() { + //Maybe exec was not yet set + int trial = 0; + while( exec == null && trial < 5 ) { + try { + Thread.sleep( (long) (100 * Math.pow( 2, trial )) ); + } catch ( InterruptedException e ) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } + trial++; + } + if ( exec != null ) { + exec.close(); + } + } + + @Override + public void onClose( int exitCode, String reason ) { + if ( !finished ) { + log.error( "Copy task was not finished, but closed. ExitCode: " + exitCode + " Reason: " + reason ); + scheduler.copyTaskFinished( copyTask, exitCode == 0 ); + } + scheduler.informResourceChange(); + } + + @Override + public void onFailure( Throwable t, Response failureResponse ) { + log.info( name + " failed, output: ", t ); + log.info( name + " Exec Output: {} ", out ); + log.info( name + " Exec Error Output: {} ", error ); + close(); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "failed" ); + } + + @Override + public void onExit( int exitCode, Status reason ) { + finished = true; + if ( exitCode != 0 ) { + log.info( name + " was finished exitCode = {}, reason = {}", exitCode, reason ); + log.info( name + " Exec Output: {} ", out ); + log.info( name + " Exec Error Output: {} ", error ); + } else { + log.info( name + " was finished successfully" ); + log.debug( name + " Exec Output: {} ", out ); + log.debug( name + " Exec Error Output: {} ", error ); + } + if ( copyTask.getTask() instanceof cws.k8s.scheduler.model.cluster.CopyTask ) { + cws.k8s.scheduler.model.cluster.CopyTask task = (cws.k8s.scheduler.model.cluster.CopyTask) copyTask.getTask(); + task.finished(); + } + scheduler.copyTaskFinished( copyTask, exitCode == 0 ); + close(); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "finished(" + exitCode + ")" ); + } +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/ShellCopy.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/ShellCopy.java new file mode 100644 index 00000000..2cbcd936 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/copystrategy/ShellCopy.java @@ -0,0 +1,47 @@ +package cws.k8s.scheduler.scheduler.la2.copystrategy; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import cws.k8s.scheduler.client.CWSKubernetesClient; +import cws.k8s.scheduler.scheduler.LocationAwareSchedulerV2; +import cws.k8s.scheduler.util.CopyTask; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskFilesAlignment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.file.Path; + +@Slf4j +@RequiredArgsConstructor +public class ShellCopy implements CopyRunner { + + private final CWSKubernetesClient client; + private final LocationAwareSchedulerV2 scheduler; + private final LogCopyTask logCopyTask; + + @Override + public void startCopyTasks( final CopyTask copyTask, final NodeTaskFilesAlignment nodeTaskFilesAlignment ) { + final String nodeName = nodeTaskFilesAlignment.node.getName().replace( " ", "_" ); + String copyTaskIdentifier = nodeName + "-" + copyTask.getTask().getCurrentCopyTaskId(); + String filename = ".command.init." + copyTaskIdentifier + ".json"; + String[] command = new String[3]; + command[0] = "/bin/bash"; + command[1] = "-c"; + command[2] = "cd " + nodeTaskFilesAlignment.task.getWorkingDir() + " && "; + try { + new ObjectMapper().writeValue( Path.of( nodeTaskFilesAlignment.task.getWorkingDir(), filename ).toFile(), copyTask.getInputs() ); + } catch ( JsonProcessingException e ) { + throw new RuntimeException( e ); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + command[2] += "/code/ftp.py false \"" + copyTaskIdentifier + "\" \"" + filename + "\""; + String name = nodeTaskFilesAlignment.task.getConfig().getName() + "-copy-" + nodeTaskFilesAlignment.node.getName(); + log.info( "Starting {} to node {}", nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName() ); + logCopyTask.copy( nodeTaskFilesAlignment.task.getConfig().getName(), nodeTaskFilesAlignment.node.getName(), copyTask.getInputFiles().size(), "start" ); + client.execCommand( scheduler.getDaemonNameOnNode( copyTask.getNodeLocation().getIdentifier() ), scheduler.getNamespace(), command, new LaListener( copyTask, name, nodeTaskFilesAlignment, scheduler, logCopyTask ) ); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java new file mode 100644 index 00000000..221aed25 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/OptimalReadyToRunToNode.java @@ -0,0 +1,136 @@ +package cws.k8s.scheduler.scheduler.la2.ready2run; + +import com.google.ortools.Loader; +import com.google.ortools.sat.*; +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; +import cws.k8s.scheduler.util.score.CalculateScore; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.math.BigDecimal; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +public class OptimalReadyToRunToNode implements ReadyToRunToNode { + + private final BigDecimal MILLION = new BigDecimal(1_000_000); + private CalculateScore calculateScore = null; + + @Setter + private LogCopyTask logger; + + @Override + public void init( CalculateScore calculateScore ) { + Loader.loadNativeLibraries(); + this.calculateScore = calculateScore; + } + + /** + * Creates an optimal alignment for tasks with all data on node. + */ + @Override + public List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ) { + + if ( taskWithAllData.isEmpty() || availableByNode.isEmpty() ){ + log.info( "No tasks can be scheduled on any node. (No node has all data)" ); + return Collections.emptyList(); + } + + long start = System.currentTimeMillis(); + final LinkedList taskNodeBoolVars = new LinkedList<>(); + + CpModel model = new CpModel(); + + Map memUsed = new HashMap<>(); + Map cpuUsed = new HashMap<>(); + for ( NodeWithAlloc node : availableByNode.keySet() ) { + memUsed.put( node, LinearExpr.newBuilder() ); + cpuUsed.put( node, LinearExpr.newBuilder() ); + } + + LinearExprBuilder objective = LinearExpr.newBuilder(); + + int index = 0; + for ( TaskInputsNodes taskInputsNodes : taskWithAllData ) { + List onlyOnOneNode = new ArrayList<>(); + final Requirements request = taskInputsNodes.getTask().getPlanedRequirements(); + final long ram = request.getRam().longValue(); + final long cpu = request.getCpu().multiply( MILLION ).longValue(); + for ( NodeWithAlloc node : taskInputsNodes.getNodesWithAllData() ) { + final Requirements availableOnNode = availableByNode.get( node ); + //Can schedule task on node? + if ( availableOnNode != null && availableOnNode.higherOrEquals( request ) ) { + final BoolVar boolVar = model.newBoolVar( "x_" + index + "_" + node ); + onlyOnOneNode.add( boolVar ); + memUsed.get(node).addTerm( boolVar, ram ); + cpuUsed.get(node).addTerm( boolVar, cpu ); + final long score = calculateScore.getScore( taskInputsNodes.getTask(), node, taskInputsNodes.getTaskSize() ); + objective.addTerm( boolVar, score ); + taskNodeBoolVars.add( new TaskNodeBoolVar( taskInputsNodes, node, boolVar ) ); + } + } + if ( !onlyOnOneNode.isEmpty() ) { + model.addAtMostOne(onlyOnOneNode); + } + index++; + } + + if ( taskNodeBoolVars.isEmpty() ) { + log.info( "No tasks can be scheduled on any node. Not enough resources available." ); + return Collections.emptyList(); + } + + for ( Map.Entry entry : memUsed.entrySet() ) { + model.addLessOrEqual( entry.getValue(), availableByNode.get( entry.getKey() ).getRam().longValue() ); + model.addLessOrEqual( cpuUsed.get( entry.getKey() ), availableByNode.get( entry.getKey() ).getCpu().multiply( MILLION ).longValue() ); + } + + log.info( "Model created in " + (System.currentTimeMillis() - start) + "ms ( " + taskNodeBoolVars.size() + " vars )" ); + model.maximize( objective ); + CpSolver solver = new CpSolver(); + solver.getParameters().setMaxTimeInSeconds( 10 ); + final CpSolverStatus solve = solver.solve( model ); + final String message = "Solved in " + (System.currentTimeMillis() - start) + "ms ( " + taskNodeBoolVars.size() + " vars ) + solution is: " + solve; + log.info( message ); + logger.log( message ); + log.info("Total packed value: " + solver.objectiveValue()); + if ( solve != CpSolverStatus.OPTIMAL && solve != CpSolverStatus.FEASIBLE ) { + return Collections.emptyList(); + } + + return taskNodeBoolVars.stream() + .filter( taskNodeBoolVar -> solver.booleanValue( taskNodeBoolVar.getBoolVar() ) ) + .map( TaskNodeBoolVar::createAlignment ) + .collect( Collectors.toList() ); + } + + @Getter + @RequiredArgsConstructor + private class TaskNodeBoolVar { + private final TaskInputsNodes taskInputsNodes; + private final NodeWithAlloc node; + private final BoolVar boolVar; + + NodeTaskLocalFilesAlignment createAlignment(){ + return new NodeTaskLocalFilesAlignment( + node, + taskInputsNodes.getTask(), + taskInputsNodes.getInputsOfTask().getSymlinks(), + taskInputsNodes.getInputsOfTask().allLocationWrapperOnLocation( node.getNodeLocation() ) + ); + } + + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java new file mode 100644 index 00000000..c7d6b560 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/RandomReadyToRunToNode.java @@ -0,0 +1,70 @@ +package cws.k8s.scheduler.scheduler.la2.ready2run; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; +import cws.k8s.scheduler.util.score.CalculateScore; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +@Slf4j +public class RandomReadyToRunToNode implements ReadyToRunToNode { + + + @Setter + private LogCopyTask logger; + + + @Override + public void init( CalculateScore calculateScore ) {} + + /** + * Select the first node that fits the requirements. + * + * @param taskWithAllData + * @param availableByNode + * @return + */ + @Override + public List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ) { + long start = System.currentTimeMillis(); + final List alignment = new LinkedList<>(); + final Iterator iterator = taskWithAllData.iterator(); + while( iterator.hasNext() ) { + final TaskInputsNodes taskInputsNodes = iterator.next(); + final Requirements taskRequest = taskInputsNodes.getTask().getPod().getRequest(); + for ( NodeWithAlloc nodeWithAll : taskInputsNodes.getNodesWithAllData() ) { + final Requirements availableOnNode = availableByNode.get( nodeWithAll ); + if ( availableOnNode != null && + availableOnNode.higherOrEquals( taskRequest ) ) { + alignment.add( + new NodeTaskLocalFilesAlignment( + nodeWithAll, + taskInputsNodes.getTask(), + taskInputsNodes.getInputsOfTask().getSymlinks(), + taskInputsNodes.getInputsOfTask().allLocationWrapperOnLocation( nodeWithAll.getNodeLocation() ) + ) + ); + availableOnNode.subFromThis( taskRequest ); + iterator.remove(); + break; + } + } + } + final String message = "Solved in " + (System.currentTimeMillis() - start) + "ms ( " + taskWithAllData.size() + " vars )"; + log.info( message ); + logger.log( message ); + return alignment; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java new file mode 100644 index 00000000..0ed60a39 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/la2/ready2run/ReadyToRunToNode.java @@ -0,0 +1,24 @@ +package cws.k8s.scheduler.scheduler.la2.ready2run; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Requirements; +import cws.k8s.scheduler.scheduler.data.TaskInputsNodes; +import cws.k8s.scheduler.util.LogCopyTask; +import cws.k8s.scheduler.util.NodeTaskLocalFilesAlignment; +import cws.k8s.scheduler.util.score.CalculateScore; + +import java.util.List; +import java.util.Map; + +public interface ReadyToRunToNode { + + void init( CalculateScore calculateScore ); + + void setLogger( LogCopyTask logger ); + + List createAlignmentForTasksWithAllDataOnNode( + List taskWithAllData, + Map availableByNode + ); + +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java index c8417132..55a5a093 100644 --- a/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java +++ b/src/main/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritize.java @@ -2,7 +2,6 @@ import cws.k8s.scheduler.model.Task; -import java.util.Comparator; import java.util.List; public class RankMaxPrioritize implements Prioritize { diff --git a/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/InputEntry.java b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/InputEntry.java new file mode 100644 index 00000000..469b1e3e --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/InputEntry.java @@ -0,0 +1,20 @@ +package cws.k8s.scheduler.scheduler.schedulingstrategy; + +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +@RequiredArgsConstructor +public class InputEntry implements Comparable { + + public final String currentIP; + public final String node; + public final List files; + public final long size; + + @Override + public int compareTo(@NotNull InputEntry o) { + return Long.compare( size, o.size ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/Inputs.java b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/Inputs.java new file mode 100644 index 00000000..af266906 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/scheduler/schedulingstrategy/Inputs.java @@ -0,0 +1,37 @@ +package cws.k8s.scheduler.scheduler.schedulingstrategy; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import lombok.RequiredArgsConstructor; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@RequiredArgsConstructor +public class Inputs { + + public final String dns; + public final String execution; + public final List data = new LinkedList<>(); + public final List symlinks = new LinkedList<>(); + public final String syncDir; + public final String hash; + public final Map> waitForFilesOfTask = new ConcurrentHashMap<>(); + public final int speed; + + public void waitForTask( Map waitForTask ){ + for (Map.Entry e : waitForTask.entrySet()) { + final String taskHash = e.getValue().getConfig().getRunName(); + final List listOfPaths = waitForFilesOfTask.computeIfAbsent( taskHash, k -> new LinkedList<>() ); + listOfPaths.add( e.getKey() ); + } + } + + public void sortData(){ + data.sort(Collections.reverseOrder()); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/AlignmentWrapper.java b/src/main/java/cws/k8s/scheduler/util/AlignmentWrapper.java new file mode 100644 index 00000000..951dd650 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/AlignmentWrapper.java @@ -0,0 +1,40 @@ +package cws.k8s.scheduler.util; + +import lombok.Getter; +import lombok.ToString; + +import java.util.LinkedList; +import java.util.List; + +@Getter +@ToString +public class AlignmentWrapper { + + private final List filesToCopy = new LinkedList<>(); + private long toCopySize = 0; + private final List waitFor = new LinkedList<>(); + private long toWaitSize = 0; + private double cost = 0; + + public void addAlignmentToCopy( FilePath filePath, double cost, long size ) { + filesToCopy.add( filePath ); + toCopySize += size; + this.cost = cost; + } + + public void addAlignmentToWaitFor( FilePathWithTask filePath, long size ) { + toWaitSize += size; + waitFor.add( filePath ); + } + + public boolean empty() { + return filesToCopy.isEmpty() && waitFor.isEmpty(); + } + + public List getAll() { + List newList = new LinkedList<>(filesToCopy); + newList.addAll(waitFor); + return newList; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/CopyTask.java b/src/main/java/cws/k8s/scheduler/util/CopyTask.java new file mode 100644 index 00000000..13b381aa --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/CopyTask.java @@ -0,0 +1,36 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskInputFileLocationWrapper; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.scheduler.schedulingstrategy.Inputs; +import cws.k8s.scheduler.util.copying.CurrentlyCopyingOnNode; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +@Getter +@RequiredArgsConstructor +public class CopyTask { + + @Getter(AccessLevel.NONE) + private static final AtomicLong copyTaskId = new AtomicLong(0); + private final long id = copyTaskId.getAndIncrement(); + private final Inputs inputs; + private final LinkedList inputFiles; + private final CurrentlyCopyingOnNode filesForCurrentNode; + private final Task task; + + @Setter + private List allLocationWrapper; + + @Setter + private NodeLocation nodeLocation; + +} diff --git a/src/main/java/cws/k8s/scheduler/util/DaemonHolder.java b/src/main/java/cws/k8s/scheduler/util/DaemonHolder.java new file mode 100644 index 00000000..32c37cb0 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/DaemonHolder.java @@ -0,0 +1,55 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.location.NodeLocation; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class DaemonHolder { + + final Map daemonByNode = new HashMap<>(); + + public void removeDaemon(String nodeName) { + daemonByNode.remove(nodeName); + } + + public void addDaemon(String node, String daemonName, String daemonIp ) { + daemonByNode.put(node, new DaemonData( daemonName, daemonIp ) ); + } + + public String getDaemonIp(String node) { + final DaemonData daemonData = daemonByNode.get( node ); + return daemonData == null ? null : daemonData.getIp(); + } + + public String getDaemonName(String node) { + final DaemonData daemonData = daemonByNode.get( node ); + return daemonData == null ? null : daemonData.getName(); + } + + public String getDaemonIp( NodeLocation node) { + return getDaemonIp(node.getIdentifier()); + } + + public String getDaemonName(NodeLocation node) { + return getDaemonName(node.getIdentifier()); + } + + @Getter + @ToString + @AllArgsConstructor + private class DaemonData { + private String name; + private String ip; + } + + @Override + public String toString() { + return daemonByNode.toString(); + } +} diff --git a/src/main/java/cws/k8s/scheduler/util/DataOnNode.java b/src/main/java/cws/k8s/scheduler/util/DataOnNode.java new file mode 100644 index 00000000..a61134a4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/DataOnNode.java @@ -0,0 +1,34 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class DataOnNode { + + + private final Map dataOnNode = new HashMap<>(); + + @Getter + private final long overAllData; + @Getter + private final Task task; + @Getter + private final TaskInputs inputsOfTask; + + public DataOnNode( Task task, TaskInputs inputsOfTask ) { + this.task = task; + this.inputsOfTask = inputsOfTask; + this.overAllData = inputsOfTask.calculateAvgSize(); + } + + public Set getNodes() { + return dataOnNode.keySet(); + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/FileAlignment.java b/src/main/java/cws/k8s/scheduler/util/FileAlignment.java new file mode 100644 index 00000000..ced3ca95 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/FileAlignment.java @@ -0,0 +1,54 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Getter +@ToString +@RequiredArgsConstructor +public class FileAlignment { + + /* + Key: node + Value: Files from the node + */ + private final Map nodeFileAlignment; + private final List symlinks; + private final double cost; + @Setter + private double weight = 1.0; + + /** + * Check if data is copied from at least one node + * @param node this node is not checked + * @return + */ + public boolean copyFromSomewhere( Location node ) { + return nodeFileAlignment + .entrySet() + .stream() + .anyMatch( a -> a.getKey() != node && !a.getValue().getFilesToCopy().isEmpty() ); + } + + public List getAllLocationWrappers(){ + return nodeFileAlignment + .entrySet() + .parallelStream() + .flatMap( l -> l + .getValue() + .getAll() + .parallelStream() + .map(FilePath::getLocationWrapper) + ) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/cws/k8s/scheduler/util/FilePath.java b/src/main/java/cws/k8s/scheduler/util/FilePath.java new file mode 100644 index 00000000..ddb5437a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/FilePath.java @@ -0,0 +1,30 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.location.hierachy.RealHierarchyFile; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class FilePath { + + private final PathFileLocationTriple pathFileLocationTriple; + private final LocationWrapper locationWrapper; + + public String getPath() { + return pathFileLocationTriple.path.toString(); + } + + public RealHierarchyFile getFile() { + return pathFileLocationTriple.file; + } + + public LocationWrapper getLocationWrapper() { + return locationWrapper; + } + + @Override + public String toString() { + return getPath(); + } +} diff --git a/src/main/java/cws/k8s/scheduler/util/FilePathWithTask.java b/src/main/java/cws/k8s/scheduler/util/FilePathWithTask.java new file mode 100644 index 00000000..060672f9 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/FilePathWithTask.java @@ -0,0 +1,18 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import lombok.Getter; + +public class FilePathWithTask extends FilePath { + + @Getter + private final Task task; + + public FilePathWithTask(PathFileLocationTriple pathFileLocationTriple, LocationWrapper locationWrapper, Task task ) { + super(pathFileLocationTriple, locationWrapper); + this.task = task; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/LogCopyTask.java b/src/main/java/cws/k8s/scheduler/util/LogCopyTask.java new file mode 100644 index 00000000..3add5a4d --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/LogCopyTask.java @@ -0,0 +1,53 @@ +package cws.k8s.scheduler.util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +public class LogCopyTask { + + private final BufferedWriter writer; + + public LogCopyTask() { + try { + final String pathname = "/input/data/scheduler/"; + new File( pathname ).mkdirs(); + this.writer = new BufferedWriter( new FileWriter( pathname + "copytasks.csv" ) ); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + + public void log( String text ) { + try { + writer.write( text ); + writer.newLine(); + writer.flush(); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + + public void copy( String task, String node, int filesToCopy, String additional ) { + String str = "\"" + task + "\";\"" + node + "\";\"" + filesToCopy+ "\";\"" + additional + "\";" + System.currentTimeMillis(); + synchronized ( writer ) { + try { + writer.write( str ); + writer.newLine(); + writer.flush(); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + } + + public void close() { + try { + writer.close(); + } catch ( IOException e ) { + throw new RuntimeException( e ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/MyExecListner.java b/src/main/java/cws/k8s/scheduler/util/MyExecListner.java new file mode 100644 index 00000000..c6767d6b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/MyExecListner.java @@ -0,0 +1,15 @@ +package cws.k8s.scheduler.util; + +import io.fabric8.kubernetes.client.dsl.ExecListener; +import io.fabric8.kubernetes.client.dsl.ExecWatch; + +import java.io.ByteArrayOutputStream; + +public interface MyExecListner extends ExecListener { + + void setExec( ExecWatch exec); + void setError( ByteArrayOutputStream error ); + void setOut( ByteArrayOutputStream out ); + + +} diff --git a/src/main/java/cws/k8s/scheduler/util/NodeTaskFilesAlignment.java b/src/main/java/cws/k8s/scheduler/util/NodeTaskFilesAlignment.java new file mode 100644 index 00000000..a3b0bc30 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/NodeTaskFilesAlignment.java @@ -0,0 +1,22 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import lombok.Getter; +import lombok.Setter; + +public class NodeTaskFilesAlignment extends NodeTaskAlignment { + + public final FileAlignment fileAlignment; + public final int prio; + + @Getter + @Setter + private boolean removeInit = false; + + public NodeTaskFilesAlignment( NodeWithAlloc node, Task task, FileAlignment fileAlignment, int prio ) { + super(node, task); + this.fileAlignment = fileAlignment; + this.prio = prio; + } +} diff --git a/src/main/java/cws/k8s/scheduler/util/NodeTaskLocalFilesAlignment.java b/src/main/java/cws/k8s/scheduler/util/NodeTaskLocalFilesAlignment.java new file mode 100644 index 00000000..3d773ab4 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/NodeTaskLocalFilesAlignment.java @@ -0,0 +1,21 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; + +import java.util.List; + +public class NodeTaskLocalFilesAlignment extends NodeTaskAlignment { + + public final List symlinks; + public final List locationWrappers; + + public NodeTaskLocalFilesAlignment( NodeWithAlloc node, Task task, List symlinks, List locationWrappers ) { + super(node, task); + this.symlinks = symlinks; + this.locationWrappers = locationWrappers; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/SortedList.java b/src/main/java/cws/k8s/scheduler/util/SortedList.java new file mode 100644 index 00000000..6cd162a5 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/SortedList.java @@ -0,0 +1,51 @@ +package cws.k8s.scheduler.util; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; + +public class SortedList> extends LinkedList { + + public SortedList( Collection collection ) { + super( collection ); + this.sort( Comparable::compareTo ); + } + + @Override + public boolean add( T elem ) { + int insertionIndex = Collections.binarySearch(this, elem ); + if (insertionIndex < 0) { + insertionIndex = -(insertionIndex + 1); + } + super.add( insertionIndex, elem ); + return true; + } + + @Override + public void add( int index, T element ) { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public boolean addAll( Collection c ) { + for ( T t : c ) { + this.add( t ); + } + return true; + } + + @Override + public boolean addAll( int index, Collection c ) { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public void addFirst( T t ) { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public void addLast( T t ) { + throw new UnsupportedOperationException( "Not supported" ); + } +} diff --git a/src/main/java/cws/k8s/scheduler/util/TaskNodeStats.java b/src/main/java/cws/k8s/scheduler/util/TaskNodeStats.java new file mode 100644 index 00000000..2773cd0b --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/TaskNodeStats.java @@ -0,0 +1,26 @@ +package cws.k8s.scheduler.util; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class TaskNodeStats { + + final long sizeRemaining; + final long sizeCurrentlyCopying; + final long sizeOnNode; + + public long getTaskSize() { + return sizeRemaining + sizeCurrentlyCopying + sizeOnNode; + } + + public boolean allOnNode() { + return sizeRemaining == 0 && sizeCurrentlyCopying == 0; + } + + public boolean allOnNodeOrCopying() { + return sizeRemaining == 0; + } + +} \ No newline at end of file diff --git a/src/main/java/cws/k8s/scheduler/util/TaskStats.java b/src/main/java/cws/k8s/scheduler/util/TaskStats.java new file mode 100644 index 00000000..8aaca2cd --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/TaskStats.java @@ -0,0 +1,33 @@ +package cws.k8s.scheduler.util; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.scheduler.la2.TaskStat; +import cws.k8s.scheduler.scheduler.la2.TaskStatComparator; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class TaskStats { + + private final Map taskStats = new HashMap<>(); + + public TaskStat get( Task task ) { + return this.taskStats.get( task ); + } + + public void add( TaskStat taskStat ) { + this.taskStats.put( taskStat.getTask(), taskStat ); + } + + public Collection getTaskStats() { + return this.taskStats.values(); + } + + public void setComparator( TaskStatComparator comparator ) { + for ( TaskStat taskStat : taskStats.values() ) { + taskStat.setComparator( comparator ); + } + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/Tuple.java b/src/main/java/cws/k8s/scheduler/util/Tuple.java index 6aea4dd5..3eaa73b9 100644 --- a/src/main/java/cws/k8s/scheduler/util/Tuple.java +++ b/src/main/java/cws/k8s/scheduler/util/Tuple.java @@ -2,8 +2,10 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.ToString; @Getter +@ToString @RequiredArgsConstructor public class Tuple { diff --git a/src/main/java/cws/k8s/scheduler/util/copying/CopySource.java b/src/main/java/cws/k8s/scheduler/util/copying/CopySource.java new file mode 100644 index 00000000..109a527a --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/copying/CopySource.java @@ -0,0 +1,15 @@ +package cws.k8s.scheduler.util.copying; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class CopySource { + + private final Task task; + private final Location location; + +} diff --git a/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopying.java b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopying.java new file mode 100644 index 00000000..a306e915 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopying.java @@ -0,0 +1,121 @@ +package cws.k8s.scheduler.util.copying; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.util.AlignmentWrapper; +import cws.k8s.scheduler.util.FilePath; +import lombok.ToString; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Keeps track of what is currently being copied to all node and which copy tasks are running. + * This is used to avoid copying the same data to a node multiple times and to overload a single node with copy tasks. + */ +@ToString +public class CurrentlyCopying { + + private final Map copyingToNode = new ConcurrentHashMap<>(); + private final Map> taskOnNodes = new HashMap<>(); + private final Map> nodesForTask = new HashMap<>(); + + public void add( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { + if ( currentlyCopyingOnNode == null || currentlyCopyingOnNode.isEmpty() ) { + return; + } + copyingToNode.compute( nodeLocation, ( node, currentlyCopying ) -> { + if ( currentlyCopying == null ) { + return currentlyCopyingOnNode; + } else { + currentlyCopying.add( currentlyCopyingOnNode ); + return currentlyCopying; + } + } ); + copyToNode( task, nodeLocation ); + } + + public CurrentlyCopyingOnNode get( NodeLocation nodeLocation ) { + return copyingToNode.computeIfAbsent( nodeLocation, node -> new CurrentlyCopyingOnNode() ); + } + + public void remove( Task task, NodeLocation nodeLocation, CurrentlyCopyingOnNode currentlyCopyingOnNode ) { + if ( currentlyCopyingOnNode == null || currentlyCopyingOnNode.isEmpty() ) { + return; + } + copyingToNode.compute( nodeLocation, ( node, currentlyCopying ) -> { + if ( currentlyCopying == null ) { + return null; + } else { + currentlyCopying.remove( currentlyCopyingOnNode ); + return currentlyCopying; + } + } ); + finishedCopyToNode( task, nodeLocation ); + } + + private void copyToNode( Task task, NodeLocation node ){ + synchronized ( this ) { + List tasks = taskOnNodes.computeIfAbsent( node, x -> new LinkedList<>() ); + tasks.add( task ); + + List nodes = nodesForTask.computeIfAbsent( task, x -> new LinkedList<>() ); + nodes.add( node ); + } + } + + private void finishedCopyToNode( Task task, NodeLocation node ){ + synchronized ( this ) { + List tasks = taskOnNodes.get( node ); + if ( tasks != null ) { + tasks.remove( task ); + } + + List nodes = nodesForTask.get( task ); + if ( nodes != null ) { + nodes.remove( node ); + } + } + } + + public List getTasksOnNode( NodeLocation nodeLocation ) { + synchronized ( this ) { + return taskOnNodes.getOrDefault( nodeLocation, new LinkedList<>() ); + } + } + + public int getNumberOfNodesForTask( Task task ) { + final List nodeLocations; + synchronized ( this ) { + nodeLocations = nodesForTask.get( task ); + } + return nodeLocations == null ? 0 : nodeLocations.size(); + } + + public Map getCurrentlyCopyingTasksOnNode() { + Map result = new HashMap<>(); + for ( Map.Entry> entry : taskOnNodes.entrySet() ) { + result.put( entry.getKey(), entry.getValue().size() ); + } + return result; + } + + public void addAlignment( final Map nodeFileAlignment, Task task, NodeWithAlloc node ) { + for (Map.Entry entry : nodeFileAlignment.entrySet()) { + final CurrentlyCopyingOnNode map = get(node.getNodeLocation()); + for ( FilePath filePath : entry.getValue().getFilesToCopy()) { + if ( entry.getKey() != node.getNodeLocation() ) { + map.add( filePath.getPath(), task, entry.getKey() ); + } + } + } + } + + + +} diff --git a/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopyingOnNode.java b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopyingOnNode.java new file mode 100644 index 00000000..d98c669d --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/copying/CurrentlyCopyingOnNode.java @@ -0,0 +1,70 @@ +package cws.k8s.scheduler.util.copying; + +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.location.Location; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Keeps track of what is currently being copied to a node. + * This is used to avoid copying the same data to a node multiple times. + */ +public class CurrentlyCopyingOnNode { + + private final Map< String, CopySource> currentlyCopying = new HashMap<>(); + + public void add( String path, Task task, Location location ) { + synchronized ( this.currentlyCopying ) { + if ( !this.currentlyCopying.containsKey( path ) ) { + this.currentlyCopying.put( path, new CopySource( task, location ) ); + } else { + throw new IllegalStateException( "Already copying " + path ); + } + } + } + + public boolean isCurrentlyCopying( String path ) { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.containsKey( path ); + } + } + + public CopySource getCopySource( String path ) { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.get( path ); + } + } + + void add ( CurrentlyCopyingOnNode currentlyCopying ) { + synchronized ( this.currentlyCopying ) { + this.currentlyCopying.putAll( currentlyCopying.currentlyCopying ); + } + } + + void remove ( CurrentlyCopyingOnNode currentlyCopying ) { + synchronized ( this.currentlyCopying ) { + this.currentlyCopying.keySet().removeAll( currentlyCopying.currentlyCopying.keySet() ); + } + } + + public Set getAllFilesCurrentlyCopying() { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.keySet(); + } + } + + public boolean isEmpty() { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.isEmpty(); + } + } + + @Override + public String toString() { + synchronized ( this.currentlyCopying ) { + return this.currentlyCopying.keySet().toString(); + } + } +} diff --git a/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java new file mode 100644 index 00000000..5b8eb523 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/score/CalculateScore.java @@ -0,0 +1,14 @@ +package cws.k8s.scheduler.util.score; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; + +public interface CalculateScore { + + /** + * Score must be higher than 0 + * @return + */ + long getScore( Task task, NodeWithAlloc location, long inputSize ); + +} diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java new file mode 100644 index 00000000..79694528 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeRankScore.java @@ -0,0 +1,16 @@ +package cws.k8s.scheduler.util.score; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; + +public class FileSizeRankScore extends FileSizeScore { + + @Override + public long getScore( Task task, NodeWithAlloc location, long size ) { + //Add one to avoid becoming zero + final int rank = task.getProcess().getRank() + 1; + final long rankFactor = 100_000_000_000_000L * rank; // long would allow a rank of 92233 + return super.getScore( task, location, size ) + rankFactor ; + } + +} diff --git a/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java new file mode 100644 index 00000000..52c5da88 --- /dev/null +++ b/src/main/java/cws/k8s/scheduler/util/score/FileSizeScore.java @@ -0,0 +1,13 @@ +package cws.k8s.scheduler.util.score; + +import cws.k8s.scheduler.model.NodeWithAlloc; +import cws.k8s.scheduler.model.Task; + +public class FileSizeScore implements CalculateScore { + + @Override + public long getScore( Task task, NodeWithAlloc location, long size ) { + //add one to prefer two tasks which sum up to the same score otherwise + return size + 1; + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 00000000..bdca2cf6 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,4 @@ +logging.level.root=INFO +logging.level.fonda=INFO +server.port=8080 +server.servlet.context-path=/ \ No newline at end of file diff --git a/src/main/resources/copystrategies/ftp.py b/src/main/resources/copystrategies/ftp.py new file mode 100644 index 00000000..da9191a0 --- /dev/null +++ b/src/main/resources/copystrategies/ftp.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +import ftplib +import json +import logging as log +import os +import shutil +import signal +import sys +import time +import urllib.request +import urllib.parse +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from time import sleep + +exitIfFileWasNotFound = True +CLOSE = False +UNEXPECTED_ERROR = "Unexpected error" +EXIT = 0 +log.basicConfig( + format='%(levelname)s: %(message)s', + level=log.DEBUG, + handlers=[ + log.FileHandler(".command.init.log"), + log.StreamHandler() + ] +) +trace = {} +traceFilePath = ".command.scheduler.trace" +errors = 0 + + +def myExit(code): + global EXIT + EXIT = code + global CLOSE + CLOSE = True + writeTrace(trace) + exit(EXIT) + + +def close(signalnum, syncFile): + log.info("Killed: %s", str(signalnum)) + closeWithWarning(50, syncFile) + + +def closeWithWarning(errorCode, syncFile): + syncFile.write('##FAILURE##\n') + syncFile.flush() + syncFile.close() + myExit(errorCode) + + +def getIP(node, dns, execution): + ip = urllib.request.urlopen(dns + "daemon/" + execution + "/" + node).read() + return str(ip.decode("utf-8")) + + +# True if the file was deleted or did not exist +def clearLocation(path, dst=None): + if os.path.exists(path): + log.debug("Delete %s", path) + if os.path.islink(path): + if dst is not None and os.readlink(path) == dst: + return False + else: + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + return True + + +def getFTP(node, currentIP, dns, execution, syncFile): + global errors + connectionProblem = 0 + while connectionProblem < 8: + try: + if currentIP is None: + log.info("Request ip for node: %s", node) + ip = getIP(node, dns, execution) + else: + ip = currentIP + log.info("Try to connect to %s", ip) + ftp = ftplib.FTP(ip, timeout=10) + ftp.login("root", "password") + ftp.set_pasv(True) + ftp.encoding = 'utf-8' + log.info("Connection established") + return ftp + except ConnectionRefusedError: + errors += 1 + log.warning("Connection refused! Try again...") + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + connectionProblem += 1 + time.sleep(2 ** connectionProblem) + closeWithWarning(8, syncFile) + + +def closeFTP(ftp): + global errors + if ftp is None: + return + try: + ftp.quit() + ftp.close() + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + + +def downloadFile(ftp, filename, size, index, node, syncFile): + global errors + log.info("Download %s [%s/%s] - %s", node, str(index).rjust(len(str(size))), str(size), filename) + try: + syncFile.write("S-" + filename + '\n') + clearLocation(filename) + Path(filename[:filename.rindex("/")]).mkdir(parents=True, exist_ok=True) + start = time.time() + ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write, 102400) + end = time.time() + sizeInMB = os.path.getsize(filename) / 1000000 + delta = (end - start) + log.info("Speed: %.3f Mbit/s", sizeInMB / delta) + return sizeInMB, delta + except ftplib.error_perm as err: + errors += 1 + if str(err) == "550 Failed to open file.": + log.warning("File not found node: %s file: %s", node, filename) + if exitIfFileWasNotFound: + closeWithWarning(40, syncFile) + except FileNotFoundError: + errors += 1 + log.warning("File not found node: %s file: %s", node, filename) + if exitIfFileWasNotFound: + closeWithWarning(41, syncFile) + except EOFError: + errors += 1 + log.warning("It seems the connection was lost! Try again...") + return None + except BaseException: + errors += 1 + log.exception(UNEXPECTED_ERROR) + return None + return 0, 0 + + +def download(node, currentIP, files, dns, execution, syncFile): + ftp = None + size = len(files) + global CLOSE + sizeInMB = 0 + downloadTime = 0 + while not CLOSE and len(files) > 0: + if ftp is None: + ftp = getFTP(node, currentIP, dns, execution, syncFile) + currentIP = None + filename = files[0] + index = size - len(files) + 1 + result = downloadFile(ftp, filename, size, index, node, syncFile) + if result is None: + ftp = None + continue + sizeInMB += result[0] + downloadTime += result[1] + files.pop(0) + syncFile.write("F-" + filename + '\n') + closeFTP(ftp) + return node, sizeInMB / downloadTime + + +def waitForFiles(syncFilePath, files, startTime): + # wait max. 60 seconds + while True: + if startTime + 60 < time.time(): + return False + if os.path.isfile(syncFilePath): + break + log.debug("Wait for file creation") + time.sleep(0.1) + + # Read file live + with open(syncFilePath, 'r') as syncFileTask: + current = [] + while len(files) > 0: + data = syncFileTask.read() + if not data: + time.sleep(0.3) + else: + for d in data: + if d != "\n": + current.append(d) + else: + text = ''.join(current) + current = [] + if text.startswith("S-"): + continue + if text == "##FAILURE##": + log.debug("Read FAILURE in %s", syncFilePath) + myExit(51) + if text == "##FINISHED##": + log.debug("Read FINISHED in " + syncFilePath + " before all files were found") + myExit(52) + log.debug("Look for " + text[:2] + " with " + text[2:] + " in " + str(files)) + if text[:2] == "F-" and text[2:] in files: + files.remove(text[2:]) + if len(files) == 0: + return True + return len(files) == 0 + + +def loadConfig(configFilePath): + if not os.path.isfile(configFilePath): + log.error("Config file not found: %s", configFilePath) + myExit(102) + + with open(configFilePath, 'r') as configFile: + config = json.load(configFile) + + os.makedirs(config["syncDir"], exist_ok=True) + return config + + +def registerSignal(syncFile): + signal.signal(signal.SIGINT, lambda signalnum, handler: close(signalnum, syncFile)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: close(signalnum, syncFile)) + + +def registerSignal2(): + signal.signal(signal.SIGINT, lambda signalnum, handler: myExit(1)) + signal.signal(signal.SIGTERM, lambda signalnum, handler: myExit(1)) + + +def generateSymlinks(symlinks): + for s in symlinks: + src = s["src"] + dst = s["dst"] + if clearLocation(src, dst): + Path(src[:src.rindex("/")]).mkdir(parents=True, exist_ok=True) + try: + os.symlink(dst, src) + except FileExistsError: + log.warning("File exists: %s -> %s", src, dst) + + +def downloadAllData(data, dns, execution, syncFile): + global trace + throughput = [] + with ThreadPoolExecutor(max_workers=max(10, len(data))) as executor: + futures = [] + for d in data: + files = d["files"] + node = d["node"] + currentIP = d["currentIP"] + futures.append(executor.submit(download, node, currentIP, files, dns, execution, syncFile)) + lastNum = -1 + while len(futures) > 0: + if lastNum != len(futures): + log.info("Wait for %d threads to finish", len(futures)) + lastNum = len(futures) + for f in futures[:]: + if f.done(): + throughput.append(f.result()) + futures.remove(f) + sleep(0.1) + trace["scheduler_init_throughput"] = "\"" + ",".join("{}:{:.3f}".format(*x) for x in throughput) + "\"" + + +def waitForDependingTasks(waitForFilesOfTask, startTime, syncDir): + # Now check for files of other tasks + for waitForTask in waitForFilesOfTask: + waitForFilesSet = set(waitForFilesOfTask[waitForTask]) + if not waitForFiles(syncDir + waitForTask, waitForFilesSet, startTime): + log.error(syncDir + waitForTask + " was not successful") + myExit(200) + + +def writeTrace(dataMap): + if sys.argv[1] == 'true': + global errors + if len(dataMap) == 0 or errors > 0: + return + with open(traceFilePath, "a") as traceFile: + for d in dataMap: + traceFile.write(d + "=" + str(dataMap[d]) + "\n") + traceFile.write("scheduler_init_errors=" + str(errors) + "\n") + + +def finishedDownload(dns, execution, taskname): + try: + dns = dns + "downloadtask/" + execution + log.info("Request: %s", dns) + urllib.request.urlopen(dns, taskname.encode( "utf-8" )) + except BaseException as err: + log.exception(err) + myExit(100) + + +def run(): + global trace + startTime = time.time() + log.info("Start to setup the environment") + config = loadConfig(".command.inputs.json") + + dns = config["dns"] + execution = config["execution"] + data = config["data"] + symlinks = config["symlinks"] + taskname = config["hash"] + + with open(config["syncDir"] + config["hash"], 'w') as syncFile: + registerSignal(syncFile) + syncFile.write('##STARTED##\n') + syncFile.flush() + startTimeSymlinks = time.time() + generateSymlinks(symlinks) + trace["scheduler_init_symlinks_runtime"] = int((time.time() - startTimeSymlinks) * 1000) + syncFile.write('##SYMLINKS##\n') + syncFile.flush() + startTimeDownload = time.time() + downloadAllData(data, dns, execution, syncFile) + trace["scheduler_init_download_runtime"] = int((time.time() - startTimeDownload) * 1000) + if CLOSE: + log.debug("Closed with code %s", str(EXIT)) + exit(EXIT) + log.info("Finished Download") + syncFile.write('##FINISHED##\n') + registerSignal2() + + finishedDownload(dns, execution, taskname) + + startTimeDependingTasks = time.time() + waitForDependingTasks(config["waitForFilesOfTask"], startTime, config["syncDir"]) + trace["scheduler_init_depending_tasks_runtime"] = int((time.time() - startTimeDependingTasks) * 1000) + log.info("Waited for all tasks") + + runtime = int((time.time() - startTime) * 1000) + trace["scheduler_init_runtime"] = runtime + writeTrace(trace) + + +if __name__ == '__main__': + run() diff --git a/src/main/resources/copystrategies/nothing.sh b/src/main/resources/copystrategies/nothing.sh new file mode 100644 index 00000000..03538c00 --- /dev/null +++ b/src/main/resources/copystrategies/nothing.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/dag/DAGTest.java b/src/test/java/cws/k8s/scheduler/dag/DAGTest.java index 4b2777c1..ed8a1575 100644 --- a/src/test/java/cws/k8s/scheduler/dag/DAGTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/DAGTest.java @@ -1,31 +1,32 @@ package cws.k8s.scheduler.dag; import lombok.extern.slf4j.Slf4j; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.*; import java.util.stream.Collectors; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertThrows; @Slf4j -public class DAGTest { +class DAGTest { private void compare(int uid, List vertices, int[] ancestorIds, int[] descendantsIds ){ //noinspection OptionalGetWithoutIsPresent final Vertex vertex = vertices.stream().filter(v -> v.getUid() == uid).findFirst().get(); if( vertex.getAncestors() == null ){ - assertNull( ancestorIds ); + Assertions.assertNull( ancestorIds ); } else { final int[] a = vertex.getAncestors().stream().mapToInt( Vertex::getUid ).sorted().toArray(); - assertArrayEquals("Compare Ancestors for uid: " + uid, ancestorIds, a); + Assertions.assertArrayEquals( ancestorIds, a, "Compare Ancestors for uid: " + uid ); } if( vertex.getDescendants() == null ){ - assertNull( descendantsIds ); + Assertions.assertNull( descendantsIds ); } else { final int[] d = vertex.getDescendants().stream().mapToInt(Vertex::getUid).sorted().toArray(); - assertArrayEquals("Compare Descendants for uid: " + uid, descendantsIds, d); + Assertions.assertArrayEquals( descendantsIds, d, "Compare Descendants for uid: " + uid ); } } @@ -53,16 +54,16 @@ private HashSet setTo(DAG dag, int idA, int[]... otherIds){ private void isSameEdge( HashSet expectedIn, HashSet expectedOut, Vertex vertex ){ final String[] in = setToArrayEdges(expectedIn); final String[] out = setToArrayEdges(expectedOut); - assertArrayEquals( "In of " + vertex.getLabel(), in, setToArrayEdges(vertex.getIn()) ); - assertArrayEquals( "Out of " + vertex.getLabel(), out, setToArrayEdges(vertex.getOut()) ); + Assertions.assertArrayEquals( in, setToArrayEdges(vertex.getIn()), "In of " + vertex.getLabel() ); + Assertions.assertArrayEquals( out, setToArrayEdges(vertex.getOut()), "Out of " + vertex.getLabel() ); } private void isSameProc( HashSet expectedAncestors, HashSet expectedDescendants, Vertex vertex, int rank ){ final String[] descendants = setToArrayProcesses(expectedDescendants); final String[] ancestors = setToArrayProcesses(expectedAncestors); - assertArrayEquals( "Descendants of " + vertex.getLabel(), descendants, setToArrayProcesses(vertex.getDescendants()) ); - assertArrayEquals( "Ancestors of " + vertex.getLabel(), ancestors, setToArrayProcesses(vertex.getAncestors()) ); - assertEquals( rank, vertex.getRank() ); + Assertions.assertArrayEquals( descendants, setToArrayProcesses(vertex.getDescendants()), "Descendants of " + vertex.getLabel() ); + Assertions.assertArrayEquals( ancestors, setToArrayProcesses(vertex.getAncestors()), "Ancestors of " + vertex.getLabel() ); + Assertions.assertEquals( rank, vertex.getRank() ); } private String[] setToArrayProcesses(Set set) { @@ -132,7 +133,7 @@ public List genEdgeList(){ } @Test - public void testRelations() { + void testRelations() { for (int q = 0; q < 500 ; q++) { final DAG dag = new DAG(); List vertexList = genVertexList(); @@ -144,7 +145,7 @@ public void testRelations() { } @Test - public void testRelations2() { + void testRelations2() { for (int q = 0; q < 500 ; q++) { final DAG dag = new DAG(); List vertexList = genVertexList(); @@ -164,7 +165,7 @@ public void testRelations2() { } @Test - public void smallTest(){ + void smallTest(){ final DAG dag = new DAG(); @@ -188,26 +189,26 @@ public void smallTest(){ dag.registerVertices( vertexList ); dag.registerEdges( inputEdges ); - assertEquals( new HashSet<>(), a.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), a.getAncestors() ); final HashSet descA = new HashSet<>(); descA.add( b ); - assertEquals( descA, a.getDescendants() ); + Assertions.assertEquals( descA, a.getDescendants() ); final HashSet ancB = new HashSet<>(); ancB.add( a ); - assertEquals( new HashSet<>(), b.getDescendants() ); - assertEquals( ancB, b.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), b.getDescendants() ); + Assertions.assertEquals( ancB, b.getAncestors() ); - assertEquals( ancB, filter.getAncestors() ); - assertEquals( descA, filter.getDescendants() ); + Assertions.assertEquals( ancB, filter.getAncestors() ); + Assertions.assertEquals( descA, filter.getDescendants() ); - assertEquals( new HashSet<>(),o.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), o.getAncestors() ); final HashSet descO = new HashSet<>(); descO.add( a ); descO.add( b ); - assertEquals( descO, o.getDescendants() ); + Assertions.assertEquals( descO, o.getDescendants() ); } @@ -223,7 +224,7 @@ public void smallTest(){ * */ @Test - public void deleteTest(){ + void deleteTest(){ final DAG dag = new DAG(); @@ -258,37 +259,37 @@ public void deleteTest(){ dag.removeVertices( filter2.getUid() ); } - assertEquals( new HashSet<>(), a.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), a.getAncestors() ); if ( i == 0 ) { - assertEquals( new HashSet<>( Arrays.asList( new Edge(2,a,filter), new Edge(3,a,filter2) ) ), a.getOut() ); + Assertions.assertEquals( new HashSet<>( Arrays.asList( new Edge(2,a,filter), new Edge(3,a,filter2) ) ), a.getOut() ); } else { - assertEquals( new HashSet<>(List.of(new Edge(2, a, filter))), a.getOut() ); + Assertions.assertEquals( new HashSet<>( List.of(new Edge(2, a, filter))), a.getOut() ); } final HashSet descA = new HashSet<>(); descA.add( b ); - assertEquals( descA, a.getDescendants() ); + Assertions.assertEquals( descA, a.getDescendants() ); final HashSet ancB = new HashSet<>(); ancB.add( a ); - assertEquals( new HashSet<>(), b.getDescendants() ); - assertEquals( ancB, b.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), b.getDescendants() ); + Assertions.assertEquals( ancB, b.getAncestors() ); if ( i == 0 ) { - assertEquals( new HashSet<>( Arrays.asList( new Edge(4,filter,b), new Edge(5,filter2,b) ) ), b.getIn() ); + Assertions.assertEquals( new HashSet<>( Arrays.asList( new Edge(4,filter,b), new Edge(5,filter2,b) ) ), b.getIn() ); } else { - assertEquals( new HashSet<>(List.of(new Edge(4, filter, b))), b.getIn() ); + Assertions.assertEquals( new HashSet<>( List.of(new Edge(4, filter, b))), b.getIn() ); } - assertEquals( ancB, filter.getAncestors() ); - assertEquals( descA, filter.getDescendants() ); + Assertions.assertEquals( ancB, filter.getAncestors() ); + Assertions.assertEquals( descA, filter.getDescendants() ); - assertEquals( new HashSet<>(),o.getAncestors() ); + Assertions.assertEquals( new HashSet<>(), o.getAncestors() ); final HashSet descO = new HashSet<>(); descO.add( a ); descO.add( b ); - assertEquals( descO, o.getDescendants() ); + Assertions.assertEquals( descO, o.getDescendants() ); } } @@ -297,7 +298,7 @@ public void deleteTest(){ @Test - public void deleteTest2() { + void deleteTest2() { final DAG dag = createDag(); dag.removeEdges( 3 ); @@ -325,7 +326,7 @@ public void deleteTest2() { } @Test - public void deleteTest3() { + void deleteTest3() { final DAG dag = createDag(); dag.removeVertices( dag.getByProcess("c").getUid(), dag.getByProcess("e").getUid() ); @@ -360,7 +361,7 @@ public void deleteTest3() { * e */ @Test - public void deleteTest4() { + void deleteTest4() { final DAG dag = createDag(); final Process a = new Process("a", 1); @@ -415,7 +416,7 @@ public void deleteTest4() { * e */ @Test - public void deleteTest5() { + void deleteTest5() { final DAG dag = new DAG(); final Process a = new Process("a", 1); diff --git a/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java index 7b2ab3f4..c33f9ab3 100644 --- a/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java +++ b/src/test/java/cws/k8s/scheduler/dag/VertexDeserializerTest.java @@ -2,28 +2,25 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import org.junit.Test; -import org.springframework.boot.test.autoconfigure.json.JsonTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; -import static org.junit.Assert.assertEquals; - -@JsonTest -public class VertexDeserializerTest { +class VertexDeserializerTest { @Test - public void testDeserialize() throws IOException { + void testDeserialize() throws IOException { String json = "{\"label\":\"a\", \"type\":\"PROCESS\", \"uid\":0}"; final ObjectMapper objectMapper = new ObjectMapper(); SimpleModule module = new SimpleModule(); module.addDeserializer(Vertex.class, new VertexDeserializer()); objectMapper.registerModule(module); final Vertex process = objectMapper.readValue(json, Vertex.class); - assertEquals( "a", process.getLabel() ); - assertEquals( Type.PROCESS, process.getType() ); - assertEquals( 0, process.getUid() ); + Assertions.assertEquals( "a", process.getLabel() ); + Assertions.assertEquals( Type.PROCESS, process.getType() ); + Assertions.assertEquals( 0, process.getUid() ); System.out.println( process ); } diff --git a/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java new file mode 100644 index 00000000..0497831e --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/InputFileCollectorTest.java @@ -0,0 +1,233 @@ +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.InputEdge; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.*; +import cws.k8s.scheduler.model.taskinputs.PathFileLocationTriple; +import cws.k8s.scheduler.model.taskinputs.SymlinkInput; +import cws.k8s.scheduler.model.taskinputs.TaskInputs; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +@Slf4j +class InputFileCollectorTest { + + private LocationWrapper location11; + private LocationWrapper location12; + private LocationWrapper location21; + private LocationWrapper location23; + private LocationWrapper location34; + private LocationWrapper location35; + private LocationWrapper location44; + private LocationWrapper location45; + + @BeforeEach + public void init(){ + location11 = new LocationWrapper( NodeLocation.getLocation("Node1"), 1, 100); + location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 1, 101); + location21 = new LocationWrapper(NodeLocation.getLocation("Node1"), 1, 102); + location23 = new LocationWrapper(NodeLocation.getLocation("Node3"), 1, 103); + location34 = new LocationWrapper(NodeLocation.getLocation("Node4"), 1, 104); + location35 = new LocationWrapper(NodeLocation.getLocation("Node5"), 1, 105); + location44 = new LocationWrapper(NodeLocation.getLocation("Node4"), 1, 106); + location45 = new LocationWrapper(NodeLocation.getLocation("Node5"), 1, 107); + } + + @Test + void getInputsOfTaskTest() throws NoAlignmentFoundException { + + final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; + final String root2 = "/workdir/01/db62d739d658b839f07a1a77d877d1/"; + final String root3 = "/workdir/02/db62d739d658b839f07a1a77d877d2/"; + + final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); + + final Path path1 = Paths.get(root + "a.txt"); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) hierarchyWrapper.getFile(path1), List.of(location11, location12)); + + final Path path2 = Paths.get(root + "a/b.txt"); + Assertions.assertNotNull( hierarchyWrapper.addFile(path2, location21) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path2, location23) ); + final PathFileLocationTriple file2 = new PathFileLocationTriple(path2, (RealHierarchyFile) hierarchyWrapper.getFile(path2), List.of(location21, location23)); + + final Path path3 = Paths.get(root2 + "a/b.txt"); + Assertions.assertNotNull( hierarchyWrapper.addFile(path3, location34) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path3, location35) ); + final PathFileLocationTriple file3 = new PathFileLocationTriple(path3, (RealHierarchyFile) hierarchyWrapper.getFile(path3), List.of(location34, location35)); + + final Path path4 = Paths.get(root3 + "b/c.txt"); + Assertions.assertNotNull( hierarchyWrapper.addFile(path4, location44) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path4, location45) ); + final PathFileLocationTriple file4 = new PathFileLocationTriple(path4, (RealHierarchyFile) hierarchyWrapper.getFile(path4), List.of(location44, location45)); + + final Path path5 = Paths.get(root2 + "b/c.txt"); + Assertions.assertTrue( hierarchyWrapper.addSymlink( path5, path4 ) ); + final SymlinkInput symlink = new SymlinkInput( path5, path4 ); + + + InputFileCollector inputFileCollector = new InputFileCollector( hierarchyWrapper ); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + dag.registerVertices(vertexList); + + final TaskConfig taskConfig = new TaskConfig("processA"); + final List> fileInputs = taskConfig.getInputs().fileInputs; + final InputParam a = new InputParam<>("a", new FileHolder(null, root, null)); + final InputParam b = new InputParam<>("b", new FileHolder(null, root2 + "a/b.txt", null)); + + fileInputs.add( a ); + fileInputs.add( b ); + final Task task = new Task(taskConfig, dag); + + final Set expected = new HashSet<>(List.of(file1, file2, file3)); + + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); + List inputsOfTaskFiles = inputsOfTask.getFiles(); + Assertions.assertEquals( 3, inputsOfTaskFiles.size() ); + Assertions.assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); + + //Not existing file + final InputParam c = new InputParam<>("c", new FileHolder(null, root2 + "a.txt", null)); + fileInputs.add( c ); + + inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); + inputsOfTaskFiles = inputsOfTask.getFiles(); + Assertions.assertEquals( 3, inputsOfTaskFiles.size() ); + Assertions.assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); + + //Symlink + final InputParam d = new InputParam<>("d", new FileHolder(null, root2 + "b/c.txt", null)); + fileInputs.add( d ); + expected.add( file4 ); + + inputsOfTask = inputFileCollector.getInputsOfTask(task, Integer.MAX_VALUE); + inputsOfTaskFiles = inputsOfTask.getFiles(); + Assertions.assertEquals( 4, inputsOfTaskFiles.size() ); + Assertions.assertEquals( expected, new HashSet<>(inputsOfTaskFiles) ); + Assertions.assertEquals( Set.of( symlink ), new HashSet<>(inputsOfTask.getSymlinks()) ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); + + + } + + @Test + void getInputsOfTaskTestExcludeNodes() throws NoAlignmentFoundException { + + final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; + + final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); + + InputFileCollector inputFileCollector = new InputFileCollector( hierarchyWrapper ); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + vertexList.add(new Process("processC", 3)); + dag.registerVertices(vertexList); + dag.registerEdges( List.of( new InputEdge(1,1,2),new InputEdge(2,2,3) ) ); + + final TaskConfig taskConfigA = new TaskConfig("processA"); + final Task taskA = new Task(taskConfigA, dag); + + final TaskConfig taskConfigB = new TaskConfig("processB"); + final Task taskB = new Task(taskConfigB, dag); + + final TaskConfig taskConfigC = new TaskConfig("processC"); + final List> fileInputs = taskConfigC.getInputs().fileInputs; + final InputParam a = new InputParam<>("a.txt", new FileHolder(null, root, null)); + fileInputs.add( a ); + final Task taskC = new Task(taskConfigC, dag); + + LocationWrapper location13 = new LocationWrapper(NodeLocation.getLocation("Node3"), 2, 102, taskB); + LocationWrapper location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 2, 102, taskC); + LocationWrapper location11 = new LocationWrapper(NodeLocation.getLocation("Node1"), 2, 102, taskA); + location11.use(); + + final Path path1 = Paths.get(root + "a.txt"); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location13) ); + final HierarchyFile file = hierarchyWrapper.getFile(path1); + Assertions.assertNotNull( file ); + final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); + + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC, Integer.MAX_VALUE); + List inputsOfTaskFiles = inputsOfTask.getFiles(); + Assertions.assertEquals( 1, inputsOfTaskFiles.size() ); + Assertions.assertEquals( file1, inputsOfTaskFiles.get(0) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertTrue( inputsOfTask.hasExcludedNodes() ); + Assertions.assertFalse( inputsOfTask.canRunOnLoc( NodeLocation.getLocation( "Node1" ) ) ); + } + + @Test + void getInputsOfTaskTestNotExcludeNodes() throws NoAlignmentFoundException { + + final String root = "/workdir/00/db62d739d658b839f07a1a77d877df/"; + + final HierarchyWrapper hierarchyWrapper = new HierarchyWrapper("/workdir/"); + + InputFileCollector inputFileCollector = new InputFileCollector( hierarchyWrapper ); + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + vertexList.add(new Process("processC", 3)); + dag.registerVertices(vertexList); + dag.registerEdges( List.of( new InputEdge(1,1,2),new InputEdge(2,2,3) ) ); + + final TaskConfig taskConfigA = new TaskConfig("processA"); + final Task taskA = new Task(taskConfigA, dag); + + final TaskConfig taskConfigB = new TaskConfig("processB"); + final Task taskB = new Task(taskConfigB, dag); + + final TaskConfig taskConfigC = new TaskConfig("processC"); + final List> fileInputs = taskConfigC.getInputs().fileInputs; + final InputParam a = new InputParam<>("a.txt", new FileHolder(null, root, null)); + fileInputs.add( a ); + final Task taskC = new Task(taskConfigC, dag); + + LocationWrapper location13 = new LocationWrapper(NodeLocation.getLocation("Node3"), 2, 102, taskB); + LocationWrapper location12 = new LocationWrapper(NodeLocation.getLocation("Node2"), 2, 102, taskC); + LocationWrapper location11 = new LocationWrapper(NodeLocation.getLocation("Node1"), 2, 102, taskA); + location13.use(); + + final Path path1 = Paths.get(root + "a.txt"); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location11) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location12) ); + Assertions.assertNotNull( hierarchyWrapper.addFile(path1, location13) ); + final HierarchyFile file = hierarchyWrapper.getFile(path1); + Assertions.assertNotNull( file ); + final PathFileLocationTriple file1 = new PathFileLocationTriple(path1, (RealHierarchyFile) file, List.of(location12, location13)); + log.info(file1.toString()); + + TaskInputs inputsOfTask = inputFileCollector.getInputsOfTask(taskC, Integer.MAX_VALUE); + List inputsOfTaskFiles = inputsOfTask.getFiles(); + Assertions.assertEquals( 1, inputsOfTaskFiles.size() ); + Assertions.assertEquals( file1, inputsOfTaskFiles.get(0) ); + Assertions.assertTrue( inputsOfTask.getSymlinks().isEmpty() ); + Assertions.assertFalse( inputsOfTask.hasExcludedNodes() ); + + } + +} \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java b/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java new file mode 100644 index 00000000..9f0716bf --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/RequirementsTest.java @@ -0,0 +1,220 @@ +package cws.k8s.scheduler.model; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.HashSet; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +class RequirementsTest { + + @Test + void ZERO() { + final Requirements zero = ImmutableRequirements.ZERO; + assertEquals( zero, new Requirements( BigDecimal.ZERO, BigDecimal.ZERO ) ); + assertEquals( BigDecimal.ZERO, zero.getCpu() ); + assertEquals( BigDecimal.ZERO, zero.getRam() ); + } + + @Test + void addToThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 7 ), BigDecimal.valueOf( 8 ) ); + Requirements c = a.addToThis( b ); + assertSame( a, c ); + assertEquals( BigDecimal.valueOf( 12 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 14 ), c.getRam() ); + } + + @Test + void add() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 7 ), BigDecimal.valueOf( 8 ) ); + Requirements c = a.add( b ); + assertNotSame( a, c ); + assertEquals( BigDecimal.valueOf( 12 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 14 ), c.getRam() ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + @Test + void addRAMtoThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + a.addRAMtoThis( BigDecimal.valueOf( 8 ) ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 14 ), a.getRam() ); + } + + @Test + void addCPUtoThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + a.addCPUtoThis( BigDecimal.valueOf( 8 ) ); + assertEquals( BigDecimal.valueOf( 13 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + @Test + void subFromThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + Requirements c = a.subFromThis( b ); + assertSame( a, c ); + assertEquals( BigDecimal.valueOf( 2 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 1 ), c.getRam() ); + } + + @Test + void sub() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + Requirements c = a.sub( b ); + assertNotSame( a, c ); + assertEquals( BigDecimal.valueOf( 2 ), c.getCpu() ); + assertEquals( BigDecimal.valueOf( 1 ), c.getRam() ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + @Test + void multiply() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = a.multiply( BigDecimal.valueOf( 3 ) ); + assertNotSame( a, b ); + assertEquals( BigDecimal.valueOf( 15 ) , b.getCpu() ); + assertEquals( BigDecimal.valueOf( 18 ) , b.getRam() ); + assertEquals( BigDecimal.valueOf( 5 ) , a.getCpu() ); + assertEquals( BigDecimal.valueOf( 6 ) , a.getRam() ); + } + + @Test + void multiplyToThis() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = a.multiplyToThis( BigDecimal.valueOf( 3 ) ); + assertSame( a, b ); + assertEquals( BigDecimal.valueOf( 15 ), b.getCpu() ); + assertEquals( BigDecimal.valueOf( 18 ), b.getRam() ); + } + + @Test + void testHigherOrEquals() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertTrue( a.higherOrEquals( b ) ); + assertFalse( b.higherOrEquals( a ) ); + assertTrue( a.higherOrEquals( a ) ); + assertTrue( b.higherOrEquals( b ) ); + } + + @Test + void testClone() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = a.clone(); + assertNotSame( a, b ); + assertEquals( a.getCpu(), b.getCpu() ); + assertEquals( a.getRam(), b.getRam() ); + } + + + @Test + void smaller() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smaller( b ) ); + assertTrue( b.smaller( a ) ); + assertFalse( a.smaller( a ) ); + assertFalse( b.smaller( b ) ); + + Requirements c = new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smaller( c ) ); + assertFalse( c.smaller( a ) ); + assertFalse( c.smaller( c ) ); + } + + @Test + void smallerEquals() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smallerEquals( b ) ); + assertTrue( b.smallerEquals( a ) ); + assertTrue( a.smallerEquals( a ) ); + assertTrue( b.smallerEquals( b ) ); + + Requirements c = new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 5 ) ); + assertFalse( a.smallerEquals( c ) ); + assertFalse( c.smallerEquals( a ) ); + assertTrue( c.smallerEquals( c ) ); + + Requirements d = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertTrue( a.smallerEquals( d ) ); + assertTrue( d.smallerEquals( a ) ); + assertTrue( d.smallerEquals( d ) ); + } + + @Test + void getCpu() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertEquals( BigDecimal.valueOf( 5 ), a.getCpu() ); + } + + @Test + void getRam() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertEquals( BigDecimal.valueOf( 6 ), a.getRam() ); + } + + + @Test + void atLeastOneBigger() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 3 ), BigDecimal.valueOf( 5 ) ); + assertTrue( a.atLeastOneBigger( b ) ); + assertFalse( b.atLeastOneBigger( a ) ); + assertFalse( a.atLeastOneBigger( a ) ); + assertFalse( b.atLeastOneBigger( b ) ); + + Requirements c = new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 5 ) ); + assertTrue( a.atLeastOneBigger( c ) ); + assertTrue( c.atLeastOneBigger( a ) ); + assertFalse( c.atLeastOneBigger( c ) ); + + Requirements d = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertFalse( a.atLeastOneBigger( d ) ); + assertFalse( d.atLeastOneBigger( a ) ); + assertFalse( d.atLeastOneBigger( d ) ); + } + + @Test + void hashSetTest() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + final HashSet requirements = new HashSet<>(); + requirements.add( a ); + requirements.add( b ); + assertEquals( 1, requirements.size() ); + requirements.add( a.add( b ) ); + assertEquals( 2, requirements.size() ); + requirements.add( a.add( b ).sub( new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ) ) ); + assertEquals( 2, requirements.size() ); + } + + @Test + void equalsTest() { + Requirements a = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + Requirements b = new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 6 ) ); + assertEquals( a, b ); + assertEquals( b, a ); + assertEquals( a, a ); + assertEquals( b, b ); + assertNotEquals( null, a ); + assertNotEquals( a, new Object() ); + assertNotEquals( a, new Requirements( BigDecimal.valueOf( 5 ), BigDecimal.valueOf( 7 ) ) ); + assertNotEquals( a, new Requirements( BigDecimal.valueOf( 6 ), BigDecimal.valueOf( 6 ) ) ); + assertNotSame( a, b ); + } + + +} \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java b/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java new file mode 100644 index 00000000..5dbdf32a --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/TaskResultParserTest.java @@ -0,0 +1,321 @@ +package cws.k8s.scheduler.model; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.location.NodeLocation; +import cws.k8s.scheduler.model.location.hierachy.LocationWrapper; +import cws.k8s.scheduler.model.outfiles.OutputFile; +import cws.k8s.scheduler.model.outfiles.PathLocationWrapperPair; +import cws.k8s.scheduler.model.outfiles.SymlinkOutput; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; + +@Slf4j +class TaskResultParserTest { + + private final String NL = "\n"; + + private Path storeData( String[] inputdata, String[] outputdata ){ + Path tmpDir = null; + + try { + + tmpDir = Files.createTempDirectory( "results" ); + log.info("Path: {}", tmpDir ); + BufferedWriter pw = new BufferedWriter( new FileWriter( tmpDir.resolve(".command.infiles").toString() ) ); + for (String s : inputdata) { + pw.write( s ); + pw.write( '\n' ); + } + pw.close(); + + pw = new BufferedWriter( new FileWriter( tmpDir.resolve(".command.outfiles").toString() ) ); + for (String s : outputdata) { + pw.write( s ); + pw.write( '\n' ); + } + pw.close(); + } catch ( Exception ignored ){} + return tmpDir; + } + + private DAG dag; + + @BeforeEach + public void before(){ + dag = new DAG(); + List vertexList = new LinkedList<>(); + final Process a = new Process("P1", 2); + vertexList.add(a); + dag.registerVertices(vertexList); + } + + + @Test + void test1(){ + + String[] infiles = { + "1636549091222254911", + "/tmp/nxf.3iuGDWr6Id;1;;4096;directory", + "file.txt;1;/pvcdata/testfile.txt;6;regular file", + ".command.err;1;;0;regular empty file", + ".command.out;1;;0;regular empty file" + }; + + String[] outfiles = { + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt;1;/pvcdata/testfile.txt;13;regular file;-;1636549091230400136;1636549091230935687", + "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt;1;/pvcdata/testfile.txt;13;regular file;-;1636549091230676133;1636549091230588932", + }; + + final Path path = storeData(infiles, outfiles); + + final Task task = new Task(new TaskConfig("P1"), dag); + + final TaskResultParser taskResultParser = new TaskResultParser(); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, NodeLocation.getLocation("Node1"), false, task); + + log.info("{}", newAndUpdatedFiles); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/pvcdata/testfile.txt"), new LocationWrapper(NodeLocation.getLocation("Node1"), 1636549091230L, 13, task)) ); + expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file.txt", "/pvcdata/testfile.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/1e/249602b469f33100bb4a65203cb650/file1.txt", "/pvcdata/testfile.txt") ); + + Assertions.assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + void test2(){ + + String[] infiles = { + "1636720962171455407", + "/tmp/nxf.IANFIlM3Kv;1;;4096;directory", + "file.txt;1;/pvcdata/testfile.txt;0;regular empty file", + ".command.err;1;;0;regular empty file", + ".command.out;1;;0;regular empty file" + }; + + String[] outfiles = { + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t;1;;4096;directory;-;1636720962223377564;1636720962223927449", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt;1;;0;regular empty file;-;1636720962223648023;1636720962223304687", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt;1;;2;regular file;-;1636720962223821586;1636720962223246135", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt;1;;2;regular file;-;1636720962223035735;1636720962223994896", + "/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt;1;;2;regular file;-;1636720962223163152;1636720962223785315" + }; + + final Path path = storeData(infiles, outfiles); + + final TaskResultParser taskResultParser = new TaskResultParser(); + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Task task = new Task( new TaskConfig("P1"), dag); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); + + log.info("{}", newAndUpdatedFiles.toArray()); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/filenew.txt"), new LocationWrapper(node1, 1636720962223L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/b.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/c.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/ac/fbbbb38f79bf684ddd54a3e190e8fa/t/a.txt"), new LocationWrapper(node1, 1636720962223L, 2, task )) ); + + Assertions.assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + void test3(){ + + final TaskResultParser taskResultParser = new TaskResultParser(); + + String[] infiles = { + "1636728138941757518", + "/tmp/nxf.9J6Y5mcXRD;1;;directory", + "t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;directory", + "t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;regular empty file", + "t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;regular file", + "t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;regular file", + "t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;regular file", + ".command.err;1;;regular empty file", + ".command.out;1;;regular empty file" + }; + + String[] outfiles = { + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t;4096;directory;-;1636728137009129688;1636728136973190140", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt;0;regular empty file;-;1636728136969424493;1636728136969738095", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt;2;regular file;-;1636728136973929300;1636728136973771278", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt;2;regular file;-;1636728136973979318;1636728136973691162", + "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt;1;/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt;4;regular file;-;1636728136973542646;1636728140993004364" + }; + + final Path path = storeData(infiles, outfiles); + + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Task task = new Task( new TaskConfig("P1"), dag ); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); + + log.info("{}", newAndUpdatedFiles); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt"), new LocationWrapper( node1, 1636728140993L, 4, task )) ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/b.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/b.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/c.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/c.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/a.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/a.txt") ); + expected.add( new SymlinkOutput( "/localdata/localwork/a2/f105825376b35dd6918824136adbf6/t/filenew.txt", "/localdata/localwork/3c/b1c1be1266dfd66b81a9942383e266/t/filenew.txt") ); + + Assertions.assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + void test4(){ + + String[] infiles = { + "---", + "---", + "wvdb;1;/input/FORCE2NXF-Rangeland/inputdata/wvdb;0;directory", + "wvdb/WVP_2020-12-16.txt;1;;420379;regular file", + }; + + final TaskResultParser taskResultParser = new TaskResultParser(); + Set inputData = new HashSet(); + taskResultParser.processInput(Arrays.stream(infiles),inputData); + final HashSet expected = new HashSet<>(); + log.info("{}", inputData); + } + + @Test + void test5(){ + + final TaskResultParser taskResultParser = new TaskResultParser(); + + String[] infiles = { + "1656925624971202680", + "/localdata/localwork/scratch/nxf.VX0s8jr100/;1;;directory", + ".command.err;1;;regular file", + ".command.out;1;;regular file" + }; + + String[] outfiles = { + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5;1;;4096;directory;1656925625099630891;1656925625099376329;1656925625219623172", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5;1;;4096;directory;1656925625099699272;1656925625099839483;1656925625219393367", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a;1;;4096;directory;1656925625151870586;1656925625151443598;1656925625207978673", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/file.txt;1;;0;regular empty file;1656925625019383511;1656925625019813106;1656925625019125236", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b;1;;4096;directory;1656925625163388439;1656925625163351190;1656925625195325883", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/file.txt;1;;0;regular empty file;1656925625023099305;1656925625023645725;1656925625023215656", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c;1;;4096;directory;1656925625171561519;1656925625171633246;1656925625175783837", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/file.txt;1;;0;regular empty file;1656925625023206366;1656925625023824131;1656925625023970434", + "/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c/file.txt;1;;0;regular empty file;1656925625023995796;1656925625023981665;1656925625023053106" + }; + + final Path path = storeData(infiles, outfiles); + + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Task task = new Task( new TaskConfig("P1"), dag ); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); + + log.info("{}", newAndUpdatedFiles); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/file.txt"), new LocationWrapper( node1, 1656925625019L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/9c/33932e89127a7f1bb09bc2ca0453c5/a/b/c/file.txt"), new LocationWrapper( node1, 1656925625023L, 0, task )) ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + void test6(){ + + final TaskResultParser taskResultParser = new TaskResultParser(); + + String[] infiles = { + "1658328555702440734", + "/localdata/localwork/scratch/nxf.QqsTpl7jcy/", + ".command.err;1;;0;regular file;1658328555691575355;1658328555691575355;1658328555691575355", + "JK2782_TGGCCGATCAACGA_L008_R2_001.fastq.gz.tengrand.fq.gz;1;/pvcdata/work/stage/da/08d4344c92af6ca44284ba0e17b484/JK2782_TGGCCGATCAACGA_L008_R2_001.fastq.gz.tengrand.fq.gz;111;symbolic link;1658328555659575265;1658328555659575265;1658328555659575265", + ".command.out;1;;0;regular file;1658328555691575355;1658328555691575355;1658328555691575355", + "JK2782_TGGCCGATCAACGA_L008_R1_001.fastq.gz.tengrand.fq.gz;1;/pvcdata/work/stage/db/a8a5b9f6d77f6fa9878977a278bc69/JK2782_TGGCCGATCAACGA_L008_R1_001.fastq.gz.tengrand.fq.gz;111;symbolic link;1658328555651575241;1658328555651575241;1658328555651575241", + + }; + + String[] outfiles = { + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9;1;;4096;directory;1658328570467617203;1658328570355616887;1658328570467617203", + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/a;1;;249228;regular file;1658328570427617091;1658328569675614962;1658328569683614985", + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/b;1;;260036;regular file;1658328570467617203;1658328559663586610;1658328569659614917", + "/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/c;1;;265581;regular file;1658328570391616989;1658328559231585386;1658328569707615053", + }; + + Set inputData = new HashSet(); + taskResultParser.processInput(Arrays.stream(infiles),inputData); + log.info("{}", inputData); + + final Path path = storeData(infiles, outfiles); + + final NodeLocation node1 = NodeLocation.getLocation("Node1"); + final Task task = new Task( new TaskConfig("P1"), dag ); + final Set newAndUpdatedFiles = taskResultParser.getNewAndUpdatedFiles(path, node1, false, task); + + log.info("{}", newAndUpdatedFiles); + + final HashSet expected = new HashSet<>(); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/a"), new LocationWrapper( node1, 1658328569683L, 249228, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/b"), new LocationWrapper( node1, 1658328569659L, 260036, task )) ); + expected.add( new PathLocationWrapperPair(Path.of("/localdata/localwork/30/adb97e8cffa8a086608565fb4c4ea9/c"), new LocationWrapper( node1, 1658328569707L, 265581, task )) ); + Assertions.assertEquals( expected, newAndUpdatedFiles ); + + } + + @Test + void test7(){ + long a = System.currentTimeMillis(); + long b = (long) (1658328570467617203l / 1.0E6); + log.info("{} {}", a - b, b); + } + + + @Test + void fileWalker() throws IOException { + FileVisitor visitor = new FileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + log.info("preVisitDirectory: {}", dir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + log.info("visitFile: {}", file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + log.info("visitFileFailed: {}", file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + log.info("postVisitDirectory: {}", dir); + return FileVisitResult.CONTINUE; + } + }; + log.info("{}", Files.walkFileTree(Paths.get("C:\\Users\\Fabian Lehmann\\Documents\\VBox\\Paper\\test"), EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor)); + } + +} \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java b/src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java new file mode 100644 index 00000000..24a51cdd --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/cluster/GroupClusterTest.java @@ -0,0 +1,37 @@ +package cws.k8s.scheduler.model.cluster; + +import org.apache.commons.collections4.map.HashedMap; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GroupClusterTest { + + @Test + void jaccard() { + Set a = Set.of("a", "b", "c"); + Set b = Set.of("a", "b", "c"); + assertEquals(1.0, GroupCluster.calculateJaccardSimilarityCoefficient(a, b)); + + a = Set.of("a", "b", "c"); + b = Set.of("a", "b", "d"); + assertEquals(0.6666666666666666, GroupCluster.calculateJaccardSimilarityCoefficient(a, b)); + + a = Set.of("a", "b", "c"); + b = Set.of("d", "e", "f"); + assertEquals(0.01, GroupCluster.calculateJaccardSimilarityCoefficient(a, b)); + } + + @Test + void testTasksNotOnNode() { + Map labels = new HashedMap<>(); + for (int i = 0; i < 10; i++) { + System.out.println( labels.merge( "a", 1, Integer::sum ) ); + } + + } + +} \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java new file mode 100644 index 00000000..5a429d63 --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/HierarchyWrapperTest.java @@ -0,0 +1,255 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.model.location.NodeLocation; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +class HierarchyWrapperTest { + + final String workdir = "/folder/localworkdir/"; + HierarchyWrapper hw; + List files; + Collection result; + final String temporaryDir = workdir + "/ab/abcdasdasd/test/./abcd/"; + LocationWrapper node1; + DAG dag; + + private LocationWrapper getLocationWrapper( String location ){ + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, new Task( new TaskConfig("processA"), dag) ); + } + + @BeforeEach + public void init() { + dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + dag.registerVertices(vertexList); + + node1 = getLocationWrapper("Node1"); + + hw = new HierarchyWrapper(workdir); + files = new LinkedList<>(); + + files.add( Paths.get(temporaryDir + "test" )); + files.add( Paths.get(temporaryDir + "file.abc" )); + files.add( Paths.get(temporaryDir + "a/test.abc" )); + files.add( Paths.get(temporaryDir + "a/file.abc" )); + files.add( Paths.get(temporaryDir + "d/test" )); + files.add( Paths.get(temporaryDir + "d/e/file.txt" )); + files.add( Paths.get(temporaryDir + "b/c/test.abc" )); + files.add( Paths.get(temporaryDir + "bc/file.abc" )); + + files.parallelStream().forEach( x -> Assertions.assertNotNull( hw.addFile(x, node1) ) ); + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); + compare( files, result); + } + + private void compare( List a, Collection b){ + Assertions.assertEquals( new HashSet<>(a.stream().map( Path::normalize).collect( Collectors.toList())), new HashSet<>(b) ); + Assertions.assertEquals( a.size(), b.size() ); + } + + + @Test + void getAllFilesInDir() { + log.info("{}", this.result); + Collection result; + + result = hw.getAllFilesInDir( Paths.get(temporaryDir + "b/" )).keySet(); + compare(List.of(Paths.get(temporaryDir + "b/c/test.abc")), result); + + log.info("{}", result); + + result = hw.getAllFilesInDir(Paths.get( temporaryDir + "b" )).keySet(); + compare(List.of(Paths.get(temporaryDir + "b/c/test.abc")), result); + + log.info("{}", result); + } + + @Test + void getAllFilesInFile() { + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(temporaryDir + "test/" ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/" ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(temporaryDir + "d/test/c/" ) ) ); + } + + @Test + void getAllFilesOutOfScrope() { + Assertions.assertNull( hw.getAllFilesInDir( Paths.get("/somewhere/" ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/" ) ) ); + } + + @Test + void getAllFilesinAllWorkdirs() { + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(workdir ) ) ); + Assertions.assertNull( hw.getAllFilesInDir( Paths.get(workdir + "/ab/" ) ) ); + } + + @Test + void createFileinFile() { + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "test/b.txt"), node1) ); + + files.remove( Paths.get(temporaryDir + "test" )); + files.add( Paths.get(temporaryDir + "test/b.txt") ); + + result = hw.getAllFilesInDir( Paths.get(temporaryDir) ).keySet(); + compare( files, result); + } + + @Test + void createFileinWorkdir() { + Assertions.assertNull( hw.addFile( Paths.get(workdir + "ab/b.txt"), node1) ); + + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); + compare( files, result); + } + + @Test + void createFileOutOfScope() { + Assertions.assertNull( hw.addFile( Paths.get("/somewhere/test.txt"), node1) ); + Assertions.assertNull( hw.addFile( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/test.txt"), node1) ); + + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); + compare( files, result); + } + + @Test + void createFileTwice() { + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "bc/file.abc"), node1) ); + + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); + compare( files, result); + } + + @Test + void createFileButWasFolder() { + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "bc"),node1) ); + + files.remove( Paths.get(temporaryDir + "bc/file.abc") ); + files.add( Paths.get(temporaryDir + "bc") ); + + result = hw.getAllFilesInDir(Paths.get(temporaryDir)).keySet(); + compare( files, result); + + } + + @Test + void testParallelAdd() { + System.gc(); + long intialMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + + List files = new LinkedList<>(); + String wd = workdir + "ab/abcdefg/"; + Map> m = new HashMap<>(); + + int iters = 1_000_000; + + for (int i = 0; i < iters; i++) { + String p = wd + (i % 3) + "/" + (i % 4); + Path file = Paths.get(p + "/" + "file-" + i); + files.add( file ); + final List currentData = m.getOrDefault(p, new LinkedList<>()); + currentData.add(file); + m.put(p, currentData); + } + + System.gc(); + + long finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + + log.info( "Memory input: {}mb", (finalMem - intialMem) / 1024 / 1024 ); + + intialMem = finalMem; + + files.parallelStream().forEach( x -> Assertions.assertNotNull( hw.addFile(x, getLocationWrapper("Node1")) ) ); + + finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + log.info( "Memory hierachy: {}mb", (finalMem - intialMem) / 1024 / 1024 ); + System.gc(); + finalMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + log.info( "Memory hierachy after gc: {}mb", (finalMem - intialMem) / 1024 / 1024 ); + log.info( "Memory per entry: {}b", (finalMem - intialMem) / iters ); + + result = hw.getAllFilesInDir( Paths.get( wd ) ).keySet(); + compare(files, result); + + for (Map.Entry> entry : m.entrySet()) { + result = hw.getAllFilesInDir( Paths.get( entry.getKey() )).keySet(); + compare( entry.getValue(), result); + } + } + + @Test + void testGetFile() { + + hw = new HierarchyWrapper(workdir); + files = new LinkedList<>(); + + files.add( Paths.get( temporaryDir + "test" ) ); + files.add( Paths.get( temporaryDir + "file.abc" ) ); + files.add( Paths.get( temporaryDir + "a/test.abc" ) ); + files.add( Paths.get( temporaryDir + "a/file.abc" ) ); + files.add( Paths.get( temporaryDir + "d/test" ) ); + files.add( Paths.get( temporaryDir + "d/e/file.txt" ) ); + files.add( Paths.get( temporaryDir + "b/c/test.abc" ) ); + + int index = 0; + LocationWrapper[] lw = new LocationWrapper[ files.size() ]; + for ( Path file : files) { + lw[index] = getLocationWrapper( "Node" + index ); + Assertions.assertNotNull( hw.addFile(file,lw[index++]) ); + } + + result = hw.getAllFilesInDir( Paths.get(temporaryDir) ).keySet(); + compare( files, result); + + index = 0; + for ( Path file : files) { + RealHierarchyFile realFile = (RealHierarchyFile) hw.getFile(file); + + final LocationWrapper[] locations = realFile.getLocations(); + Assertions.assertEquals( lw[index++], locations[0] ); + Assertions.assertEquals( 1, locations.length ); + } + } + + @Test + void testGetFileOutOfScope() { + Assertions.assertNull( hw.getFile( Paths.get("/file.txt" ) ) ); + Assertions.assertNull( hw.getFile( Paths.get("/somewhere/on/the/machine/very/deep/hierarchy/file.txt" ) ) ); + } + + @Test + void testGetFileWorkdir() { + Assertions.assertNull( hw.getFile( Paths.get(workdir + "ab/test.txt" )) ); + } + + @Test + void testGetFileButIsDir() { + final HierarchyFile file = hw.getFile(Paths.get(temporaryDir + "d")); + Assertions.assertNotNull( file ); + Assertions.assertTrue( file.isDirectory() ); + } + + @Test + void testFileIsNowDir(){ + Assertions.assertNotNull( hw.addFile( Paths.get(temporaryDir + "d"), getLocationWrapper("nodeXY") ) ); + result = hw.getAllFilesInDir( Paths.get(temporaryDir ) ).keySet(); + Assertions.assertNotNull( hw.getFile( Paths.get(temporaryDir + "d" )) ); + Assertions.assertNull( hw.getFile( Paths.get(temporaryDir + "d/e/file.txt" )) ); + } +} \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java new file mode 100644 index 00000000..098cd273 --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/model/location/hierachy/RealHierarchyFileTest.java @@ -0,0 +1,434 @@ +package cws.k8s.scheduler.model.location.hierachy; + +import cws.k8s.scheduler.dag.DAG; +import cws.k8s.scheduler.dag.InputEdge; +import cws.k8s.scheduler.dag.Process; +import cws.k8s.scheduler.dag.Vertex; +import cws.k8s.scheduler.model.Task; +import cws.k8s.scheduler.model.TaskConfig; +import cws.k8s.scheduler.model.location.LocationType; +import cws.k8s.scheduler.model.location.NodeLocation; +import org.apache.commons.collections4.iterators.PermutationIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +class RealHierarchyFileTest { + + private Task processA; + private Task processB; + private Task processC; + private Task processD; + private Task processE; + + @BeforeEach + public void before(){ + DAG dag = new DAG(); + List vertexList = new LinkedList<>(); + vertexList.add(new Process("processA", 1)); + vertexList.add(new Process("processB", 2)); + vertexList.add(new Process("processC", 3)); + vertexList.add(new Process("processD", 4)); + vertexList.add(new Process("processE", 5)); + dag.registerVertices(vertexList); + List edgeList = new LinkedList<>(); + edgeList.add( new InputEdge(1,1,2) ); + edgeList.add( new InputEdge(2,2,4) ); + edgeList.add( new InputEdge(3,4,5) ); + edgeList.add( new InputEdge(4,1,3) ); + dag.registerEdges(edgeList); + processA = new Task( new TaskConfig("processA"), dag); + processB = new Task( new TaskConfig("processB"), dag); + processC = new Task( new TaskConfig("processC"), dag); + processD = new Task( new TaskConfig("processD"), dag); + processE = new Task( new TaskConfig("processE"), dag); + } + + private LocationWrapper getLocationWrapper( String location ){ + return new LocationWrapper( NodeLocation.getLocation(location), 0, 100, processA ); + } + + + @Test + void addLocation() { + + List locations = new LinkedList<>(); + + + final LocationWrapper node1 = getLocationWrapper("Node1"); + locations.add(node1); + final RealHierarchyFile realFile = new RealHierarchyFile(node1); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + final LocationWrapper node2 = getLocationWrapper("Node2"); + locations.add(node2); + realFile.addOrUpdateLocation( false, node2); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + final LocationWrapper node3 = getLocationWrapper("Node3"); + locations.add(node3); + realFile.addOrUpdateLocation( false, node3); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + final LocationWrapper node1New = getLocationWrapper("Node1"); + realFile.addOrUpdateLocation( false, node1New); + Assertions.assertArrayEquals( locations.toArray(), realFile.getLocations() ); + + } + + @Test + void addEmptyLocation() { + final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("node1") ); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( false, null )); + assertThrows(IllegalArgumentException.class, () -> realFile.addOrUpdateLocation( true, null )); + } + + @Test + void addInParallel() { + final LocationWrapper node0 = getLocationWrapper("Node0"); + final RealHierarchyFile realFile = new RealHierarchyFile(node0); + + List locations = new LinkedList<>(); + + for (int i = 1; i < 10_000; i++) { + locations.add( getLocationWrapper( "Node" + i ) ); + } + + Collections.shuffle(locations); + + locations.parallelStream().forEach( r -> realFile.addOrUpdateLocation( false, r ) ); + + locations.add( node0 ); + Assertions.assertEquals( new HashSet<>(locations), new HashSet<>( Arrays.asList(realFile.getLocations())) ); + + } + + @Test + void changeFile() { + + final LocationWrapper node0 = getLocationWrapper("Node0"); + final RealHierarchyFile realFile = new RealHierarchyFile(node0); + final LocationWrapper node1 = getLocationWrapper("Node1"); + realFile.addOrUpdateLocation( false, node1); + final LocationWrapper node2 = getLocationWrapper("Node2"); + realFile.addOrUpdateLocation( false, node2); + final LocationWrapper node3 = getLocationWrapper("Node3"); + realFile.addOrUpdateLocation( false, node3); + + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("NodeNew"), 5, 120, processB ); + realFile.addOrUpdateLocation( false, nodeNew ); + LocationWrapper[] expected = { node0, node1, node2, node3, nodeNew }; + Assertions.assertArrayEquals( expected, realFile.getLocations() ); + + } + + @Test + void overwriteFile() { + final LinkedList results = new LinkedList<>(); + final LocationWrapper node0 = getLocationWrapper("Node0"); + results.add( node0 ); + Assertions.assertTrue( node0.isActive() ); + final RealHierarchyFile realFile = new RealHierarchyFile(node0); + Assertions.assertArrayEquals( results.toArray(), realFile.getLocations() ); + Assertions.assertTrue( node0.isActive() ); + final LocationWrapper node1 = getLocationWrapper("Node1"); + results.add( node1 ); + realFile.addOrUpdateLocation( true, node1); + Assertions.assertArrayEquals( results.toArray(), realFile.getLocations() ); + Assertions.assertFalse( node0.isActive() ); + Assertions.assertTrue( node1.isActive() ); + final LocationWrapper node2 = getLocationWrapper("Node2"); + Assertions.assertArrayEquals( results.toArray(), realFile.getLocations() ); + results.add( node2 ); + realFile.addOrUpdateLocation( true, node2); + Assertions.assertFalse( node0.isActive() ); + Assertions.assertFalse( node1.isActive() ); + Assertions.assertTrue( node2.isActive() ); + } + + @Test + void changeFileOnExistingLocation() { + + final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("Node0") ); + + final LocationWrapper nodeNew = new LocationWrapper(NodeLocation.getLocation("Node0"), 5, 120, processA ); + realFile.addOrUpdateLocation( false, nodeNew ); + LocationWrapper[] expected = { nodeNew }; + Assertions.assertArrayEquals( expected, realFile.getLocations() ); + + final LocationWrapper nodeNew2 = new LocationWrapper(NodeLocation.getLocation("Node0"), 6, 170, processB ); + realFile.addOrUpdateLocation( false, nodeNew2 ); + LocationWrapper[] expected2 = { nodeNew2 }; + Assertions.assertArrayEquals( expected2, realFile.getLocations() ); + + } + + @Test + void isFile() { + final RealHierarchyFile realFile = new RealHierarchyFile( getLocationWrapper("Node0") ); + Assertions.assertFalse( realFile.isDirectory() ); + Assertions.assertFalse( realFile.isSymlink() ); + } + + + @Test + void getFilesForTaskTest() throws InterruptedException, NoAlignmentFoundException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc1, loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc2, loc3, loc4)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); + } + + locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + realFile.addOrUpdateLocation(false, next.get(4)); + + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( List.of(loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); + } + } + + @Test + void getFilesForTaskTestInitFiles() throws InterruptedException, NoAlignmentFoundException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + realFile.addOrUpdateLocation(false, next.get(4)); + + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); + } + + } + + @Test + void getFilesForTaskTestMultipleInitFiles() throws InterruptedException, NoAlignmentFoundException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis() - 2, 2, processA ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, processB ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node5"), System.currentTimeMillis() - 5, 2, null); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis() - 5, 2, processB ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node4"), System.currentTimeMillis() - 2, 2, processC ); + + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealHierarchyFile realFile = new RealHierarchyFile(next.get(0)); + realFile.addOrUpdateLocation(false, next.get(1)); + realFile.addOrUpdateLocation(false, next.get(2)); + realFile.addOrUpdateLocation(false, next.get(3)); + realFile.addOrUpdateLocation(false, next.get(4)); + + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processA).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processB).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processC).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processD).getMatchingLocations()) ); + Assertions.assertEquals( new HashSet<>( Arrays.asList(loc3, loc4, loc5)), new HashSet<>(realFile.getFilesForTask(processE).getMatchingLocations()) ); + } + } + + @Test + void getFilesForTaskTestDifferentAncestors() throws InterruptedException, NoAlignmentFoundException { + + final CountDownLatch waiter = new CountDownLatch(1); + + final NodeLocation location1 = NodeLocation.getLocation("Node1"); + LocationWrapper loc1 = new LocationWrapper( location1, System.currentTimeMillis() - 2, 2, null ); + loc1.use(); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis() - 1, 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + final NodeLocation location3 = NodeLocation.getLocation("Node3"); + LocationWrapper loc3 = new LocationWrapper( location3, System.currentTimeMillis() - 5, 2, null ); + loc3.use(); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + final NodeLocation location4 = NodeLocation.getLocation("Node4"); + LocationWrapper loc4 = new LocationWrapper( location4, System.currentTimeMillis() - 2, 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + final NodeLocation location5 = NodeLocation.getLocation("Node5"); + LocationWrapper loc5 = new LocationWrapper( location5, System.currentTimeMillis() - 5, 2, null ); + loc5.use(); + + List locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + PermutationIterator permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealHierarchyFile realFile = new RealHierarchyFile( next.get(0) ); + realFile.addOrUpdateLocation( false, next.get(1) ); + realFile.addOrUpdateLocation( false, next.get(2) ); + realFile.addOrUpdateLocation( false, next.get(3) ); + realFile.addOrUpdateLocation( false, next.get(4) ); + + for (Task task : List.of(processA, processB, processC, processD, processE)) { + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask( task ); + Assertions.assertEquals( new HashSet<>( List.of( loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); + Assertions.assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); + } + + } + + + while( loc4.getCreateTime() != loc5.getCreateTime() ){ + loc4 = new LocationWrapper( location4, System.currentTimeMillis() - 2, 2, null ); + loc5 = new LocationWrapper( location5, System.currentTimeMillis() - 5, 2, null ); + } + + locationWrapperList = List.of(loc1, loc2, loc3, loc4, loc5); + + permutationIterator = new PermutationIterator<>(locationWrapperList); + while ( permutationIterator.hasNext() ) { + final List next = permutationIterator.next(); + RealHierarchyFile realFile = new RealHierarchyFile( next.get(0) ); + realFile.addOrUpdateLocation( false, next.get(1) ); + realFile.addOrUpdateLocation( false, next.get(2) ); + realFile.addOrUpdateLocation( false, next.get(3) ); + realFile.addOrUpdateLocation( false, next.get(4) ); + + for (Task task : List.of(processA, processB, processC, processD, processE)) { + final RealHierarchyFile.MatchingLocationsPair filesForTask = realFile.getFilesForTask( task ); + Assertions.assertEquals( new HashSet<>( List.of( loc4, loc5 ) ), new HashSet<>( filesForTask.getMatchingLocations() ) ); + Assertions.assertEquals( new HashSet<>( List.of( location1, location3 ) ), new HashSet<>( filesForTask.getExcludedNodes() ) ); + } + + } + + } + + @Test + void getLastLocationTest() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); + + RealHierarchyFile realFile = new RealHierarchyFile( loc1 ); + Assertions.assertEquals( loc1, realFile.getLastUpdate( LocationType.NODE ) ); + realFile.addOrUpdateLocation(false, loc4 ); + Assertions.assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE ) ); + realFile.addOrUpdateLocation(false, loc2 ); + Assertions.assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE ) ); + realFile.addOrUpdateLocation(false, loc3 ); + Assertions.assertEquals( loc4, realFile.getLastUpdate( LocationType.NODE ) ); + realFile.addOrUpdateLocation(false, loc5 ); + Assertions.assertEquals( loc5, realFile.getLastUpdate( LocationType.NODE ) ); + + realFile.addOrUpdateLocation(true, loc2 ); + Assertions.assertEquals( loc2, realFile.getLastUpdate( LocationType.NODE ) ); + + } + + @Test + void getLocationWrapperForNodeTest() throws InterruptedException { + + final CountDownLatch waiter = new CountDownLatch(1); + + LocationWrapper loc1 = new LocationWrapper( NodeLocation.getLocation("Node1"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc2 = new LocationWrapper( NodeLocation.getLocation("Node2"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc3 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc4 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); + Assertions.assertFalse( waiter.await(2, TimeUnit.MILLISECONDS ) ); + LocationWrapper loc5 = new LocationWrapper( NodeLocation.getLocation("Node3"), System.currentTimeMillis(), 2, null ); + + RealHierarchyFile realFile = new RealHierarchyFile(loc1); + Assertions.assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); + realFile.addOrUpdateLocation(false, loc4 ); + Assertions.assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + realFile.addOrUpdateLocation(false, loc3 ); + Assertions.assertEquals( loc4, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + realFile.addOrUpdateLocation(false, loc2 ); + Assertions.assertEquals( loc2, realFile.getLocationWrapper( NodeLocation.getLocation("Node2") ) ); + realFile.addOrUpdateLocation(false, loc5 ); + Assertions.assertEquals( loc5, realFile.getLocationWrapper( NodeLocation.getLocation("Node3") ) ); + Assertions.assertEquals( loc1, realFile.getLocationWrapper( NodeLocation.getLocation("Node1") ) ); + + final NodeLocation node99 = NodeLocation.getLocation("Node99"); + assertThrows( RuntimeException.class, () -> realFile.getLocationWrapper( node99 ) ); + + } + + + +} \ No newline at end of file diff --git a/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java b/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java index 47b93ee8..25567e41 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/MemoryScalerTest.java @@ -5,7 +5,7 @@ import cws.k8s.scheduler.model.SchedulerConfig; import cws.k8s.scheduler.model.TaskMetrics; import cws.k8s.scheduler.prediction.predictor.TestTask; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; diff --git a/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java b/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java index 7f4d0c1c..e60165c2 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/offset/VarianceOffsetTest.java @@ -2,7 +2,7 @@ import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class VarianceOffsetTest { diff --git a/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java b/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java index 81295ac7..2cdde0f4 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/predictor/LinearPredictorSquaredLossTest.java @@ -1,42 +1,40 @@ package cws.k8s.scheduler.prediction.predictor; import org.jetbrains.annotations.NotNull; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class LinearPredictorSquaredLossTest { +class LinearPredictorSquaredLossTest { @Test - public void testOneTask() { + void testOneTask() { LinearPredictorSquaredLoss lp = getLinearPredictor(); lp.addTask( new TestTask( 1d, 1d ) ); - assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testTwoTasks() { + void testTwoTasks() { LinearPredictorSquaredLoss lp = getLinearPredictor(); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); - assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testPredict() { + void testPredict() { LinearPredictorSquaredLoss lp = getLinearPredictor(); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); lp.addTask( new TestTask( 3d, 3d ) ); - assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); - assertEquals( (Double) 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ) ); + Assertions.assertEquals( (Double) 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertEquals( (Double) 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ) ); } @Test - public void noData() { + void noData() { LinearPredictorSquaredLoss lp = getLinearPredictor(); - assertNull(lp.queryPrediction( new TestTask( 4d, 4d ) )); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @NotNull diff --git a/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java b/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java index 7b4a8c05..4e8b07bd 100644 --- a/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java +++ b/src/test/java/cws/k8s/scheduler/prediction/predictor/PolynomialPredictorTest.java @@ -3,58 +3,56 @@ import cws.k8s.scheduler.prediction.Predictor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; @Slf4j -public class PolynomialPredictorTest { +class PolynomialPredictorTest { @Test - public void testOneTask() { + void testOneTask() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); - assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testTwoTasks() { + void testTwoTasks() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); - assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void testThreeTasksLinear() { + void testThreeTasksLinear() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); lp.addTask( new TestTask( 3d, 3d ) ); - assertEquals( 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ), 0.0001 ); - assertEquals( 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 4d, lp.queryPrediction( new TestTask( 4d, 4d ) ), 0.0001 ); + Assertions.assertEquals( 0d, lp.queryPrediction( new TestTask( 0d, 0d ) ), 0.0001 ); } @Test - public void testThreeTasksPoly() { + void testThreeTasksPoly() { Predictor lp = getPolyPredictor(2); lp.addTask( new TestTask( 1d, 1d ) ); lp.addTask( new TestTask( 2d, 2d ) ); lp.addTask( new TestTask( 3d, 9d ) ); - assertEquals( 1d, lp.queryPrediction( new TestTask( 1d, 0d ) ), 0.0001 ); - assertEquals( 2d, lp.queryPrediction( new TestTask( 2d, 0d ) ), 0.0001 ); - assertEquals( 9d, lp.queryPrediction( new TestTask( 3d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 1d, lp.queryPrediction( new TestTask( 1d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 2d, lp.queryPrediction( new TestTask( 2d, 0d ) ), 0.0001 ); + Assertions.assertEquals( 9d, lp.queryPrediction( new TestTask( 3d, 0d ) ), 0.0001 ); } @Test - public void noData() { + void noData() { Predictor lp = getPolyPredictor(2); - assertNull(lp.queryPrediction( new TestTask( 4d, 4d ) )); + Assertions.assertNull( lp.queryPrediction( new TestTask( 4d, 4d ) ) ); } @Test - public void measure(){ + void measure(){ int iterations = 1000; Predictor lp1 = getPolyPredictor(2); TestTask[] tasks = new TestTask[iterations]; @@ -69,7 +67,7 @@ public void measure(){ } @Test - public void compareLinear(){ + void compareLinear(){ int iterations = 100; Predictor lp = getPolyPredictor(2); TestTask[] tasks = new TestTask[iterations]; @@ -83,12 +81,12 @@ public void compareLinear(){ if ( prediction1 == null ) { continue; } - assertEquals( task.y, lp.queryPrediction( task ), 1 ); + Assertions.assertEquals( task.y, lp.queryPrediction( task ), 1 ); } } @Test - public void comparePoly(){ + void comparePoly(){ int iterations = 1000; Predictor lp = getPolyPredictor(2); TestTask[] tasks = new TestTask[iterations]; @@ -103,11 +101,11 @@ public void comparePoly(){ if ( prediction1 == null ) { continue; } - assertEquals( task.y, lp.queryPrediction( task ), 1 ); + Assertions.assertEquals( task.y, lp.queryPrediction( task ), 1 ); } for ( int x = 0; x < 1000; x++ ) { final TestTask task = new TestTask( x, 0d ); - assertEquals( getY( x ), lp.queryPrediction( task ), 1 ); + Assertions.assertEquals( getY( x ), lp.queryPrediction( task ), 1 ); } } diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java index 9617757c..e16f13dc 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/FifoPrioritizeTest.java @@ -6,7 +6,8 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; class FifoPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java index 463414fc..699798ab 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/LeastFinishedFirstPrioritizeTest.java @@ -6,7 +6,7 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class LeastFinishedFirstPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java index 9067985c..bc036806 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/MaxInputPrioritizeTest.java @@ -6,7 +6,8 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; class MaxInputPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java index a4e20e0b..a4c539d6 100644 --- a/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java +++ b/src/test/java/cws/k8s/scheduler/scheduler/prioritize/RankMaxPrioritizeTest.java @@ -6,7 +6,7 @@ import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class RankMaxPrioritizeTest { diff --git a/src/test/java/cws/k8s/scheduler/util/SortedListTest.java b/src/test/java/cws/k8s/scheduler/util/SortedListTest.java new file mode 100644 index 00000000..ac2768a3 --- /dev/null +++ b/src/test/java/cws/k8s/scheduler/util/SortedListTest.java @@ -0,0 +1,138 @@ +package cws.k8s.scheduler.util; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +class SortedListTest { + + @Test + void constructorTest() { + final List integers = Arrays.asList( 8, 4, 1, 5 ); + final SortedList sortedList = new SortedList<>( integers ); + assertEquals( 1, sortedList.get( 0 ) ); + assertEquals( 4, sortedList.get( 1 ) ); + assertEquals( 5, sortedList.get( 2 ) ); + assertEquals( 8, sortedList.get( 3 ) ); + } + + @Test + void add() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + log.info( sortedList.toString() ); + sortedList.add( 8 ); + log.info( sortedList.toString() ); + assertEquals( 8, sortedList.get( 3 ) ); + sortedList.add( 3 ); + log.info( sortedList.toString() ); + assertEquals( 3, sortedList.get( 1 ) ); + sortedList.add( 1 ); + log.info( sortedList.toString() ); + assertEquals( 1, sortedList.get( 1 ) ); + sortedList.add( 5 ); + log.info( sortedList.toString() ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 3, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + + } + + @Test + void addWithDuplicates() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + sortedList.add( 8 ); + assertEquals( 8, sortedList.get( 3 ) ); + sortedList.add( 4 ); + assertEquals( 4, sortedList.get( 1 ) ); + assertEquals( 4, sortedList.get( 2 ) ); + sortedList.add( 1 ); + assertEquals( 1, sortedList.get( 1 ) ); + sortedList.add( 5 ); + log.info( sortedList.toString() ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 4, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + + } + + @Test + void addToEmptyList() { + final SortedList sortedList = new SortedList<>( new LinkedList< Integer >() ); + sortedList.add( 4 ); + log.info( sortedList.toString() ); + assertEquals( 4, sortedList.get( 0 ) ); + } + + @Test + void addAtEnd() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 5 ) ); + sortedList.add( 6 ); + log.info( sortedList.toString() ); + assertEquals( 5, sortedList.get( 0 ) ); + assertEquals( 6, sortedList.get( 1 ) ); + } + + @Test + void addAtBeginning() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 5 ) ); + sortedList.add( 4 ); + log.info( sortedList.toString() ); + assertEquals( 4, sortedList.get( 0 ) ); + assertEquals( 5, sortedList.get( 1 ) ); + } + + + @Test + void addAtMiddle() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 4, 6 ) ); + sortedList.add( 5 ); + log.info( sortedList.toString() ); + assertEquals( 4, sortedList.get( 0 ) ); + assertEquals( 5, sortedList.get( 1 ) ); + assertEquals( 6, sortedList.get( 2 ) ); + } + + @Test + void addAll() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + sortedList.addAll( Arrays.asList( 8, 3, 1, 5 ) ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 3, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + } + + @Test + void addAllWithDuplicates() { + final SortedList sortedList = new SortedList<>( Arrays.asList( 9, 0, 4, 7 ) ); + sortedList.addAll( Arrays.asList( 8, 4, 1, 5 ) ); + assertEquals( 0, sortedList.get( 0 ) ); + assertEquals( 1, sortedList.get( 1 ) ); + assertEquals( 4, sortedList.get( 2 ) ); + assertEquals( 4, sortedList.get( 3 ) ); + assertEquals( 5, sortedList.get( 4 ) ); + assertEquals( 7, sortedList.get( 5 ) ); + assertEquals( 8, sortedList.get( 6 ) ); + assertEquals( 9, sortedList.get( 7 ) ); + } +} \ No newline at end of file