Commit e90b8e18 authored by Daniel Seybold's avatar Daniel Seybold

Merge branch 'tsbs_integration_2' into 'master'

Tsbs integration

See merge request !2
parents cc53fde6 14f0f3df
Pipeline #62322 passed with stage
in 16 minutes and 54 seconds
package de.uulm.omi.dbms.workload.api;
import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
import de.uulm.omi.dbms.workload.api.WorkloadApiService;
import de.uulm.omi.dbms.workload.api.factories.WorkloadApiServiceFactory;
import io.swagger.annotations.ApiParam;
import io.swagger.jaxrs.*;
import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
import java.io.File;
import de.uulm.omi.cloudiator.dbms.evaluator.model.TPCCWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.TSBSLoadWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
import java.util.Map;
import java.util.List;
import de.uulm.omi.dbms.workload.api.NotFoundException;
import java.io.InputStream;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import javax.servlet.ServletConfig;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.*;
import javax.validation.constraints.*;
import java.io.File;
import java.io.IOException;
@Path("/workload")
......@@ -120,7 +102,7 @@ public class WorkloadApi {
public Response workloadTsbsLoadPost(@ApiParam(value = "the id of the workload execution task, used as prefix for result files",required=true) @QueryParam("taskId") String taskId
,@ApiParam(value = "the TSBS workload configuration" ,required=true) TSBSLoadWorkload workload
,@Context SecurityContext securityContext)
throws NotFoundException {
throws NotFoundException, IOException, InterruptedException {
return delegate.workloadTsbsLoadPost(taskId,workload,securityContext);
}
@POST
......
......@@ -13,6 +13,7 @@ import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
import java.io.IOException;
import java.util.List;
import de.uulm.omi.dbms.workload.api.NotFoundException;
......@@ -26,6 +27,6 @@ public abstract class WorkloadApiService {
public abstract Response workloadResultGet( @NotNull String taskId, @NotNull String workloadType,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadStatusGet( String applicationInstanceId,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadTpccPost( @NotNull String taskId,TPCCWorkload workload,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException, IOException, InterruptedException;
public abstract Response workloadYcsbPost( @NotNull String taskId,YCSBWorkload workload,SecurityContext securityContext) throws NotFoundException;
}
package de.uulm.omi.dbms.workload.api.impl;
import de.uulm.omi.cloudiator.dbms.evaluator.model.ProcessStatus;
import de.uulm.omi.cloudiator.dbms.evaluator.model.TPCCWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.TSBSLoadWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
import de.uulm.omi.dbms.workload.api.ApiResponseMessage;
import de.uulm.omi.dbms.workload.api.NotFoundException;
import de.uulm.omi.dbms.workload.api.WorkloadApiService;
......@@ -13,17 +8,20 @@ import de.uulm.omi.utils.TextFileLoader;
import de.uulm.omi.utils.WorkloadApiProperties;
import de.uulm.omi.workload.WorkloadProcess;
import de.uulm.omi.workload.tpcc.TPCCCommandBuilder;
import de.uulm.omi.workload.tsbs.TSBSCommandBuilder;
import de.uulm.omi.workload.ycsb.YCSBCommandBuilder;
import java.io.File;
import java.io.FileNotFoundException;
import java.math.BigDecimal;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.validation.constraints.NotNull;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.SecurityContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.Properties;
@javax.annotation.Generated(value = "io.swagger.codegen.languages.JavaJerseyServerCodegen", date = "2018-01-16T09:59:22.348Z")
public class WorkloadApiServiceImpl extends WorkloadApiService {
......@@ -46,7 +44,9 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
private static final String TPCC_BINARY = "./loadgen.sh";
//TSBS constants
private final String TSBS_RESULT_FOLDER = "TSBS/";
private final String TSBS_RESULT_FOLDER = "/tmp/"; // TODO CHANGE
private static final String TSBS_BINARY = "/opt/workloads/tsbs/bin/";
private static final String TSBS_SCRIPTS = "/opt/workloads/tsbs/src/github.com/timescale/tsbs/scripts/";
......@@ -151,7 +151,7 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
@Override
public Response workloadTsbsLoadPost(String taskId, TSBSLoadWorkload workload,
SecurityContext securityContext) throws NotFoundException {
SecurityContext securityContext) throws NotFoundException, IOException, InterruptedException {
//check if already a workloadProcess running
......@@ -167,22 +167,87 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
Properties properties = WorkloadApiProperties.getInstance().getProperties();
Long workloadTimestamp = System.currentTimeMillis();
//TODO: implement the load phase call for TSBS similar to the existing TPC and YCSB implementations
//TODO: check for the static constants as defined in the beginning
//TODO: check for additional static properties in the workloadAPI.proeprties file, properties can be accessed via the proeprties object
//TODO: based on the provided data two calls should be executed:
//TODO: first call should be the bash call tsbs_generate_data
workload.getTsbsDataConfig();
//TODO: second call should be NUM_WORKERS=2 BATCH_SIZE=30 BULK_DATA_DIR=/tmp scripts/load_influx.sh > /tmp/load_metrics.csv
workload.getDbEndpoints();
workload.getTsbsLoadConfig();
//TODO: either wait and check the state of the first call or combine both calls in a oneliner using &&
return null;
//Generate Data
String generateDataCommand= "";
generateDataCommand = TSBSCommandBuilder.generateDataCommand(workload.getTsbsDataConfig());
String resultFileName =
//properties.getProperty(WorkloadApiProperties.WORKLOAD_RESULTS_FOLDER) +
TSBS_RESULT_FOLDER
//+ taskId + "-"
+ workload.getTsbsDataConfig().getFormat() + "-data.gz";
generateDataCommand += " | gzip > " + resultFileName;
ProcessBuilder genDataProcess= new ProcessBuilder();
genDataProcess.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateDataCommand);
Process processData = genDataProcess.start();
System.out.println(generateDataCommand);
while (processData.isAlive()) {
Thread.sleep(1000);
}
System.out.println("finished data generation");
String fileToLoad = TSBS_RESULT_FOLDER /*+ taskId + "-"*/ + workload.getTsbsDataConfig().getFormat() + "-data.gz";
String queriesToExecute = TSBS_RESULT_FOLDER /*+ taskId + "-" + workload.getTsbsDataConfig().getFormat() + "-" + */ + workload.getTsbsQueryConfig().getQueryType() + ".gz";
//Generate Queries
String generateQueriesCommand = "";
generateQueriesCommand = TSBSCommandBuilder.generateQueries(workload.getTsbsQueryConfig());
String resultQueriesName =
TSBS_RESULT_FOLDER
//+ taskId + "-"
+ workload.getTsbsQueryConfig().getQueryType() + ".gz";
generateQueriesCommand += "| gzip > " + resultQueriesName;
System.out.println(generateQueriesCommand);
ProcessBuilder genQueryProcess = new ProcessBuilder();
genQueryProcess.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateQueriesCommand);
Process processGenQueries = genQueryProcess.start();
while (processGenQueries.isAlive()) {
Thread.sleep(1000);
}
System.out.println("finished query generation");
// Benchmarking insert/write performance
String loadCommand = "";
loadCommand = TSBSCommandBuilder.measurePerformance(workload.getTsbsDataConfig(), workload.getTsbsLoadConfig(), TSBS_RESULT_FOLDER, TSBS_SCRIPTS);
loadCommand += " > " + TSBS_RESULT_FOLDER //+ taskId + "_"
+ "load_metrics.csv";
System.out.println(loadCommand);
ProcessBuilder loadProcess = new ProcessBuilder();
loadProcess.command(UNIX_SHELL, UNIX_COMMAND_PARAM, loadCommand);
Process processLoad = loadProcess.start();
while (processLoad.isAlive()) {
Thread.sleep(1000);
}
System.out.println("finished loading");
// Execute Queries
String exeQueries = "cat " + queriesToExecute + " | gunzip | ";
exeQueries += TSBSCommandBuilder.executeQueries(workload.getTsbsDataConfig(), workload.getTsbsQueryExecution()/*, queriesToExecute*/);
exeQueries += " > " + TSBS_RESULT_FOLDER + "query_execution.csv";
System.out.println(exeQueries);
workloadProcess
.startWorkload(TSBS_BINARY, UNIX_SHELL, UNIX_COMMAND_PARAM, exeQueries, taskId,
workloadTimestamp);
WorkloadProcessInstance workloadProcessInstance = new WorkloadProcessInstance();
workloadProcessInstance.setTimestamp(BigDecimal.valueOf(workloadProcess.getStartTimestamp()));
workloadProcessInstance.setProcessId(workloadProcess.getProcessId());
if (workloadProcess.isRunning()) {
LOGGER.debug("WorkloadPrcess started!");
workloadProcessInstance.setProcessStatus(ProcessStatus.RUNNING);
return Response.ok().entity(workloadProcessInstance).build();
} else {
if (workloadProcess.getExitCode() != 0) {
LOGGER.debug("workloadPrcess exited with " + workloadProcess.getExitCode()
+ " Incorrect command provided? Command: " + "TODO");
workloadProcessInstance.setProcessStatus(ProcessStatus.ERROR);
return Response.ok().entity(workloadProcessInstance).build();
} else {
LOGGER.debug(
"workloadPrcess already finished, incorrect command provided? Command: " + "TODO");
workloadProcessInstance.setProcessStatus(ProcessStatus.IDLE);
return Response.ok().entity(workloadProcessInstance).build();
}
}
}
......
package de.uulm.omi.workload.tsbs;
import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
public class TSBSCommandBuilder {
private static final Logger LOGGER = LoggerFactory.getLogger(TSBSCommandBuilder.class);
public static String generateDataCommand(TSBSDataConfig dataConfig) {
checkNotNull(dataConfig);
boolean fail = false;
String generateDataCommand = "tsbs_generate_data ";
if (dataConfig.getUsecase() != null){
generateDataCommand += "-use-case=\"" + dataConfig.getUsecase().toLowerCase()+ "\" ";
}
else fail=true;
if (dataConfig.getSeed() != null) {
generateDataCommand += "-seed=" + dataConfig.getSeed().toString().toLowerCase() + " ";
}
else fail=true;
if (dataConfig.getScale() != null) {
generateDataCommand += "-scale=" + dataConfig.getScale().toString().toLowerCase() + " ";
}
else fail=true;
if (dataConfig.getStartTime() != null) {
generateDataCommand += "-timestamp-start=\"" + dataConfig.getStartTime() + "\" ";
}
else fail=true;
if (dataConfig.getEndTime() != null) {
generateDataCommand += "-timestamp-end=\"" + dataConfig.getEndTime() + "\" ";
}
else fail=true;
if (dataConfig.getLogInterval() != null) {
generateDataCommand += "-log-interval=\"" + dataConfig.getLogInterval() + "\" ";
}
else fail=true;
if (dataConfig.getFormat() != null) {
generateDataCommand += "-format=\"" + dataConfig.getFormat()+ "\"";
} else fail=true;
if (fail) {
LOGGER.error("Required arguments are missing\n" + "One of the following:");
LOGGER.info("Arguments needed: \n" +
"-use-case\n" +
"-seed\n" +
"-scale\n" +
"-timestamp-start\n" +
"-timestamp-end\n" +
"-log-interval\n" +
"-format");
return "";
}
else return generateDataCommand;
}
public static String generateQueries(TSBSQueryConfig queryConfig) {
checkNotNull(queryConfig);
boolean fail = false;
String generateQueryCommand = "tsbs_generate_queries ";
if(queryConfig.getUsecase() != null) {
generateQueryCommand += "-use-case=" + "\"" + queryConfig.getUsecase() + "\" ";
} else fail = true;
if(queryConfig.getSeed() != null) {
generateQueryCommand += "-seed=" + queryConfig.getSeed() + " ";
} else fail = true;
if(queryConfig.getScale() != null) {
generateQueryCommand += "-scale=" + queryConfig.getScale() + " ";
} else fail = true;
if(queryConfig.getStartTime() != null) {
generateQueryCommand += "-timestamp-start=" + "\"" + queryConfig.getStartTime() + "\" ";
} else fail = true;
if(queryConfig.getEndTime() != null) {
generateQueryCommand += "-timestamp-end=" + "\"" + queryConfig.getEndTime() + "\" ";
} else fail = true;
if(queryConfig.getQueries().intValue() > 0) {
generateQueryCommand += "-queries=" + queryConfig.getQueries() + " ";
} else fail= true;
if(queryConfig.getQueryType() != null) {
generateQueryCommand += "-query-type=" + "\"" + queryConfig.getQueryType() + "\" ";
} else fail=true;
if(queryConfig.getFormat() != null) {
generateQueryCommand += "-format=" + "\"" + queryConfig.getFormat() + "\" ";
} else fail = true;
if (fail) {
LOGGER.error("Required arguments are missing\n" + "One of the following:");
LOGGER.info("Arguments needed: \n" +
"-use-case\n" +
"-seed\n" +
"-scale\n" +
"-timestamp-start\n" +
"-timestamp-end\n" +
"-queries\n" +
"-query-type\n" +
"-format");
return "";
}
else return generateQueryCommand;
}
@Deprecated
public static String loadCommand(TSBSDataConfig dataConfig, TSBSLoadConfig loadConfig, String fileToLoad) {
checkNotNull(dataConfig);
boolean fail = false;
String loadingCommand = "tsbs_load_";
loadingCommand += dataConfig.getFormat().toString().toLowerCase() + " ";
if(loadConfig.getBatchSize().intValue() > 0) {
loadingCommand += "-batch-size=" + loadConfig.getBatchSize().toString() + " ";
} else fail = true;
if(loadConfig.getNumberOfWorkers().intValue() > 0) {
loadingCommand += "-workers=" + loadConfig.getNumberOfWorkers().toString() + " ";
} else fail=true;
if(loadConfig.getDbmsPorperties().size() > 0) {
for(TSDBMSDBMSProperties prop : loadConfig.getDbmsPorperties()) {
loadingCommand += prop.getName() + "=" + "\"" + prop.getValue() + "\" ";
}
} else fail = true;
loadingCommand += "-file=" + "\"" + fileToLoad + "\"";
if (fail) {
LOGGER.error("Loading command cannot be executed.");
return "";
}
else return loadingCommand;
}
public static String measurePerformance(TSBSDataConfig dataConfig, TSBSLoadConfig loadConfig, String TSBSResultFolder, String ScriptFolder) {
checkNotNull(dataConfig);
checkNotNull(loadConfig);
String loadingCommand = "";
boolean fail=false;
if(loadConfig.getNumberOfWorkers().intValue() > 0 ) {
loadingCommand += "NUM_WORKERS=" + loadConfig.getNumberOfWorkers() + " ";
} else fail=true;
if(loadConfig.getBatchSize().intValue() > 0) {
loadingCommand += "BATCH_SIZE=" + loadConfig.getBatchSize() + " ";
} else fail=true;
loadingCommand += "BULK_DATA_DIR=" + TSBSResultFolder + " ";
loadingCommand += ScriptFolder + "load_" + dataConfig.getFormat() + ".sh";
if (fail) {
LOGGER.error("Loading command cannot be executed.");
return "";
}
else return loadingCommand;
}
public static String executeQueries(TSBSDataConfig dataConfig, TSBSQueryExecution quExe/*, String queriesToExecute*/) {
String exeQueries = "tsbs_run_queries_" + dataConfig.getFormat() + " ";
boolean fail=false;
if(quExe.getWorkers().intValue() > 0) {
exeQueries += "--workers=" + quExe.getWorkers() + " ";
} else fail = true;
if(quExe.getDbmsPorperties().size() > 0 ) {
for(TSDBMSDBMSProperties prop : quExe.getDbmsPorperties()) {
exeQueries += prop.getName() + "=\"" + prop.getValue() + "\" ";
}
} else fail = true;
if (fail) {
LOGGER.error("Query execution cannot be executed.");
return "";
}else {
//exeQueries += "-file=" + "\"" + queriesToExecute + "\"";
return exeQueries;
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment