Commit 4011284d authored by Julia Abramenko's avatar Julia Abramenko

- added new command generation (ongoing)

parent 5e0149aa
Pipeline #58341 failed with stage
in 18 minutes and 37 seconds
......@@ -167,6 +167,12 @@
<scope>compile</scope>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
</dependencies>
<repositories>
<repository>
......
package de.uulm.omi.dbms.workload.api;
import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
import de.uulm.omi.dbms.workload.api.WorkloadApiService;
import de.uulm.omi.dbms.workload.api.factories.WorkloadApiServiceFactory;
import io.swagger.annotations.ApiParam;
import io.swagger.jaxrs.*;
import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
import java.io.File;
import de.uulm.omi.cloudiator.dbms.evaluator.model.TPCCWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.TSBSLoadWorkload;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
import java.util.Map;
import java.util.List;
import de.uulm.omi.dbms.workload.api.NotFoundException;
import java.io.InputStream;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import javax.servlet.ServletConfig;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.*;
import javax.validation.constraints.*;
import java.io.File;
import java.io.IOException;
@Path("/workload")
......@@ -120,7 +102,7 @@ public class WorkloadApi {
public Response workloadTsbsLoadPost(@ApiParam(value = "the id of the workload execution task, used as prefix for result files",required=true) @QueryParam("taskId") String taskId
,@ApiParam(value = "the TSBS workload configuration" ,required=true) TSBSLoadWorkload workload
,@Context SecurityContext securityContext)
throws NotFoundException {
throws NotFoundException, IOException {
return delegate.workloadTsbsLoadPost(taskId,workload,securityContext);
}
@POST
......
......@@ -13,6 +13,7 @@ import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
import java.io.IOException;
import java.util.List;
import de.uulm.omi.dbms.workload.api.NotFoundException;
......@@ -26,6 +27,6 @@ public abstract class WorkloadApiService {
public abstract Response workloadResultGet( @NotNull String taskId, @NotNull String workloadType,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadStatusGet( String applicationInstanceId,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadTpccPost( @NotNull String taskId,TPCCWorkload workload,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException;
public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException, IOException;
public abstract Response workloadYcsbPost( @NotNull String taskId,YCSBWorkload workload,SecurityContext securityContext) throws NotFoundException;
}
......@@ -13,9 +13,11 @@ import de.uulm.omi.utils.TextFileLoader;
import de.uulm.omi.utils.WorkloadApiProperties;
import de.uulm.omi.workload.WorkloadProcess;
import de.uulm.omi.workload.tpcc.TPCCCommandBuilder;
import de.uulm.omi.workload.tsbs.TSBSCommandBuilder;
import de.uulm.omi.workload.ycsb.YCSBCommandBuilder;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.Properties;
import javax.validation.constraints.NotNull;
......@@ -46,7 +48,8 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
private static final String TPCC_BINARY = "./loadgen.sh";
//TSBS constants
private final String TSBS_RESULT_FOLDER = "TSBS/";
private final String TSBS_RESULT_FOLDER = "/tmp/"; // TODO CHANGE
private static final String TSBS_BINARY = "/opt/workload/";
......@@ -151,7 +154,7 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
@Override
public Response workloadTsbsLoadPost(String taskId, TSBSLoadWorkload workload,
SecurityContext securityContext) throws NotFoundException {
SecurityContext securityContext) throws NotFoundException, IOException {
//check if already a workloadProcess running
......@@ -167,22 +170,78 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
Properties properties = WorkloadApiProperties.getInstance().getProperties();
Long workloadTimestamp = System.currentTimeMillis();
//TODO: implement the load phase call for TSBS similar to the existing TPC and YCSB implementations
//TODO: check for the static constants as defined in the beginning
//TODO: check for additional static properties in the workloadAPI.proeprties file, properties can be accessed via the proeprties object
//TODO: based on the provided data two calls should be executed:
//TODO: first call should be the bash call tsbs_generate_data
workload.getTsbsDataConfig();
//TODO: second call should be NUM_WORKERS=2 BATCH_SIZE=30 BULK_DATA_DIR=/tmp scripts/load_influx.sh > /tmp/load_metrics.csv
workload.getDbEndpoints();
workload.getTsbsLoadConfig();
//TODO: either wait and check the state of the first call or combine both calls in a oneliner using &&
return null;
//Generate Data
String generateDataCommand= "";
generateDataCommand = TSBSCommandBuilder.generateDataCommand(workload.getTsbsDataConfig());
String resultFileName =
//properties.getProperty(WorkloadApiProperties.WORKLOAD_RESULTS_FOLDER) +
TSBS_RESULT_FOLDER
+ taskId + "-" + workload.getTsbsDataConfig().getFormat() + "-data.gz";
generateDataCommand += "| gzip > " + resultFileName;
ProcessBuilder processBuilder = new ProcessBuilder();
processBuilder.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateDataCommand);
Process processData = processBuilder.start();
while (processData.isAlive()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
//Generate Queries
String generateQueriesCommand = "";
generateQueriesCommand = TSBSCommandBuilder.generateQueries(workload.getTsbsQueryConfig());
String resultQueriesName =
TSBS_RESULT_FOLDER
+ taskId + "-" + workload.getTsbsQueryConfig().getQueryType() + ".gz";
generateQueriesCommand += " gzip > " + resultQueriesName;
/*processBuilder.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateQueriesCommand);
Process processGenQueries = processBuilder.start();
while (processGenQueries.isAlive()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}*/
// Benchmarking insert/write performance
/*String loadCommand = "";
loadCommand = TSBSCommandBuilder.loadCommand(workload.getTsbsDataConfig(), workload.getTsbsLoadConfig(), TSBS_BINARY);
processBuilder.command(UNIX_SHELL, UNIX_COMMAND_PARAM, loadCommand);
Process processLoad = processBuilder.start();
while (processLoad.isAlive()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}*/
workloadProcess
.startWorkload(TSBS_BINARY, UNIX_SHELL, UNIX_COMMAND_PARAM, generateQueriesCommand, taskId,
workloadTimestamp);
WorkloadProcessInstance workloadProcessInstance = new WorkloadProcessInstance();
workloadProcessInstance.setTimestamp(BigDecimal.valueOf(workloadProcess.getStartTimestamp()));
workloadProcessInstance.setProcessId(workloadProcess.getProcessId());
if (workloadProcess.isRunning()) {
LOGGER.debug("WorkloadPrcess started!");
workloadProcessInstance.setProcessStatus(ProcessStatus.RUNNING);
return Response.ok().entity(workloadProcessInstance).build();
} else {
if (workloadProcess.getExitCode() != 0) {
LOGGER.debug("workloadPrcess exited with " + workloadProcess.getExitCode()
+ " Incorrect command provided? Command: " + "TODO");
workloadProcessInstance.setProcessStatus(ProcessStatus.ERROR);
return Response.ok().entity(workloadProcessInstance).build();
} else {
LOGGER.debug(
"workloadPrcess already finished, incorrect command provided? Command: " + "TODO");
workloadProcessInstance.setProcessStatus(ProcessStatus.IDLE);
return Response.ok().entity(workloadProcessInstance).build();
}
}
}
......
package de.uulm.omi.workload.tsbs;
import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
public class TSBSCommandBuilder {
private static final Logger LOGGER = LoggerFactory.getLogger(TSBSCommandBuilder.class);
public static String generateDataCommand(TSBSDataConfig dataConfig) {
checkNotNull(dataConfig);
boolean fail = false;
String generateDataCommand = "tsbs_generate_data ";
if (dataConfig.getUsecase() != null){
generateDataCommand += "-use-case=\"" + dataConfig.getUsecase().toLowerCase()+ "\" ";
}
else fail=true;
if (dataConfig.getSeed() != null) {
generateDataCommand += "-seed=" + dataConfig.getSeed().toString().toLowerCase() + " ";
}
else fail=true;
if (dataConfig.getScale() != null) {
generateDataCommand += "-scale=" + dataConfig.getScale().toString().toLowerCase() + " ";
}
else fail=true;
if (dataConfig.getStartTime() != null) {
generateDataCommand += "-timestamp-start=\"" + dataConfig.getStartTime() + "\" ";
}
else fail=true;
if (dataConfig.getEndTime() != null) {
generateDataCommand += "-timestamp-end=\"" + dataConfig.getEndTime() + "\" ";
}
else fail=true;
if (dataConfig.getLogInterval() != null) {
generateDataCommand += "-log-interval=\"" + dataConfig.getLogInterval() + "\" ";
}
else fail=true;
if (dataConfig.getFormat() != null) {
generateDataCommand += "-format=\"" + dataConfig.getFormat()+ "\" ";
}
else fail=true;
// save resulting output in a gzip file
//assert dataConfig.getFormat() != null;
//generateDataCommand += "| gzip > " + tsbsResultFolder + dataConfig.getFormat() + "-data.gz";
if (fail) {
LOGGER.error("Required arguments are missing\n" + "One of the following:");
LOGGER.info("Arguments needed: \n" +
"-use-case\n" +
"-seed\n" +
"-scale\n" +
"-timestamp-start\n" +
"-timestamp-end\n" +
"-log-interval\n" +
"-format");
return "";
}
else return generateDataCommand;
}
public static String generateQueries(TSBSQueryConfig queryConfig) {
checkNotNull(queryConfig);
boolean fail = false;
String generateQueryCommand = "tsbs_generate_queries ";
if(queryConfig.getUsecase() != null) {
generateQueryCommand += "-use-case=" + "\"" + queryConfig.getUsecase() + "\" ";
} else fail = true;
if(queryConfig.getSeed() != null) {
generateQueryCommand += "-seed=" + queryConfig.getSeed() + " ";
} else fail = true;
if(queryConfig.getScale() != null) {
generateQueryCommand += "-scale=" + queryConfig.getScale() + " ";
} else fail = true;
if(queryConfig.getStartTime() != null) {
generateQueryCommand += "-timestamp-start=" + "\"" + queryConfig.getStartTime() + "\" ";
} else fail = true;
if(queryConfig.getEndTime() != null) {
generateQueryCommand += "-timestamp-end=" + "\"" + queryConfig.getEndTime() + "\" ";
} else fail = true;
if(queryConfig.getQueries().intValue() > 0) {
generateQueryCommand += "-queries=" + queryConfig.getQueries() + " ";
} else fail= true;
if(queryConfig.getQueryType() != null) {
generateQueryCommand += "-query-type=" + "\"" + queryConfig.getQueryType() + "\" ";
} else fail=true;
if(queryConfig.getFormat() != null) {
generateQueryCommand += "-format=" + "\"" + queryConfig.getFormat() + "\" ";
}
if (fail) {
LOGGER.error("Required arguments are missing\n" + "One of the following:");
LOGGER.info("Arguments needed: \n" +
"-use-case\n" +
"-seed\n" +
"-scale\n" +
"-timestamp-start\n" +
"-timestamp-end\n" +
"-queries\n" +
"-query-type\n" +
"-format");
return "";
}
else return generateQueryCommand;
}
public static String loadCommand(TSBSDataConfig dataConfig, TSBSLoadConfig loadConfig, String tsbsBinary) {
checkNotNull(dataConfig);
checkNotNull(loadConfig);
boolean fail = false;
String loadingCommand = "";
//NUM_WORKERS=2 BATCH_SIZE=30 BULK_DATA_DIR=/tmp scripts/load_influx.sh > /tmp/load_metrics.csv
if (loadConfig.getNumberOfWorkers().intValue() > 0) {
loadingCommand += "NUM_WORKERS=" + loadConfig.getNumberOfWorkers().toString() + " ";
} else fail=true;
if (loadConfig.getBatchSize().intValue() > 0) {
loadingCommand += "BATCH_SIZE=" + loadConfig.getBatchSize().toString() + " ";
} else fail=true;
loadingCommand += "BULK_DATA_DIR=" + tsbsBinary + "/load_" + dataConfig.getFormat().toString().toLowerCase()
+ ".sh > /tmp/load_metrics.csv ";
if (loadConfig.getDbmsPorperties().size() > 0) {
for(TSDBMSDBMSProperties key : loadConfig.getDbmsPorperties()) {
loadingCommand += "" + key.getName().toLowerCase() + " " + "\"" +key.getValue().toLowerCase() + "\" ";
}
} else fail=true;
if (fail) {
LOGGER.error("Loading command cannot be executed.");
return "";
}
else return loadingCommand;
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment