Skip to content

GitLab

W
workload-API
Commit e90b8e18 authored by Daniel Seybold's avatar Daniel Seybold
Browse files
Options

Merge branch 'tsbs_integration_2' into 'master' 

Tsbs integration

See merge request !2
parents cc53fde6 14f0f3df
Pipeline #62322 passed with stage
    in 16 minutes and 54 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 275 additions and 53 deletions
    src/gen/java/de/uulm/omi/dbms/workload/api/WorkloadApi.java
    View file @ e90b8e18
    package de.uulm.omi.dbms.workload.api;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
    import de.uulm.omi.dbms.workload.api.WorkloadApiService;
    import de.uulm.omi.dbms.workload.api.factories.WorkloadApiServiceFactory;
    import io.swagger.annotations.ApiParam;
    import io.swagger.jaxrs.*;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
    import java.io.File;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.TPCCWorkload;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.TSBSLoadWorkload;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
    import java.util.Map;
    import java.util.List;
    import de.uulm.omi.dbms.workload.api.NotFoundException;
    import java.io.InputStream;
    import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
    import org.glassfish.jersey.media.multipart.FormDataParam;
    import javax.servlet.ServletConfig;
    import javax.ws.rs.*;
    import javax.ws.rs.core.Context;
    import javax.ws.rs.core.Response;
    import javax.ws.rs.core.SecurityContext;
    import javax.ws.rs.*;
    import javax.validation.constraints.*;
    import java.io.File;
    import java.io.IOException;
    @Path("/workload")
    ......@@ -120,7 +102,7 @@ public class WorkloadApi {
    public Response workloadTsbsLoadPost(@ApiParam(value = "the id of the workload execution task, used as prefix for result files",required=true) @QueryParam("taskId") String taskId
    ,@ApiParam(value = "the TSBS workload configuration" ,required=true) TSBSLoadWorkload workload
    ,@Context SecurityContext securityContext)
    throws NotFoundException {
    throws NotFoundException, IOException, InterruptedException {
    return delegate.workloadTsbsLoadPost(taskId,workload,securityContext);
    }
    @POST
    ......
    src/gen/java/de/uulm/omi/dbms/workload/api/WorkloadApiService.java
    View file @ e90b8e18
    ......@@ -13,6 +13,7 @@ import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
    import java.io.IOException;
    import java.util.List;
    import de.uulm.omi.dbms.workload.api.NotFoundException;
    ......@@ -26,6 +27,6 @@ public abstract class WorkloadApiService {
    public abstract Response workloadResultGet( @NotNull String taskId, @NotNull String workloadType,SecurityContext securityContext) throws NotFoundException;
    public abstract Response workloadStatusGet( String applicationInstanceId,SecurityContext securityContext) throws NotFoundException;
    public abstract Response workloadTpccPost( @NotNull String taskId,TPCCWorkload workload,SecurityContext securityContext) throws NotFoundException;
    public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException;
    public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException, IOException, InterruptedException;
    public abstract Response workloadYcsbPost( @NotNull String taskId,YCSBWorkload workload,SecurityContext securityContext) throws NotFoundException;
    }
    src/main/java/de/uulm/omi/dbms/workload/api/impl/WorkloadApiServiceImpl.java
    View file @ e90b8e18
    package de.uulm.omi.dbms.workload.api.impl;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.ProcessStatus;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.TPCCWorkload;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.TSBSLoadWorkload;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
    import de.uulm.omi.dbms.workload.api.ApiResponseMessage;
    import de.uulm.omi.dbms.workload.api.NotFoundException;
    import de.uulm.omi.dbms.workload.api.WorkloadApiService;
    ......@@ -13,17 +8,20 @@ import de.uulm.omi.utils.TextFileLoader;
    import de.uulm.omi.utils.WorkloadApiProperties;
    import de.uulm.omi.workload.WorkloadProcess;
    import de.uulm.omi.workload.tpcc.TPCCCommandBuilder;
    import de.uulm.omi.workload.tsbs.TSBSCommandBuilder;
    import de.uulm.omi.workload.ycsb.YCSBCommandBuilder;
    import java.io.File;
    import java.io.FileNotFoundException;
    import java.math.BigDecimal;
    import java.util.Properties;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import javax.validation.constraints.NotNull;
    import javax.ws.rs.core.Response;
    import javax.ws.rs.core.Response.Status;
    import javax.ws.rs.core.SecurityContext;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import java.io.File;
    import java.io.FileNotFoundException;
    import java.io.IOException;
    import java.math.BigDecimal;
    import java.util.Properties;
    @javax.annotation.Generated(value = "io.swagger.codegen.languages.JavaJerseyServerCodegen", date = "2018-01-16T09:59:22.348Z")
    public class WorkloadApiServiceImpl extends WorkloadApiService {
    ......@@ -46,7 +44,9 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
    private static final String TPCC_BINARY = "./loadgen.sh";
    //TSBS constants
    private final String TSBS_RESULT_FOLDER = "TSBS/";
    private final String TSBS_RESULT_FOLDER = "/tmp/"; // TODO CHANGE
    private static final String TSBS_BINARY = "/opt/workloads/tsbs/bin/";
    private static final String TSBS_SCRIPTS = "/opt/workloads/tsbs/src/github.com/timescale/tsbs/scripts/";
    ......@@ -151,7 +151,7 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
    @Override
    public Response workloadTsbsLoadPost(String taskId, TSBSLoadWorkload workload,
    SecurityContext securityContext) throws NotFoundException {
    SecurityContext securityContext) throws NotFoundException, IOException, InterruptedException {
    //check if already a workloadProcess running
    ......@@ -167,22 +167,87 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
    Properties properties = WorkloadApiProperties.getInstance().getProperties();
    Long workloadTimestamp = System.currentTimeMillis();
    //TODO: implement the load phase call for TSBS similar to the existing TPC and YCSB implementations
    //TODO: check for the static constants as defined in the beginning
    //TODO: check for additional static properties in the workloadAPI.proeprties file, properties can be accessed via the proeprties object
    //TODO: based on the provided data two calls should be executed:
    //TODO: first call should be the bash call tsbs_generate_data
    workload.getTsbsDataConfig();
    //TODO: second call should be NUM_WORKERS=2 BATCH_SIZE=30 BULK_DATA_DIR=/tmp scripts/load_influx.sh > /tmp/load_metrics.csv
    workload.getDbEndpoints();
    workload.getTsbsLoadConfig();
    //TODO: either wait and check the state of the first call or combine both calls in a oneliner using &&
    return null;
    //Generate Data
    String generateDataCommand= "";
    generateDataCommand = TSBSCommandBuilder.generateDataCommand(workload.getTsbsDataConfig());
    String resultFileName =
    //properties.getProperty(WorkloadApiProperties.WORKLOAD_RESULTS_FOLDER) +
    TSBS_RESULT_FOLDER
    //+ taskId + "-"
    + workload.getTsbsDataConfig().getFormat() + "-data.gz";
    generateDataCommand += " | gzip > " + resultFileName;
    ProcessBuilder genDataProcess= new ProcessBuilder();
    genDataProcess.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateDataCommand);
    Process processData = genDataProcess.start();
    System.out.println(generateDataCommand);
    while (processData.isAlive()) {
    Thread.sleep(1000);
    }
    System.out.println("finished data generation");
    String fileToLoad = TSBS_RESULT_FOLDER /*+ taskId + "-"*/ + workload.getTsbsDataConfig().getFormat() + "-data.gz";
    String queriesToExecute = TSBS_RESULT_FOLDER /*+ taskId + "-" + workload.getTsbsDataConfig().getFormat() + "-" + */ + workload.getTsbsQueryConfig().getQueryType() + ".gz";
    //Generate Queries
    String generateQueriesCommand = "";
    generateQueriesCommand = TSBSCommandBuilder.generateQueries(workload.getTsbsQueryConfig());
    String resultQueriesName =
    TSBS_RESULT_FOLDER
    //+ taskId + "-"
    + workload.getTsbsQueryConfig().getQueryType() + ".gz";
    generateQueriesCommand += "| gzip > " + resultQueriesName;
    System.out.println(generateQueriesCommand);
    ProcessBuilder genQueryProcess = new ProcessBuilder();
    genQueryProcess.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateQueriesCommand);
    Process processGenQueries = genQueryProcess.start();
    while (processGenQueries.isAlive()) {
    Thread.sleep(1000);
    }
    System.out.println("finished query generation");
    // Benchmarking insert/write performance
    String loadCommand = "";
    loadCommand = TSBSCommandBuilder.measurePerformance(workload.getTsbsDataConfig(), workload.getTsbsLoadConfig(), TSBS_RESULT_FOLDER, TSBS_SCRIPTS);
    loadCommand += " > " + TSBS_RESULT_FOLDER //+ taskId + "_"
    + "load_metrics.csv";
    System.out.println(loadCommand);
    ProcessBuilder loadProcess = new ProcessBuilder();
    loadProcess.command(UNIX_SHELL, UNIX_COMMAND_PARAM, loadCommand);
    Process processLoad = loadProcess.start();
    while (processLoad.isAlive()) {
    Thread.sleep(1000);
    }
    System.out.println("finished loading");
    // Execute Queries
    String exeQueries = "cat " + queriesToExecute + " | gunzip | ";
    exeQueries += TSBSCommandBuilder.executeQueries(workload.getTsbsDataConfig(), workload.getTsbsQueryExecution()/*, queriesToExecute*/);
    exeQueries += " > " + TSBS_RESULT_FOLDER + "query_execution.csv";
    System.out.println(exeQueries);
    workloadProcess
    .startWorkload(TSBS_BINARY, UNIX_SHELL, UNIX_COMMAND_PARAM, exeQueries, taskId,
    workloadTimestamp);
    WorkloadProcessInstance workloadProcessInstance = new WorkloadProcessInstance();
    workloadProcessInstance.setTimestamp(BigDecimal.valueOf(workloadProcess.getStartTimestamp()));
    workloadProcessInstance.setProcessId(workloadProcess.getProcessId());
    if (workloadProcess.isRunning()) {
    LOGGER.debug("WorkloadPrcess started!");
    workloadProcessInstance.setProcessStatus(ProcessStatus.RUNNING);
    return Response.ok().entity(workloadProcessInstance).build();
    } else {
    if (workloadProcess.getExitCode() != 0) {
    LOGGER.debug("workloadPrcess exited with " + workloadProcess.getExitCode()
    + " Incorrect command provided? Command: " + "TODO");
    workloadProcessInstance.setProcessStatus(ProcessStatus.ERROR);
    return Response.ok().entity(workloadProcessInstance).build();
    } else {
    LOGGER.debug(
    "workloadPrcess already finished, incorrect command provided? Command: " + "TODO");
    workloadProcessInstance.setProcessStatus(ProcessStatus.IDLE);
    return Response.ok().entity(workloadProcessInstance).build();
    }
    }
    }
    ......
    src/main/java/de/uulm/omi/workload/tsbs/TSBSCommandBuilder.java 0 → 100644
    View file @ e90b8e18
    package de.uulm.omi.workload.tsbs;
    import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import static com.google.common.base.Preconditions.checkNotNull;
    public class TSBSCommandBuilder {
    private static final Logger LOGGER = LoggerFactory.getLogger(TSBSCommandBuilder.class);
    public static String generateDataCommand(TSBSDataConfig dataConfig) {
    checkNotNull(dataConfig);
    boolean fail = false;
    String generateDataCommand = "tsbs_generate_data ";
    if (dataConfig.getUsecase() != null){
    generateDataCommand += "-use-case=\"" + dataConfig.getUsecase().toLowerCase()+ "\" ";
    }
    else fail=true;
    if (dataConfig.getSeed() != null) {
    generateDataCommand += "-seed=" + dataConfig.getSeed().toString().toLowerCase() + " ";
    }
    else fail=true;
    if (dataConfig.getScale() != null) {
    generateDataCommand += "-scale=" + dataConfig.getScale().toString().toLowerCase() + " ";
    }
    else fail=true;
    if (dataConfig.getStartTime() != null) {
    generateDataCommand += "-timestamp-start=\"" + dataConfig.getStartTime() + "\" ";
    }
    else fail=true;
    if (dataConfig.getEndTime() != null) {
    generateDataCommand += "-timestamp-end=\"" + dataConfig.getEndTime() + "\" ";
    }
    else fail=true;
    if (dataConfig.getLogInterval() != null) {
    generateDataCommand += "-log-interval=\"" + dataConfig.getLogInterval() + "\" ";
    }
    else fail=true;
    if (dataConfig.getFormat() != null) {
    generateDataCommand += "-format=\"" + dataConfig.getFormat()+ "\"";
    } else fail=true;
    if (fail) {
    LOGGER.error("Required arguments are missing\n" + "One of the following:");
    LOGGER.info("Arguments needed: \n" +
    "-use-case\n" +
    "-seed\n" +
    "-scale\n" +
    "-timestamp-start\n" +
    "-timestamp-end\n" +
    "-log-interval\n" +
    "-format");
    return "";
    }
    else return generateDataCommand;
    }
    public static String generateQueries(TSBSQueryConfig queryConfig) {
    checkNotNull(queryConfig);
    boolean fail = false;
    String generateQueryCommand = "tsbs_generate_queries ";
    if(queryConfig.getUsecase() != null) {
    generateQueryCommand += "-use-case=" + "\"" + queryConfig.getUsecase() + "\" ";
    } else fail = true;
    if(queryConfig.getSeed() != null) {
    generateQueryCommand += "-seed=" + queryConfig.getSeed() + " ";
    } else fail = true;
    if(queryConfig.getScale() != null) {
    generateQueryCommand += "-scale=" + queryConfig.getScale() + " ";
    } else fail = true;
    if(queryConfig.getStartTime() != null) {
    generateQueryCommand += "-timestamp-start=" + "\"" + queryConfig.getStartTime() + "\" ";
    } else fail = true;
    if(queryConfig.getEndTime() != null) {
    generateQueryCommand += "-timestamp-end=" + "\"" + queryConfig.getEndTime() + "\" ";
    } else fail = true;
    if(queryConfig.getQueries().intValue() > 0) {
    generateQueryCommand += "-queries=" + queryConfig.getQueries() + " ";
    } else fail= true;
    if(queryConfig.getQueryType() != null) {
    generateQueryCommand += "-query-type=" + "\"" + queryConfig.getQueryType() + "\" ";
    } else fail=true;
    if(queryConfig.getFormat() != null) {
    generateQueryCommand += "-format=" + "\"" + queryConfig.getFormat() + "\" ";
    } else fail = true;
    if (fail) {
    LOGGER.error("Required arguments are missing\n" + "One of the following:");
    LOGGER.info("Arguments needed: \n" +
    "-use-case\n" +
    "-seed\n" +
    "-scale\n" +
    "-timestamp-start\n" +
    "-timestamp-end\n" +
    "-queries\n" +
    "-query-type\n" +
    "-format");
    return "";
    }
    else return generateQueryCommand;
    }
    @Deprecated
    public static String loadCommand(TSBSDataConfig dataConfig, TSBSLoadConfig loadConfig, String fileToLoad) {
    checkNotNull(dataConfig);
    boolean fail = false;
    String loadingCommand = "tsbs_load_";
    loadingCommand += dataConfig.getFormat().toString().toLowerCase() + " ";
    if(loadConfig.getBatchSize().intValue() > 0) {
    loadingCommand += "-batch-size=" + loadConfig.getBatchSize().toString() + " ";
    } else fail = true;
    if(loadConfig.getNumberOfWorkers().intValue() > 0) {
    loadingCommand += "-workers=" + loadConfig.getNumberOfWorkers().toString() + " ";
    } else fail=true;
    if(loadConfig.getDbmsPorperties().size() > 0) {
    for(TSDBMSDBMSProperties prop : loadConfig.getDbmsPorperties()) {
    loadingCommand += prop.getName() + "=" + "\"" + prop.getValue() + "\" ";
    }
    } else fail = true;
    loadingCommand += "-file=" + "\"" + fileToLoad + "\"";
    if (fail) {
    LOGGER.error("Loading command cannot be executed.");
    return "";
    }
    else return loadingCommand;
    }
    public static String measurePerformance(TSBSDataConfig dataConfig, TSBSLoadConfig loadConfig, String TSBSResultFolder, String ScriptFolder) {
    checkNotNull(dataConfig);
    checkNotNull(loadConfig);
    String loadingCommand = "";
    boolean fail=false;
    if(loadConfig.getNumberOfWorkers().intValue() > 0 ) {
    loadingCommand += "NUM_WORKERS=" + loadConfig.getNumberOfWorkers() + " ";
    } else fail=true;
    if(loadConfig.getBatchSize().intValue() > 0) {
    loadingCommand += "BATCH_SIZE=" + loadConfig.getBatchSize() + " ";
    } else fail=true;
    loadingCommand += "BULK_DATA_DIR=" + TSBSResultFolder + " ";
    loadingCommand += ScriptFolder + "load_" + dataConfig.getFormat() + ".sh";
    if (fail) {
    LOGGER.error("Loading command cannot be executed.");
    return "";
    }
    else return loadingCommand;
    }
    public static String executeQueries(TSBSDataConfig dataConfig, TSBSQueryExecution quExe/*, String queriesToExecute*/) {
    String exeQueries = "tsbs_run_queries_" + dataConfig.getFormat() + " ";
    boolean fail=false;
    if(quExe.getWorkers().intValue() > 0) {
    exeQueries += "--workers=" + quExe.getWorkers() + " ";
    } else fail = true;
    if(quExe.getDbmsPorperties().size() > 0 ) {
    for(TSDBMSDBMSProperties prop : quExe.getDbmsPorperties()) {
    exeQueries += prop.getName() + "=\"" + prop.getValue() + "\" ";
    }
    } else fail = true;
    if (fail) {
    LOGGER.error("Query execution cannot be executed.");
    return "";
    }else {
    //exeQueries += "-file=" + "\"" + queriesToExecute + "\"";
    return exeQueries;
    }
    }
    }
    \ No newline at end of file
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment