- added new command generation (ongoing)

pom.xml

@@ -167,6 +167,12 @@
         <scope>compile</scope>
     </dependency>
 
+    <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+      <version>2.3.1</version>
+    </dependency>
+
   </dependencies>
   <repositories>
     <repository>

src/gen/java/de/uulm/omi/dbms/workload/api/WorkloadApi.java

 package de.uulm.omi.dbms.workload.api;
 
+import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
 import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
-import de.uulm.omi.dbms.workload.api.WorkloadApiService;
 import de.uulm.omi.dbms.workload.api.factories.WorkloadApiServiceFactory;
-
 import io.swagger.annotations.ApiParam;
-import io.swagger.jaxrs.*;
-
-import de.uulm.omi.cloudiator.dbms.evaluator.model.Error;
-import java.io.File;
-import de.uulm.omi.cloudiator.dbms.evaluator.model.TPCCWorkload;
-import de.uulm.omi.cloudiator.dbms.evaluator.model.TSBSLoadWorkload;
-import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
-import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
-import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
-
-import java.util.Map;
-import java.util.List;
-import de.uulm.omi.dbms.workload.api.NotFoundException;
-
-import java.io.InputStream;
-
-import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.glassfish.jersey.media.multipart.FormDataParam;
 
 import javax.servlet.ServletConfig;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.SecurityContext;
-import javax.ws.rs.*;
-import javax.validation.constraints.*;
+import java.io.File;
+import java.io.IOException;
 
 @Path("/workload")
 
@@ -120,7 +102,7 @@ public class WorkloadApi  {
     public Response workloadTsbsLoadPost(@ApiParam(value = "the id of the workload execution task, used as prefix for result files",required=true) @QueryParam("taskId") String taskId
 ,@ApiParam(value = "the TSBS workload configuration" ,required=true) TSBSLoadWorkload workload
 ,@Context SecurityContext securityContext)
-    throws NotFoundException {
+            throws NotFoundException, IOException {
         return delegate.workloadTsbsLoadPost(taskId,workload,securityContext);
     }
     @POST

src/gen/java/de/uulm/omi/dbms/workload/api/WorkloadApiService.java

@@ -13,6 +13,7 @@ import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessInstance;
 import de.uulm.omi.cloudiator.dbms.evaluator.model.WorkloadProcessSubmissionError;
 import de.uulm.omi.cloudiator.dbms.evaluator.model.YCSBWorkload;
 
+import java.io.IOException;
 import java.util.List;
 import de.uulm.omi.dbms.workload.api.NotFoundException;
 
@@ -26,6 +27,6 @@ public abstract class WorkloadApiService {
     public abstract Response workloadResultGet( @NotNull String taskId, @NotNull String workloadType,SecurityContext securityContext) throws NotFoundException;
     public abstract Response workloadStatusGet( String applicationInstanceId,SecurityContext securityContext) throws NotFoundException;
     public abstract Response workloadTpccPost( @NotNull String taskId,TPCCWorkload workload,SecurityContext securityContext) throws NotFoundException;
-    public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException;
+    public abstract Response workloadTsbsLoadPost( @NotNull String taskId,TSBSLoadWorkload workload,SecurityContext securityContext) throws NotFoundException, IOException;
     public abstract Response workloadYcsbPost( @NotNull String taskId,YCSBWorkload workload,SecurityContext securityContext) throws NotFoundException;
 }

src/main/java/de/uulm/omi/dbms/workload/api/impl/WorkloadApiServiceImpl.java

@@ -13,9 +13,11 @@ import de.uulm.omi.utils.TextFileLoader;
 import de.uulm.omi.utils.WorkloadApiProperties;
 import de.uulm.omi.workload.WorkloadProcess;
 import de.uulm.omi.workload.tpcc.TPCCCommandBuilder;
+import de.uulm.omi.workload.tsbs.TSBSCommandBuilder;
 import de.uulm.omi.workload.ycsb.YCSBCommandBuilder;
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.math.BigDecimal;
 import java.util.Properties;
 import javax.validation.constraints.NotNull;
@@ -46,7 +48,8 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
   private static final String TPCC_BINARY = "./loadgen.sh";
 
   //TSBS constants
-  private final String TSBS_RESULT_FOLDER = "TSBS/";
+  private final String TSBS_RESULT_FOLDER = "/tmp/"; // TODO CHANGE
+  private static final String TSBS_BINARY = "/opt/workload/";
 
 
 
@@ -151,7 +154,7 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
 
   @Override
   public Response workloadTsbsLoadPost(String taskId, TSBSLoadWorkload workload,
-      SecurityContext securityContext) throws NotFoundException {
+      SecurityContext securityContext) throws NotFoundException, IOException {
 
 
     //check if already a workloadProcess running
@@ -167,22 +170,78 @@ public class WorkloadApiServiceImpl extends WorkloadApiService {
     Properties properties = WorkloadApiProperties.getInstance().getProperties();
     Long workloadTimestamp = System.currentTimeMillis();
 
-    //TODO: implement the load phase call for TSBS similar to the existing TPC and YCSB implementations
-
-    //TODO: check for the static constants as defined in the beginning
-    //TODO: check for additional static properties in the workloadAPI.proeprties file, properties can be accessed via the proeprties object
-
-    //TODO: based on the provided data two calls should be executed:
-    //TODO: first call should be the bash call tsbs_generate_data
-    workload.getTsbsDataConfig();
-    //TODO: second call should be NUM_WORKERS=2 BATCH_SIZE=30 BULK_DATA_DIR=/tmp scripts/load_influx.sh > /tmp/load_metrics.csv
-    workload.getDbEndpoints();
-    workload.getTsbsLoadConfig();
-    //TODO: either wait and check the state of the first call or combine both calls in a oneliner using &&
-
-    return null;
+    //Generate Data
+    String generateDataCommand= "";
+    generateDataCommand = TSBSCommandBuilder.generateDataCommand(workload.getTsbsDataConfig());
+    String resultFileName =
+            //properties.getProperty(WorkloadApiProperties.WORKLOAD_RESULTS_FOLDER) +
+            TSBS_RESULT_FOLDER
+                    + taskId + "-" +  workload.getTsbsDataConfig().getFormat() + "-data.gz";
+    generateDataCommand += "| gzip > " + resultFileName;
+    ProcessBuilder processBuilder = new ProcessBuilder();
+    processBuilder.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateDataCommand);
+    Process processData = processBuilder.start();
+    while (processData.isAlive()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
 
+    //Generate Queries
+    String generateQueriesCommand = "";
+    generateQueriesCommand = TSBSCommandBuilder.generateQueries(workload.getTsbsQueryConfig());
+    String resultQueriesName =
+            TSBS_RESULT_FOLDER
+                    + taskId + "-" +  workload.getTsbsQueryConfig().getQueryType() + ".gz";
+    generateQueriesCommand += " gzip > " + resultQueriesName;
+    /*processBuilder.command(UNIX_SHELL, UNIX_COMMAND_PARAM, generateQueriesCommand);
+    Process processGenQueries = processBuilder.start();
+    while (processGenQueries.isAlive()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }*/
+
+    // Benchmarking insert/write performance
+    /*String loadCommand = "";
+    loadCommand = TSBSCommandBuilder.loadCommand(workload.getTsbsDataConfig(), workload.getTsbsLoadConfig(), TSBS_BINARY);
+    processBuilder.command(UNIX_SHELL, UNIX_COMMAND_PARAM, loadCommand);
+    Process processLoad = processBuilder.start();
+    while (processLoad.isAlive()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }*/
 
+    workloadProcess
+            .startWorkload(TSBS_BINARY, UNIX_SHELL, UNIX_COMMAND_PARAM, generateQueriesCommand, taskId,
+                    workloadTimestamp);
+    WorkloadProcessInstance workloadProcessInstance = new WorkloadProcessInstance();
+    workloadProcessInstance.setTimestamp(BigDecimal.valueOf(workloadProcess.getStartTimestamp()));
+    workloadProcessInstance.setProcessId(workloadProcess.getProcessId());
+    if (workloadProcess.isRunning()) {
+      LOGGER.debug("WorkloadPrcess started!");
+      workloadProcessInstance.setProcessStatus(ProcessStatus.RUNNING);
+      return Response.ok().entity(workloadProcessInstance).build();
+    } else {
+      if (workloadProcess.getExitCode() != 0) {
+        LOGGER.debug("workloadPrcess exited with " + workloadProcess.getExitCode()
+                + " Incorrect command provided? Command: " + "TODO");
+        workloadProcessInstance.setProcessStatus(ProcessStatus.ERROR);
+        return Response.ok().entity(workloadProcessInstance).build();
+      } else {
+        LOGGER.debug(
+                "workloadPrcess already finished, incorrect command provided? Command: " + "TODO");
+        workloadProcessInstance.setProcessStatus(ProcessStatus.IDLE);
+        return Response.ok().entity(workloadProcessInstance).build();
+      }
+    }
   }
 
 

src/main/java/de/uulm/omi/workload/tsbs/TSBSCommandBuilder.java

+package de.uulm.omi.workload.tsbs;
+
+import de.uulm.omi.cloudiator.dbms.evaluator.model.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class TSBSCommandBuilder {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(TSBSCommandBuilder.class);
+
+    public static String generateDataCommand(TSBSDataConfig dataConfig) {
+
+        checkNotNull(dataConfig);
+        boolean fail = false;
+
+        String generateDataCommand = "tsbs_generate_data ";
+        if (dataConfig.getUsecase() != null){
+            generateDataCommand += "-use-case=\"" + dataConfig.getUsecase().toLowerCase()+ "\" ";
+        }
+        else fail=true;
+        if (dataConfig.getSeed() != null) {
+            generateDataCommand += "-seed=" + dataConfig.getSeed().toString().toLowerCase() + " ";
+        }
+        else fail=true;
+        if (dataConfig.getScale() != null) {
+            generateDataCommand += "-scale=" + dataConfig.getScale().toString().toLowerCase() + " ";
+        }
+        else fail=true;
+        if (dataConfig.getStartTime() != null) {
+            generateDataCommand += "-timestamp-start=\"" + dataConfig.getStartTime() + "\" ";
+        }
+        else fail=true;
+        if (dataConfig.getEndTime() != null) {
+            generateDataCommand += "-timestamp-end=\"" + dataConfig.getEndTime() + "\" ";
+        }
+        else fail=true;
+        if (dataConfig.getLogInterval() != null) {
+            generateDataCommand += "-log-interval=\"" + dataConfig.getLogInterval() + "\" ";
+        }
+        else fail=true;
+        if (dataConfig.getFormat() != null) {
+            generateDataCommand += "-format=\"" + dataConfig.getFormat()+ "\" ";
+        }
+        else fail=true;
+
+        // save resulting output in a gzip file
+        //assert dataConfig.getFormat() != null;
+        //generateDataCommand += "| gzip > " + tsbsResultFolder  + dataConfig.getFormat() + "-data.gz";
+
+        if (fail) {
+            LOGGER.error("Required arguments are missing\n" + "One of the following:");
+            LOGGER.info("Arguments needed: \n" +
+                    "-use-case\n" +
+                    "-seed\n" +
+                    "-scale\n" +
+                    "-timestamp-start\n" +
+                    "-timestamp-end\n" +
+                    "-log-interval\n" +
+                    "-format");
+            return "";
+        }
+
+        else return generateDataCommand;
+
+    }
+
+    public static String generateQueries(TSBSQueryConfig queryConfig) {
+        checkNotNull(queryConfig);
+        boolean fail = false;
+        String generateQueryCommand = "tsbs_generate_queries ";
+        if(queryConfig.getUsecase() != null) {
+            generateQueryCommand += "-use-case=" + "\"" + queryConfig.getUsecase() + "\" ";
+        } else fail = true;
+        if(queryConfig.getSeed() != null) {
+            generateQueryCommand += "-seed=" + queryConfig.getSeed() + " ";
+        } else fail = true;
+        if(queryConfig.getScale() != null) {
+            generateQueryCommand += "-scale=" + queryConfig.getScale() + " ";
+        } else fail = true;
+        if(queryConfig.getStartTime() != null) {
+            generateQueryCommand += "-timestamp-start=" + "\"" + queryConfig.getStartTime() + "\" ";
+        } else fail = true;
+        if(queryConfig.getEndTime() != null) {
+            generateQueryCommand += "-timestamp-end=" + "\"" + queryConfig.getEndTime() + "\" ";
+        } else fail = true;
+        if(queryConfig.getQueries().intValue() > 0) {
+            generateQueryCommand += "-queries=" + queryConfig.getQueries() + " ";
+        } else fail= true;
+        if(queryConfig.getQueryType() != null) {
+            generateQueryCommand += "-query-type=" + "\"" + queryConfig.getQueryType() + "\" ";
+        } else fail=true;
+        if(queryConfig.getFormat() != null) {
+            generateQueryCommand += "-format=" + "\"" + queryConfig.getFormat() + "\" ";
+        }
+        if (fail) {
+            LOGGER.error("Required arguments are missing\n" + "One of the following:");
+            LOGGER.info("Arguments needed: \n" +
+                    "-use-case\n" +
+                    "-seed\n" +
+                    "-scale\n" +
+                    "-timestamp-start\n" +
+                    "-timestamp-end\n" +
+                    "-queries\n" +
+                    "-query-type\n" +
+                    "-format");
+            return "";
+        }
+
+        else return generateQueryCommand;
+    }
+
+    public static String loadCommand(TSBSDataConfig dataConfig, TSBSLoadConfig loadConfig, String tsbsBinary) {
+        checkNotNull(dataConfig);
+        checkNotNull(loadConfig);
+        boolean fail = false;
+        String loadingCommand = "";
+
+        //NUM_WORKERS=2 BATCH_SIZE=30 BULK_DATA_DIR=/tmp scripts/load_influx.sh > /tmp/load_metrics.csv
+        if (loadConfig.getNumberOfWorkers().intValue() > 0) {
+            loadingCommand += "NUM_WORKERS=" + loadConfig.getNumberOfWorkers().toString() + " ";
+        } else fail=true;
+        if (loadConfig.getBatchSize().intValue() > 0) {
+            loadingCommand += "BATCH_SIZE=" + loadConfig.getBatchSize().toString() + " ";
+        } else fail=true;
+        loadingCommand += "BULK_DATA_DIR=" + tsbsBinary + "/load_" + dataConfig.getFormat().toString().toLowerCase()
+                + ".sh > /tmp/load_metrics.csv ";
+        if (loadConfig.getDbmsPorperties().size() > 0) {
+            for(TSDBMSDBMSProperties key : loadConfig.getDbmsPorperties()) {
+                loadingCommand += "" +  key.getName().toLowerCase()  + " " + "\"" +key.getValue().toLowerCase() + "\" ";
+            }
+        } else fail=true;
+
+        if (fail) {
+            LOGGER.error("Loading command cannot be executed.");
+            return "";
+        }
+        else return loadingCommand;
+
+    }
+
+}
\ No newline at end of file