Commit 63c4d8ef authored by Daniel Seybold's avatar Daniel Seybold

spark extensions + first mesos draft

parent c98d96f6
......@@ -6,8 +6,8 @@ scriptsFolder = ./scripts/
scriptsFileExtension = .sh
scriptServiceWildcard = SCRIPT_SERVICE_ENDPOINT
scriptServiceIp = 134.60.30.87
scriptServicePort = 8080
scriptServiceIp = 134.60.64.68
scriptServicePort = 9001
colosseumUri = http://localhost:9000/api
......
......@@ -13,3 +13,8 @@
osname : "UBUNTU"
osversion : "14.04.2 AMD64 LTS Java8"
login : "ubuntu"
-
uuid : "d20201de-4b53-11e7-a919-92ebcb67fe33/Ulm/83f41918-2d73-49dd-8f42-4983fbdbf705"
osname : "UBUNTU"
osversion : "Ubuntu Server 14.04 RAW"
login : "ubuntu"
\ No newline at end of file
#Mesos Master installation
#based on: https://www.digitalocean.com/community/tutorials/how-to-configure-a-production-ready-mesosphere-cluster-on-ubuntu-14-04
#TODO: set hostname to 0.0.0.0
##init
rm /etc/hosts
sudo echo 127.0.0.1 localhost.localdomain localhost `hostname` | sudo tee /etc/hosts
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
CODENAME=$(lsb_release -cs)
echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list
sudo apt-get -y update
sudo apt-get install mesosphere -y
##zookeeper
#modify zk config
sudo nano /etc/mesos/zk
# create unique zk id
nano /etc/zookeeper/conf/myid
#update zookeeper config
nano /etc/zookeeper/conf/zoo.cfg
#optional: modify quorum
nano /etc/mesos-master/quorum
##mesos master
#set ip addresses
LOCAL_ADDRESS=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
echo ${LOCAL_ADDRESS} | sudo tee /etc/mesos-master/ip
sudo cp /etc/mesos-master/ip /etc/mesos-master/hostname
##marathon
sudo mkdir -p /etc/marathon/conf
sudo cp /etc/mesos-master/hostname /etc/marathon/conf
sudo cp /etc/mesos/zk /etc/marathon/conf/master
sudo cp /etc/marathon/conf/master /etc/marathon/conf/zk
# configure zk for marathon
echo "zk://"${LOCAL_ADDRESS}":2181/marathon" > /etc/marathon/conf/zk
#optional, ensure that no slave process is running
sudo stop mesos-slave
echo manual | sudo tee /etc/init/mesos-slave.override
\ No newline at end of file
#Mesos Slave installation
#based on: https://www.digitalocean.com/community/tutorials/how-to-configure-a-production-ready-mesosphere-cluster-on-ubuntu-14-04
#TODO: set hostname to 0.0.0.0
##init
rm /etc/hosts
sudo echo 127.0.0.1 localhost.localdomain localhost `hostname` | sudo tee /etc/hosts
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
CODENAME=$(lsb_release -cs)
echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list
sudo apt-get -y update
sudo apt-get install mesosphere -y
#stop zookeeper
sudo stop zookeeper
echo manual | sudo tee /etc/init/zookeeper.override
# stop mesos master (shouldn't be running anyway)
echo manual | sudo tee /etc/init/mesos-master.override
sudo stop mesos-master
# configure ZK to resolve mesos master
echo zk://192.168.0.226:2181/mesos | sudo tee /etc/mesos/zk
#optional: enable docker containers
wget https://raw.githubusercontent.com/cloudiator/lance/master/install/docker_retry_fix_version.sh
chmod +x docker...
./dock...
echo 'docker,mesos' > /etc/mesos-slave/containerizers
echo '10mins' > /etc/mesos-slave/executor_registration_timeout
\ No newline at end of file
......@@ -36,5 +36,9 @@ wget https://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop${HA
tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
#download a sample file for the wordcount example job
wget -O /home/ubuntu/words.txt https://raw.githubusercontent.com/melphi/spark-examples/master/first-example/src/test/resources/loremipsum.txt
# 10KB sample file
# wget -O /home/ubuntu/words.txt https://raw.githubusercontent.com/melphi/spark-examples/master/first-example/src/test/resources/loremipsum.txt
# 6MB sample
# wget -O /home/ubuntu/words.txt http://norvig.com/big.txt
# 100MB sample
wget -O /home/ubuntu/words.txt https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/jquery-speedtest/100MB.txt
\ No newline at end of file
package io.swagger.api.impl;
import com.github.ywilkof.sparkrestclient.ClusterMode;
import com.github.ywilkof.sparkrestclient.FailedSparkRequestException;
import com.github.ywilkof.sparkrestclient.SparkRestClient;
import io.swagger.api.ApiResponseMessage;
......@@ -28,6 +29,7 @@ public class SparkJobApiServiceImpl extends SparkJobApiService {
SparkRestClient sparkRestClient = SparkRestClient.builder()
.masterHost(job.getMasterUrl())
.sparkVersion("2.1.0")
.clusterMode(ClusterMode.spark)
.build();
LOGGER.debug("submitting job");
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment