-
Notifications
You must be signed in to change notification settings - Fork 336
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support Samza on Kubernetes. Contributed by Weiqing Yang &Jian He
- Loading branch information
Showing
17 changed files
with
1,147 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| @if "%DEBUG%" == "" @echo off | ||
| @rem ########################################################################## | ||
| @rem | ||
| @rem Gradle startup script for Windows | ||
| @rem | ||
| @rem ########################################################################## | ||
|
|
||
| @rem Set local scope for the variables with windows NT shell | ||
| if "%OS%"=="Windows_NT" setlocal | ||
|
|
||
| set DIRNAME=%~dp0 | ||
| if "%DIRNAME%" == "" set DIRNAME=. | ||
| set APP_BASE_NAME=%~n0 | ||
| set APP_HOME=%DIRNAME% | ||
|
|
||
| @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. | ||
| set DEFAULT_JVM_OPTS= | ||
|
|
||
| @rem Find java.exe | ||
| if defined JAVA_HOME goto findJavaFromJavaHome | ||
|
|
||
| set JAVA_EXE=java.exe | ||
| %JAVA_EXE% -version >NUL 2>&1 | ||
| if "%ERRORLEVEL%" == "0" goto init | ||
|
|
||
| echo. | ||
| echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. | ||
| echo. | ||
| echo Please set the JAVA_HOME variable in your environment to match the | ||
| echo location of your Java installation. | ||
|
|
||
| goto fail | ||
|
|
||
| :findJavaFromJavaHome | ||
| set JAVA_HOME=%JAVA_HOME:"=% | ||
| set JAVA_EXE=%JAVA_HOME%/bin/java.exe | ||
|
|
||
| if exist "%JAVA_EXE%" goto init | ||
|
|
||
| echo. | ||
| echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% | ||
| echo. | ||
| echo Please set the JAVA_HOME variable in your environment to match the | ||
| echo location of your Java installation. | ||
|
|
||
| goto fail | ||
|
|
||
| :init | ||
| @rem Get command-line arguments, handling Windows variants | ||
|
|
||
| if not "%OS%" == "Windows_NT" goto win9xME_args | ||
|
|
||
| :win9xME_args | ||
| @rem Slurp the command line arguments. | ||
| set CMD_LINE_ARGS= | ||
| set _SKIP=2 | ||
|
|
||
| :win9xME_args_slurp | ||
| if "x%~1" == "x" goto execute | ||
|
|
||
| set CMD_LINE_ARGS=%* | ||
|
|
||
| :execute | ||
| @rem Setup the command line | ||
|
|
||
| set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar | ||
|
|
||
| @rem Execute Gradle | ||
| "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% | ||
|
|
||
| :end | ||
| @rem End local scope for the variables with windows NT shell | ||
| if "%ERRORLEVEL%"=="0" goto mainEnd | ||
|
|
||
| :fail | ||
| rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of | ||
| rem the _cmd.exe /c_ return code! | ||
| if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 | ||
| exit /b 1 | ||
|
|
||
| :mainEnd | ||
| if "%OS%"=="Windows_NT" endlocal | ||
|
|
||
| :omega |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| # samzaJarsFolder includes all the Samza jars (you needs to make sure all the samza jars are there.) | ||
| # You can build Samza image by: | ||
| # docker build -t dockerHubAccount/samza:versionNumber . | ||
| # Then Samza user can use the Samza image as base image to build their application image. | ||
| # | ||
|
|
||
| FROM ubuntu:latest | ||
|
|
||
| RUN apt-get update -y && apt-get upgrade -y && apt-get install -y openjdk-8-jdk | ||
|
|
||
| ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 | ||
| ENV PATH $PATH:$JAVA_HOME/bin | ||
|
|
||
| RUN mkdir -p /opt/samza | ||
| WORKDIR /opt/samza/ | ||
| COPY samzaJarsFolder/ /opt/samza/ |
57 changes: 57 additions & 0 deletions
57
samza-kubernetes/src/main/java/org/apache/samza/config/KubeConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.samza.config; | ||
|
|
||
| public class KubeConfig { | ||
|
|
||
| // the image name of samza | ||
| public static final String APP_IMAGE = "kube.app.image"; | ||
| public static final String DEFAULT_IMAGE = "weiqingyang/samza:v0"; | ||
|
|
||
| // The directory path inside which the log will be stored. | ||
| public static final String SAMZA_MOUNT_DIR = "kube.app.pod.mnt.path"; | ||
| public static final String K8S_API_NAMESPACE = "kube.app.namespace"; | ||
| public static final String STREAM_PROCESSOR_CONTAINER_NAME_PREFIX = "sp"; | ||
| public static final String JC_CONTAINER_NAME_PREFIX = "jc"; | ||
| public static final String POD_RESTART_POLICY = "OnFailure"; | ||
| public static final String JC_POD_NAME_FORMAT = "%s-%s-%s"; // jc-appName-appId | ||
| public static final String TASK_POD_NAME_FORMAT = "%s-%s-%s-%s"; // sp-appName-appId-containerId | ||
|
|
||
| // Environment variable | ||
| public static final String COORDINATOR_POD_NAME = "COORDINATOR_POD_NAME"; | ||
| public static final String AZURE_REMOTE_VOLUME_ENABLED = "kube.app.volume.azure.file-share.enabled"; | ||
| public static final String AZURE_SECRET = "kube.app.volume.azure-secret"; | ||
| public static final String AZURE_FILESHARE = "kube.app.volume.azure.file-share"; | ||
|
|
||
| private Config config; | ||
| public KubeConfig(Config config) { | ||
| this.config = config; | ||
| } | ||
|
|
||
| public static KubeConfig validate(Config config) throws ConfigException { | ||
| KubeConfig kc = new KubeConfig(config); | ||
| kc.validate(); | ||
| return kc; | ||
| } | ||
|
|
||
| private void validate() throws ConfigException { | ||
| // TODO | ||
| } | ||
| } |
40 changes: 40 additions & 0 deletions
40
samza-kubernetes/src/main/java/org/apache/samza/job/kubernetes/KubeClientFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.samza.job.kubernetes; | ||
|
|
||
| import io.fabric8.kubernetes.client.Config; | ||
| import io.fabric8.kubernetes.client.ConfigBuilder; | ||
| import io.fabric8.kubernetes.client.DefaultKubernetesClient; | ||
| import io.fabric8.kubernetes.client.KubernetesClient; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
|
|
||
| public class KubeClientFactory { | ||
| private static final Logger LOG = LoggerFactory.getLogger(KubeClientFactory.class); | ||
|
|
||
| public static KubernetesClient create() { | ||
| ConfigBuilder builder = new ConfigBuilder(); | ||
| Config config = builder.build(); | ||
| KubernetesClient client = new DefaultKubernetesClient(config); | ||
| LOG.info("Kubernetes client created. "); | ||
| return client; | ||
| } | ||
| } |
Oops, something went wrong.