-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
杜文凯
committed
Jul 16, 2022
1 parent
3ad70d2
commit 9a2e5d0
Showing
22 changed files
with
753 additions
and
209 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| # 分布式数据同步机制 | ||
|
|
||
| https://lotabout.me/2019/Data-Synchronization-in-Distributed-System/ | ||
|
|
||
|
|
||
| https://highlyscalable.wordpress.com/2013/08/20/in-stream-big-data-processing/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| import numpy as np | ||
| import matplotlib.pylot as plt | ||
|
|
||
|
|
||
| class ACO: | ||
| def __init__(self, parameters): | ||
| """ | ||
| Ant Colony 蚁周模型调整 | ||
| parameter: [NGEN, pop_size, var_num_min, var_num_max] | ||
| """ | ||
| self.NGEN = parameters[0] | ||
| self.pop_size = parameters[1] | ||
| self.var_num = len(parameters[2]) | ||
| self.bound = [] | ||
| self.bound.append(parameters[2]) | ||
| self.bound.append(parameters[3]) | ||
|
|
||
| self.pop_x = np.zeros((self.pop_size, self.var_num)) | ||
| self.g_best = np.zeros((1, self.var_num)) | ||
|
|
||
| temp = -1 | ||
| for i in range(self.pop_size): | ||
| for j in range(self.var_num): | ||
| self.pop_x[i][j] = np.random.uniform(self.bound[0][j], self.bound[0][j]) | ||
| fit = self.fitness(self.pop_x[i]) | ||
| if fit > temp: | ||
| self.g_best = self.pop_x[i] | ||
| temp = fit | ||
|
|
||
| def fitness(slef, index_var): | ||
| x1 = index_var[0] | ||
| x2 = index_var[1] | ||
| x3 = index_var[2] | ||
| x4 = index_var[3] | ||
| y = x1 ** 2 + x2 **2 + x3 ** 3 + x4 ** 4 | ||
| return y |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| # 四大启发式算法 | ||
|
|
||
| # 蚁群算法 | ||
|
|
||
|
|
||
| 蚁周模型(Ant-Cycle) | ||
| 蚁量模型(Ant-Quantity) | ||
| 蚁密模型(Ant-Density) | ||
|
|
||
| https://bbs.huaweicloud.com/blogs/354067 | ||
|
|
||
|
|
||
| https://finthon.com/python-aco/ | ||
|
|
||
| # A*寻路算法 | ||
|
|
||
| https://blog.csdn.net/u014361280/article/details/104740876 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # spark on yarn 调度 | ||
|
|
||
|
|
||
|
|
||
| https://blog.coderap.com/article/309 | ||
|
|
||
| https://www.cnblogs.com/xia520pi/p/8695141.html |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| 1. API接口层获取sql语句 | ||
|
|
||
| ```scala | ||
| @At(path = Array("/run/script"), types = Array(GET, POST)){ | ||
| val sparkSession = getSession | ||
|
|
||
| accessAuth(sparkSession) | ||
| val jobInfo = JobManager.getJobInfo( | ||
| param("owner"), param("jobType", MLSQLJobType.SCRIPT), param("jobName"), param("sql"), | ||
| paramAsLong("timeout", -1L) | ||
| ) | ||
| val context = createScriptSQLExecListener(sparkSession, jobInfo.groupId) | ||
|
|
||
| def query = { | ||
| if (paramAsBoolean("async", false)) { | ||
| JobManager.asyncRun(sparkSession, jobInfo, () => { | ||
| val urlString = param("callback") | ||
| val maxTries = Math.max(0, paramAsInt("maxRetries", -1)) + 1 | ||
| try { | ||
| ScriptSQLExec.parse(param("sql"), context, | ||
| skipInclude = paramAsBoolean("skipInclude", false), | ||
| skipAuth = paramAsBoolean("skipAuth", true), | ||
| skipPhysicalJob = paramAsBoolean("skipPhysicalJob", false), | ||
| skipGrammarValidate = paramAsBoolean("skipGrammarValidate", true)) | ||
|
|
||
| outputResult = getScriptResult(context, sparkSession) | ||
|
|
||
| executeWithRetrying[HttpResponse](maxTries)( | ||
| RestUtils.httpClientPost(urlString, | ||
| Map("stat" -> s"""succeeded""", | ||
| "res" -> outputResult, | ||
| "jobInfo" -> JSONTool.toJsonStr(jobInfo))), | ||
| HttpStatus.SC_OK == _.getStatusLine.getStatusCode, | ||
| response => logger.error(s"Succeeded SQL callback request failed after ${maxTries} attempts, " + | ||
| s"the last response status is: ${response.getStatusLine.getStatusCode}.") | ||
| ) | ||
| } | ||
| ``` | ||
| 2. 创建脚本SQL执行监听器 | ||
| ```scala | ||
| private def createScriptSQLExecListener(sparkSession: SparkSession, groupId: String) = { | ||
|
|
||
| val allPathPrefix = fromJson(param("allPathPrefix", "{}"), classOf[Map[String, String]]) | ||
| val defaultPathPrefix = param("defaultPathPrefix", "") | ||
| val context = new ScriptSQLExecListener(sparkSession, defaultPathPrefix, allPathPrefix) | ||
| val ownerOption = if (params.containsKey("owner")) Some(param("owner")) else None | ||
| val userDefineParams = params.toMap.filter(f => f._1.startsWith("context.")).map(f => (f._1.substring("context.".length), f._2)) | ||
| ScriptSQLExec.setContext(new MLSQLExecuteContext(context, param("owner"), context.pathPrefix(None), groupId, | ||
| userDefineParams ++ Map("__PARAMS__" -> JSONTool.toJsonStr(params().toMap)) | ||
| )) | ||
| context.initFromSessionEnv | ||
| context.addEnv("SKIP_AUTH", param("skipAuth", "true")) | ||
| context.addEnv("HOME", context.pathPrefix(None)) | ||
| context.addEnv("OWNER", ownerOption.getOrElse("anonymous")) | ||
| context | ||
| } | ||
| ``` |
Oops, something went wrong.