diff --git a/DevOps/airflow.md b/DevOps/airflow.md new file mode 100644 index 0000000..d079497 --- /dev/null +++ b/DevOps/airflow.md @@ -0,0 +1,4 @@ +# 使用airflow来做etl + + +https://gtoonstra.github.io/etl-with-airflow/principles.html diff --git "a/DevOps/\346\225\217\346\215\267\345\274\200\345\217\221/JIRA.md" "b/DevOps/\346\225\217\346\215\267\345\274\200\345\217\221/JIRA.md" new file mode 100644 index 0000000..ee255dc --- /dev/null +++ "b/DevOps/\346\225\217\346\215\267\345\274\200\345\217\221/JIRA.md" @@ -0,0 +1,12 @@ +# JIRA + + +https://www.atlassian.com/zh/agile/tutorials/creating-your-agile-board + +https://webeye.atlassian.net/secure/ShowConstantsHelp.jspa?decorator=popup#IssueTypes + +https://support.atlassian.com/jira-software-cloud/docs/manage-epics-on-the-roadmap/ + +## 自动化 + +https://support.atlassian.com/jira-software-cloud/docs/automate-your-jira-cloud-processes-and-workflows/ diff --git a/Q&A/python-q&a.md b/Q&A/python-q&a.md index f910eca..cf58219 100644 --- a/Q&A/python-q&a.md +++ b/Q&A/python-q&a.md @@ -81,3 +81,13 @@ https://stackoverflow.com/questions/12047847/super-object-not-calling-getattr ## __getattr__ https://www.cnblogs.com/xybaby/p/6280313.html + + +## 65192 Task was destroyed but it is pending! +https://stackoverflow.com/questions/33505066/python3-asyncio-task-was-destroyed-but-it-is-pending-with-some-specific-condit + + +https://stackoverflow.com/questions/27402796/multiple-loops-with-asyncio + + +GCP 9 10 2天数据 diff --git "a/Q&A/\345\205\250\351\223\276\350\267\257\347\233\221\346\216\247.md" "b/Q&A/\345\205\250\351\223\276\350\267\257\347\233\221\346\216\247.md" new file mode 100644 index 0000000..fa0cd5c --- /dev/null +++ "b/Q&A/\345\205\250\351\223\276\350\267\257\347\233\221\346\216\247.md" @@ -0,0 +1,120 @@ +## 日志标准提炼 + + +请求日志标准 + +date, user_id, 登录人，部门，业务类型， 登录人邮箱，请求实际ip，实际serverip，请求方法, 请求url，请求参数， 请求状态码， 错误traceback, 标签用于事件分类 + + +https://mp.weixin.qq.com/s?__biz=MzU0OTE4MzYzMw==&mid=2247488015&idx=2&sn=563ae34159fa560cfcdaee83cfbdb263&chksm=fbb29bf1ccc512e7d5447ffa316358ae8e780fb19f0242ae8b2b77264bd1353a7239a1d830cd&mpshare=1&scene=1&srcid=&sharer_sharetime=1577937494737&sharer_shareid=2a07c120ead5e9a5e0c0fc8b3f05a6dd&key=e941abab93429f934cb25f2a26a2d6f05d55d8c9ceb509ca986152450bffd4677e85a8d338510e479bec763923844e2cfe495e80d3ad7d4861b4f0e344a0167b82baabc013a6ff8407921e6e2c16a89c&ascene=1&uin=MTE5MjQ0MzcwOQ%3D%3D&devicetype=Windows+10&version=62080079&lang=zh_CN&exportkey=A%2BG28iXG5s6DdrKpoSLy%2Fzo%3D&pass_ticket=n3xbx4dv1fXXVF5kGO5UxxxuczfB4o6g7YUqOLOtgRRwmY9p10xNZYgXjc8oCif3 + +## 全链路监控 + +OpenTracing + + +### Dapper + +https://bigbully.github.io/Dapper-translation/ + +https://research.google/pubs/pub36356/ + +https://imfox.io/2017/10/14/hunter-1/ + +## 蚂蚁金服 SOFATracer + +https://www.lizenghai.com/archives/47135.html + +## 阿里鹰眼 + +https://www.infoq.cn/article/kMPZTgJqs7VJC5vkVCR2 + + +http://jm.taobao.org/about/ + +## zipkin + +https://www.cnblogs.com/zhangs1986/p/8879744.html + +https://carey.akhack.com/2018/10/12/django%E5%88%86%E5%B8%83%E5%BC%8F%E9%93%BE%E8%B7%AF%E7%9B%91%E6%8E%A7%E4%B9%8BZipkin/ + + +https://riboseyim.github.io/2018/05/18/DevOps-OpenTracing/ + +https://www.sofastack.tech/projects/sofa-tracer/traceid-generated-rule/ + +https://ixyzero.com/blog/archives/4438.html + +https://leancloudblog.com/terms/full-link-tracing/ + + +## google Stackdriver Trace + +https://cloud.google.com/trace/docs/viewing-details + + + +## 业务全链路智能化监控 + + +https://juejin.im/entry/5ba06ee4f265da0ab87372ee + + +https://www.sofastack.tech/projects/sofa-tracer/traceid-generated-rule/ + + +### 数据采集标准化 + +采集日志规范规范 +- 基础信息 + - 时间 + - traceId + - ip + - 应用名 +- 调用信息 + - 业务码 + - 接口 + - 文件名称 + - 方法名称 + - 延时 + - 是否成功 +- 错误信息 + - 错误码 + - 类型 + - 摘要 +- 用户信息 +- 设备信息 + +参考 +https://juejin.im/entry/5ba06ee4f265da0ab87372ee + + +##### traceID + +https://twitter.github.io/finagle/docs/com/twitter/finagle/tracing/TraceId.html + + +## jager + +https://pjw.io/articles/2018/05/18/jaeger-tutorial/ + +## opencensus google + +https://opencensus.io/ + +https://opencensus.io/community/talks/ + +## jager和opencensus对比 + +https://medium.com/jaegertracing/jaeger-and-opentelemetry-1846f701d9f2 + +https://www.infoq.cn/article/pb3QOmLVoBN6IZS8hBZy + + +## 亚马逊 + +https://aws.amazon.com/cn/xray/ + +https://docs.aws.amazon.com/xray-sdk-for-python/latest/reference/ + +https://www.sofastack.tech/projects/sofa-tracer/traceid-generated-rule/ diff --git "a/Q&A/\345\221\212\350\255\246\347\233\221\346\216\247.md" "b/Q&A/\345\221\212\350\255\246\347\233\221\346\216\247.md" new file mode 100644 index 0000000..0ba23a3 --- /dev/null +++ "b/Q&A/\345\221\212\350\255\246\347\233\221\346\216\247.md" @@ -0,0 +1,169 @@ +## 告警监控 + +- 收集 +- 存储 +- 报警规则 +- 报警行为 + +https://zhuanlan.zhihu.com/p/27382099 + +https://dbaplus.cn/news-141-2038-1.html + + +http://fex.baidu.com/blog/2014/05/build-performance-monitor-in-7-days/ + + + +https://dbaplus.cn/news-141-2038-1.html + + +## 告警监控系统设计 + +https://juejin.im/post/5cf0e202f265da1b8b2b44fa + + +http://www.yunweipai.com/archives/23202.html + +https://juejin.im/entry/5ba06ee4f265da0ab87372ee + + +## 重要监控系统设计文章 + + + +http://download.xuliangwei.com/jiankong.html + +https://testerhome.com/topics/19188 + +https://segmentfault.com/a/1190000015308746 + + +## 设计实现 + +### 设计目标 + +- 故障分析定位 +应用系统出问题，首要想到的是，通过监控系统从复杂的系统间找到问题，定位问题所在。 +- 系统问题预警 +该预警一般要求实时性，同时一般会考虑性能与实时性的中和。所起到的目的就是在客户反馈问题前，能做到预警，做的更高级一点的就是能够根据预警做到自动处理，取代以前的人工干预的部分。 +- 提供历史趋势分析 +这个比较好理解，随着时间历史监控数据的增多，根据历史数据做一些监控预测分析一类的简单工作还是比较容易的，比如：下个月的交易量，下个促销日的并发量，下个版本发布后，能够承受的uv等以及衍生出来的CPU，memory，磁盘使用量等预测信息。 +- 提供可视化报表等，辅助分析 +通过可视化的仪表盘能够直接获取系统运行状态，资源使用情况，以及服务运行状态等直观信息。 + +### 数据采集标准化 + +采集日志规范规范 +- 基础信息 + - 时间 + - traceId + - ip + - 应用名 +- 调用信息 + - 业务码 + - 接口 + - 文件名称 + - 方法名称 + - 方法 + - 延时 + - 是否成功 +- 错误信息 + - 错误码 + - 类型 + - 摘要 +- 扩展信息 + - 压测 + - 业务身份 + +参考: + +https://juejin.im/entry/5ba06ee4f265da0ab87372ee + +### 数据采集 + +Open Census 是一组用于各种语言的库，使您可以收集应用程序指标和分布式跟踪，然后将数据实时传输到您选择的后端。开发人员和管理员可以分析此数据，以了解应用程序的运行状况并调试问题. + +如何在我的项目中使用OpenCensus？ +我们提供用于Go，Java，C＃，Node.js，C ++，Ruby，Erlang / Elixir，Python，Scala和PHP的库。 + +支持的后端包括Azure Monitor，Datadog，Instana，Jaeger，New Relic，SignalFX，Google Cloud Monitoring + Trace和Zipkin。您还可以添加对其他后端的支持。 + +#### opencensus 使用 + +```sh +pipenv install opencensus opencensus-ext-django +``` + +https://opencensus.io/ + +https://opentelemetry.io/ + +https://opencensus.io/quickstart/python/ + +https://cloud.google.com/monitoring/custom-metrics/open-census?hl=zh-cn + +https://docs.microsoft.com/en-us/azure/azure-monitor/app/opencensus-python-request + +https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors + +https://stackoverflow.com/tags/opencensus/hot?filter=all + + +### 数据存储 + + +### 数据分析 + +Zipkin展示分析 + + +架构 + +https://zipkin.io/pages/architecture.html + + +``` +sudo docker run -d -p 9411:9411 openzipkin/zipkin +$ sudo docker logs -f 86861199ee03 + + oo + oooo + oooooo + oooooooo + oooooooooo + oooooooooooo + ooooooo ooooooo + oooooo ooooooo + oooooo ooooooo + oooooo o o oooooo + oooooo oo oo oooooo + ooooooo oooo oooo ooooooo + oooooo ooooo ooooo ooooooo + oooooo oooooo oooooo ooooooo + oooooooo oo oo oooooooo + ooooooooooooo oo oo ooooooooooooo + oooooooooooo oooooooooooo + oooooooo oooooooo + oooo oooo + + ________ ____ _ _____ _ _ + |__ /_ _| _ \| |/ /_ _| \ | | + / / | || |_) | ' / | || \| | + / /_ | || __/| . \ | || |\ | + |____|___|_| |_|\_\___|_| \_| + +:: version 2.20.1 :: commit 7cbe4d0 :: + +2020-03-13 03:44:49.589 INFO 1 --- [ main] z.s.ZipkinServer : Starting ZipkinServer on 86861199ee03 with PID 1 (/zipkin/BOOT-INF/classes started by zipkin in /zipkin) +2020-03-13 03:44:49.594 INFO 1 --- [ main] z.s.ZipkinServer : The following profiles are active: shared +2020-03-13 03:44:50.397 WARN 1 --- [ main] i.m.c.i.b.j.JvmGcMetrics : GC notifications will not be available because com.sun.management.GarbageCollectionNotificationInfo is not present +2020-03-13 03:44:50.486 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.verboseExceptions: rate-limit=10 (default) +2020-03-13 03:44:50.494 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.verboseSocketExceptions: false (default) +2020-03-13 03:44:50.494 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.verboseResponses: false (default) +2020-03-13 03:44:50.522 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.useEpoll: true (default) +2020-03-13 03:44:50.523 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.maxNumConnections: 2147483647 (default) +2020-03-13 03:44:50.523 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.numCommonWorkers: 8 (default) +2020-03-13 03:44:50.524 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.numCommonBlockingTaskThreads: 200 (default) +2020-03-13 03:44:50.525 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.defaultMaxRequestLength: 10485760 (default) +2020-03-13 03:44:50.526 INFO 1 --- [ main] c.l.a.c.Flags : com.linecorp.armeria.defaultMaxResponseLength: 10485760 (default) +``` diff --git "a/Q&A/\346\231\256\347\275\227\347\261\263\344\277\256\346\226\257.md" "b/Q&A/\346\231\256\347\275\227\347\261\263\344\277\256\346\226\257.md" new file mode 100644 index 0000000..dce70cf --- /dev/null +++ "b/Q&A/\346\231\256\347\275\227\347\261\263\344\277\256\346\226\257.md" @@ -0,0 +1,18 @@ +# 普罗米修斯 + + +https://yunlzheng.gitbook.io/prometheus-book/parti-prometheus-ji-chu/promql/prometheus-promql-best-praticase + +https://prometheus.io/docs/prometheus/latest/getting_started/ + +https://www.qikqiak.com/post/alertmanager-of-prometheus-in-practice/ + +https://www.infoq.cn/article/ObvBIBflGNgboDBNk4Iy + +https://blog.csdn.net/lijiaocn/article/details/81865120 + +https://aleiwu.com/post/prometheus-bp/ + +## Alertmanager + +https://www.jianshu.com/p/fd0b018539cd diff --git "a/Q&A/\351\224\231\350\257\257\347\233\221\346\216\247\345\271\263\345\217\260.md" "b/Q&A/\351\224\231\350\257\257\347\233\221\346\216\247\345\271\263\345\217\260.md" new file mode 100644 index 0000000..b2cbed5 --- /dev/null +++ "b/Q&A/\351\224\231\350\257\257\347\233\221\346\216\247\345\271\263\345\217\260.md" @@ -0,0 +1,336 @@ +[](https://www.g2.com/products/sentry/competitors/alternatives) + + +https://airbrake.io/ + + +https://alternative.me/sentry + +https://sourceforge.net/software/?q=sentry + + +https://sourceforge.net/software/?q=sentry + + +异常监控对比 + +https://blog.csdn.net/uikoo9/article/details/97286762 + + +https://www.open-open.com/lib/view/open1436018241732.html + + + +stackdriver + +https://cloud.google.com/error-reporting?hl=zh-cn + +sentry + +https://sentry.io/features/ + +# Sentry + +http://www.yunweipai.com/archives/22303.html + + +## sentry install + +https://www.jianshu.com/p/cea2d22fbb32 + +https://github.com/getsentry/onpremise + +```sh +sudo yum install docker-ce docker-ce-cli containerd.io -y + +sudo curl -L "https://github.com/docker/compose/releases/download/1.25.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + +sudo chmod +x /usr/local/bin/docker-compose + +sudo chown hedwig:hedwig sentry/ + +git clone https://github.com/getsentry/onpremise.git + + +mkdir sentry-data sentry-postgres -p + +sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose + +$ sudo docker-compose run --rm web config generate-secret-key +ERROR: Volume sentry-data declared as external, but could not be found. Please create the volume manually using `docker volume create --name=sentry-data` and try again. + +$ sudo docker volume create --name=sentry-data +sentry-data +$ sudo docker volume create --name=sentry-postgres +sentry-postgres +sudo docker volume create --name=sentry-redis + + +sudo docker volume create --name=sentry-zookeeper +sudo docker volume create --name=sentry-kafka +sudo docker volume create --name=sentry-clickhouse +sudo docker volume create --name=sentry-symbolicator + +$ sudo docker-compose run --rm web config generate-secret-key +Starting sentry_onpremise_clickhouse_1 ... done +Starting sentry_onpremise_redis_1 ... done +Starting sentry_onpremise_smtp_1 ... +Starting sentry_onpremise_postgres_1 ... done +Starting sentry_onpremise_memcached_1 ... done +Starting sentry_onpremise_smtp_1 ... done +Starting sentry_onpremise_kafka_1 ... done +Starting sentry_onpremise_snuba-consumer_1 ... done +Starting sentry_onpremise_snuba-api_1 ... done +Starting sentry_onpremise_snuba-replacer_1 ... done +密钥: +#y&(2tsx_dpl!dad0#*1+f*f5jiiu*ii%k@c=pi+f8adu(-7z* + +``` + +依赖: +- redis +- postgres +- memcached +- smtp +- snuba-api +- snuba-consumer +- snuba-replacer +- symbolicator +- kafka + +## ERROR: Couldn't connect to Docker daemon at http+docker://localhost - is it running? + +https://www.awaimai.com/2587.html + +https://stackoverflow.com/questions/38775954/sudo-docker-compose-command-not-found + +https://blog.csdn.net/xiojing825/article/details/79494408 + +## ERROR sentry + +执行这个生成前端静态文件, sentry的源码路径 `/usr/local/lib/python2.7/site-packages/sentry/` +> sentry django collectstatic + +``` + + +
+
+
+
+
+

Please wait while we load an obnoxious amount of JavaScript.

+

You may need to disable adblocking extensions to load Sentry.

+
+
+
+ + + + + + + +/usr/local/lib/python2.7/site-packages/sentry/static +``` +https://cloud.tencent.com/developer/article/1173949 + + +https://docs.sentry.io/server/ + +https://docs.sentry.io/server/nginx/ + +https://thinkhard.tech/2019/05/25/docker-sentry-deploy/ + +https://juejin.im/post/5c9f4639e51d452724598c07 + +https://blog.frognew.com/2017/11/using-docker-compose-install-sentry.html + +https://www.chenng.cn/post/sentry-in-practice.html + + + +# sentry 的客户端 raven + +https://github.com/getsentry/raven-python + +# sentryt命令 + +```sh +root@e6c9ff197ee6:/usr/src/sentry# sentry --help +Usage: sentry [OPTIONS] COMMAND [ARGS]... + + Sentry is cross-platform crash reporting built with love. + + The configuration file is looked up in the `~/.sentry` config directory but this can be + overridden with the `SENTRY_CONF` environment variable or be explicitly provided through the + `--config` parameter. + +Options: + --config PATH Path to configuration files. + --version Show the version and exit. + --help Show this message and exit. + +Commands: + cleanup Delete a portion of trailing data based on... + config Manage runtime config options. + createuser Create a new user. + devserver Starts a lightweight web server for... + devservices Manage dependent development services... + django Execute Django subcommands. + exec Execute a script. + export Exports core metadata for the Sentry... + files Manage files from filestore. + help Show this message and exit. + import Imports data from a Sentry export. + init Initialize new configuration directory. + permissions Manage Permissions for Users. + plugins Manage Sentry plugins. + queues Manage Sentry queues. + repair Attempt to repair any invalid data. + run Run a service. + shell Run a Python interactive interpreter. + start DEPRECATED see `sentry run` instead. + tsdb Tools for interacting with the time series... + upgrade Perform any pending database migrations and... + + +root@e6c9ff197ee6:/usr/src/sentry# sentry django help +03:29:03 [WARNING] sentry.utils.geo: settings.GEOIP_PATH_MMDB not configured. +03:29:07 [INFO] sentry.plugins.github: apps-not-configured + +Type 'sentry django help ' for help on a specific subcommand. + +Available subcommands: + +[auth] + changepassword + createsuperuser + +[contenttypes] + remove_stale_contenttypes + +[django] + check + compilemessages + createcachetable + dbshell + diffsettings + dumpdata + flush + inspectdb + loaddata + makemessages + migrate + sendtestemail + shell + showmigrations + sqlflush + sqlmigrate + sqlsequencereset + squashmigrations + startapp + startproject + test + testserver + +[sentry] + check_notifications + collectstatic + create_sample_event + generate_reset_password_link + makemigrations + merge_users + send_fake_data + serve_normalize + +[sessions] + clearsessions + +[staticfiles] + findstatic + runserver +``` + + +```sh +# nginx conf +server { + listen 9001; + server_name 35.185.90.193; + + access_log /var/log/nginx/sentry-access.log; + + add_header X-Cache-Status $upstream_cache_status; + include /etc/nginx/mime.types; + + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_redirect off; + + # keepalive + raven.js is a disaster + keepalive_timeout 0; + + # use very aggressive timeouts + proxy_read_timeout 5s; + proxy_send_timeout 5s; + send_timeout 5s; + resolver_timeout 5s; + client_body_timeout 5s; + + # buffer larger messages + client_body_buffer_size 1024M; + client_max_body_size 1024M; + + location / { + proxy_pass http://localhost:9000; + } +} + + +server { + listen 8080 default; + server_name 35.185.90.193; + access_log /var/log/nginx/hedwig-old-access.log; + + add_header X-Cache-Status $upstream_cache_status; + include /etc/nginx/mime.types; + + client_body_buffer_size 1024M; + client_max_body_size 1024M; + + gzip on; + gzip_min_length 1k; + gzip_buffers 4 16k; + gzip_disable "MSIE [1-6]\."; + gzip_comp_level 3; + gzip_types application/json text/plain application/x-javascript text/css application/xml text/javascript application/javascript; + + root /opt/webeye/hedwig/src/admin; + + location ~ ^/static/ { + access_log off; + try_files $uri @proxied; + } + + location / { + try_files $uri @proxied; + } + + location @proxied { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://127.0.0.1:17070; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + } +} + +``` diff --git "a/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221.md" "b/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221.md" index b585f86..e9de94b 100644 --- "a/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221.md" +++ "b/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221/\345\256\211\345\205\250\346\212\200\350\203\275\346\240\221.md" @@ -7,3 +7,6 @@ [evicos 安全技能树](https://evilcos.me/security_skill_tree_basic/index.html) +https://www.concise-courses.com/hacking-tools/top-ten/ + +http://sourceware.org/gdb/wiki/GDB%20Front%20Ends diff --git "a/\345\270\270\347\224\250\345\221\275\344\273\244/\347\263\273\347\273\237\345\270\270\347\224\250\345\221\275\344\273\244/Linux/Linux\345\270\270\347\224\250\345\221\275\344\273\244.md" "b/\345\270\270\347\224\250\345\221\275\344\273\244/\347\263\273\347\273\237\345\270\270\347\224\250\345\221\275\344\273\244/Linux/Linux\345\270\270\347\224\250\345\221\275\344\273\244.md" index 4f8ef50..d81f1db 100644 --- "a/\345\270\270\347\224\250\345\221\275\344\273\244/\347\263\273\347\273\237\345\270\270\347\224\250\345\221\275\344\273\244/Linux/Linux\345\270\270\347\224\250\345\221\275\344\273\244.md" +++ "b/\345\270\270\347\224\250\345\221\275\344\273\244/\347\263\273\347\273\237\345\270\270\347\224\250\345\221\275\344\273\244/Linux/Linux\345\270\270\347\224\250\345\221\275\344\273\244.md" @@ -301,3 +301,10 @@ https://stackoverflow.com/questions/8802131/execute-multiline-mysql-in-shellscri https://stackoverflow.com/questions/8149545/pass-array-to-mysql-stored-routine https://blog.basilediougoant.com/2016/02/06/how-to-pass-an-array-to-mysql-stored-procedure/ + +## mysql export remote database data +```sql +mysql -umysqlusername -pmysqlpass databasename -B -e "select * from \`tablename\`;" | sed 's/\t/","/g;s/^/"/;s/$/"/;s/\n//g' > mysql_exported_table.csv +``` + +https://stackoverflow.com/questions/16834880/mysql-export-resultset-as-csv-from-remote-server diff --git "a/\345\270\270\350\247\201\345\215\217\350\256\256/ProtocolBuffers.md" "b/\345\270\270\350\247\201\345\215\217\350\256\256/ProtocolBuffers.md" new file mode 100644 index 0000000..9be6dc5 --- /dev/null +++ "b/\345\270\270\350\247\201\345\215\217\350\256\256/ProtocolBuffers.md" @@ -0,0 +1,10 @@ +# protocol buffers + + +- Define message formats in a `.proto` file. +- Use the protocol buffer compiler +- Use the Python protocol buffer API to write and read message. + + + +https://developers.google.com/protocol-buffers/docs/tutorials diff --git "a/\346\223\215\344\275\234\347\263\273\347\273\237/\345\244\232\350\267\257\345\244\215\347\224\250\346\234\272\345\210\266.md" "b/\346\223\215\344\275\234\347\263\273\347\273\237/\345\244\232\350\267\257\345\244\215\347\224\250\346\234\272\345\210\266.md" index 21baff9..6c449b4 100644 --- "a/\346\223\215\344\275\234\347\263\273\347\273\237/\345\244\232\350\267\257\345\244\215\347\224\250\346\234\272\345\210\266.md" +++ "b/\346\223\215\344\275\234\347\263\273\347\273\237/\345\244\232\350\267\257\345\244\215\347\224\250\346\234\272\345\210\266.md" @@ -6,4 +6,14 @@ I/O多路复用就通过一种机制,可以监视多个描述符,一旦某个描 参考: -[select, poll, epoll机制](http://www.cnblogs.com/Anker/p/3265058.html) \ No newline at end of file +[select, poll, epoll机制](http://www.cnblogs.com/Anker/p/3265058.html) + +[CPU](https://notbe.cn/2019/05/12/YPL_ZML_90138301.html) + +https://stackoverflow.com/questions/38193596/asyncio-multiprocessing-unix + +https://stackoverflow.com/questions/46074841/why-coroutines-cannot-be-used-with-run-in-executor + +https://pymotw.com/3/asyncio/executors.html + +https://stackoverflow.com/questions/46074841/why-coroutines-cannot-be-used-with-run-in-executor diff --git "a/\346\225\260\346\215\256\345\272\223/bigquery.md" "b/\346\225\260\346\215\256\345\272\223/bigquery.md" new file mode 100644 index 0000000..6bd3505 --- /dev/null +++ "b/\346\225\260\346\215\256\345\272\223/bigquery.md" @@ -0,0 +1,6 @@ + +https://cloud.google.com/bigquery/docs/how-to + +费用 + +https://cloud.google.com/bigquery/quotas#load_jobs diff --git "a/\346\225\260\346\215\256\345\272\223/mysql/mysql.md" "b/\346\225\260\346\215\256\345\272\223/mysql/mysql.md" index 7a22b46..774aa18 100644 --- "a/\346\225\260\346\215\256\345\272\223/mysql/mysql.md" +++ "b/\346\225\260\346\215\256\345\272\223/mysql/mysql.md" @@ -84,5 +84,69 @@ mysqldump db tb1 tb2 > path ['like % 查询'] (https://dataedo.com/kb/query/mysql/find-tables-which-names-start-with-specific-prefix) +## mysql Insert ignore into,insert into on duplicate,replace into -## 避免笛卡尔积 +整体上我分为两个大的部分，会分别测试这三种数据写入场景。 + +> 第一部分基于id,name的数据列，其中id为主键，自增 + +这三种场景的结果从自增列的处理方式来看是完全对等的，但是对于重复数据的处理方式还是存在差异。 + +相比而言，**replace into**和**insert into on duplicate**存在本质的区别，**replace into**是覆盖写，即删除原来的，写入新的。不光是主键列，其他列也会保持一致 + +**insert into on duplicate**则可以根据自己的需求来定制重复数据的处理策略，不会主动改变数据。 + +**insert ignore into** 在这种场景下最为通用，而且对于数据的侵入性最小。 + +所以如果要保证源端的数据基于主键完全一致，不管非主键列的数据是否一致，都需要完全覆盖，选择**replace into**是一种好的方法。 + +否则采用**insert into on duplcate**或者**insert ignore into** + +> 第二部分基于id,xid,name的数据列，其中id为主键，自增，xid为唯一性索引 + +**insert ignore into**如果不指定自增列，尽管没有写入数据，但是自增列依然会自增 + +**replace into**如果不指定自增列，会看到数据重新写入的效果已经非常明显，而且自增列始终会自动维护。 + +**insert into on duplicate**对于重复数据依然会消耗自增列值，实现相对更加灵活。 + +https://mp.weixin.qq.com/s/T4sXXPn-9rXSSag0hP9zwg + + +## mysql 死锁 + +```mysql +mysql> show variables like '%innodb%lock%timeout%'; ++--------------------------+-------+ +| Variable_name | Value | ++--------------------------+-------+ +| innodb_lock_wait_timeout | 50 | ++--------------------------+-------+ +1 row in set (0.01 sec) + +mysql> show variables like '%innodb%deadlock%'; ++----------------------------+-------+ +| Variable_name | Value | ++----------------------------+-------+ +| innodb_print_all_deadlocks | OFF | ++----------------------------+-------+ +1 row in set (0.02 sec) +``` + +InnoDb 的监控主要分为四种：标准监控（Standard InnoDB Monitor）、锁监控（InnoDB Lock Monitor）、表空间监控（InnoDB Tablespace Monitor）和表监控（InnoDB Table Monitor）。后两种监控已经基本上废弃了，关于各种监控的作用可以参考 MySQL 的官方文档 Enabling InnoDB Monitors [https://dev.mysql.com/doc/refman/5.6/en/innodb-enabling-monitors.html] 或者 [http://yeshaoting.cn/article/database/%E5%BC%80%E5%90%AFInnoDB%E7%9B%91%E6%8E%A7/] 这篇文章. + +常见锁的类型: +- 记录锁（LOCK_REC_NOT_GAP）: lock_mode X locks rec but not gap +- 间隙锁（LOCK_GAP）: lock_mode X locks gap before rec +- Next-key 锁（LOCK_ORNIDARY）: lock_mode X +- 插入意向锁（LOCK_INSERT_INTENTION）: lock_mode X locks gap before rec insert intention + +https://ketao1989.github.io/2014/10/09/Mysql-Delete-Insert-Deadlock-Analyse/ + +https://juejin.im/post/5d04b33e6fb9a07ee1692653 + +https://leokongwq.github.io/2017/07/06/mysql-deadlock-detection.html + +https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html + +https://www.aneasystone.com/archives/2018/04/solving-dead-locks-four.html diff --git "a/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/Django\345\256\241\350\256\241.md" "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/Django\345\256\241\350\256\241.md" new file mode 100644 index 0000000..8d92c4a --- /dev/null +++ "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/Django\345\256\241\350\256\241.md" @@ -0,0 +1,59 @@ +# Django 审计调研 + + +https://stackoverflow.com/questions/11694804/django-auditing-models-and-recording-admin-usage + +https://github.com/soynatan/django-easy-audit + +https://github.com/jjkester/django-auditlog + +https://djangopackages.org/grids/g/model-audit/ + + +## django 中间件的原理 + +https://docs.djangoproject.com/en/3.0/topics/http/middleware/ + + +## 审计 + +easy-audit 使用django的signal来监控 + +https://docs.djangoproject.com/en/3.0/topics/signals/ + +## 静态检测模块 + +https://docs.djangoproject.com/en/3.0/topics/checks/ + + +# 安全中间件 csrf + +https://www.secpulse.com/archives/76204.html + + + +## django signal 源码解读 + +https://dreamgoing.github.io/Django%E6%BA%90%E7%A0%81%E9%98%85%E8%AF%BB(%E4%BA%8C)-Signal.html + + +https://django-advanced-training.readthedocs.io/en/latest/features/signals/ + +## django 的user 为 SimpleLazyObject +https://www.hongweipeng.com/index.php/archives/1787/ + + + +## copy wsgi.input + +https://stackoverflow.com/questions/1783383/how-do-i-copy-wsgi-input-if-i-want-to-process-post-data-more-than-once + +https://wsgi.readthedocs.io/en/latest/specifications/handling_post_forms.html + +## django 中间件原理 + +https://www.jianshu.com/p/eacea630e078 + +https://www.liujiangblog.com/blog/45/ + +https://simpleisbetterthancomplex.com/tutorial/2016/07/18/how-to-create-a-custom-django-middleware.html diff --git "a/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/django.md" "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/django.md" index c17af9a..e53a571 100644 --- "a/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/django.md" +++ "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/Django/django.md" @@ -133,3 +133,39 @@ https://stackoverflow.com/questions/16854659/exclude-field-from-values-or-values https://blog.knownsec.com/2016/10/django-csrf-bypass_cve-2016-7401/ https://strcpy.me/index.php/archives/753/ https://www.leavesongs.com/HTML/zhihu-xss-worm.html + +## 不能添加外键约束 + +https://stackoverflow.com/questions/15534977/mysql-cannot-add-foreign-key-constraint +https://stackoverflow.com/questions/18391034/cannot-resolve-table-name-close-to +https://www.percona.com/blog/2017/04/06/dealing-mysql-error-code-1215-cannot-add-foreign-key-constraint/ + +https://stackoverflow.com/questions/10028214/add-foreign-key-to-existing-table?rq=1 + +https://stackoverflow.com/questions/14122031/how-to-remove-constraints-from-my-mysql-table + +https://stackoverflow.com/questions/18930084/mysql-error-1215-hy000-cannot-add-foreign-key-constraint + + +## django filter != +https://stackoverflow.com/questions/844556/filtering-for-empty-or-null-names-in-a-queryset + + +## Django debug panel + +https://www.bedjango.com/blog/how-debug-django-useful-tips/ + + +## django signals +https://docs.djangoproject.com/en/3.0/topics/signals/ + + +## django backends log + +https://docs.djangoproject.com/en/3.0/topics/logging/ + + +## django 的traceback机制 + + +https://pycoders-weekly-chinese.readthedocs.io/en/latest/issue8/Notes-on-tracing-code-execution-in-Django-and-Python.html diff --git "a/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python-snippet.md" "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python-snippet.md" index fb1d018..9de8d7c 100644 --- "a/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python-snippet.md" +++ "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python-snippet.md" @@ -228,3 +228,18 @@ context.proxy_server.start() context.proxy = context.proxy_server.create_proxy() print("proxy start... {}".format(context.proxy_server)) ``` + +# threadpoolexecutor map和 submit的区别 + +https://stackoverflow.com/questions/20838162/how-does-threadpoolexecutor-map-differ-from-threadpoolexecutor-submit + + +# less-copies-in-python-with-the-buffer-protocol-and-memoryviews + +https://eli.thegreenplace.net/2011/11/28/less-copies-in-python-with-the-buffer-protocol-and-memoryviews + +https://julien.danjou.info/high-performance-in-python-with-zero-copy-and-the-buffer-protocol/ + +https://bugs.python.org/issue30193 + +https://forum.micropython.org/viewtopic.php?t=103 diff --git "a/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python.md" "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python.md" new file mode 100644 index 0000000..70c129a --- /dev/null +++ "b/\347\274\226\347\250\213\350\257\255\350\250\200/Python/python.md" @@ -0,0 +1,59 @@ +# python + +## python深入探究 + +https://devguide.python.org/appendix/#language-development-in-depth + + +## python获取子类 + +### __subclasses__ + +https://xz.aliyun.com/t/2308 + +https://ctf-wiki.github.io/ctf-wiki/pwn/linux/sandbox/python-sandbox-escape/ + +https://pyzh.readthedocs.io/en/latest/python-magic-methods-guide.html + +https://stackoverflow.com/questions/35165669/is-there-a-pythonic-way-to-skip-decoration-on-a-subclass-method + +https://help.semmle.com/wiki/display/PYTHON/Overwriting+attribute+in+super-class+or+sub-class + +### 反射 + +https://stackoverflow.com/questions/34623610/how-to-use-the-subclasscheck-magic-method + +`__init_subclass__` + +https://stackoverflow.com/questions/9008444/how-to-warn-about-class-name-deprecation + +https://www.hongweipeng.com/index.php/archives/1872/ + +https://stackoverflow.com/questions/45400284/understanding-init-subclass + +https://www.python.org/dev/peps/pep-0487/ + +https://python-history.blogspot.com/search?updated-min=2009-01-01T00:00:00-08:00&updated-max=2010-01-01T00:00:00-08:00&max-results=19 + +`__instancecheck__` + +https://stackoverflow.com/questions/47740953/why-isnt-instancecheck-being-called + + +### 抽象基类 + +https://docs.python.org/2/library/abc.html + + + +https://null-byte.wonderhowto.com/how-to/hack-like-pro-python-scripting-for-aspiring-hacker-part-2-0159319/ + + +## python 获取class里的functions + +https://stackoverflow.com/questions/1911281/how-do-i-get-list-of-methods-in-a-python-class + +inspect.getmembers(Command, predicate=inspect.ismethod) + + +https://jfine-python-classes.readthedocs.io/en/latest/call-a-class.html diff --git "a/\350\265\204\346\272\220/\345\245\275\347\224\250\347\232\204\350\275\257\344\273\266.md" "b/\350\265\204\346\272\220/\345\245\275\347\224\250\347\232\204\350\275\257\344\273\266.md" index 2d4a6f1..fe76fc2 100644 --- "a/\350\265\204\346\272\220/\345\245\275\347\224\250\347\232\204\350\275\257\344\273\266.md" +++ "b/\350\265\204\346\272\220/\345\245\275\347\224\250\347\232\204\350\275\257\344\273\266.md" @@ -1,3 +1,5 @@ # 任务管理类 https://www.jianshu.com/p/0db06f66f97d + +https://www.52pojie.cn/thread-815048-1-1.html