Crontab定时任务使用virtualenv/conda会出现无法激活环境
需要使用特殊参数
"$@"
来传递命令的后续参数
手动编写一个wrapper.sh，包含如下内容
#! /bin/bash cd /some/work/dir source /some/virtualenv/dir/bin/activate "$@" |
测试使用
#bash /some/work/dir/wrapper.sh python xxx.py |
#crontab */5 8-20 * * * /some/work/dir/wrapper.sh python xxx.py |
FastAPI 使用JWT认证的中间件
fastapi的中间件还是太少，单独开发JWT需要，starlette本身提供认证相关实现，只需要自定义一个AuthenticationBackend即可，本次我们实现使用中间价方式拆包JWT的令牌，获取payload里面的用户信息
私有定义的payload内容格式如下
{
"usid": "SkDQBhEjUfygRSeEBech", //UUID Short
"uname": "test user name", //Username
"mid":"700010001" // Member ID
} |
调用代码
app = FastAPI() app.add_middleware(AuthenticationMiddleware,backend=JWTAuthenticationBackend(secret_key="YOUR_SECRET_KEY")) |
完整的代码
import jwt from starlette.authentication import ( AuthenticationBackend, AuthenticationError, BaseUser, AuthCredentials, UnauthenticatedUser ) class JWTUser(BaseUser): def __init__(self, user_id_short: str, member_number: str, user_name: str,token: str, payload: dict) -> None: self.user_name = user_name self.user_id_short = user_id_short self.member_number = member_number self.token = token self.payload = payload @property def is_authenticated(self) -> bool: return True @property def display_name(self) -> str: return self.user_name ## 会员名处理，加*？ @property def display_user_id_short(self) -> str: return self.user_id_short ## 会员id处理 @property def display_member_number(self) -> str: return self.member_number ## 会员号处理，加*？ class JWTAuthenticationBackend(AuthenticationBackend): def __init__(self, secret_key: str, algorithm: str = 'HS256', prefix: str = 'JWT', user_name_field:str = 'uname' , user_id_field: str = 'usid',member_number_field: str = 'mid'): self.secret_key = secret_key self.algorithm = algorithm self.prefix = prefix self.user_name_field = user_name_field self.user_id_field = user_id_field self.member_number_field = member_number_field @classmethod def get_token_from_header(cls, authorization: str, prefix: str): """ Parses the Authorization header and returns only the token :param authorization: :return: """ try: scheme, token = authorization.split() except ValueError: raise AuthenticationError('Could not separate Authorization scheme and token') if scheme.lower() != prefix.lower(): raise AuthenticationError(f'Authorization scheme {scheme} is not supported') return token async def authenticate(self, request): if "Authorization" not in request.headers: return None auth = request.headers["Authorization"] token = self.get_token_from_header(authorization=auth, prefix=self.prefix) try: payload = jwt.decode(token, key=self.secret_key, algorithms=self.algorithm) except jwt.InvalidTokenError as e: raise AuthenticationError(str(e)) return AuthCredentials(["jwt_authenticated"]), JWTUser(user_id_short=payload[self.user_id_field], member_number=payload[self.member_number_field],user_name=payload[self.user_name_field],token=token, payload=payload) class JWTWebSocketAuthenticationBackend(AuthenticationBackend): def __init__(self, secret_key: str, algorithm: str = 'HS256', query_param_name: str = 'jwt', user_name_field:str = 'uname' , user_id_field: str = 'usid',member_number_field: str = 'mid'): self.secret_key = secret_key self.algorithm = algorithm self.query_param_name = query_param_name self.user_name_field = user_name_field self.user_id_field = user_id_field self.member_number_field = member_number_field async def authenticate(self, request): if self.query_param_name not in request.query_params: return AuthCredentials(), UnauthenticatedUser() token = request.query_params[self.query_param_name] try: payload = jwt.decode(token, key=self.secret_key, algorithms=self.algorithm) except jwt.InvalidTokenError as e: raise AuthenticationError(str(e)) return AuthCredentials(["jwt_authenticated"]), JWTUser(user_id_short=payload[self.user_id_field], member_number=payload[self.member_number_field],user_name=payload[self.user_name_field],token=token, payload=payload) |
Nvidia CUDA开发环境 Docker容器启用显卡
1.准备docker>19.03 环境，配置好nvidia-container-toolkit
2.确定本机已安装的显卡驱动版本，匹配需要的容器版本
3.Pull基础docker镜像，可以从官方或者dockerhub下载
https://ngc.nvidia.com/catalog/containers/nvidia:cuda/tags
https://gitlab.com/nvidia/container-images/cuda
cuda10-py36-conda的Dockerfile
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 MAINTAINER Limc <limc@limc.com.cn> #close frontend ENV DEBIAN_FRONTEND noninteractive # add cuda user # --disabled-password = Don't assign a password # using root group for OpenShift compatibility ENV CUDA_USER_NAME=cuda10 ENV CUDA_USER_GROUP=root # add user RUN adduser --system --group --disabled-password --no-create-home --disabled-login $CUDA_USER_NAME RUN adduser $CUDA_USER_NAME $CUDA_USER_GROUP # Install basic dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cmake \ git \ wget \ libopencv-dev \ libsnappy-dev \ python-dev \ python-pip \ #tzdata \ vim # Install conda for python RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh # Set locale ENV LANG C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH RUN ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc && \ find /opt/conda/ -follow -type f -name '*.a' -delete && \ find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ /opt/conda/bin/conda clean -afy # copy entrypoint.sh #COPY ./entrypoint.sh /entrypoint.sh # install #ENTRYPOINT ["/entrypoint.sh"] # Initialize workspace COPY ./app /app # make workdir WORKDIR /app # update pip if nesseary #RUN pip install --upgrade --no-cache-dir pip # install gunicorn # RUN pip install --no-cache-dir -r ./requirements.txt # install use conda #RUN conda install --yes --file ./requirements.txt RUN while read requirement; do conda install --yes $requirement; done < requirements.txt # copy entrypoint.sh COPY ./entrypoint.sh /entrypoint.sh # install ENTRYPOINT ["/entrypoint.sh"] # switch to non-root user USER $CUDA_USER_NAME |
运行容器Makefile
IMG:=`cat Name` GPU_OPT:=all MOUNT_ETC:= MOUNT_LOG:= MOUNT_APP:=-v `pwd`/work/app:/app MOUNT:=$(MOUNT_ETC) $(MOUNT_LOG) $(MOUNT_APP) EXT_VOL:= PORT_MAP:= LINK_MAP:= RESTART:=no CONTAINER_NAME:=docker-cuda10-py36-hello echo: echo $(IMG) run: docker rm $(CONTAINER_NAME) || echo docker run -d --gpus $(GPU_OPT) --name $(CONTAINER_NAME) $(LINK_MAP) $(PORT_MAP) --restart=$(RESTART) \ $(EXT_VOL) $(MOUNT) $(IMG) run_i: docker rm $(CONTAINER_NAME) || echo docker run -i -t --gpus $(GPU_OPT) --name $(CONTAINER_NAME) $(LINK_MAP) $(PORT_MAP) \ $(EXT_VOL) $(MOUNT) $(IMG) /bin/bash exec_i: docker exec -i -t --name $(CONTAINER_NAME) /bin/bash stop: docker stop $(CONTAINER_NAME) rm: stop docker rm $(CONTAINER_NAME) |
Entrypoint.sh
set -e # Add python as command if needed if [ "${1:0:1}" = '-' ]; then set -- python "$@" fi # Drop root privileges if we are running gunicorn # allow the container to be started with `--user` if [ "$1" = 'python' -a "$(id -u)" = '0' ]; then # Change the ownership of user-mutable directories to gunicorn for path in \ /app \ /usr/local/cuda/ \ ; do chown -R cuda10:root "$path" done set -- su-exec python "$@" #exec su-exec elasticsearch "$BASH_SOURCE" "$@" fi # As argument is not related to gunicorn, # then assume that user wants to run his own process, # for example a `bash` shell to explore this image exec "$@" |
几个注意点
1.显卡运行需要root用户权限，否则会出现以下，
docker: Error response from daemon: OCI runtime create failed: container_linux.go:345
考虑安全性可以在容器内创建新用户并加入到root组
2.本机显卡驱动和CUDA必须匹配官方容器的版本，cudnn则不需要匹配，可以使用多个不同版本的cudnn，但是必须满足显卡要求的使用范围
3.docker运行容器非正常结束时会占用显卡，如果卡死，会造成容器外部无法使用，重启docker-daemon也无效，这时只能重启电脑
完整的源代码
https://github.com/limccn/ultrasound-nerve-segmentation-in-tensorflow/commit/d7de1cbeb641d2fae4f5a78ff590a0254667b398
参考
https://gitlab.com/nvidia/container-images/cuda
1.Tensorboard需要对比多个Training的数据
常规的做法现实单次训练的数据，–logdir=path
tensorboard --logdir=path |
多个Training的数据对比，则需要规定各个run的名称
tensorboard --logdir=run1:“path1",run2:“path2",run3:“path3"
2.Tensorboard强制使用CPU
GPU被占满的时候，只能使用CPU来渲染Tensorboard的数据，具体做法是在启动命令前增加CUDA_VISIBLE_DEVICES=“”
CUDA_VISIBLE_DEVICES=“" tensorboard --logdir=path |
3.Tensorboard提示No dashboards are active for the current data
一般发生在Tensorboard切换版本后，–logdir=后面不要接带单引号的路径，如果路径包含空格，转意或使用双引号
# 错误 tensorboard —logdir=‘path' # 正确 tensorboard --logdir=path
tensorboard完整的命令参数
usage: tensorboard [-h] [--helpfull] [--logdir PATH] [--host ADDR] [--port PORT] [--purge_orphaned_data BOOL] [--reload_interval SECONDS] [--db URI] [--db_import] [--db_import_use_op] [--inspect] [--version_tb] [--tag TAG] [--event_file PATH] [--path_prefix PATH] [--window_title TEXT] [--max_reload_threads COUNT] [--reload_task TYPE] [--samples_per_plugin SAMPLES_PER_PLUGIN] [--debugger_data_server_grpc_port PORT] [--debugger_port PORT] [--master_tpu_unsecure_channel ADDR] |
PostgreSql 习惯上会将特殊数据类型的各个节点按字典/JSON类型存储
程序中需要获得完整的数据信息的时候，需要对这个节点进行自展开。
以下使用global id方式进行展开，一般适用于SQL+NoSQL结合的系统使用
import sys, os import numpy as np def get_object_by_gid(id): for dict in data["json"]: if dict["gid"] == id: return dict.copy() def self_exact_node(key): dict = get_object_by_gid(key) for k,v in dict.items(): if k == "sub_item" : item_arr = [] for id in v["gids"]: item_arr.append(self_exact_node(id)) v["item_arr"] = item_arr.copy() return dict def demo(): data_exact = data.copy() for d in data_exact["json"]: d = self_exact_node(d["gid"]) def main(): demo() if __name__ == '__main__’: sys.exit(main()) |
其他玩法
# Global ID方式 "gid": "大分类2:中分类2:子分类2", # 复合ID "id":{"l1_cat":"大分类1","l2_cat":"中分类2","l3_cat":"子分类2"} # 链表式 "chain":{"next_gid":"","pre_gid":"","head_gid":"","tail_gid":""} # 二叉树式 "btree":{"next_sibling":"","child":”"} |
使用applyColorMap可以对单个通道的图像进行伪彩色处理和热力图
OpenCV的定义了12种colormap常数，选择一个需要的即可
cv2.applyColorMap(heatmap_g, cv2.COLORMAP_JET)
图像可以使用addWeighted进行叠加处理
cv2.addWeighted(heatmap_img, alpha, merge_img, 1-alpha, 0, merge_img) # 将热度图覆盖到原图
def heatmap_overlay(image,heatmap): # 灰度化heatmap heatmap_g = heatmap.astype(np.uint8) # 热力图伪彩色 heatmap_color = cv2.applyColorMap(heatmap_g, cv2.COLORMAP_JET) # overlay热力图 merge_img = image.copy() heatmap_img = heatmap_color.copy() overlay = image.copy() alpha = 0.25 # 设置覆盖图片的透明度 #cv2.rectangle(overlay, (0, 0), (merge_img.shape[1], merge_img.shape[0]), (0, 0, 0), -1) # 设置蓝色为热度图基本色 cv2.addWeighted(overlay, alpha, merge_img, 1-alpha, 0, merge_img) # 将背景热度图覆盖到原图 cv2.addWeighted(heatmap_img, alpha, merge_img, 1-alpha, 0, merge_img) # 将热度图覆盖到原图 return merge_img |
参考：
https://blog.csdn.net/u013381011/article/details/78341861
将视频按FPS拆解成单张图片
使用cv2.VideoCapture
cv2.VideoCapture(video_path)
计算FPS使用，注意部分压缩视频FPS存在丢帧情况，需要进行跳帧处理
fps = int(vidcap.get(cv2.CAP_PROP_FPS))
def video_split(): video_path = 'test/video/video_01.mp4' video_name = video_path[:-4] vidcap = cv2.VideoCapture(video_path) success,image = vidcap.read() fps = int(vidcap.get(cv2.CAP_PROP_FPS)) count = 0 while success: image = image_process(image) cv2.imwrite("%s/%d.jpg" % (video_name, count), image) #if count % fps == 0: # cv2.imwrite("%s/%d.jpg" % (video_name, int(count / fps)), image) print('Process %dth seconds: ' % int(count / fps), success) success,image = vidcap.read() count += 1 |
Recent Comments