Python画出心目中的自己（Python画出心目中的自己）

作者 | 李秋键

责编 | 晋兆雨

头图 | CSDN下载自视觉中国

引言：人脸图像的生成在各个行业有着重要应用，例如刑事调查、人物设计、教育培训等。然而一幅逼真的人脸肖像，对于职业画家也要至少数小时才能绘制出来；对于从未接触过绘画的新手，就更是难如登天了。新手绘制出来的人脸草图往往非常简陋抽象，甚至有不匀称、不完整。但如果使用智能人脸画板，无疑是有如神助。

本项目主要来源于中科院和香港城市大学的一项研究DeepFaceDrawing，论文标题是《DeepFaceDrawing: DeepGeneration of Face Images from sketches》

具体效果如下图可见：

实验前的准备

首先我们使用的python版本是3.6.5所用到的模块如下：

Pyqt5模块：PyQt5是基于Digia公司强大的图形程式框架Qt5的python接口，由一组python模块构成。PyQt5本身拥有超过620个类和6000函数及方法。在可以运行于多个平台，包括：Unix, Windows, and Mac OS。

• opencv是将用来进行图像处理和生成。

• numpy模块用来处理矩阵运算。

• Jittor模块国内清华大学开源的深度学习框架。

• _thread是多线程库。

网络模型的定义和训练

首先这个图像合成模块采用了一种利用发生器和鉴别器的GAN结构。从融合的特征图生成真实的人脸图像。鉴别器采用多尺度鉴别方式:对输入进行尺度划分，特征图和生成的图像在三个不同的层次上，经过三个不同的过程。：

（1）权重网络层和损失定义：

def weights_init_normal(m):classname = m.__class__.__name__ifclassname.find("Conv") != -1:jt.init.gauss_(m.weight,0.0, 0.02)elifclassname.find("BatchNorm") != -1:jt.init.gauss_(m.weight,1.0, 0.02)jt.init.constant_(m.bias,0.0)def get_norm_layer(norm_type='instance'):if (norm_type == 'batch'):norm_layer = nn.BatchNormelif (norm_type == 'instance'):norm_layer =nn.InstanceNorm2delse:raiseNotImplementedError(('normalization layer [%s] is not found' % norm_type))return norm_layerclass MSELoss:def __init__(self):passdef __call__(self, output,target):from jittor.nn importmse_lossreturn mse_loss(output,target)class BCELoss:def __init__(self):passdef __call__(self, output,target):from jittor.nn importbce_lossreturn bce_loss(output,target)

（2）模型特征编解码：

特征匹配模块包含5个译码网络，以compact作为输入由分量流形得到的特征向量，并将其转换为对应的特征向量为后续生成的特征图的大小。

def define_part_encoder(model='mouth', norm='instance', input_nc=1,latent_dim=512):norm_layer =get_norm_layer(norm_type=norm)image_size = 512if 'eye' in model:image_size = 128elif 'mouth' in model:image_size = 192elif 'nose' in model:image_size = 160elif 'face' in model:image_size = 512else:print("Whole Image!!")net_encoder =EncoderGenerator_Res(norm_layer,image_size,input_nc, latent_dim) # input longsize 256 to 512*4*4print("net_encoder of part" model " is:",image_size)return net_encoderdef define_part_decoder(model='mouth', norm='instance', output_nc=1,latent_dim=512):norm_layer =get_norm_layer(norm_type=norm)image_size = 512if 'eye' in model:image_size = 128elif 'mouth' in model:image_size = 192elif 'nose' in model:image_size = 160else:print("Whole Image!!")net_decoder =DecoderGenerator_image_Res(norm_layer,image_size,output_nc, latent_dim) # input longsize 256 to 512*4*4print("net_decoder to imageof part " model " is:",image_size)return net_decoderdef define_feature_decoder(model='mouth', norm='instance', output_nc=1,latent_dim=512):norm_layer =get_norm_layer(norm_type=norm)image_size = 512if 'eye' in model:image_size = 128elif 'mouth' in model:image_size = 192elif 'nose' in model:image_size = 160else:print("Whole Image!!")net_decoder =DecoderGenerator_feature_Res(norm_layer,image_size,output_nc, latent_dim) # input longsize 256 to 512*4*4print("net_decoder to imageof part " model " is:",image_size)# print(net_decoder)return net_decoderdef define_G(input_nc, output_nc, ngf, n_downsample_global=3,n_blocks_global=9, norm='instance'):norm_layer =get_norm_layer(norm_type=norm)netG = GlobalGenerator(input_nc,output_nc, ngf, n_downsample_global, n_blocks_global, norm_layer)return netG

图形界面的定义

在这篇论文中，作者一方面将人脸关键区域（双眼、鼻、嘴和其他区域）作为面元，学习其特征嵌入，将输入草图的对应部分送到由数据库样本中面元的特征向量构成的流形空间进行校准。另一方面，参考 pix2pixHD [5]的网络模型设计，使用 conditional GAN 来学习从编码的面元特征到真实图像的映射生成结果。

（1）鼠标绘制函数的定义：

class OutputGraphicsScene(QGraphicsScene):def__init__(self, parent=None):QGraphicsScene.__init__(self, parent)# self.modes = mode_listself.mouse_clicked = Falseself.prev_pt = Noneself.setSceneRect(0,0,self.width,self.height)# self.masked_image = Noneself.selectMode = 0# save the history of editself.history = self.ori_img = np.ones((512,512, 3),dtype=np.uint8)*255self.mask_put = 1 # 1 marksuse brush while 0 user eraseself.convert = False# self.setPos(0 ,0)self.firstDisplay = Trueself.convert_on = Falsedefreset(self):self.convert = Falseself.ori_img = np.ones((512,512, 3),dtype=np.uint8)*255self.updatePixmap(True)self.prev_pt = NonedefsetSketchImag(self,sketch_mat, mouse_up=False):self.ori_img =sketch_mat.copyself.image_list = self.image_list.append(self.ori_img.copy )defmousePressEvent(self,event):if not self.mask_put orself.selectMode == 1:self.mouse_clicked =Trueself.prev_pt = Noneelse:self.make_sketch(event.scenePos)defmake_sketch_Eraser(self,pts):if len(pts)>0:for pt in pts:cv2.line(self.color_img,pt['prev'],pt['curr'],self.paint_color,self.paint_size)cv2.line(self.mask_img,pt['prev'],pt['curr'],(0,0,0),self.paint_size )self.updatePixmapdefmodify_sketch(self, pts):if len(pts)>0:for pt in pts:cv2.line(self.ori_img,pt['prev'],pt['curr'],self.paint_color,self.paint_size)self.updatePixmapdefget_stk_color(self, color):self.stk_color = colordeferase_prev_pt(self):self.prev_pt = Nonedefreset_items(self):for i inrange(len(self.items)):item = self.items[0]self.removeItem(item)defundo(self):iflen(self.image_list)>1:num =len(self.image_list)-2self.ori_img =self.image_list[num].copyself.image_list.pop(num 1)self.updatePixmap(True)defgetImage(self):returnself.ori_img*(1-self.mask_img) self.color_img*self.mask_imgdefupdatePixmap(self,mouse_up=False):sketch = self.ori_imgqim = QImage(sketch.data,sketch.shape[1], sketch.shape[0], QImage.Format_RGB888)if self.firstDisplay :self.reset_itemsself.imItem =self.addPixmap(QPixmap.fromImage(qim))self.firstDispla = Falseelse:self.imItem.setPixmap(QPixmap.fromImage(qim))deffresh_board(self):print('======================================================')while(True):if(self.convert_on):print('======================================================')time.sleep(100)iter_start_time =time.timeself.updatePixmapprint('TimeSketch:',time.time - iter_start_time)

（2）GUI界面：其核心思路并非直接用输入草图作为网络生成条件，而是将人脸进行分块操作后利用数据驱动的思想对抽象的草图特征空间进行隐式建模，并在这个流形空间中找到输入草图特征的近邻组合来重构特征，进而合成人脸图像。

class WindowUI(QtWidgets.QMainWindow,Ui_SketchGUI):def__init__(self):super(WindowUI,self).__init__self.setupUi(self)self.setEventsself._translate =QtCore.QCoreApplication.translateself.output_img = Noneself.brush_size =self.BrushSize.valueself.eraser_size =self.EraseSize.valueself.modes = [0,1,0] #0marks the eraser, 1 marks the brushself.Modify_modes = [0,1,0]#0 marks the eraser, 1 marks the brushself.output_scene =OutputGraphicsSceneself.output.setScene(self.output_scene)self.output.setAlignment(Qt.AlignTop | Qt.AlignLeft)self.output.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)self.output.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)self.output_view =QGraphicsView(self.output_scene)#self.output_view.fitInView(self.output_scene.updatePixmap)self.input_scene =InputGraphicsScene(self.modes, self.brush_size,self.output_scene)self.input.setScene(self.input_scene)self.input.setAlignment(Qt.AlignTop | Qt.AlignLeft)self.input.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)self.input.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)self.input_scene.convert_on= self.RealTime_checkBox.isCheckedself.output_scene.convert_on= self.RealTime_checkBox.isCheckedself.BrushNum_label.setText(self._translate("SketchGUI",str(self.brush_size)))self.EraserNum_label.setText(self._translate("SketchGUI",str(self.eraser_size)))self.start_time =time.time# self.# try:# # thread.start_new_thread(self.output_scene.fresh_board,)# thread.start_new_thread(self.input_scene.thread_shadow,)# except:# print("Error: unable to startthread")# print("Finish")defsetEvents(self):self.Undo_Button.clicked.connect(self.undo)self.Brush_Button.clicked.connect(self.brush_mode)self.BrushSize.valueChanged.connect(self.brush_change)self.Clear_Button.clicked.connect(self.clear)self.Eraser_Button.clicked.connect(self.eraser_mode)self.EraseSize.valueChanged.connect(self.eraser_change)self.Save_Button.clicked.connect(self.saveFile)#weight barself.part0_Slider.valueChanged.connect(self.changePart)self.part1_Slider.valueChanged.connect(self.changePart)self.part2_Slider.valueChanged.connect(self.changePart)self.part3_Slider.valueChanged.connect(self.changePart)self.part4_Slider.valueChanged.connect(self.changePart)self.part5_Slider.valueChanged.connect(self.changAllPart)self.Load_Button.clicked.connect(self.open)self.Convert_Sketch.clicked.connect(self.convert)self.RealTime_checkBox.clicked.connect(self.convert_on)self.Shadow_checkBox.clicked.connect(self.shadow_on)self.Female_Button.clicked.connect(self.choose_Gender)self.Man_Button.clicked.connect(self.choose_Gender)self.actionSave.triggered.connect(self.saveFile)defmode_select(self, mode):for i inrange(len(self.modes)):self.modes[i] =0self.modes[mode] =1defbrush_mode(self):self.mode_select(1)self.brush_changeself.statusBar.showMessage("Brush")deferaser_mode(self):self.mode_select(0)self.eraser_changeself.statusBar.showMessage("Eraser")defundo(self):self.input_scene.undoself.output_scene.undodefbrush_change(self):self.brush_size =self.BrushSize.valueself.BrushNum_label.setText(self._translate("SketchGUI",str(self.brush_size)))if self.modes[1]:self.input_scene.paint_size = self.brush_sizeself.input_scene.paint_color = (0,0,0)self.statusBar.showMessage("Change Brush Size in ",self.brush_size)deferaser_change(self):self.eraser_size =self.EraseSize.valueself.EraserNum_label.setText(self._translate("SketchGUI",str(self.eraser_size)))if self.modes[0]:print( self.eraser_size)self.input_scene.paint_size = self.eraser_sizeself.input_scene.paint_color = (1,1,1)self.statusBar.showMessage("Change Eraser Size in ",self.eraser_size)defchangePart(self):self.input_scene.part_weight['eye1'] = self.part0_Slider.value/100self.input_scene.part_weight['eye2']= self.part1_Slider.value/100self.input_scene.part_weight['nose'] = self.part2_Slider.value/100self.input_scene.part_weight['mouth'] = self.part3_Slider.value/100self.input_scene.part_weight[''] = self.part4_Slider.value/100self.input_scene.start_Shadow#self.input_scene.updatePixmapdefchangAllPart(self):value =self.part5_Slider.valueself.part0_Slider.setProperty("value", value)self.part1_Slider.setProperty("value", value)self.part2_Slider.setProperty("value", value)self.part3_Slider.setProperty("value", value)self.part4_Slider.setProperty("value", value)self.changePartdefclear(self):self.input_scene.resetself.output_scene.resetself.start_time =time.timeself.input_scene.start_Shadowself.statusBar.showMessage("Clear Drawing Board")defconvert(self):self.statusBar.showMessage("Press Convert")self.input_scene.convert_RGBself.output_scene.updatePixmapdefopen(self):fileName, _ =QFileDialog.getOpenFileName(self, "Open File",QDir.currentPath,"Images Files (*.*)") #jpg;*.jpeg;*.pngiffileName:image =QPixmap(fileName)mat_img =cv2.imread(fileName)mat_img = cv2.resize(mat_img,(512, 512), interpolation=cv2.INTER_CUBIC)mat_img =cv2.cvtColor(mat_img, cv2.COLOR_RGB2BGR)if image.is:QMessageBox.information(self, "Image Viewer","Cannotload %s." % fileName)return#cv2.imshow('open',mat_img)self.input_scene.start_Shadowself.input_scene.setSketchImag(mat_img)defsaveFile(self):cur_time =strftime("%Y-%m-%d-%H-%M-%S", gmtime)file_dir ='./saveImage/' cur_timeif notos.path.isdir(file_dir) :os.makedirs(file_dir)cv2.imwrite(file_dir '/hand-draw.jpg',self.input_scene.sketch_img*255)cv2.imwrite(file_dir '/colorized.jpg',cv2.cvtColor(self.output_scene.ori_img,cv2.COLOR_BGR2RGB))print(file_dir)defconvert_on(self):# ifself.RealTime_checkBox.isCheched:print('self.RealTime_checkBox',self.input_scene.convert_on)self.input_scene.convert_on= self.RealTime_checkBox.isCheckedself.output_scene.convert_on= self.RealTime_checkBox.isCheckeddefshadow_on(self):_translate =QtCore.QCoreApplication.translateself.input_scene.shadow_on =not self.input_scene.shadow_onself.input_scene.updatePixmapifself.input_scene.shadow_on:self.statusBar.showMessage("Shadow ON")else:self.statusBar.showMessage("Shadow OFF")defchoose_Gender(self):ifself.Female_Button.isChecked:self.input_scene.sex =1else:self.input_scene.sex =0self.input_scene.start_Shadow

总结

这里给出模型的体验网址：

http://www.geometrylearning.com:3000/index_621.html

该方法核心亮点之一，便是以多通道特征图作为中间结果来改善信息流。从本质上看，这是将输入草图作为软约束来替代传统方法中的硬约束，因此能够用粗糙甚至不完整的草图来生成高质量的完整人脸图像。

反思DeepFaceDrawing

1）画不出丑脸：

从图中可以看出，即使给出丑陋的草图，输出的也会是平均来说漂亮的人脸，这大概是因为所用的训练数据集都是名人，平均“颜值”较高，因此神经网络学到了一种漂亮的平均；这能算是一种在“颜值上的”数据不平衡问题吗。

2）安全问题

比如人脸支付场景中，可能存在利用该项技术盗刷的问题。随着人脸活体检测技术的发展，这种隐患应该能得以有效避免。

3）技术攻击性

相比于Deepfake，本文的DeepFaceDrawing应该算是相对无害的。

4）商业价值

如论文作者所说，这项技术在犯罪侦查、人物设计、教育培训等方面都可以有所作为。期待有一天这项技术更加通用，这样一来其商业价值会更大。

完整代码：

链接：https:///s/1ARIzPEbUSNzAIdPsRl6h-A

提取码：4llk

作者简介

李秋键，CSDN 博客专家，CSDN达人课作者。硕士在读于中国矿业大学，开发有taptap安卓武侠游戏一部，vip视频解析，文意转换工具，写作机器人等项目，发表论文若干，多次高数竞赛获奖等等。

,