1.爬取指定关键字图片
'''
爬取指定关键字图片
'''
import re # 正则表达式，解析网页
import requests # 请求网页
import traceback
import os
def dowmloadPic(html, keyword, startNum):
headers = {'user-agent': 'Mozilla/5.0'} # 浏览器伪装，因为有的网站会反爬虫，通过该headers可以伪装成浏览器访问，否则user-agent中的代理信息为python
pic_url = re.findall('"objURL":"(.*?)",', html, re.S) # 找到符合正则规则的目标网站
num = len(pic_url)
i = startNum
subroot = root + '/' + word
# txtpath = subroot + '/download_detail.txt'
print('找到关键词:' + keyword + '的图片，现在开始下载图片...')
for each in pic_url:
a = '第' + str(i + 1) + '张图片，图片地址:' + str(each) + '\n'
b = '正在下载' + a
print(b)
path = subroot + '/' + str(i + 1)
try:
if not os.path.exists(subroot):
os.mkdir(subroot)
if not os.path.exists(path):
pic = requests.get(each, headers=headers, timeout=10)
with open(path + '.jpg', 'wb') as f:
f.write(pic.content)
f.close()
# with open(txtpath, 'a') as f:
# f.write(a)
# f.close()
except:
continue
# traceback.print_exc()
# print('【错误】当前图片无法下载')
# os.remove(pic.content)
# pass
# continue
i += 1
return i
if __name__ == '__main__':
headers = {'user-agent': 'Mozilla/5.0'}
# words = ['人拉行李箱', '行李箱和人', '人托行李箱']
# words = ['黄色行李箱','黄色包包','黄色双肩包','黄色腰包','黄色手提包','黄色单肩包','黄色手提袋']
words = ['人拉灰色行李箱','人背灰色包包','人背灰色双肩包','人背灰色腰包','人拿灰色手提包','人背灰色单肩包','人拿灰色手提袋']
# words = ['粉色行李箱','粉色包包','粉色双肩包','粉色腰包','粉色手提包','粉色单肩包','粉色手提袋']
# words = [ ‘蓝色行李箱','棕色包包','棕色双肩包','棕色腰包','棕色手提包','棕色单肩包','棕色手提袋']
# words = ['绿色行李箱','绿色包包','绿色双肩包','绿色腰包','绿色手提包','绿色单肩包','绿色手提袋']
# words = ['紫色行李箱','紫色包包','紫色双肩包','紫色腰包','紫色手提包','紫色单肩包','紫色手提袋']
# words = ['棕色行李箱','棕色包包','棕色双肩包','棕色腰包','棕色手提包','棕色单肩包','棕色手提袋']
# words = ['花色行李箱','花色包包','花色双肩包','花色腰包','花色手提包','花色单肩包','花色手提袋']
# words = ['青色行李箱','青色包包','青色双肩包','青色腰包','青色手提包','青色单肩包','青色手提袋']
# words = ['白色行李箱','白色包包','白色双肩包','白色腰包','白色手提包','白色单肩包','白色手提袋']
# words为一个列表，可以自动保存多个关键字的图片
root = './download_images'
for word in words:
root = root + word + '&'
if not os.path.exists(root):
os.mkdir(root)
for word in words:
lastNum = 0
# word = input("Input key word: ")
if word.strip() == "exit":
break
pageId = 0
# 此处的参数为需爬取的页数，设置为30页
for i in range(15): #获取10*60张图
url = 'http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word=' + word + "&pn=" + str(
pageId) + "&gsm=?&ct=&ic=0&lm=-1&width=0&height=0"
pageId += 20
html = requests.get(url, headers=headers)
# print(html.text) #打印网页源码，相当于在网页中右键查看源码内容
lastNum = dowmloadPic(html.text, word, lastNum, ) # 本条语句执行一次获取60张图
类似生成如下文件夹，每个文件夹都会有显示
2、每个文件夹进行合并成一个大的文件夹
import os
import shutil
#目标文件夹，此处为绝对路径，也可以是相对路径
determination =r'G:\pachong\1white'
if not os.path.exists(determination):
os.makedirs(determination)
#源文件夹路径，根目录
path = r'G:\pachong\white'
#根目录下的所有一级目录，以列表形式赋给first_dir
first_dir = os.listdir(path)
#遍历每一个一级目录
for first in first_dir:
#一级目录绝对路径
dir = path + '/' + str(first)
#得到一级目录下的二级目录
# second_dir = os.listdir(dir)
#遍历每一个二级目录
# for second in second_dir:
# # 二级目录绝对路径
# source = dir + '/' + str(second)
# 二级目录绝对路径下所有图片
imgs = os.listdir(dir)
for img in imgs:
source_img = dir + '/' + str(img)
deter = determination + '/' + str(first) + '_' + '_' + str(img)
shutil.copyfile(source_img, deter)
3.合并后进行筛选、转化三通道RGB
###############################################2.转通道、重命名、删小图等
import os.path
from PIL import Image
from PIL import ImageFile
MAGES = True
import cv2
from tqdm import tqdm
from PIL import Image
import os
ImageFile.LOAD_TRUNCATED_IMAGES = True
import glob
path = r"F:\1213bag\all_kind\111colour_all\bag_colour_all_v3\666\white"
files = os.listdir(path)
# print(files)
b = 0
i = 0
a = 0
for file in files:
original = path + os.sep + files[b]
new = path + os.sep + "whiteo" + str(b+1) + ".jpg" ###改颜色
os.rename(original,new)
b +=1
for pic in tqdm(glob.glob("F:/1213bag/all_kind/111colour_all/bag_colour_all_v3/666/white/*.jpg")): ########## 改地址
# # # basename = os.path.basename(image_name)
# before_name = os.path.splitext(pic)[0]
# txt_name = os.path.splitext(before_name)[0] + ".txt"
# txt_name = os.path.join(save_path,txt_name)
# f = open(txt_name, "w").
# if pic.endwith('.txt'):
# os.remove(pic)
try:
img = Image.open(pic)
# if img is None:
# os.remove(os.path.join(path, pic))
# img.close()
# print(pic)
# print(img.getbands()) # ('P',) 这种是有彩色的，而L是没有彩色的
# print(img.size)
# Img = np.array(img)
# a = np.unique(Img)
# print(a) #看像素值
if len(img.getbands()) != 3:
img = img.convert("RGB")
pic_new = os.path.join(pic)
img.save(pic_new)
a +=1
i += 1
size = img.size
w = size[0] # 宽度
h = size[1] # 高度
if w<100 or h<100:
img.close()
os.remove(pic) #去除小图，爬虫下在下来的小图会有问题，必须img.close() 否则会报错，这张图正在使用进程
# if w == 640:
# if h == 480:
# c += 1
except:
print('read image failed已删除!')
# img = Image.open(pic)
# img.close()
# break
os.remove(pic)
print('图像image的总数量: ', i)
print('总图像不是RGB的数量：', a)
# print('总图像是640*480的数量：', c)
4、去除重复图像
import shutil
import numpy as np
from PIL import Image
import os
def 比较图片大小(dir_image1, dir_image2):
with open(dir_image1, "rb") as f1:
size1 = len(f1.read())
with open(dir_image2, "rb") as f2:
size2 = len(f2.read())
if (size1 == size2):
result = "大小相同"
else:
result = "大小不同"
return result
def 比较图片尺寸(dir_image1, dir_image2):
image1 = Image.open(dir_image1)
image2 = Image.open(dir_image2)
if (image1.size == image2.size):
result = "尺寸相同"
else:
result = "尺寸不同"
return result
def 比较图片内容(dir_image1, dir_image2):
image1 = np.array(Image.open(dir_image1))
image2 = np.array(Image.open(dir_image2))
if (np.array_equal(image1, image2)):
result = "内容相同"
else:
result = "内容不同"
return result
def 比较两张图片是否相同(dir_image1, dir_image2):
# 比较两张图片是否相同
# 第一步：比较大小是否相同
# 第二步：比较长和宽是否相同
# 第三步：比较每个像素是否相同
# 如果前一步不相同，则两张图片必不相同
result = "两张图不同"
大小 = 比较图片大小(dir_image1, dir_image2)
if (大小 == "大小相同"):
尺寸 = 比较图片尺寸(dir_image1, dir_image2)
if (尺寸 == "尺寸相同"):
内容 = 比较图片内容(dir_image1, dir_image2)
if (内容 == "内容相同"):
result = "两张图相同"
return result
if __name__ == '__main__':
load_path = r'F:\1213bag\all_kind\111colour_all\bag_colour_all_v3\666\white' # 要去重的文件夹
save_path = r'G:\pachong\13411sf' # 空文件夹，用于存储检测到的重复的照片
os.makedirs(save_path, exist_ok=True)
# 获取图片列表 file_map，字典{文件路径filename : 文件大小image_size}
file_map = {}
image_size = 0
# 遍历filePath下的文件、文件夹（包括子目录）
for parent, dirnames, filenames in os.walk(load_path):
# for dirname in dirnames:
# print('parent is %s, dirname is %s' % (parent, dirname))
for filename in filenames:
# print('parent is %s, filename is %s' % (parent, filename))
# print('the full name of the file is %s' % os.path.join(parent, filename))
image_size = os.path.getsize(os.path.join(parent, filename))
file_map.setdefault(os.path.join(parent, filename), image_size)
# 获取的图片列表按 文件大小image_size 排序
file_map = sorted(file_map.items(), key=lambda d: d[1], reverse=False)
file_list = []
for filename, image_size in file_map:
file_list.append(filename)
# 取出重复的图片
file_repeat = []
for currIndex, filename in enumerate(file_list):
dir_image1 = file_list[currIndex]
dir_image2 = file_list[currIndex + 1]
result = 比较两张图片是否相同(dir_image1, dir_image2)
if (result == "两张图相同"):
file_repeat.append(file_list[currIndex + 1])
print("\n相同的图片：", file_list[currIndex], file_list[currIndex + 1])
else:
print('\n不同的图片：', file_list[currIndex], file_list[currIndex + 1])
currIndex += 1
if currIndex >= len(file_list) - 1:
break
# 将重复的图片移动到新的文件夹，实现对原文件夹降重
for image in file_repeat:
shutil.move(image, save_path)
print("正在移除重复照片：", image)
处理前会有重复图片和一些小图：
还有一些下载失败无法显示的图，点开无法显示
经过以上一系列处理之后，则如下图所示：
