[人工智能]处理数据集用到的脚本文件

1.图片归一化

import numpy as np
from PIL import Image
import os
from numpy import *
import cv2  #导入openCV
input_dir = './road/JPEGImages/'  #文件夹名字后一定要加/，才能识别为打开文件夹中的内容
out_dir1 = './After_graying_'  #进行灰度化后的图片保存在该文件夹下
out_dir2 = './'  #进行灰度化和归一化后的图片保存在该文件夹下

#3.1灰度化
#a = os.listdir(input_dir)
#for i in a:    
#    print(i)
#    Img = Image.open(input_dir + i)  #用PIL的库来逐个读取图片
#    Img_gray = Img.convert('L')  #灰度化L
#    Img_gray.save('./After_graying' + out_dir1 + i)  #用PIL的库来逐个保存图片到指定路径下
#    print('~~~~~~~~~~~~~This is a picture after graying~~~~~~~~~~')
#    print(Img_gray)

#3.2对上述灰度化后的Img_gray进行归一化
b = os.listdir('./road/JPEGImages/')
for j in b:
    print(j)
    photo = cv2.imread("./road/JPEGImages/" + j)
    #cv2.imread()接口读图像，读进来直接是BGR 格式数据格式在 0~255，通道格式为(W,H,C)
    result = np.zeros(photo.shape, dtype=np.float32) 
    cv2.normalize(photo, result, 0, 1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F, mask=None)
    print('~~~~~~~~~~This is data of BGR after normalizing : ~~~~~~~~~~~~~~~~~~')
    print(result)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    Img_output = Image.fromarray(np.uint8(result*255))  # 将array恢复成图片 Img_output，不能直接把result保存在文件夹里。
    Img_output.save('./After_normalizing/' + j)  #将归一化后的图片保存在指定路径下
    
    #3.3下面将以上每张图归一化后的数据保存下项目文件夹下的 txtfile.txt 中
    #fname = open("/media/yms/KINGSTON/808/车道线数据集/txtfile.txt", 'a') #a是向txt中追加写入
    #fname.write("Name of the Picture:" + str(j) + '\n')
    #fname.write("图像的形状,返回一个图像的(行数,列数,通道数):" + str(result.shape) + '\n')
    #fname.write("图像的像素数目:"+str(result.size)+'\n')
    #fname.write("图像的数据类型:"+str(result.dtype)+'\n')
    #Xlenth = result.shape[1]  #图片列数
    #Ylenth = result.shape[0]  #图片行数
    
    #m = 1
    #for p in range(Ylenth):  #外循环是行
    #    fname.write(str(m) + ':'+'\n')
    #    for q in range(Xlenth):   #内循环是列
    #        fname.write(str(result[p][q])+' ')
    #    m += 1
    #    fname.write('\n')
    #fname.close()

2.修改xml中类别属性的长宽高

import math
import os
from PIL import Image, ImageDraw
import numpy as np
import xml.etree.ElementTree as ET

#读取原xml中的box，并返回
def read_xml(xml_name):
    etree = ET.parse(xml_name)
    root = etree.getroot()
    box = []
    for obj in root.iter('object'):
        xmin, ymin, xmax, ymax = (x.text for x in obj.find('bndbox'))
        box.append([xmin, ymin, xmax, ymax])
    return box


#xml_name--原xml   save_name--保存的xml
def write_xml(xml_name, save_name):
    etree = ET.parse(xml_name)
    root = etree.getroot()
    # 修改box的值
    for obj in root.iter('object'):
        name = obj.find('name')
        if name=='head' or name=='tail':
            xmlbox = obj.find('bndbox')
            b0 = float(xmlbox.find('xmin').text)
            b1 = float(xmlbox.find('xmax').text)
            b2 = float(xmlbox.find('ymin').text)
            b3 = float(xmlbox.find('ymax').text)
            print(b0)
            x = (b1 + b0) / 2.0
            y = (b2 + b3) / 2.0
            w = b3 - b2
            h = b1 - b0
            if w>=20 or h>=20:
                xmin, ymin, xmax, ymax = int(x - 2 * w / 5), int(y - 2 * h / 5), int(x + 2 * w / 5), int(y + 2 * h / 5)
                xmlbox.find('xmin').text = str(xmin)
                xmlbox.find('ymin').text = str(ymin)
                xmlbox.find('xmax').text = str(xmax)
                xmlbox.find('ymax').text = str(ymax)

    etree.write(save_name)


def start(sourceDir, targetDir):
    for root, dir1, filenames in os.walk(sourceDir):
        for filename in filenames:
            file = os.path.splitext(filename)[0]
            if os.path.splitext(filename)[1] == '.xml':
                filename_jpg = os.path.join(root, filename)
                xml_name = os.path.join(root, file + '.xml')
                box = read_xml(xml_name)
                save_xml = os.path.join(targetDir, file + '.xml')
                write_xml(xml_name, save_xml)


if __name__ == "__main__":
    # 源文件夹
    sourceDir = r"/media/yms/KINGSTON/808/车道线数据集/Annotations"
    # 目标文件夹
    targetDir = r"/media/yms/KINGSTON/808/车道线数据集/Annotations1"
    start(sourceDir, targetDir)

3.将coco数据集转为voc数据集
3.1 获取所要的类别

#-*- coding:utf-8-*-
import json
className = {
    3: 'car',
    6: 'bus',
    8: 'truck'
}
classNum = [3,6,8]
cocojson="/home/yms/下载/annotations_trainval2017/annotations/instances_val2017.json"#coco数据集的标签地址（json地址）
def writeNum(Num):
    with open("COCO_val.json", "a+") as f:
        f.write(str(Num))
inputfile = []
inner = {}
with open(cocojson, "r+") as f:
    allData = json.load(f)
    data =allData["annotations"]
    print(data[1])
    print("read ready")
for i in data:
    if (i['category_id'] in classNum):
        inner = {
            "filename":str(i["image_id"]).zfill(12),
            "name":className[i["category_id"]],
            "bndbox":i["bbox"]
        }
        inputfile.append(inner)
inputfile = json.dumps(inputfile)
writeNum(inputfile)

3.2从官方给出的coco数据集中筛选出含有上述类别的图片及相应的标签文件，保存到指定文件夹下

import json
import os
import cv2

nameStr = []
with open("COCO_val.json", "r+") as f:
    data = json.load(f)
    print("read ready")
for i in data:
    imgName = str(i["filename"]) + ".jpg"
    nameStr.append(imgName)
nameStr = set(nameStr)
print(nameStr)
print(len(nameStr))

path = '/home/yms/下载/val2017/'#coco数据集的验证（or训练）数据集的地址
savePath = "/home/yms/下载/val/"#你想把图片存放的地址
count = 0
for file in nameStr:
    img = cv2.imread(path + file)
    cv2.imwrite(savePath + file, img)
    count = count + 1
    print('num: ' + count.__str__() + '     ' + file + '\n')

3.3将文件生成xml文件，用于训练

# -*- coding:utf-8-*-

import xml.dom
import xml.dom.minidom
import os
# from PIL import Image
import cv2
import json

# xml文件规范定义


_IMAGE_PATH = '/home/yms/下载/val2017'#测试数据集的地址

_INDENT = '' * 4
_NEW_LINE = '\n'
_FOLDER_NODE = 'COCO2014'
_ROOT_NODE = 'annotation'
_DATABASE_NAME = 'LOGODection'
_ANNOTATION = 'COCO2014'
_AUTHOR = 'SyGoing_CSDN'
_SEGMENTED = '0'
_DIFFICULT = '0'
_TRUNCATED = '0'
_POSE = 'Unspecified'

# _IMAGE_COPY_PATH= 'JPEGImages'
_ANNOTATION_SAVE_PATH = '/home/yms/下载/label/'#标签存放地址


# _IMAGE_CHANNEL= 3

# 封装创建节点的过程
def createElementNode(doc, tag, attr):  # 创建一个元素节点
    element_node = doc.createElement(tag)

    # 创建一个文本节点
    text_node = doc.createTextNode(attr)

    # 将文本节点作为元素节点的子节点
    element_node.appendChild(text_node)

    return element_node


# 封装添加一个子节点
def createChildNode(doc, tag, attr, parent_node):
    child_node = createElementNode(doc, tag, attr)

    parent_node.appendChild(child_node)


# object节点比较特殊
def createObjectNode(doc, attrs):
    object_node = doc.createElement('object')

    midname = attrs['name']

    #if midname != 'person':
     #   midname = 'car'

    createChildNode(doc, 'name', midname,
                    object_node)

    # createChildNode(doc, 'name',attrs['name'],
    #                object_node)

    createChildNode(doc, 'pose',
                    _POSE, object_node)

    createChildNode(doc, 'truncated',
                    _TRUNCATED, object_node)

    createChildNode(doc, 'difficult',
                    _DIFFICULT, object_node)

    bndbox_node = doc.createElement('bndbox')

    createChildNode(doc, 'xmin', str(int(attrs['bndbox'][0])),
                    bndbox_node)

    createChildNode(doc, 'ymin', str(int(attrs['bndbox'][1])),
                    bndbox_node)

    createChildNode(doc, 'xmax', str(int(attrs['bndbox'][0] + attrs['bndbox'][2])),
                    bndbox_node)

    createChildNode(doc, 'ymax', str(int(attrs['bndbox'][1] + attrs['bndbox'][3])),
                    bndbox_node)

    object_node.appendChild(bndbox_node)

    return object_node


# 将documentElement写入XML文件
def writeXMLFile(doc, filename):
    tmpfile = open('tmp.xml', 'w')

    doc.writexml(tmpfile, addindent='' * 4, newl='\n', encoding='utf-8')

    tmpfile.close()

    # 删除第一行默认添加的标记

    fin = open('tmp.xml')
    # print(filename)
    fout = open(filename, 'w')
    # print(os.path.dirname(fout))

    lines = fin.readlines()

    for line in lines[1:]:

        if line.split():
            fout.writelines(line)

            # new_lines =''.join(lines[1:])

        # fout.write(new_lines)

    fin.close()

    fout.close()


if __name__ == "__main__":
    ##读取图片列表
    img_path = "/home/yms/下载/val/"#刚刚存放想要的图片的地址
    fileList = os.listdir(img_path)
    if fileList == 0:
        os._exit(-1)

    with open("COCO_val.json", "r") as f:
        ann_data = json.load(f)

current_dirpath = os.path.dirname(os.path.abspath('__file__'))

if not os.path.exists(_ANNOTATION_SAVE_PATH):
    os.mkdir(_ANNOTATION_SAVE_PATH)

    # if notos.path.exists(_IMAGE_COPY_PATH):
#    os.mkdir(_IMAGE_COPY_PATH)

for imageName in fileList:
    saveName = imageName.strip(".jpg")
    print(saveName)
    # pos =fileList[xText].rfind(".")
    # textName =fileList[xText][:pos]

    # ouput_file = open(_TXT_PATH +'/' + fileList[xText])
    # ouput_file =open(_TXT_PATH)

    # lines = ouput_file.readlines()

    xml_file_name = os.path.join(_ANNOTATION_SAVE_PATH, (saveName + '.xml'))
    # withopen(xml_file_name,"w") as f:
    #     pass

    img = cv2.imread(os.path.join(img_path, imageName))
    print(os.path.join(img_path, imageName))
    # cv2.imshow(img)
    height, width, channel = img.shape
    print(height, width, channel)

    my_dom = xml.dom.getDOMImplementation()

    doc = my_dom.createDocument(None, _ROOT_NODE, None)

    # 获得根节点
    root_node = doc.documentElement

    # folder节点

    createChildNode(doc, 'folder', _FOLDER_NODE, root_node)

    # filename节点

    createChildNode(doc, 'filename', saveName + '.jpg', root_node)

    # source节点

    source_node = doc.createElement('source')

    # source的子节点

    createChildNode(doc, 'database', _DATABASE_NAME, source_node)

    createChildNode(doc, 'annotation', _ANNOTATION, source_node)

    createChildNode(doc, 'image', 'flickr', source_node)

    createChildNode(doc, 'flickrid', 'NULL', source_node)

    root_node.appendChild(source_node)

# owner节点

    owner_node = doc.createElement('owner')

# owner的子节点

    createChildNode(doc, 'flickrid', 'NULL', owner_node)

    createChildNode(doc, 'name', _AUTHOR, owner_node)

    root_node.appendChild(owner_node)

# size节点

    size_node = doc.createElement('size')

    createChildNode(doc, 'width', str(width), size_node)

    createChildNode(doc, 'height', str(height), size_node)

    createChildNode(doc, 'depth', str(channel), size_node)

    root_node.appendChild(size_node)

# segmented节点

    createChildNode(doc, 'segmented', _SEGMENTED, root_node)

    for ann in ann_data:
        imgName =  str(ann["filename"])
        cname = saveName;
        if (saveName == imgName):
            # object节点
            object_node = createObjectNode(doc, ann)
            root_node.appendChild(object_node)

        else:
            continue

        # 构建XML文件名称

    print(xml_file_name)

    # 创建XML文件

    # createXMLFile(attrs, width,height, xml_file_name)

    # # 写入文件
    #
    writeXMLFile(doc, xml_file_name)

4.批量提取，根据一个文件，从另一个文件提取相同前缀名并保存
4.1获取文件夹下图片的前缀名，并将其保存到txt文件中

import os
class ReadImageName():
    def __init__(self):
        self.path='.'
    def readname(self):
        filenames=os.listdir(self.path)
        filelist=[]
        for item in filenames:
            if item.endswith('.jpg'):
                itemname=os.path.join(self.path,item)
                itemname=itemname[-21:]
                filelist.append(itemname)

        fo=open('imagename.txt',"w")
        for item in filelist:
            fo.write(str(item)+ "\n")

if __name__=="__main__":
    log=ReadImageName()
    log.readname()

4.2根据txt中文件的名字批量提取对应的文件名并保存到另一个文件夹

# -*- coding: UTF-8 -*- 
# !/usr/bin/env python
import shutil
 
# 根据txt中文件的名字批量提取对应的文件名并保存到另一个文件夹
 
data = []
for line in open("/media/yms/KINGSTON/808/bdd100k/JPEGImages/imagename.txt", "r"):  # 设置文件对象并读取每一行文件
    data.append(line)
 
for a in data:
    src = '/media/yms/KINGSTON/808/bdd100k/Annotations1/{}.xml'.format(a[:-1])
    dst = '/media/yms/KINGSTON/808/bdd100k/Annotations/{}.xml'.format(a[:-1])
    shutil.move(src, dst)

5.修改xml文件，进行字符替换（比如将Difficult改为difficult）

import os
import io
#paths文件目录，files保存文件路径以及名字
paths='/media/yms/KINGSTON/kk/Annotations/'
files=[]
oldStr='Difficult'
newStr='difficult'
for file in os.listdir(paths):
    if file.endswith('.xml'):
        files.append(paths+file)
for file in files:
    with open(file, "r", encoding="utf-8") as f:
        lines=f.readlines()#将文件内容保存到内存
    with open(file, "w", encoding="utf-8") as f_w:
        for line in lines:#将内存中的文件逐行读取
            if oldStr in line:
                line=line.replace(oldStr,newStr)#新内容代替旧内容
            f_w.write(line)