data_preprocessing_script.py

# -*- coding: utf-8 -*-
"""data-preprocessing-script.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/16-k5IL9GYypFAux7FCtOx9RtziSObV3f

# Data Preprocessing Script to Yolo Formate

### The following code is for the Script to prepare this ROW dataset into A YOLO object detection configeration dataset.
### We go into a series of steps from data preprocessing to finally training the model.
> 1. Download the dataset into your local device
> 2. Run this script and assert that `src`, `dst` paths are set to the local folder paths
> 3. this will transfere this row data into a ready to train dataset using object detection yolo formate
> 4. Build the yolomodel and run the notebook model here : ""

### Run This Script to :
> * Convert the .XML Labels Files Format to YOLO Formate (class xCenter yCenter width height)
> * Build The YOLO Folders Architecture

### This cell convert .xml to yolo formate by detecting the <object> to count the number of objects detected in the image ,then store the class number and encode it to be [0, 1, 2, 3, 4, 5]
    Note that you can change the dataset path and the distenation through the first 2 lines of 
    code
"""

# #########################################################
Root_src = r"/kaggle/input/egyptian-currency"
Dst_src = r"/kaggle/working/"
# #########################################################

import os

def convert(size, box):
    dw = 1./(size[0])
    dh = 1./(size[1])
    x = (box[0] + box[1])/2.0 - 1
    y = (box[2] + box[3])/2.0 - 1
    w = box[1] - box[0]
    h = box[3] - box[2]
    x = x*dw
    w = w*dw
    y = y*dh
    h = h*dh
    return (x,y,w,h)

def find_between(s, first, last):
    try:
        start = s.index(first) + len(first)
        end = s.index(last, start)
        return s[start:end]
    except ValueError:
        return ""

def remove(data, section):
    data.replace(section,' ')
    return data


def freq(str):
    str = str.split()
    str2 = []
    for i in str:
        if i not in str2:
            str2.append(i)
    for i in range(0, len(str2)):
            if str2[i] == "</object>":
                return str.count(str2[i])

# #ReadyLabelsFolder


dst = os.path.join(Dst_src,"YOLO Labels")
os.makedirs(dst)

for foldername in os.listdir(Root_src):
  src = os.path.join(Root_src, foldername)

   # iterate over files in
   # that directory
  for filename in os.listdir(src):
      file = os.path.join(src, filename)
      # checking if it is a file
      if os.path.isfile(file) and file.lower().endswith('.xml'):

          f = open(file, "r")
          data = f.read()

          numOfClasses = freq(data)

          for i in range(0, numOfClasses):
              Class = find_between(data, "<name>", "</name>")
              if Class == "5Egp":
                  Class = '0'
              elif Class == "10Egp":
                  Class = '1'
              elif Class == "20Egp":
                  Class = '2'
              elif Class == "50Egp":
                  Class = '3'
              elif Class == "100Egp":
                  Class = '4'
              elif Class == "200Egp":
                  Class = '5'
              xmin = find_between(data, "<xmin>", "</xmin>")
              xmax = find_between(data, "<xmax>", "</xmax>")
              ymin = find_between(data, "<ymin>", "</ymin>")
              ymax = find_between(data, "<ymax>", "</ymax>")
              width = find_between(data, "<width>", "</width>")
              height = find_between(data, "<height>", "</height>")

              ret = convert([int(width), int(height)], [float(xmin), float(xmax), float(ymin), float(ymax)])

              newFileName = filename.split('.')[0]
              newFileName += '.txt'
              newFolder = os.path.join(dst,newFileName)
              f = open(newFolder, "w")
              f.write(Class + ' ' + str(ret[0]) + ' ' + str(ret[1]) + ' ' + str(ret[2]) + ' ' + str(ret[3]) + '\n')
              rmSection = find_between(data, "<object>", "</object>")
              data = data.replace(rmSection, " ", 1)

          f.close()

print("Labels Moved to Yolo Labels")

"""### This cell Selects the .JPG files from the 10 Folders of the dataset and store them all in a single Folder named "YOLO Images" to be easy in dividing the data to train, val and test"""

import shutil
import os

#Images from original folder to my folder
img_dst = os.path.join(Dst_src,"YOLO Images")
os.makedirs(img_dst)

images_count = 0

for foldername in os.listdir(Root_src):
    src = os.path.join(Root_src, foldername)
    print (src, "Start")
    if(foldername != "YOLO Images"):
        for filename in os.listdir(src):
            file = os.path.join(src, filename)
            # checking if it is a file
            if os.path.isfile(file) and file.lower().endswith('.jpg'):
                images_count = images_count + 1
                dst_path = os.path.join(img_dst,filename)

                shutil.copy(file, dst_path)

print("Images moved to YOLO Images")
print(images_count)

"""### Here we start building the architecture by creating Train and Test Folders, then images and labels Folders"""

train_path = os.path.join(Dst_src,"Train")
os.makedirs(train_path)
label_path = os.path.join(train_path,"labels")
os.makedirs(label_path)
images_path = os.path.join(train_path,"images")
os.makedirs(images_path)

TN_path = os.path.join(label_path,"train")
VL_path = os.path.join(label_path,"val")

os.makedirs(TN_path)
os.makedirs(VL_path)

TN_path_ = os.path.join(images_path,"train")
VL_path_ = os.path.join(images_path,"val")

os.makedirs(TN_path_)
os.makedirs(VL_path_)


test_path = os.path.join(Dst_src,"Test")
os.makedirs(test_path)

label_path_ = os.path.join(test_path,"labels")
os.makedirs(label_path_)

images_path_ = os.path.join(test_path,"images")
os.makedirs(images_path_)

"""### Here we start to divid the data to 80% Training, 10% Validation and 10% Testing

1) Images
"""

counter = 0
for filename in os.listdir(img_dst):
    counter += 1
    s = os.path.join(img_dst,filename)

    if counter > 0 and counter < images_count * 0.8:
      d = os.path.join(TN_path_,filename)
      shutil.move(s, d)
    elif counter > images_count * 0.8 and counter < images_count * 0.9:
      d = os.path.join(VL_path_,filename)
      shutil.move(s, d)
    else:
      d = os.path.join(images_path_,filename)
      shutil.move(s, d)

"""2) Labels"""

counter = 0
for filename in os.listdir(dst):
    counter = counter + 1
    s = os.path.join(dst,filename)

    if counter > 0 and counter < images_count * 0.8:
      d = os.path.join(TN_path,filename)
      shutil.move(s, d)
    elif counter > images_count * 0.8 and counter < images_count * 0.9:
      d = os.path.join(VL_path,filename)
      shutil.move(s, d)
    else:
      d = os.path.join(label_path_,filename)
      shutil.move(s, d)

print("All Done !!")