Kaggle대회 얼굴 키포인트 탐지(Facial Keypoints Detection)증강

본 작업은 코랩에서 진행하였고 5년 전 Kaggle에서 진행한 얼굴 키포인트 탐지의 사용자레퍼런스를 활용했습니다.
test,train,idlookup 파일 및 api링크를 원하시는 분들은 댓글 남겨주시면 감사하겠습니다. (질문도 받아요)

Module 임폴트

import numpy as np  #넘파이
import pandas as pd #csv파일 받아들이기 위한 판다스
import seaborn as sns #그래프확인 시본
import matplotlib.pyplot as plt #matplot
%matplotlib inline
import cv2 #opencv
from math import sin, cos, pi #sin,cos,pi import

from keras.applications.mobilenet_v2 import MobileNetV2  #mobilenet이용
from keras.layers import Conv2D, LeakyReLU, GlobalAveragePooling2D, Dropout, Dense #model layer
from keras.models import Sequential #순차적 모델

def plot_sample(image, keypoint, axis, title):
    image = image.reshape(96,96) #96x96으로 reshape
    axis.imshow(image, cmap='gray')#axis이미지보여주기.cmap은 gray
    axis.scatter(keypoint[0::2], keypoint[1::2], marker='x', s=20)#키포인트 마커로 찍어주기
    plt.title(title)

train_file = pd.read_csv('/content/training.csv')  #각 csv파일 읽어들이기(pandas사용)
test_file = pd.read_csv('/content/test.csv')
idlookup_file = pd.read_csv('/content/IdLookupTable.csv')

train_file.head() #train_file 헤드 확인

test_file.head() #testfile 확인

idlookup_file.head() #idlookup 확인

train_file.isnull().sum() #결측값 개수의 합 구하기

clean_train_file = train_file.dropna() # 결측값 제거(증강에사용)
train_file = train_file.fillna(method='ffill')

train_file.isnull().sum() #제거 후 확인

Load images and keypoints

def load_images(image_data):
    images = [] #리스트 내포 사용
    for idx, sample in image_data.iterrows(): #image의 특정값을 바꾸기 떄문에 iterrows사용
        image = np.array(sample['Image'].split(' '), dtype=int) #np.array로 sampleimage를 배열로 바꾸고 int로 바꿈,부분문자열사용
        image = np.reshape(image, (96,96,1))#96*96인 1차원행렬로 바꿈
        images.append(image) #리스트에 추가
    images = np.array(images)/255.
    return images

def load_keypoints(keypoint_data):
    keypoint_data = keypoint_data.drop(['Image'], axis=1)
    keypoint_features = []
    for idx, features in keypoint_data.iterrows():
        keypoint_features.append(features)
    keypoint_features = np.array(keypoint_features, dtype=float)
    return keypoint_features

train_images = load_images(train_file)
images = load_images(clean_train_file)
train_keypoints = load_keypoints(train_file)
keypoints = load_keypoints(clean_train_file)
test_images = load_images(test_file)

Augmentation

class aug_config:
    rotation_augmentation = True
    brightness_augmentation = True
    shift_augmentation = True
    random_noise_augmentation = True
    rotation_angles = [12]
    pixel_shifts = [12]

Rotation

def rotate_augmentation(images, keypoints, rotation_angles): #회전시킨 이미지
    rotated_images = []
    rotated_keypoints = []
    for angle in rotation_angles:
        for angle in [angle, -angle]:
            M = cv2.getRotationMatrix2D((48,48), angle, 1.)
            angle_rad = -angle*pi/180.
            for image in images:
                rotated_image = cv2.warpAffine(image, M, (96,96), flags=cv2.INTER_CUBIC)
                rotated_images.append(rotated_image)
            for keypoint in keypoints:
                rotated_keypoint = keypoint - 48.
                for idx in range(0, len(rotated_keypoint), 2):
                    rotated_keypoint[idx] = rotated_keypoint[idx]*cos(angle_rad)-rotated_keypoint[idx+1]*sin(angle_rad)
                    rotated_keypoint[idx+1] = rotated_keypoint[idx]*sin(angle_rad)+rotated_keypoint[idx+1]*cos(angle_rad)
                rotated_keypoint += 48.   
                rotated_keypoints.append(rotated_keypoint)
            
    return np.reshape(rotated_images,(-1,96,96,1)), rotated_keypoints

if aug_config.rotation_augmentation:
    rotated_train_images, rotated_train_keypoints = rotate_augmentation(images, keypoints, aug_config.rotation_angles)
    train_images = np.concatenate((train_images, rotated_train_images))
    train_keypoints = np.concatenate((train_keypoints, rotated_train_keypoints))
    fig, axis = plt.subplots()
    plot_sample(rotated_train_images[19], rotated_train_keypoints[19], axis, "Rotation Augmentation")

Change Brightness

def alter_brightness(images, keypoints):
    altered_brightness_images = []
    inc_brightness_images = np.clip(images*1.2, 0.0, 1.0)    
    dec_brightness_images = np.clip(images*0.6, 0.0, 1.0)    
    altered_brightness_images.extend(inc_brightness_images)
    altered_brightness_images.extend(dec_brightness_images)
    return altered_brightness_images, np.concatenate((keypoints, keypoints))

if aug_config.brightness_augmentation:
    altered_brightness_images, altered_brightness_keypoints = alter_brightness(images, keypoints)
    train_images = np.concatenate((train_images, altered_brightness_images))
    train_keypoints = np.concatenate((train_keypoints, altered_brightness_keypoints))
    fig, axis = plt.subplots()
    plot_sample(altered_brightness_images[19], altered_brightness_keypoints[19], axis, "Alter Brightness Augmentation")

def shift_images(images, keypoints, pixel_shifts): #이미지 시프트
    shifted_images = []
    shifted_keypoints = []
    for shift in pixel_shifts:    
        for (shift_x,shift_y) in [(-shift,-shift),(-shift,shift),(shift,-shift),(shift,shift)]:
            M = np.float32([[1,0,shift_x],[0,1,shift_y]])
            for image, keypoint in zip(images, keypoints):
                shifted_image = cv2.warpAffine(image, M, (96,96), flags=cv2.INTER_CUBIC)
                shifted_keypoint = np.array([(point+shift_x) if idx%2==0 else (point+shift_y) for idx, point in enumerate(keypoint)])
                if np.all(0.0<shifted_keypoint) and np.all(shifted_keypoint<96.0):
                    shifted_images.append(shifted_image.reshape(96,96,1))
                    shifted_keypoints.append(shifted_keypoint)
    shifted_keypoints = np.clip(shifted_keypoints,0.0,96.0)
    return shifted_images, shifted_keypoints

if aug_config.shift_augmentation:
    shifted_train_images, shifted_train_keypoints = shift_images(images, keypoints, aug_config.pixel_shifts)
    train_images = np.concatenate((train_images, shifted_train_images))
    train_keypoints = np.concatenate((train_keypoints, shifted_train_keypoints))
    fig, axis = plt.subplots()
    plot_sample(shifted_train_images[19], shifted_train_keypoints[19], axis, "Shift Augmentation")

def add_noise(images): #노이즈 이미지
    noisy_images = []
    for image in images:
        noisy_image = cv2.add(image, 0.008*np.random.randn(96,96,1))    # 입력 이미지에 임의 일반 노이즈 추가 및 [-1,1] 사이에 노이즈가 발생하는 이미지 클리핑
        noisy_images.append(noisy_image.reshape(96,96,1))
    return noisy_images

if aug_config.random_noise_augmentation:
    noisy_train_images = add_noise(images)
    train_images = np.concatenate((train_images, noisy_train_images))
    train_keypoints = np.concatenate((train_keypoints, keypoints))
    fig, axis = plt.subplots()
    plot_sample(noisy_train_images[19], keypoints[19], axis, "Random Noise Augmentation")

print(train_images.shape)
print(train_keypoints.shape)

Modeling

model = Sequential()
pretrained_model = MobileNetV2(input_shape=(96,96,3), include_top=False, weights='imagenet')
pretrained_model.trainable = True

model.add(Conv2D(3, (1,1), padding='same', input_shape=(96,96,1)))
model.add(LeakyReLU(alpha=0.1))
model.add(pretrained_model)
model.add(GlobalAveragePooling2D())
model.add(Dropout(0.1))
model.add(Dense(30))
model.summary()

from keras.callbacks import EarlyStopping, ReduceLROnPlateau
earlyStopping = EarlyStopping(monitor='loss', patience=30, mode='min',
                             baseline=None)

rlp = ReduceLROnPlateau(monitor='val_loss', factor=0.7, patience=5, min_lr=1e-15, mode='min', verbose=1)

model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy'])

history = model.fit(train_images, train_keypoints, epochs=200, batch_size=64, validation_split=0.15, callbacks=[earlyStopping, rlp])

sns.set_style('darkgrid')

fig, ax = plt.subplots(2, 1, figsize=(20, 10))
df = pd.DataFrame(history.history)
df[['loss', 'val_loss']].plot(ax=ax[0])
df[['accuracy', 'val_accuracy']].plot(ax=ax[1])
ax[0].set_title('Model Loss', fontsize=12)
ax[1].set_title('Model Acc', fontsize=12)
fig.suptitle('Model Metrics', fontsize=18);

Predicting on Test Set and Submission

test_preds = model.predict(test_images)

feature_names = list(idlookup_file['FeatureName'])
image_ids = list(idlookup_file['ImageId']-1)
row_ids = list(idlookup_file['RowId'])

feature_list = []
for feature in feature_names:
    feature_list.append(feature_names.index(feature))
    
predictions = []
for x,y in zip(image_ids, feature_list):
    predictions.append(test_preds[x][y])
    
row_ids = pd.Series(row_ids, name = 'RowId')
locations = pd.Series(predictions, name = 'Location')
locations = locations.clip(0.0,96.0)
submission_result = pd.concat([row_ids,locations],axis = 1)
submission_result.to_csv('submission.csv',index = False)