OT for image color adaptation with mapping estimation

OT for domain adaptation with image color adaptation [6] with mapping estimation [8].

[6] Ferradans, S., Papadakis, N., Peyre, G., & Aujol, J. F. (2014). Regularized discrete optimal transport. SIAM Journal on Imaging Sciences, 7(3), 1853-1882.

[8] M. Perrot, N. Courty, R. Flamary, A. Habrard, “Mapping estimation for discrete optimal transport”, Neural Information Processing Systems (NIPS), 2016.

# Authors: Remi Flamary <remi.flamary@unice.fr>
#          Stanislas Chambon <stan.chambon@gmail.com>
#
# License: MIT License

# sphinx_gallery_thumbnail_number = 3

import numpy as np
import matplotlib.pylab as pl
import ot

r = np.random.RandomState(42)


def im2mat(img):
    """Converts and image to matrix (one pixel per line)"""
    return img.reshape((img.shape[0] * img.shape[1], img.shape[2]))


def mat2im(X, shape):
    """Converts back a matrix to an image"""
    return X.reshape(shape)


def minmax(img):
    return np.clip(img, 0, 1)

Generate data

# Loading images
I1 = pl.imread('../../data/ocean_day.jpg').astype(np.float64) / 256
I2 = pl.imread('../../data/ocean_sunset.jpg').astype(np.float64) / 256


X1 = im2mat(I1)
X2 = im2mat(I2)

# training samples
nb = 1000
idx1 = r.randint(X1.shape[0], size=(nb,))
idx2 = r.randint(X2.shape[0], size=(nb,))

Xs = X1[idx1, :]
Xt = X2[idx2, :]

Domain adaptation for pixel distribution transfer

# EMDTransport
ot_emd = ot.da.EMDTransport()
ot_emd.fit(Xs=Xs, Xt=Xt)
transp_Xs_emd = ot_emd.transform(Xs=X1)
Image_emd = minmax(mat2im(transp_Xs_emd, I1.shape))

# SinkhornTransport
ot_sinkhorn = ot.da.SinkhornTransport(reg_e=1e-1)
ot_sinkhorn.fit(Xs=Xs, Xt=Xt)
transp_Xs_sinkhorn = ot_sinkhorn.transform(Xs=X1)
Image_sinkhorn = minmax(mat2im(transp_Xs_sinkhorn, I1.shape))

ot_mapping_linear = ot.da.MappingTransport(
    mu=1e0, eta=1e-8, bias=True, max_iter=20, verbose=True)
ot_mapping_linear.fit(Xs=Xs, Xt=Xt)

X1tl = ot_mapping_linear.transform(Xs=X1)
Image_mapping_linear = minmax(mat2im(X1tl, I1.shape))

ot_mapping_gaussian = ot.da.MappingTransport(
    mu=1e0, eta=1e-2, sigma=1, bias=False, max_iter=10, verbose=True)
ot_mapping_gaussian.fit(Xs=Xs, Xt=Xt)

X1tn = ot_mapping_gaussian.transform(Xs=X1)  # use the estimated mapping
Image_mapping_gaussian = minmax(mat2im(X1tn, I1.shape))

Out:

It.  |Loss        |Delta loss
--------------------------------
    0|3.680534e+02|0.000000e+00
    1|3.592503e+02|-2.391783e-02
    2|3.590686e+02|-5.058263e-04
    3|3.589747e+02|-2.616618e-04
    4|3.589166e+02|-1.616220e-04
    5|3.588771e+02|-1.102962e-04
    6|3.588481e+02|-8.062176e-05
    7|3.588260e+02|-6.155952e-05
    8|3.588087e+02|-4.834273e-05
    9|3.587946e+02|-3.927426e-05
   10|3.587829e+02|-3.256575e-05
   11|3.587731e+02|-2.721097e-05
   12|3.587648e+02|-2.321165e-05
   13|3.587576e+02|-2.020205e-05
   14|3.587513e+02|-1.741284e-05
   15|3.587458e+02|-1.540330e-05
   16|3.587409e+02|-1.371068e-05
   17|3.587365e+02|-1.226889e-05
   18|3.587325e+02|-1.103162e-05
   19|3.587322e+02|-9.876653e-07
It.  |Loss        |Delta loss
--------------------------------
    0|3.784871e+02|0.000000e+00
    1|3.646491e+02|-3.656142e-02
    2|3.642975e+02|-9.642655e-04
    3|3.641626e+02|-3.702413e-04
    4|3.640888e+02|-2.026301e-04
    5|3.640419e+02|-1.289607e-04
    6|3.640097e+02|-8.831646e-05
    7|3.639861e+02|-6.487612e-05
    8|3.639679e+02|-4.994063e-05
    9|3.639536e+02|-3.941436e-05
   10|3.639419e+02|-3.209753e-05

Plot original images

pl.figure(1, figsize=(6.4, 3))
pl.subplot(1, 2, 1)
pl.imshow(I1)
pl.axis('off')
pl.title('Image 1')

pl.subplot(1, 2, 2)
pl.imshow(I2)
pl.axis('off')
pl.title('Image 2')
pl.tight_layout()
Image 1, Image 2

Plot pixel values distribution

pl.figure(2, figsize=(6.4, 5))

pl.subplot(1, 2, 1)
pl.scatter(Xs[:, 0], Xs[:, 2], c=Xs)
pl.axis([0, 1, 0, 1])
pl.xlabel('Red')
pl.ylabel('Blue')
pl.title('Image 1')

pl.subplot(1, 2, 2)
pl.scatter(Xt[:, 0], Xt[:, 2], c=Xt)
pl.axis([0, 1, 0, 1])
pl.xlabel('Red')
pl.ylabel('Blue')
pl.title('Image 2')
pl.tight_layout()
Image 1, Image 2

Plot transformed images

pl.figure(2, figsize=(10, 5))

pl.subplot(2, 3, 1)
pl.imshow(I1)
pl.axis('off')
pl.title('Im. 1')

pl.subplot(2, 3, 4)
pl.imshow(I2)
pl.axis('off')
pl.title('Im. 2')

pl.subplot(2, 3, 2)
pl.imshow(Image_emd)
pl.axis('off')
pl.title('EmdTransport')

pl.subplot(2, 3, 5)
pl.imshow(Image_sinkhorn)
pl.axis('off')
pl.title('SinkhornTransport')

pl.subplot(2, 3, 3)
pl.imshow(Image_mapping_linear)
pl.axis('off')
pl.title('MappingTransport (linear)')

pl.subplot(2, 3, 6)
pl.imshow(Image_mapping_gaussian)
pl.axis('off')
pl.title('MappingTransport (gaussian)')
pl.tight_layout()

pl.show()
Im. 1, Im. 2, EmdTransport, SinkhornTransport, MappingTransport (linear), MappingTransport (gaussian)

Total running time of the script: ( 3 minutes 22.252 seconds)

Gallery generated by Sphinx-Gallery