flower_simsaim_test_1.ipynb

Code

Text

Copyright 2020 Google LLC.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Code

Text

RandBits MNIST using Contrastive Learning.

Code

Text

This notebook trains an unsupervised model on the RandBits MNIST dataset using contrastive learning, similar to the results shown in Figure 5(a) in Intriguing Properties of Contrastive Losses.

Code

Text

Imports.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
import re
import numpy as np

# import tensorflow.compat.v1 as tf
# tf.disable_eager_execution()

# @title Imports.
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()


import tensorflow_hub as hub
import tensorflow_datasets as tfds

import matplotlib
import matplotlib.pyplot as plt

import tensorflow_datasets as tfds
#@title Load class id to label text mapping from big_transfer (hidden)
# Code snippet credit: https://github.com/google-research/big_transfer



!wget https://storage.googleapis.com/bit_models/ilsvrc2012_wordnet_lemmas.txt

imagenet_int_to_str = {}

with open('ilsvrc2012_wordnet_lemmas.txt', 'r') as f:
  for i in range(1000):
    row = f.readline()
    row = row.rstrip()
    imagenet_int_to_str.update({i: row})

tf_flowers_labels = ['dandelion', 'daisy', 'tulips', 'sunflowers', 'roses']

--2024-10-01 04:11:44--  https://storage.googleapis.com/bit_models/ilsvrc2012_wordnet_lemmas.txt
Resolving storage.googleapis.com (storage.googleapis.com)... 142.251.184.207, 142.250.125.207, 209.85.200.207, ...
Connecting to storage.googleapis.com (storage.googleapis.com)|142.251.184.207|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 21675 (21K) [text/plain]
Saving to: ‘ilsvrc2012_wordnet_lemmas.txt’

ilsvrc2012_wordnet_ 100%[===================>]  21.17K  --.-KB/s    in 0s      

2024-10-01 04:11:45 (55.2 MB/s) - ‘ilsvrc2012_wordnet_lemmas.txt’ saved [21675/21675]

Code

Text

Preprocessing functions from data_util.py in SimCLR repository (hidden).

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044

#@title Preprocessing functions from data_util.py in SimCLR repository (hidden).

FLAGS_color_jitter_strength = 0.3
CROP_PROPORTION = 0.875  # Standard for ImageNet.


def random_apply(func, p, x):
  """Randomly apply function func to x with probability p."""
  return tf.cond(
      tf.less(tf.random.uniform([], minval=0, maxval=1, dtype=tf.float32),
              tf.cast(p, tf.float32)),
      lambda: func(x),
      lambda: x)


def random_brightness(image, max_delta, impl='simclrv2'):
  """A multiplicative vs additive change of brightness."""
  if impl == 'simclrv2':
    factor = tf.random.uniform(
        [], tf.maximum(1.0 - max_delta, 0), 1.0 + max_delta)
    image = image * factor
  elif impl == 'simclrv1':
    image = random_brightness(image, max_delta=max_delta)
  else:
    raise ValueError('Unknown impl {} for random brightness.'.format(impl))
  return image


def to_grayscale(image, keep_channels=True):
  image = tf.image.rgb_to_grayscale(image)
  if keep_channels:
    image = tf.tile(image, [1, 1, 3])
  return image


def color_jitter(image,
                 strength,
                 random_order=True):
  """Distorts the color of the image.
  Args:
    image: The input image tensor.
    strength: the floating number for the strength of the color augmentation.
    random_order: A bool, specifying whether to randomize the jittering order.
  Returns:
    The distorted image tensor.
  """
  brightness = 0.8 * strength
  contrast = 0.8 * strength
  saturation = 0.8 * strength
  hue = 0.2 * strength
  if random_order:
    return color_jitter_rand(image, brightness, contrast, saturation, hue)
  else:
    return color_jitter_nonrand(image, brightness, contrast, saturation, hue)


def color_jitter_nonrand(image, brightness=0, contrast=0, saturation=0, hue=0):
  """Distorts the color of the image (jittering order is fixed).
  Args:
    image: The input image tensor.
    brightness: A float, specifying the brightness for color jitter.
    contrast: A float, specifying the contrast for color jitter.
    saturation: A float, specifying the saturation for color jitter.
    hue: A float, specifying the hue for color jitter.
  Returns:
    The distorted image tensor.
  """
  with tf.name_scope('distort_color'):
    def apply_transform(i, x, brightness, contrast, saturation, hue):
      """Apply the i-th transformation."""
      if brightness != 0 and i == 0:
        x = random_brightness(x, max_delta=brightness)
      elif contrast != 0 and i == 1:
        x = tf.image.random_contrast(
            x, lower=1-contrast, upper=1+contrast)
      elif saturation != 0 and i == 2:
        x = tf.image.random_saturation(
            x, lower=1-saturation, upper=1+saturation)
      elif hue != 0:
        x = tf.image.random_hue(x, max_delta=hue)
      return x

    for i in range(4):
      image = apply_transform(i, image, brightness, contrast, saturation, hue)
      image = tf.clip_by_value(image, 0., 1.)
    return image


def color_jitter_rand(image, brightness=0, contrast=0, saturation=0, hue=0):
  """Distorts the color of the image (jittering order is random).
  Args:
    image: The input image tensor.
    brightness: A float, specifying the brightness for color jitter.
    contrast: A float, specifying the contrast for color jitter.
    saturation: A float, specifying the saturation for color jitter.
    hue: A float, specifying the hue for color jitter.
  Returns:
    The distorted image tensor.
  """
  with tf.name_scope('distort_color'):
    def apply_transform(i, x):
      """Apply the i-th transformation."""
      def brightness_foo():
        if brightness == 0:
          return x
        else:
          return random_brightness(x, max_delta=brightness)
      def contrast_foo():
        if contrast == 0:
          return x
        else:
          return tf.image.random_contrast(x, lower=1-contrast, upper=1+contrast)
      def saturation_foo():
        if saturation == 0:
          return x
        else:
          return tf.image.random_saturation(
              x, lower=1-saturation, upper=1+saturation)
      def hue_foo():
        if hue == 0:
          return x
        else:
          return tf.image.random_hue(x, max_delta=hue)
      x = tf.cond(tf.less(i, 2),
                  lambda: tf.cond(tf.less(i, 1), brightness_foo, contrast_foo),
                  lambda: tf.cond(tf.less(i, 3), saturation_foo, hue_foo))
      return x

    perm = tf.random_shuffle(tf.range(4))
    for i in range(4):
      image = apply_transform(perm[i], image)
      image = tf.clip_by_value(image, 0., 1.)
    return image


def _compute_crop_shape(
    image_height, image_width, aspect_ratio, crop_proportion):
  """Compute aspect ratio-preserving shape for central crop.
  The resulting shape retains `crop_proportion` along one side and a proportion
  less than or equal to `crop_proportion` along the other side.
  Args:
    image_height: Height of image to be cropped.
    image_width: Width of image to be cropped.
    aspect_ratio: Desired aspect ratio (width / height) of output.
    crop_proportion: Proportion of image to retain along the less-cropped side.
  Returns:
    crop_height: Height of image after cropping.
    crop_width: Width of image after cropping.
  """
  image_width_float = tf.cast(image_width, tf.float32)
  image_height_float = tf.cast(image_height, tf.float32)

  def _requested_aspect_ratio_wider_than_image():
    crop_height = tf.cast(tf.rint(
        crop_proportion / aspect_ratio * image_width_float), tf.int32)
    crop_width = tf.cast(tf.rint(
        crop_proportion * image_width_float), tf.int32)
    return crop_height, crop_width

  def _image_wider_than_requested_aspect_ratio():
    crop_height = tf.cast(
        tf.rint(crop_proportion * image_height_float), tf.int32)
    crop_width = tf.cast(tf.rint(
        crop_proportion * aspect_ratio *
        image_height_float), tf.int32)
    return crop_height, crop_width

  return tf.cond(
      aspect_ratio > image_width_float / image_height_float,
      _requested_aspect_ratio_wider_than_image,
      _image_wider_than_requested_aspect_ratio)


def center_crop(image, height, width, crop_proportion):
  """Crops to center of image and rescales to desired size.
  Args:
    image: Image Tensor to crop.
    height: Height of image to be cropped.
    width: Width of image to be cropped.
    crop_proportion: Proportion of image to retain along the less-cropped side.
  Returns:
    A `height` x `width` x channels Tensor holding a central crop of `image`.
  """
  shape = tf.shape(image)
  image_height = shape[0]
  image_width = shape[1]
  crop_height, crop_width = _compute_crop_shape(
      image_height, image_width, height / width, crop_proportion)
  offset_height = ((image_height - crop_height) + 1) // 2
  offset_width = ((image_width - crop_width) + 1) // 2
  image = tf.image.crop_to_bounding_box(
      image, offset_height, offset_width, crop_height, crop_width)

  image = tf.image.resize_bicubic([image], [height, width])[0]

  return image


def distorted_bounding_box_crop(image,
                                bbox,
                                min_object_covered=0.1,
                                aspect_ratio_range=(0.75, 1.33),
                                area_range=(0.05, 1.0),
                                max_attempts=100,
                                scope=None):
  """Generates cropped_image using one of the bboxes randomly distorted.
  See `tf.image.sample_distorted_bounding_box` for more documentation.
  Args:
    image: `Tensor` of image data.
    bbox: `Tensor` of bounding boxes arranged `[1, num_boxes, coords]`
        where each coordinate is [0, 1) and the coordinates are arranged
        as `[ymin, xmin, ymax, xmax]`. If num_boxes is 0 then use the whole
        image.
    min_object_covered: An optional `float`. Defaults to `0.1`. The cropped
        area of the image must contain at least this fraction of any bounding
        box supplied.
    aspect_ratio_range: An optional list of `float`s. The cropped area of the
        image must have an aspect ratio = width / height within this range.
    area_range: An optional list of `float`s. The cropped area of the image
        must contain a fraction of the supplied image within in this range.
    max_attempts: An optional `int`. Number of attempts at generating a cropped
        region of the image of the specified constraints. After `max_attempts`
        failures, return the entire image.
    scope: Optional `str` for name scope.
  Returns:
    (cropped image `Tensor`, distorted bbox `Tensor`).
  """
  # with tf.name_scope(scope, 'distorted_bounding_box_crop', [image, bbox]):
  if scope is None:
    scope = 'distorted_bounding_box_crop'
  else:
    scope = f"{scope}_distorted_bounding_box_crop"

  with tf.name_scope(scope):
    shape = tf.shape(image)
    sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
        shape,
        bounding_boxes=bbox,
        min_object_covered=min_object_covered,
        aspect_ratio_range=aspect_ratio_range,
        area_range=area_range,
        max_attempts=max_attempts,
        use_image_if_no_bounding_boxes=True)
    bbox_begin, bbox_size, _ = sample_distorted_bounding_box

    # Crop the image to the specified bounding box.
    offset_y, offset_x, _ = tf.unstack(bbox_begin)
    target_height, target_width, _ = tf.unstack(bbox_size)
    image = tf.image.crop_to_bounding_box(
        image, offset_y, offset_x, target_height, target_width)

    return image


def crop_and_resize(image, height, width):
  """Make a random crop and resize it to height `height` and width `width`.
  Args:
    image: Tensor representing the image.
    height: Desired image height.
    width: Desired image width.
  Returns:
    A `height` x `width` x channels Tensor holding a random crop of `image`.
  """
  bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])
  aspect_ratio = width / height
  image = distorted_bounding_box_crop(
      image,
      bbox,
      min_object_covered=0.1,
      aspect_ratio_range=(3. / 4 * aspect_ratio, 4. / 3. * aspect_ratio),
      area_range=(0.08, 1.0),
      max_attempts=100,
      scope=None)
  resized_image = tf.image.resize(image, [height, width], method='bicubic')
  return resized_image # for tf v1: tf.image.resize_bicubic([image], [height, width])[0]


def gaussian_blur(image, kernel_size, sigma, padding='SAME'):
  """Blurs the given image with separable convolution.
  Args:
    image: Tensor of shape [height, width, channels] and dtype float to blur.
    kernel_size: Integer Tensor for the size of the blur kernel. This is should
      be an odd number. If it is an even number, the actual kernel size will be
      size + 1.
    sigma: Sigma value for gaussian operator.
    padding: Padding to use for the convolution. Typically 'SAME' or 'VALID'.
  Returns:
    A Tensor representing the blurred image.
  """
  radius = tf.to_int32(kernel_size / 2)
  kernel_size = radius * 2 + 1
  x = tf.to_float(tf.range(-radius, radius + 1))
  blur_filter = tf.exp(
      -tf.pow(x, 2.0) / (2.0 * tf.pow(tf.to_float(sigma), 2.0)))
  blur_filter /= tf.reduce_sum(blur_filter)
  # One vertical and one horizontal filter.
  blur_v = tf.reshape(blur_filter, [kernel_size, 1, 1, 1])
  blur_h = tf.reshape(blur_filter, [1, kernel_size, 1, 1])
  num_channels = tf.shape(image)[-1]
  blur_h = tf.tile(blur_h, [1, 1, num_channels, 1])
  blur_v = tf.tile(blur_v, [1, 1, num_channels, 1])
  expand_batch_dim = image.shape.ndims == 3
  if expand_batch_dim:
    # Tensorflow requires batched input to convolutions, which we can fake with
    # an extra dimension.
    image = tf.expand_dims(image, axis=0)
  blurred = tf.nn.depthwise_conv2d(
      image, blur_h, strides=[1, 1, 1, 1], padding=padding)
  blurred = tf.nn.depthwise_conv2d(
      blurred, blur_v, strides=[1, 1, 1, 1], padding=padding)
  if expand_batch_dim:
    blurred = tf.squeeze(blurred, axis=0)
  return blurred


def random_crop_with_resize(image, height, width, p=1.0):
  """Randomly crop and resize an image.
  Args:
    image: `Tensor` representing an image of arbitrary size.
    height: Height of output image.
    width: Width of output image.
    p: Probability of applying this transformation.
  Returns:
    A preprocessed image `Tensor`.
  """
  def _transform(image):  # pylint: disable=missing-docstring
    image = crop_and_resize(image, height, width)
    return image
  return random_apply(_transform, p=p, x=image)


def random_color_jitter(image, p=1.0):
  def _transform(image):
    color_jitter_t = functools.partial(
        color_jitter, strength=FLAGS_color_jitter_strength)
    image = random_apply(color_jitter_t, p=0.8, x=image)
    return random_apply(to_grayscale, p=0.2, x=image)
  return random_apply(_transform, p=p, x=image)


def random_blur(image, height, width, p=1.0):
  """Randomly blur an image.
  Args:
    image: `Tensor` representing an image of arbitrary size.
    height: Height of output image.
    width: Width of output image.
    p: probability of applying this transformation.
  Returns:
    A preprocessed image `Tensor`.
  """
  del width
  def _transform(image):
    sigma = tf.random.uniform([], 0.1, 2.0, dtype=tf.float32)
    return gaussian_blur(
        image, kernel_size=height//10, sigma=sigma, padding='SAME')
  return random_apply(_transform, p=p, x=image)


def batch_random_blur(images_list, height, width, blur_probability=0.5):
  """Apply efficient batch data transformations.
  Args:
    images_list: a list of image tensors.
    height: the height of image.
    width: the width of image.
    blur_probability: the probaility to apply the blur operator.
  Returns:
    Preprocessed feature list.
  """
  def generate_selector(p, bsz):
    shape = [bsz, 1, 1, 1]
    selector = tf.cast(
        tf.less(tf.random.uniform(shape, 0, 1, dtype=tf.float32), p),
        tf.float32)
    return selector

  new_images_list = []
  for images in images_list:
    images_new = random_blur(images, height, width, p=1.)
    selector = generate_selector(blur_probability, tf.shape(images)[0])
    images = images_new * selector + images * (1 - selector)
    images = tf.clip_by_value(images, 0., 1.)
    new_images_list.append(images)

  return new_images_list

#@title Objective functions.
cls_loss_object = tf.keras.losses.CategoricalCrossentropy(
    from_logits=True,
    reduction=tf.keras.losses.Reduction.NONE)

def get_cls_loss(labels, outputs):
  return tf.reduce_mean(cls_loss_object(labels, outputs))


def D(p, z, version='simplified'):
  if version == 'original':
    z = tf.stop_gradient(z) # stop gradient
    p = tf.nn.l2_normalize(p, axis=1) # l2-normalize
    z = tf.nn.l2_normalize(z, axis=1) # l2-normalize
    return -tf.reduce_mean(tf.reduce_sum(p * z, axis=1))

  elif version == 'simplified': # same thing, much faster
    return -tf.reduce_mean(tf.keras.losses.cosine_similarity(p, tf.stop_gradient(z), axis=-1))
  else:
    raise Exception("Invalid version specified")

def get_simsiam_loss(z1, z2, p1, p2):
  L = D(p1, z2) / 2 + D(p2, z1) / 2
  return L,0

#def get_simsiam_loss(z1, z2, p1, p2):
#  batch_size = tf.shape(z1)[0]
#  dim = tf.shape(z1)[1]
#  z1 = tf.math.l2_normalize(z1, -1)
#  z2 = tf.math.l2_normalize(z2, -1)
#  p1 = tf.math.l2_normalize(p1, -1)
#  p2 = tf.math.l2_normalize(p2, -1)
#  result1 = -tf.reduce_sum(p1 * z2, axis=1)
#  result1 = tf.reduce_mean(result1)
#  result2 = -tf.reduce_sum(p2 * z1, axis=1)
#  result2 = tf.reduce_mean(result2)
#  return result1/2+result2/2, 0
  
  
#def get_contrastive_loss(z1, z2, nt_xent_temp):   # [batch_size, dim]
#  batch_size = tf.shape(z1)[0]
#  dim = tf.shape(z1)[1]
#  
#  z1 = tf.math.l2_normalize(z1, -1)
#  z2 = tf.math.l2_normalize(z2, -1)
#
#  sim = tf.matmul(z1, z2, transpose_b=True)  # [batch_size, batch_size]
#  sim /= nt_xent_temp
#
#  labels = tf.eye(batch_size)
#
#  loss = (
#      get_cls_loss(labels, sim) +
#      get_cls_loss(labels, tf.transpose(sim))
#  )
#  return tf.reduce_mean(loss), sim

#@title Model.
def dense_bn_relu(units):
  return tf.keras.Sequential([
      tf.keras.layers.Dense(
          units, use_bias=False,
          kernel_regularizer=tf.keras.regularizers.l2(1e-4)),
      tf.keras.layers.BatchNormalization(center=True, scale=True),
      tf.keras.layers.ReLU()
  ])

def conv2d_bn_relu(filters, kernel_size, strides):
  return tf.keras.Sequential([
      tf.keras.layers.Conv2D(
          filters, kernel_size, strides, use_bias=False,
          kernel_regularizer=tf.keras.regularizers.l2(1e-4)),
      tf.keras.layers.BatchNormalization(center=True, scale=True),
      tf.keras.layers.ReLU()
  ])

class ConvN(tf.keras.Model):

  def __init__(self, width_multiplier):
    super(ConvN, self).__init__()
    self.num_classes = 5
    self.latent_dim = 256 * width_multiplier
    self.proj_dim = self.latent_dim // 2

    self.enc = tf.keras.Sequential([
        conv2d_bn_relu(32 * width_multiplier, 3, 2),
        conv2d_bn_relu(64 * width_multiplier, 3, 2),
        conv2d_bn_relu(64 * width_multiplier, 3, 2),
        tf.keras.layers.Flatten(),
        dense_bn_relu(self.latent_dim)
    ])

    self.proj = tf.keras.Sequential([
        dense_bn_relu(self.latent_dim * 2),
        tf.keras.layers.Dense(
            self.proj_dim, use_bias=False, activation=None,
            kernel_regularizer=tf.keras.regularizers.l2(1e-4)),
    ])

    self.classifier = tf.keras.layers.Dense(self.num_classes)

  def call(self, inputs, training):
    y = self.enc(inputs, training)
    z = self.proj(y, training)
    pred = self.classifier(tf.stop_gradient(y))
    return y, z, pred

class ConvN2(tf.keras.Model):

  def __init__(self, width_multiplier):
    super(ConvN2, self).__init__()
    self.num_classes = 5
    self.latent_dim = 256 * width_multiplier
    self.proj_dim = self.latent_dim // 2

    self.enc = tf.keras.Sequential([
        conv2d_bn_relu(64 * width_multiplier, 3, 2),
        conv2d_bn_relu(128 * width_multiplier, 3, 2),
        conv2d_bn_relu(128 * width_multiplier, 3, 2),
        tf.keras.layers.Flatten(),
        dense_bn_relu(self.latent_dim)
    ])

    self.proj = tf.keras.Sequential([
        dense_bn_relu(self.latent_dim * 2),
        tf.keras.layers.Dense(
            self.proj_dim, use_bias=False, activation=None,
            kernel_regularizer=tf.keras.regularizers.l2(1e-4)),
    ])
    
    self.pred = tf.keras.Sequential([
        dense_bn_relu(self.latent_dim * 2),
        tf.keras.layers.Dense(
            self.proj_dim, use_bias=False, activation=None,
            kernel_regularizer=tf.keras.regularizers.l2(1e-4)),
    ])

    self.classifier = tf.keras.layers.Dense(self.num_classes)

  def call(self, inputs, training=False):
    y = self.enc(inputs, training=training)
    z = self.proj(y, training=training)
    p = self.pred(z, training=training)
    pred = self.classifier(tf.stop_gradient(y))
    return y, z, p, pred


# def preprocess_for_train(image, height, width,
#                          color_distort=True, crop=True, flip=True):
#   """Preprocesses the given image for training.
#   Args:
#     image: `Tensor` representing an image of arbitrary size.
#     height: Height of output image.
#     width: Width of output image.
#     color_distort: Whether to apply the color distortion.
#     crop: Whether to crop the image.
#     flip: Whether or not to flip left and right of an image.
#   Returns:
#     A preprocessed image `Tensor`.
#   """

#   if crop:
#     image = random_crop_with_resize(image, height, width)
#   if flip:
#     image = tf.image.random_flip_left_right(image)
#   if color_distort:
#     image = random_color_jitter(image)
#   image = tf.reshape(image, [height, width, 3])
#   image = tf.clip_by_value(image, 0., 1.)
#   return image


# def preprocess_for_eval(image, height, width, crop=True):
#   """Preprocesses the given image for evaluation.
#   Args:
#     image: `Tensor` representing an image of arbitrary size.
#     height: Height of output image.
#     width: Width of output image.
#     crop: Whether or not to (center) crop the test images.
#   Returns:
#     A preprocessed image `Tensor`.
#   """

#   if crop:
#     image = center_crop(image, height, width, crop_proportion=CROP_PROPORTION)
#   image = tf.reshape(image, [height, width, 3])
#   image = tf.clip_by_value(image, 0., 1.)
#   return image


# def preprocess_image(image, height, width, is_training=False,
#                      color_distort=True, test_crop=True):
#   """Preprocesses the given image.
#   Args:
#     image: `Tensor` representing an image of arbitrary size.
#     height: Height of output image.
#     width: Width of output image.
#     is_training: `bool` for whether the preprocessing is for training.
#     color_distort: whether to apply the color distortion.
#     test_crop: whether or not to extract a central crop of the images
#         (as for standard ImageNet evaluation) during the evaluation.
#   Returns:
#     A preprocessed image `Tensor` of range [0, 1].
#   """
#   image = tf.image.convert_image_dtype(image, dtype=tf.float32)

#   if is_training:
#     image = preprocess_for_train(image, height, width, color_distort)
#     return image
#   else:
#     image = preprocess_for_eval(image, height, width, test_crop)
#     return image


# def _preprocess_train(x, label, mnist_label):
#   print(x.shape)
#   print("image shape")
#   print(mnist_label.shape)
#   print(label.shape)
#   label = tf.cast(label, tf.int32)
#   mnist_label = tf.cast(mnist_label, tf.int32)

#   processed_image = preprocess_image(x, 224, 224, is_training=True, color_distort=False)
#   return (processed_image, label, mnist_label)

# def _preprocess_test(x, label, mnist_label):
#   label = tf.cast(label, tf.int32)
#   mnist_label = tf.cast(mnist_label, tf.int32)
#   processed_image = preprocess_image(x, 224, 224, is_training=False)

#   return (processed_image, label, mnist_label)




num_mnist_digits_used = 256 #3670

mnist = tf.keras.datasets.mnist
(mnist_images, mnist_labels), _ = mnist.load_data()
mnist_images = mnist_images[:num_mnist_digits_used]

def map_train_fn(image, label):
  # Sample one mnist image.
  # print("train label shape")
  # print(label.shape)

  i = tf.random.uniform([], maxval=num_mnist_digits_used, dtype=tf.int32)
  digit = tf.squeeze(tf.slice(mnist_images, [i, 0, 0], [1, 28, 28]))
  digit_label = tf.squeeze(tf.slice(mnist_labels, [i], [1]))
  digit = tf.image.grayscale_to_rgb(tf.expand_dims(digit, -1))
  digit = tf.image.convert_image_dtype(digit, dtype=tf.float32)
  image = tf.image.resize(image, [224, 224]) / 255.

  # size_big, size_small = 224, 28
  # images = []
  # for x_ratio in [0.15, 0.45, 0.75]:
  #   for y_ratio in [0.15, 0.45, 0.75]:
  #     pad_x, pad_y = int(size_big * x_ratio), int(size_big * y_ratio)
  #     x_max, y_max = size_big - size_small, size_big - size_small
  #     d = tf.pad(digit,
  #                [[pad_x, x_max - pad_x],
  #                [pad_y, y_max - pad_y],
  #                [0, 0]])
  #     images.append(d)
  # images.append(image)

  # resized_digit_size = 128  # New size for the digit
  # digit = tf.image.resize(digit, [resized_digit_size, resized_digit_size])

  # size_big = 224
  # size_small = resized_digit_size
  # images = []
  # #for x_ratio in [0.25, 0.5, 0.75]:
  # #    for y_ratio in [0.25, 0.5, 0.75]:
  # for x_ratio in [0.5]:
  #   for y_ratio in [0.5]:
  #         pad_x, pad_y = int(size_big * x_ratio - size_small / 2), int(size_big * y_ratio - size_small / 2)
  #         x_max, y_max = size_big - size_small, size_big - size_small
  #         d = tf.pad(digit,
  #                     [[pad_x, x_max - pad_x],
  #                     [pad_y, y_max - pad_y],
  #                     [0, 0]])
  #         images.append(d)
  # images.append(image)

  resized_digit_size = 72  # New size for the digit
  digit = tf.image.resize(digit, [resized_digit_size, resized_digit_size])

  size_big = 224
  size_small = resized_digit_size
  images = []
  #for x_ratio in [0.25, 0.5, 0.75]:
  #    for y_ratio in [0.25, 0.5, 0.75]:
  for x_ratio in [0.3, 0.7]:
      for y_ratio in [0.3, 0.7]:
          pad_x, pad_y = int(size_big * x_ratio - size_small / 2), int(size_big * y_ratio - size_small / 2)
          x_max, y_max = size_big - size_small, size_big - size_small
          d = tf.pad(digit,
                      [[pad_x, x_max - pad_x],
                      [pad_y, y_max - pad_y],
                      [0, 0]])
          images.append(d)
  images.append(image)

  # print("train mapped images shape")
  # print(len(images))
  image = tf.reduce_max(tf.stack(images, 0), 0)

  image = tf.image.convert_image_dtype(image, dtype=tf.float32)

  # print("train mapped image shape")
  # print(image.shape)

  original_image = image

  image_a = random_crop_with_resize(image, 224, 224)

  image_a = tf.image.random_flip_left_right(image)

  image_a = tf.reshape(image, [224, 224, 3])
  image_a = tf.clip_by_value(image, 0., 1.)

  image_b = random_crop_with_resize(image, 224, 224)

  image_b = tf.image.random_flip_left_right(image)

  image_b = tf.reshape(image, [224, 224, 3])
  image_b = tf.clip_by_value(image, 0., 1.)

  image = tf.stack([image_a, image_b], axis=0)  # [2, h, w, c]

  # print("train mapped image shape")
  # print(image.shape)

  label = tf.cast(label, tf.int32)
  digit_label = tf.cast(digit_label, tf.int32)


  # print("train mapped mnist label shape")
  # print(digit_label.shape)

  return image, label, digit_label, original_image


def map_test_fn(image, label):
  # Sample one mnist image.
  i = tf.random.uniform([], maxval=num_mnist_digits_used, dtype=tf.int32)
  digit = tf.squeeze(tf.slice(mnist_images, [i, 0, 0], [1, 28, 28]))
  digit_label = tf.squeeze(tf.slice(mnist_labels, [i], [1]))
  digit = tf.image.grayscale_to_rgb(tf.expand_dims(digit, -1))
  digit = tf.image.convert_image_dtype(digit, dtype=tf.float32)
  image = tf.image.resize(image, [224, 224]) / 255.

  # size_big, size_small = 224, 28
  # images = []
  # for x_ratio in [0.15, 0.45, 0.75]:
  #   for y_ratio in [0.15, 0.45, 0.75]:
  #     pad_x, pad_y = int(size_big * x_ratio), int(size_big * y_ratio)
  #     x_max, y_max = size_big - size_small, size_big - size_small
  #     d = tf.pad(digit,
  #                [[pad_x, x_max - pad_x],
  #                [pad_y, y_max - pad_y],
  #                [0, 0]])
  #     images.append(d)
  # images.append(image)

  # resized_digit_size = 128  # New size for the digit
  # digit = tf.image.resize(digit, [resized_digit_size, resized_digit_size])

  # size_big = 224
  # size_small = resized_digit_size
  # images = []
  # #for x_ratio in [0.25, 0.5, 0.75]:
  # #    for y_ratio in [0.25, 0.5, 0.75]:
  # for x_ratio in [0.5]:
  #   for y_ratio in [0.5]:
  #         pad_x, pad_y = int(size_big * x_ratio - size_small / 2), int(size_big * y_ratio - size_small / 2)
  #         x_max, y_max = size_big - size_small, size_big - size_small
  #         d = tf.pad(digit,
  #                     [[pad_x, x_max - pad_x],
  #                     [pad_y, y_max - pad_y],
  #                     [0, 0]])
  #         images.append(d)
  # images.append(image)

  resized_digit_size = 72  # New size for the digit
  digit = tf.image.resize(digit, [resized_digit_size, resized_digit_size])

  size_big = 224
  size_small = resized_digit_size
  images = []
  #for x_ratio in [0.25, 0.5, 0.75]:
  #    for y_ratio in [0.25, 0.5, 0.75]:
  for x_ratio in [0.3, 0.7]:
      for y_ratio in [0.3, 0.7]:
          pad_x, pad_y = int(size_big * x_ratio - size_small / 2), int(size_big * y_ratio - size_small / 2)
          x_max, y_max = size_big - size_small, size_big - size_small
          d = tf.pad(digit,
                      [[pad_x, x_max - pad_x],
                      [pad_y, y_max - pad_y],
                      [0, 0]])
          images.append(d)
  images.append(image)

  # print("test mapped images shape")
  # print(len(images))
  image = tf.reduce_max(tf.stack(images, 0), 0)
  # print("test mapped image shape")
  # print(image.shape)

  label = tf.cast(label, tf.int32)
  digit_label = tf.cast(digit_label, tf.int32)


  return image, label, digit_label

import tensorflow_datasets as tfds
import matplotlib.pyplot as plt
import seaborn as sns
import math
import pandas as pd

#@title Define train_and_eval() for contrastive learning.
def train_and_eval(
    batch_size=128,
    width_multiplier=1,
    extra_channel_bits=10,
    nt_xent_temp=0.1,
    learning_rate=0.001,
    epochs=10,
    log_summary_every_n_steps=100,
    eval_every_n_steps=100,
    print_output=False):
  strategy = tf.distribute.MirroredStrategy()
  #strategy = tf.distribute.get_strategy()

  # # Load dataset.
  # builder = tfds.builder('mnist')
  # builder.download_and_prepare()

  # print("batch size")
  # print(batch_size)

  # Use `tf_flowers` in this example instead of `imagenet2012` because the latter
  # needs to be downloaded manually.
  # (See https://www.tensorflow.org/datasets/catalog/imagenet2012)

  builder = tfds.builder('tf_flowers')
  builder.download_and_prepare()
  # print("Available Splits:", builder.info.splits.keys())
  # # Check for splits other than 'train'
  # if len(builder.info.splits) > 1 or 'train' not in builder.info.splits:
  #     print("There are other splits available besides 'train'.")
  # else:
  #     print("Only the 'train' split is available.")


  train_dataset = builder.as_dataset(split='train', as_supervised=True)
  total_samples = builder.info.splits['train'].num_examples
  shuffled_dataset = train_dataset.shuffle(buffer_size=total_samples, reshuffle_each_iteration=False)


  # Define the split ratio for training and testing
  train_split_ratio = 0.8
  num_train_samples = int(train_split_ratio * total_samples)

  # Create training and testing datasets
  train_dataset = shuffled_dataset.take(num_train_samples)
  test_dataset = shuffled_dataset.skip(num_train_samples)


  train_dataset = train_dataset.repeat().map(map_train_fn)
  # train_dataset = train_dataset.repeat().map(_preprocess_train)
  train_dataset = train_dataset.batch(batch_size, drop_remainder=True)

  train_iter = iter(train_dataset)

  # preprocess_train_fn, preprocess_eval_fn = get_process_fns(extra_channel_bits)
  # train_dataset = builder.as_dataset(split='train', as_supervised=True)
  # train_dataset = train_dataset.repeat().map(preprocess_train_fn)
  # train_dataset = train_dataset.batch(batch_size)

  #test_dataset = builder.as_dataset(split='test', as_supervised=True)
  test_dataset = test_dataset.map(map_test_fn)
  # test_dataset = test_dataset.repeat().map(_preprocess_test)
  test_dataset = test_dataset.batch(batch_size, drop_remainder=False)
  test_iter = iter(test_dataset)

  total_steps = int(2936 * epochs / batch_size)
  steps_per_epoch_test = math.ceil(734 / batch_size) # total: 3670

  with strategy.scope():

    # Model and optimizer.
    model = ConvN2(width_multiplier)
    global_step = tf.Variable(
        1, trainable=False, name="global_step", dtype=tf.int64)

    lr_schedule = tf.keras.optimizers.schedules.PolynomialDecay(
        initial_learning_rate=learning_rate,
        decay_steps=total_steps,
        end_learning_rate=0.)
    optimizer = tf.keras.optimizers.Adam(lr_schedule)

    # Define metrics.
    loss_metrics = [
        "train_classification_loss",
        "train_contrastive_loss",
        "train_total_loss",
    ]
    acc_metrics = ["train_accuracy", "eval_accuracy",
                  "train_contrastive_accuracy"]
    metric_list = {s: tf.keras.metrics.Mean(name=s) for s in loss_metrics}
    metric_list.update({
        s: tf.keras.metrics.SparseCategoricalAccuracy(name=s)
        for s in acc_metrics
    })

    #metric_list["rank"] = tf.keras.metrics.Mean(name="rank")
    metric_list["rank"] = tf.keras.metrics.Mean(name="rank")

    # Step functions.
    @tf.function
    def train_step(iterator):
      def step_fn(inputs):
        # print("length of the inputs")
        # print(len(inputs))
        images, labels, mnist_labels, original_image = inputs
        # print("training image size")
        # print(images.shape)
        # print("training label size")
        # print(labels.shape)
        # print("training mnist label size")
        # print(mnist_labels.shape)
        # print("training label

        batch_size = images.shape[0]
        if batch_size < 128:
            # Handle this case without slicing out of bounds
            pass

        labels_one_hot = tf.one_hot(labels, depth=5)
        # print("training label one hot size")
        # print(labels_one_hot.shape)
        images_a, images_b = tf.unstack(images, num=2, axis=1)
        # print("image a shape")
        # print(images_a.shape)
        with tf.GradientTape() as tape:
          # _, za, pred_a = model(images_a, training=True)
          # _, zb, pred_b = model(images_b, training=True)
          # _, z, pred_b = model(original_image, training=True)
          y1, z1, p1, pred_a = model(images_a, training=True)
          y2, z2, p2, pred_b = model(images_b, training=True)
          y, z, p, pred_b = model(original_image, training=True)


          #z_transposed = tf.transpose(za)

          # Compute the covariance matrix
          #c = tf.linalg.matmul(z_transposed, z_transposed, transpose_b=True)

          #r = tf.linalg.matrix_rank(c)

          #print(r.numpy())

          # print("starting loss calculation")
          # contrastive_loss, _ = (
          #     get_contrastive_loss(za, zb, nt_xent_temp))
          contrastive_loss,_ = (
              get_simsiam_loss(z1,z2,p1,p2))
          # print("loss value")
          # print(contrastive_loss)
          # print("accuracy value")
          # print(contrastive_sim)
          # print("start classification loss")
          classifier_loss = get_cls_loss(labels_one_hot, pred_a)
          # print("classifier loss output")
          # print(classifier_loss)
          # print("strtaing sum of the model loss")
          wd_loss = sum(model.losses)
          # print("sum of the model loss")
          # print(wd_loss)
          loss = classifier_loss + wd_loss + contrastive_loss

          z_transposed = tf.transpose(z)
          c = tf.linalg.matmul(z_transposed, z_transposed, transpose_b=True)

          rank = tf.linalg.matrix_rank(c)

          batch_size = tf.shape(images)[0]
          metric_list["train_contrastive_loss"].update_state(contrastive_loss)
          metric_list["train_classification_loss"].update_state(classifier_loss)
          metric_list["train_accuracy"].update_state(labels, pred_a)
          # print("metric train_contrastive_accuracy")
          # # metric_list["train_contrastive_accuracy"].update_state(
          # #     tf.range(batch_size), contrastive_sim)
          # print("metric train_total_loss")
          metric_list["train_total_loss"].update_state(loss)
          metric_list["rank"].update_state(rank)


        gradients = tape.gradient(loss, model.trainable_variables)
        optimizer.apply_gradients(zip(gradients, model.trainable_variables))

        #print(spectrum(model, original_image))

        # print(rank)


      strategy.run(step_fn, args=(next(iterator),))
      global_step.assign_add(1)



    @tf.function
    def eval_step(iterator):
      def step_fn(inputs):
        images, labels, mnist_labels = inputs
        _, _, predictions = model(images, training=False)
        metric_list["eval_accuracy"].update_state(labels, predictions)
      strategy.run(step_fn, args=(next(iterator),))

    # Train and eval loop.
    steps = []
    eval_accuracies = []
    ranks=[]
    loss=[]
    while global_step.numpy() <= total_steps:
      train_step(train_iter)
      step = global_step.numpy()

      if step % log_summary_every_n_steps == 0:
        log_msg = "Steps: {}".format(step)
        for m in loss_metrics + acc_metrics:
          if m.startswith("train"):
            log_msg += ", {}: {}".format(m, metric_list[m].result())
            metric_list[m].reset_state()
        if print_output:
          print(log_msg)

      if (step % eval_every_n_steps == 0) or (step == total_steps):
        eval_iter = iter(test_dataset)

        for m in loss_metrics + acc_metrics:
          if m.startswith("eval"):
            metric_list[m].reset_state()

        for _ in range(steps_per_epoch_test):
          eval_step(eval_iter)
        if print_output:
          print("Steps: {}, Test accuracy: {}".format(
              step, metric_list["eval_accuracy"].result()))
        steps.append(step)
        eval_accuracies.append(metric_list["eval_accuracy"].result())
      ranks.append(metric_list["rank"].result())
      loss.append(metric_list["train_total_loss"].result())

    return steps, eval_accuracies, ranks, loss, model

Code

Text

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
rows = []
cols = ['temp', 'margin', 'eval_accuracy']

# bits=0
# for temp in [1,5,10]:
margin=0.1
bits=0
# for bits in [0]:#[0, 5, 10, 15, 20]:
#   for margin in [0.05]: #[0.05,0.1,0.2]:
# for temp in [0.05, 0.1, 0.2]:
rank_matrix=[]
acc_matrix=[]
for temp in [0.2]: #, 0.2]:
  #for bits in [0, 5, 10, 15, 20]:
  steps, accuracies, ranks, los, model = train_and_eval(
      batch_size=128,
      width_multiplier=1,
      extra_channel_bits=bits,
      nt_xent_temp=temp, # [1,5,10]
      learning_rate=0.0001,
      epochs=35,
      eval_every_n_steps=5,
      print_output=True)
  rows.append([temp, margin, accuracies[-1].numpy()])
  print("temp={}, margin={}, eval_accuracy={}".format(
      temp, margin, accuracies[-1]))
  rank_matrix.append(ranks)
  acc_matrix.append(accuracies)
plot_df = pd.DataFrame.from_records(rows, columns=cols)

Steps: 5, Test accuracy: 0.25068119168281555
Steps: 10, Test accuracy: 0.24659401178359985
Steps: 15, Test accuracy: 0.23978202044963837
Steps: 20, Test accuracy: 0.2166212499141693
Steps: 25, Test accuracy: 0.173024520277977
Steps: 30, Test accuracy: 0.1634877324104309
Steps: 35, Test accuracy: 0.1621253341436386
Steps: 40, Test accuracy: 0.1648501306772232
Steps: 45, Test accuracy: 0.18801090121269226
Steps: 50, Test accuracy: 0.23297002911567688
Steps: 55, Test accuracy: 0.24931880831718445
Steps: 60, Test accuracy: 0.24931880831718445
Steps: 65, Test accuracy: 0.24931880831718445
Steps: 70, Test accuracy: 0.24931880831718445
Steps: 75, Test accuracy: 0.24931880831718445
Steps: 80, Test accuracy: 0.24931880831718445
Steps: 85, Test accuracy: 0.24931880831718445
Steps: 90, Test accuracy: 0.24931880831718445
Steps: 95, Test accuracy: 0.24931880831718445
Steps: 100, train_classification_loss: 1.6207534074783325, train_contrastive_loss: -0.7156826853752136, train_total_loss: 1.0735756158828735, train_accuracy: 0.2945864796638489, train_contrastive_accuracy: 0.0
Steps: 100, Test accuracy: 0.24931880831718445
Steps: 105, Test accuracy: 0.24931880831718445
Steps: 110, Test accuracy: 0.24795641005039215
Steps: 115, Test accuracy: 0.24795641005039215
Steps: 120, Test accuracy: 0.24795641005039215
Steps: 125, Test accuracy: 0.25068119168281555
Steps: 130, Test accuracy: 0.24931880831718445
Steps: 135, Test accuracy: 0.25476840138435364
Steps: 140, Test accuracy: 0.2602179944515228
Steps: 145, Test accuracy: 0.2670299708843231
Steps: 150, Test accuracy: 0.2724795639514923
Steps: 155, Test accuracy: 0.2711171805858612
Steps: 160, Test accuracy: 0.2738419473171234
Steps: 165, Test accuracy: 0.2833787500858307
Steps: 170, Test accuracy: 0.290190726518631
Steps: 175, Test accuracy: 0.30381470918655396
Steps: 180, Test accuracy: 0.31471389532089233
Steps: 185, Test accuracy: 0.3242506682872772
Steps: 190, Test accuracy: 0.3378746509552002
Steps: 195, Test accuracy: 0.3446866571903229
Steps: 200, train_classification_loss: 1.463657259941101, train_contrastive_loss: -0.9674503803253174, train_total_loss: 0.6623271107673645, train_accuracy: 0.36671873927116394, train_contrastive_accuracy: 0.0
Steps: 200, Test accuracy: 0.3487738370895386
Steps: 205, Test accuracy: 0.35149863362312317
Steps: 210, Test accuracy: 0.35967302322387695
Steps: 215, Test accuracy: 0.35967302322387695
Steps: 220, Test accuracy: 0.36103543639183044
Steps: 225, Test accuracy: 0.35831063985824585
Steps: 230, Test accuracy: 0.35694822669029236
Steps: 235, Test accuracy: 0.36103543639183044
Steps: 240, Test accuracy: 0.35831063985824585
Steps: 245, Test accuracy: 0.35694822669029236
Steps: 250, Test accuracy: 0.35967302322387695
Steps: 255, Test accuracy: 0.36648499965667725
Steps: 260, Test accuracy: 0.36784741282463074
Steps: 265, Test accuracy: 0.36648499965667725
Steps: 270, Test accuracy: 0.37465938925743103
Steps: 275, Test accuracy: 0.3841961920261383
Steps: 280, Test accuracy: 0.3814713954925537
Steps: 285, Test accuracy: 0.3855585753917694
Steps: 290, Test accuracy: 0.3787465989589691
Steps: 295, Test accuracy: 0.3760218024253845
Steps: 300, train_classification_loss: 1.4181119203567505, train_contrastive_loss: -0.9816875457763672, train_total_loss: 0.5991214513778687, train_accuracy: 0.37085938453674316, train_contrastive_accuracy: 0.0
Steps: 300, Test accuracy: 0.3787465989589691
Steps: 305, Test accuracy: 0.3814713954925537
Steps: 310, Test accuracy: 0.3760218024253845
Steps: 315, Test accuracy: 0.3760218024253845
Steps: 320, Test accuracy: 0.3841961920261383
Steps: 325, Test accuracy: 0.3814713954925537
Steps: 330, Test accuracy: 0.3841961920261383
Steps: 335, Test accuracy: 0.3814713954925537
Steps: 340, Test accuracy: 0.3841961920261383
Steps: 345, Test accuracy: 0.3869209885597229
Steps: 350, Test accuracy: 0.3855585753917694
Steps: 355, Test accuracy: 0.3828337788581848
Steps: 360, Test accuracy: 0.3841961920261383
Steps: 365, Test accuracy: 0.3869209885597229
Steps: 370, Test accuracy: 0.3896457850933075
Steps: 375, Test accuracy: 0.3923705816268921
Steps: 380, Test accuracy: 0.3910081684589386
Steps: 385, Test accuracy: 0.3896457850933075
Steps: 390, Test accuracy: 0.3950953781604767
Steps: 395, Test accuracy: 0.3937329649925232
Steps: 400, train_classification_loss: 1.3934378623962402, train_contrastive_loss: -0.9857189655303955, train_total_loss: 0.5669410824775696, train_accuracy: 0.38578125834465027, train_contrastive_accuracy: 0.0
Steps: 400, Test accuracy: 0.3923705816268921
Steps: 405, Test accuracy: 0.3950953781604767
Steps: 410, Test accuracy: 0.3937329649925232
Steps: 415, Test accuracy: 0.3978201746940613
Steps: 420, Test accuracy: 0.3991825580596924
Steps: 425, Test accuracy: 0.3991825580596924
Steps: 430, Test accuracy: 0.4005449712276459
Steps: 435, Test accuracy: 0.3964577615261078
Steps: 440, Test accuracy: 0.3991825580596924
Steps: 445, Test accuracy: 0.4005449712276459
Steps: 450, Test accuracy: 0.3991825580596924
Steps: 455, Test accuracy: 0.4005449712276459
Steps: 460, Test accuracy: 0.4046321511268616
Steps: 465, Test accuracy: 0.40871936082839966
Steps: 470, Test accuracy: 0.40599456429481506
Steps: 475, Test accuracy: 0.40326976776123047
Steps: 480, Test accuracy: 0.40326976776123047
Steps: 485, Test accuracy: 0.4046321511268616
Steps: 490, Test accuracy: 0.40599456429481506
Steps: 495, Test accuracy: 0.4046321511268616
Steps: 500, train_classification_loss: 1.3829106092453003, train_contrastive_loss: -0.9874333739280701, train_total_loss: 0.5515262484550476, train_accuracy: 0.3936718702316284, train_contrastive_accuracy: 0.0
Steps: 500, Test accuracy: 0.40326976776123047
Steps: 505, Test accuracy: 0.40326976776123047
Steps: 510, Test accuracy: 0.40599456429481506
Steps: 515, Test accuracy: 0.40326976776123047
Steps: 520, Test accuracy: 0.40326976776123047
Steps: 525, Test accuracy: 0.4046321511268616
Steps: 530, Test accuracy: 0.40871936082839966
Steps: 535, Test accuracy: 0.40735694766044617
Steps: 540, Test accuracy: 0.40871936082839966
Steps: 545, Test accuracy: 0.40735694766044617
Steps: 550, Test accuracy: 0.4046321511268616
Steps: 555, Test accuracy: 0.40871936082839966
Steps: 560, Test accuracy: 0.41416892409324646
Steps: 565, Test accuracy: 0.40735694766044617
Steps: 570, Test accuracy: 0.40735694766044617
Steps: 575, Test accuracy: 0.40735694766044617
Steps: 580, Test accuracy: 0.41280654072761536
Steps: 585, Test accuracy: 0.41144412755966187
Steps: 590, Test accuracy: 0.40735694766044617
Steps: 595, Test accuracy: 0.40871936082839966
Steps: 600, train_classification_loss: 1.3741754293441772, train_contrastive_loss: -0.9884025454521179, train_total_loss: 0.5391839742660522, train_accuracy: 0.39625000953674316, train_contrastive_accuracy: 0.0
Steps: 600, Test accuracy: 0.41416892409324646
Steps: 605, Test accuracy: 0.41280654072761536
Steps: 610, Test accuracy: 0.41144412755966187
Steps: 615, Test accuracy: 0.41144412755966187
Steps: 620, Test accuracy: 0.41689372062683105
Steps: 625, Test accuracy: 0.41689372062683105
Steps: 630, Test accuracy: 0.41689372062683105
Steps: 635, Test accuracy: 0.41553133726119995
Steps: 640, Test accuracy: 0.41416892409324646
Steps: 645, Test accuracy: 0.41961851716041565
Steps: 650, Test accuracy: 0.41961851716041565
Steps: 655, Test accuracy: 0.41689372062683105
Steps: 660, Test accuracy: 0.41416892409324646
Steps: 665, Test accuracy: 0.41553133726119995
Steps: 670, Test accuracy: 0.42098093032836914
Steps: 675, Test accuracy: 0.41961851716041565
Steps: 680, Test accuracy: 0.41825613379478455
Steps: 685, Test accuracy: 0.41553133726119995
Steps: 690, Test accuracy: 0.41689372062683105
Steps: 695, Test accuracy: 0.42098093032836914
Steps: 700, train_classification_loss: 1.3674238920211792, train_contrastive_loss: -0.9889559149742126, train_total_loss: 0.5299532413482666, train_accuracy: 0.40015625953674316, train_contrastive_accuracy: 0.0
Steps: 700, Test accuracy: 0.41961851716041565
Steps: 705, Test accuracy: 0.41825613379478455
Steps: 710, Test accuracy: 0.41961851716041565
Steps: 715, Test accuracy: 0.42234331369400024
Steps: 720, Test accuracy: 0.42234331369400024
Steps: 725, Test accuracy: 0.41961851716041565
Steps: 730, Test accuracy: 0.41553133726119995
Steps: 735, Test accuracy: 0.41825613379478455
Steps: 740, Test accuracy: 0.42234331369400024
Steps: 745, Test accuracy: 0.41689372062683105
Steps: 750, Test accuracy: 0.41825613379478455
Steps: 755, Test accuracy: 0.41961851716041565
Steps: 760, Test accuracy: 0.41961851716041565
Steps: 765, Test accuracy: 0.41961851716041565
Steps: 770, Test accuracy: 0.42098093032836914
Steps: 775, Test accuracy: 0.41961851716041565
Steps: 780, Test accuracy: 0.41825613379478455
Steps: 785, Test accuracy: 0.41961851716041565
Steps: 790, Test accuracy: 0.41961851716041565
Steps: 795, Test accuracy: 0.41961851716041565
Steps: 800, train_classification_loss: 1.3648165464401245, train_contrastive_loss: -0.9892807006835938, train_total_loss: 0.525963544845581, train_accuracy: 0.4046874940395355, train_contrastive_accuracy: 0.0
Steps: 800, Test accuracy: 0.41825613379478455
Steps: 802, Test accuracy: 0.41825613379478455
temp=0.2, margin=0.1, eval_accuracy=0.41825613379478455

Code

Text

Salient Map

Code

Text

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

import random
import numpy as np
import scipy.ndimage  # Importing scipy for Gaussian filter
# Set random seeds for reproducibility
random_seed = 234
tf.random.set_seed(random_seed)  # Set TensorFlow's random seed
np.random.seed(random_seed)  # Set NumPy's random seed
random.seed(random_seed)  # Set Python's random seed
num_mnist_digits_used = 256 #3670
mnist = tf.keras.datasets.mnist
(mnist_images, mnist_labels), _ = mnist.load_data()
mnist_images = mnist_images[:num_mnist_digits_used]
batch_size=128
builder = tfds.builder('tf_flowers')
builder.download_and_prepare()
train_dataset = builder.as_dataset(split='train', as_supervised=True)
total_samples = builder.info.splits['train'].num_examples
shuffled_dataset = train_dataset.shuffle(buffer_size=total_samples, reshuffle_each_iteration=False, seed=random_seed)
# Define the split ratio for training and testing
train_split_ratio = 0.8
num_train_samples = int(train_split_ratio * total_samples)
# Create training and testing datasets
train_dataset = shuffled_dataset.take(num_train_samples)
test_dataset = shuffled_dataset.skip(num_train_samples)
train_dataset = train_dataset.repeat().map(map_train_fn)
# train_dataset = train_dataset.repeat().map(_preprocess_train)
train_dataset = train_dataset.batch(batch_size)
train_iter = iter(train_dataset)
# #test_dataset = builder.as_dataset(split='test', as_supervised=True)
# test_dataset = test_dataset.map(map_test_fn)
# # test_dataset = test_dataset.repeat().map(_preprocess_test)
# test_dataset = test_dataset.batch(batch_size, drop_remainder=False)
# test_iter = iter(test_dataset)
for i in range(200):
  if(i!=129):
    images, labels, mnist_labels, original_image = train_iter.next()
    continue
  else:
    images, labels, mnist_labels, original_image = train_iter.next()
    # images=images[0,:,:,:,:]
    print(images.shape)
    labels_one_hot = tf.one_hot(labels, depth=5)
    images_a, images_b = tf.unstack(images, num=2, axis=1)
    print("image a shape")
    print(images_a.shape)
    with tf.GradientTape() as tape:
    #with tf.GradientTape(persistent=True) as tape:
        # original_img = tf.expand_dims(original_image, axis=0)
        # image_a = tf.expand_dims(images_a, axis=0)
        # image_b = tf.expand_dims(images_b, axis=0)
        # print("shape of image_a")
        # print(image_a.shape)
        # print("shape of image_b")
        # print(image_b.shape)
        tape.watch(images_a)
        #tape.watch([images_a, images_b])
        # z1, p1, pred_a = model(images_a, training=True)
        # z2, p2, pred_b = model(images_b, training=True)
        # z, p, pred_b = model(original_image, training=True)
        ya, za, pa, pred_a = model(images_a, training=True)
        yb, zb, pb, pred_b = model(images_b, training=True)
        # _, z, pred_b = model(original_image, training=True)
        # contrastive_loss, _ = (
        #     get_contrastive_loss(za, zb, 0.02))
        contrastive_loss,_ = (
            get_simsiam_loss(za,zb,pa,pb))
        
        classifier_loss = get_cls_loss(labels_one_hot, pred_a)
        wd_loss = sum(model.losses)
        loss = classifier_loss + wd_loss + contrastive_loss
  # Compute the saliency map for the input image
    gradients_a = tape.gradient(loss, images_a)
    # gradients_b = tape.gradient(loss, images_b)
    # Verify that gradients are not None
    if gradients_a is None:
        raise ValueError("Gradients are None, check if images_a is correctly linked to the loss.")
    # if gradients_b is None:
    #     raise ValueError("Gradients are None, check if images_b is correctly linked to the loss.")
    # Take the absolute value of the gradients to get the saliency map
    saliency_a = tf.reduce_max(tf.abs(gradients_a), axis=-1)
    # saliency_b = tf.reduce_max(tf.abs(gradients_b), axis=-1)
    print("labels")
    print(labels[0])
    print("prediction a")
    print(pred_a[0])
    # print("labels")
    # print(labels[1])
    # print("prediction b")
    # print(pred_b[1])
    # print(mnist_label)
    plt.figure(figsize=(4, 4))
    # plt.subplot(1, 2, 1)
    plt.imshow(images_a[0])
    plt.gca().axes.xaxis.set_ticks([])
    plt.gca().axes.yaxis.set_ticks([])
    plt.title('Original Image')
    # plt.subplot(1, 2, 2)
    plt.figure(figsize=(5, 4))
    smoothed_saliency = scipy.ndimage.gaussian_filter(saliency_a[0], sigma=1)
    vmin = 0 #np.min(smoothed_saliency)  # Set vmin to the minimum value of your data
    #print(vmin)
    vmax = np.percentile(smoothed_saliency, 99.3)  # Set vmax to the 95th percentile to limit the color range
    plt.imshow(smoothed_saliency, cmap='rainbow',interpolation='bilinear', vmin=vmin, vmax=vmax)
    plt.gca().axes.xaxis.set_ticks([])
    plt.gca().axes.yaxis.set_ticks([])
    plt.colorbar()
    plt.title('Saliency Map')
    plt.show()
    # plt.figure(figsize=(4, 4))
    # # plt.subplot(1, 2, 1)
    # plt.imshow(images_b[1])
    # plt.gca().axes.xaxis.set_ticks([])
    # plt.gca().axes.yaxis.set_ticks([])
    # plt.title('Original Image')
    # # plt.subplot(1, 2, 2)
    # plt.figure(figsize=(5, 4))
    # plt.imshow(saliency_b[1], cmap='rainbow',interpolation='bilinear')
    # plt.gca().axes.xaxis.set_ticks([])
    # plt.gca().axes.yaxis.set_ticks([])
    # plt.colorbar()
    # plt.title('Saliency Map')
    # plt.show()

Code

Text

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
import random
import numpy as np
import scipy.ndimage  # Importing scipy for Gaussian filter


# Set random seeds for reproducibility
random_seed = 3456
tf.random.set_seed(random_seed)  # Set TensorFlow's random seed
np.random.seed(random_seed)  # Set NumPy's random seed
random.seed(random_seed)  # Set Python's random seed


batch_size=128
builder = tfds.builder('tf_flowers')
builder.download_and_prepare()


train_dataset = builder.as_dataset(split='train', as_supervised=True)
total_samples = builder.info.splits['train'].num_examples
shuffled_dataset = train_dataset.shuffle(buffer_size=total_samples, reshuffle_each_iteration=False, seed=random_seed)


# Define the split ratio for training and testing
train_split_ratio = 0.8
num_train_samples = int(train_split_ratio * total_samples)

# Create training and testing datasets
train_dataset = shuffled_dataset.take(num_train_samples)
test_dataset = shuffled_dataset.skip(num_train_samples)


train_dataset = train_dataset.repeat().map(map_train_fn)
# train_dataset = train_dataset.repeat().map(_preprocess_train)
train_dataset = train_dataset.batch(batch_size)

train_iter = iter(train_dataset)

# #test_dataset = builder.as_dataset(split='test', as_supervised=True)
# test_dataset = test_dataset.map(map_test_fn)
# # test_dataset = test_dataset.repeat().map(_preprocess_test)
# test_dataset = test_dataset.batch(batch_size, drop_remainder=False)
# test_iter = iter(test_dataset)


for _ in range(5):
    images, labels, mnist_labels, original_image = train_iter.next()


    # images=images[0,:,:,:,:]
    print(images.shape)


    labels_one_hot = tf.one_hot(labels, depth=5)
    images_a, images_b = tf.unstack(images, num=2, axis=1)
    print("image a shape")
    print(images_a.shape)

    with tf.GradientTape() as tape:

    #with tf.GradientTape(persistent=True) as tape:

        # original_img = tf.expand_dims(original_image, axis=0)
        # image_a = tf.expand_dims(images_a, axis=0)
        # image_b = tf.expand_dims(images_b, axis=0)

        # print("shape of image_a")
        # print(image_a.shape)

        # print("shape of image_b")
        # print(image_b.shape)

        tape.watch(images_a)

        #tape.watch([images_a, images_b])

        # z1, p1, pred_a = model(images_a, training=True)
        # z2, p2, pred_b = model(images_b, training=True)
        # z, p, pred_b = model(original_image, training=True)


        za, pa, pred_a = model(images_a, training=True)
        zb, pb, pred_b = model(images_b, training=True)
        _, z, pred_b = model(original_image, training=True)


        # contrastive_loss, _ = (
        #     get_contrastive_loss(za, zb, 0.02))

        contrastive_loss,_ = (
            get_simsiam_loss(za,zb,pa,pb))
        
        classifier_loss = get_cls_loss(labels_one_hot, pred_a)
        wd_loss = sum(model.losses)
        loss = classifier_loss + wd_loss + contrastive_loss

  # Compute the saliency map for the input image
    gradients_a = tape.gradient(loss, images_a)
    # gradients_b = tape.gradient(loss, images_b)

    # Verify that gradients are not None
    if gradients_a is None:
        raise ValueError("Gradients are None, check if images_a is correctly linked to the loss.")
    # if gradients_b is None:
    #     raise ValueError("Gradients are None, check if images_b is correctly linked to the loss.")


    # Take the absolute value of the gradients to get the saliency map
    saliency_a = tf.reduce_max(tf.abs(gradients_a), axis=-1)
    # saliency_b = tf.reduce_max(tf.abs(gradients_b), axis=-1)
    print("labels")
    print(labels[0])
    print("prediction a")
    print(pred_a[0])

    # print("labels")
    # print(labels[1])
    # print("prediction b")
    # print(pred_b[1])

    # print(mnist_label)
    plt.figure(figsize=(4, 4))
    # plt.subplot(1, 2, 1)
    plt.imshow(images_a[0])
    plt.gca().axes.xaxis.set_ticks([])
    plt.gca().axes.yaxis.set_ticks([])
    plt.title('Original Image')

    # plt.subplot(1, 2, 2)
    plt.figure(figsize=(5, 4))

    smoothed_saliency = scipy.ndimage.gaussian_filter(saliency_a[0], sigma=1)
    vmin = 0 #np.min(smoothed_saliency)  # Set vmin to the minimum value of your data
    vmax = np.percentile(smoothed_saliency, 99)  # Set vmax to the 95th percentile to limit the color range

    plt.imshow(smoothed_saliency, cmap='rainbow',interpolation='bilinear', vmin=vmin, vmax=vmax)
    plt.gca().axes.xaxis.set_ticks([])
    plt.gca().axes.yaxis.set_ticks([])
    plt.colorbar()
    plt.title('Saliency Map')

    plt.show()

    # plt.figure(figsize=(4, 4))
    # # plt.subplot(1, 2, 1)
    # plt.imshow(images_b[1])
    # plt.gca().axes.xaxis.set_ticks([])
    # plt.gca().axes.yaxis.set_ticks([])
    # plt.title('Original Image')

    # # plt.subplot(1, 2, 2)
    # plt.figure(figsize=(5, 4))


    # plt.imshow(saliency_b[1], cmap='rainbow',interpolation='bilinear')
    # plt.gca().axes.xaxis.set_ticks([])
    # plt.gca().axes.yaxis.set_ticks([])
    # plt.colorbar()
    # plt.title('Saliency Map')

    # plt.show()

Code

Text

  1
plt.plot(ranks)

Code

Text

  1
rank_matrix[0][-1]

<tf.Tensor: shape=(), dtype=float32, numpy=101.24813>

  1
plt.plot(accuracies)

  1
max(x.numpy() for x in accuracies)

0.4482289