Finish-to-Finish Multimodal Information Augmentation and Adversarial Robustness Benchmark with AugLy for Pictures, Textual content, Audio, and PyTorch

import subprocess, sys, importlib
def _sh(cmd):
   print(f"$ {cmd}")
   subprocess.run(cmd, shell=True, test=False,
                  stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def _need(mod):
   strive:
       importlib.import_module(mod)
       return False
   besides ImportError:
       return True
if _need("augly"):
   _sh("apt-get -qq install -y libmagic1 > /dev/null 2>&1")
   _sh(f'"{sys.executable}" -m pip set up -q --no-deps augly')
   _sh(f'"{sys.executable}" -m pip set up -q "iopath>=0.1.8" "python-magic>=0.4.22" '
       f'"regex>=2021.4.4" "nlpaug==1.1.3"')
import numpy as np
from PIL import Picture, ImageDraw, ImageFont, ImageFilter
for _name, _builtin in (("float", float), ("int", int), ("bool", bool)):
   if not hasattr(np, _name):
       setattr(np, _name, _builtin)
def _size(font, textual content):
   left, high, proper, backside = font.getbbox(textual content)
   return (proper, backside)
if not hasattr(ImageFont.FreeTypeFont, "getsize"):
   ImageFont.FreeTypeFont.getsize = lambda self, t, *a, **ok: _size(self, t)
if not hasattr(ImageFont.FreeTypeFont, "getsize_multiline"):
   def _getsize_multiline(self, textual content, course=None, spacing=4, options=None,
                          language=None, stroke_width=0):
       traces = textual content.break up("n")
       w = max((_size(self, ln)[0] for ln in traces), default=0)
       h = sum(_size(self, ln)[1] for ln in traces) + spacing * (len(traces) - 1)
       return (w, h)
   ImageFont.FreeTypeFont.getsize_multiline = _getsize_multiline
import os, io, json, math, random, string, textwrap, unicodedata, warnings
from dataclasses import dataclass
from typing import Any, Dict, Checklist, Non-compulsory, Tuple
import matplotlib.pyplot as plt
import pandas as pd
import augly.picture as imaugs
import augly.textual content as textaugs
import augly.utils as augutils
from augly.picture.transforms import BaseTransform as ImageBaseTransform
warnings.filterwarnings("ignore")
pd.set_option("display.width", 160)
SEED = 1234
random.seed(SEED)
np.random.seed(SEED)
print("n" + "=" * 78)
print("AugLy ready.  assets at:", augutils.ASSETS_BASE_DIR)
print("image augs :", len([f for f in dir(imaugs) if f[0].islower()]))
print("text  augs :", len([f for f in dir(textaugs) if f[0].islower()]))
print("=" * 78 + "n")
def make_image(idx: int, w: int = 320, h: int = 240) -> Tuple[Image.Image, Tuple[int, int, int, int]]:
   """Procedurally generated 'photo' + a ground-truth bbox in pascal_voc format."""
   rng = random.Random(SEED + idx)
   img = Picture.new("RGB", (w, h), tuple(rng.randint(20, 90) for _ in vary(3)))
   d = ImageDraw.Draw(img)
   for _ in vary(70):
       x0, y0 = rng.randint(0, w), rng.randint(0, h)
       d.line([x0, y0, x0 + rng.randint(-60, 60), y0 + rng.randint(-60, 60)],
              fill=tuple(rng.randint(60, 160) for _ in vary(3)), width=rng.randint(1, 3))
   ow, oh = rng.randint(70, 130), rng.randint(60, 110)
   ox, oy = rng.randint(10, w - ow - 10), rng.randint(10, h - oh - 10)
   field = (ox, oy, ox + ow, oy + oh)
   color = tuple(rng.randint(150, 255) for _ in vary(3))
   if idx % 3 == 0:
       d.ellipse(field, fill=color, define=(255, 255, 255), width=3)
   elif idx % 3 == 1:
       d.rectangle(field, fill=color, define=(255, 255, 255), width=3)
   else:
       d.polygon([(ox + ow // 2, oy), (ox + ow, oy + oh), (ox, oy + oh)],
                 fill=color, define=(255, 255, 255))
   return img, field
N_IMAGES = 24
IMAGES, BOXES = zip(*[make_image(i) for i in range(N_IMAGES)])
IMAGES, BOXES = record(IMAGES), record(BOXES)
DEMO_IMG, DEMO_BOX = IMAGES[0], BOXES[0]
def make_text_dataset(n_per_class: int = 260):
   """Tiny sentiment corpus built from templates -> learnable but not trivial."""
   rng = random.Random(SEED)
   pos_adj = ["excellent", "delightful", "superb", "charming", "brilliant",
              "flawless", "wonderful", "outstanding", "impressive", "lovely"]
   neg_adj = ["terrible", "awful", "dreadful", "disappointing", "clumsy",
              "broken", "miserable", "useless", "painful", "sloppy"]
   subj = ["the movie", "this restaurant", "the hotel room", "their support team",
           "the new phone", "the sequel", "this laptop", "the delivery service"]
   tail_p = ["and I would recommend it to anyone", "worth every rupee",
             "I left completely satisfied", "easily the best of the year",
             "it exceeded all my expectations"]
   tail_n = ["and I want a refund", "a total waste of money",
             "I left extremely frustrated", "easily the worst of the year",
             "it failed every expectation"]
   rows = []
   for _ in vary(n_per_class):
       rows.append((f"{rng.choice(subj)} was {rng.choice(pos_adj)} {rng.choice(tail_p)}", 1))
       rows.append((f"{rng.choice(subj)} was {rng.choice(neg_adj)} {rng.choice(tail_n)}", 0))
   rng.shuffle(rows)
   return [r[0] for r in rows], [r[1] for r in rows]
TEXTS, LABELS = make_text_dataset()
DEMO_TEXT = "The quick brown fox jumps over the lazy dog near the river bank"
def make_audio(seconds: float = 2.0, sr: int = 16000) -> Tuple[np.ndarray, int]:
   """A chirp + harmonics + a little noise = something you can actually hear change."""
   t = np.linspace(0, seconds, int(sr * seconds), endpoint=False)
   f = np.linspace(220, 880, t.dimension)
   sig = 0.5 * np.sin(2 * np.pi * f * t) + 0.2 * np.sin(2 * np.pi * 2 * f * t)
   sig += 0.02 * np.random.RandomState(SEED).randn(t.dimension)
   env = np.minimal(1.0, np.minimal(t * 8, (seconds - t) * 8))
   return (sig * env).astype(np.float32), sr
AUDIO, SR = make_audio()
def show_grid(pairs, cols=4, title="", figsize_scale=2.9):
   """pairs: list of (caption, PIL.Image)."""
   rows = math.ceil(len(pairs) / cols)
   fig, axes = plt.subplots(rows, cols, figsize=(cols * figsize_scale, rows * figsize_scale))
   axes = np.atleast_1d(axes).ravel()
   for ax, (cap, im) in zip(axes, pairs):
       ax.imshow(im)
       ax.set_title(cap, fontsize=8)
       ax.axis("off")
   for ax in axes[len(pairs):]:
       ax.axis("off")
   if title:
       fig.suptitle(title, fontsize=13, y=1.0)
   plt.tight_layout()
   plt.present()
def as_str(out) -> str:
   """AugLy text augs return str for str input in some transforms, list in others."""
   return out[0] if isinstance(out, record) else out
print("n### §2  IMAGE AUGMENTATION + METADATA " + "#" * 38)
functional_result = imaugs.pixelization(DEMO_IMG, ratio=0.25)
class_result = imaugs.Pixelization(ratio=0.25, p=1.0)(DEMO_IMG)
print("functional == class:", np.array_equal(np.array(functional_result), np.array(class_result)))
IMAGE_ZOO = {
   "blur":               lambda im, m: imaugs.blur(im, radius=3.0, metadata=m),
   "brightness":         lambda im, m: imaugs.brightness(im, issue=1.7, metadata=m),
   "color_jitter":       lambda im, m: imaugs.color_jitter(im, brightness_factor=1.3,
                                                           contrast_factor=1.4,
                                                           saturation_factor=1.6, metadata=m),
   "crop":               lambda im, m: imaugs.crop(im, x1=.15, y1=.15, x2=.85, y2=.85, metadata=m),
   "encoding_quality":   lambda im, m: imaugs.encoding_quality(im, high quality=8, metadata=m),
   "grayscale":          lambda im, m: imaugs.grayscale(im, metadata=m),
   "hflip":              lambda im, m: imaugs.hflip(im, metadata=m),
   "meme_format":        lambda im, m: imaugs.meme_format(im, textual content="TOP TEXT",
                                                          caption_height=90, metadata=m),
   "opacity":            lambda im, m: imaugs.opacity(im, stage=0.45, metadata=m),
   "overlay_emoji":      lambda im, m: imaugs.overlay_emoji(im, opacity=0.9,
                                                            emoji_size=0.35, metadata=m),
   "overlay_screenshot": lambda im, m: imaugs.overlay_onto_screenshot(im, metadata=m),
   "overlay_stripes":    lambda im, m: imaugs.overlay_stripes(im, line_width=0.4,
                                                              line_opacity=0.7, metadata=m),
   "overlay_text":       lambda im, m: imaugs.overlay_text(im, opacity=0.9, metadata=m),
   "pad_square":         lambda im, m: imaugs.pad_square(im, metadata=m),
   "perspective":        lambda im, m: imaugs.perspective_transform(im, sigma=40.0, metadata=m),
   "pixelization":       lambda im, m: imaugs.pixelization(im, ratio=0.15, metadata=m),
   "random_noise":       lambda im, m: imaugs.random_noise(im, var=0.03, metadata=m),
   "rotate":             lambda im, m: imaugs.rotate(im, levels=17, metadata=m),
   "saturation":         lambda im, m: imaugs.saturation(im, issue=3.0, metadata=m),
   "scale":              lambda im, m: imaugs.scale(im, issue=0.35, metadata=m),
   "sharpen":            lambda im, m: imaugs.sharpen(im, issue=8.0, metadata=m),
   "shuffle_pixels":     lambda im, m: imaugs.shuffle_pixels(im, issue=0.15, metadata=m),
   "skew":               lambda im, m: imaugs.skew(im, skew_factor=0.35, metadata=m),
   "vflip":              lambda im, m: imaugs.vflip(im, metadata=m),
}
gallery, image_meta = [("ORIGINAL", DEMO_IMG)], []
for title, fn in IMAGE_ZOO.objects():
   m = []
   strive:
       out = fn(DEMO_IMG, m)
       gallery.append((f"{name}nintensity={m[0]['intensity']:.1f}", out))
       image_meta.append(m[0])
   besides Exception as e:
       print(f"  [skip] {name}: {type(e).__name__}: {e}")
show_grid(gallery, cols=5, title="§2  AugLy image augmentations (with AugLy's own intensity score)")
meta_df = pd.DataFrame(image_meta)[["name", "intensity", "src_width", "src_height",
                                   "dst_width", "dst_height"]]
print(meta_df.sort_values("intensity", ascending=False).head(10).to_string(index=False))