all repos — dot @ master

Shell, editor and terminal configuration, and the tools that go with them

bin/sz-rm-meta.sh (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
#!/usr/bin/env bash
set -euo pipefail

# sz-rm-meta.sh — strip metadata from image files.
#
#   sz-rm-meta.sh [options] FILE...
#
#   -l, --list        report what would be removed; change nothing
#   -o, --out DIR     write cleaned copies into DIR instead of editing in place
#       --keep-icc    keep the embedded ICC colour profile (stripped by default)
#       --keep-xattr  leave extended attributes alone (macOS: kMDItemWhereFroms,
#                     com.apple.quarantine — these record the download URL)
#   -d, --destructive re-encode the pixels as well (see "Two modes" below)
#       --quality N   JPEG quality for --destructive (default 85)
#       --scale PCT   resize percentage for --destructive (default 97)
#       --rename      rename the file to a random name, keeping the extension
#   -q, --quiet       only report failures
#
# Two modes, because "no metadata" and "not traceable" are different jobs:
#
#   default       Lossless. Rewrites the container, copies the compressed image
#                 data byte for byte. Pixels stay bit-identical, so the file is
#                 still matchable against the source by perceptual hash, and the
#                 quantization/Huffman tables still fingerprint the encoder that
#                 produced it. Right for sanitising an archive you keep.
#
#   --destructive Re-encodes: resize by --scale, re-compress at --quality, fresh
#                 quantization tables, then run the lossless pass over the
#                 result. Breaks bitstream-level linkage to the source copy.
#                 Does NOT defeat reverse-image search or perceptual hashing —
#                 nothing that preserves what the picture looks like can. Needs
#                 ImageMagick.
#
# What the lossless pass removes, per format:
#   JPEG  every APPn segment except JFIF (APP0) and Adobe (APP14), all COM
#         comments, and any trailer appended after the EOI marker. That covers
#         EXIF (camera, GPS, serial, timestamps), XMP, IPTC and the Photoshop
#         8BIM block (by-line, credit, document ancestry).
#   PNG   everything outside an allowlist of image and colour chunks. An
#         allowlist rather than a denylist because private chunks are the worst
#         offenders: Fireworks' mkBF/mkBS/mkBT embed the entire editable source
#         document, and prVw carries a full preview image.
#
# Why APP14 "Adobe" survives: it is not metadata, it declares the colour
# transform (YCbCr vs RGB vs YCCK). Dropping it turns some CMYK and RGB JPEGs
# into colour-inverted garbage. It is a 12-byte codec record written by many
# encoders, not a Photoshop watermark.
#
# Symlinks are resolved and the real file is rewritten, so cleaning a link no
# longer leaves the target dirty. A file with more than one hard link is
# rewritten in place rather than replaced, so every name sees the clean data —
# that write is not atomic, which is the honest trade for not silently breaking
# the link.
#
# Files are otherwise rewritten via a temp file in the same directory and moved
# into place, so an interrupted run leaves the original intact. A file that
# fails to parse is skipped, never half-written.
#
# Deliberately not handled: GIF, TIFF, HEIC, WebP, video. They need a real
# metadata library; this script would rather refuse than corrupt them. Install
# exiftool for those.

die() { >&2 echo "sz-rm-meta: $*"; exit 1; }
warn() { >&2 echo "sz-rm-meta: warning: $*"; }

list_only=false
out_dir=""
keep_icc=false
keep_xattr=false
destructive=false
rename=false
quality=85
scale=97
quiet=false
files=()

usage() {
    cat <<'USAGE'
sz-rm-meta.sh [options] FILE...   strip metadata from JPEG/PNG

  -l, --list         report what would be removed; change nothing
  -o, --out DIR      write cleaned copies into DIR instead of editing in place
      --keep-icc     keep the embedded ICC colour profile (stripped by default)
      --keep-xattr   leave extended attributes alone (macOS keeps the download
                     URL in kMDItemWhereFroms)
  -d, --destructive  also re-encode the pixels: breaks the bitstream fingerprint
                     linking this copy to the source. Needs ImageMagick.
      --quality N    JPEG quality for --destructive (default 85)
      --scale PCT    resize percentage for --destructive (default 97)
      --rename       rename to a random name, keeping the extension
  -q, --quiet        only report failures

Default mode is lossless: pixels stay bit-identical, which also means the file
still matches the source under reverse-image search or perceptual hashing.
Use --destructive when the goal is that a copy cannot be linked to its origin.
USAGE
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        -l|--list)        list_only=true; shift ;;
        -o|--out)         [ "$#" -ge 2 ] || die "--out needs a directory"; out_dir="$2"; shift 2 ;;
        --keep-icc)       keep_icc=true; shift ;;
        --keep-xattr)     keep_xattr=true; shift ;;
        -d|--destructive) destructive=true; shift ;;
        --quality)        [ "$#" -ge 2 ] || die "--quality needs a number"; quality="$2"; shift 2 ;;
        --scale)          [ "$#" -ge 2 ] || die "--scale needs a number"; scale="$2"; shift 2 ;;
        --rename)         rename=true; shift ;;
        -q|--quiet)       quiet=true; shift ;;
        -h|--help)        usage; exit 0 ;;
        --)               shift; files+=("$@"); break ;;
        -*)               die "unknown option '$1' (try --help)" ;;
        *)                files+=("$1"); shift ;;
    esac
done

[ "${#files[@]}" -gt 0 ] || die "no files given (try --help)"
command -v python3 >/dev/null || die "python3 not found"

case "${quality}" in ''|*[!0-9]*) die "--quality must be a number" ;; esac
case "${scale}" in ''|*[!0-9]*) die "--scale must be a number" ;; esac
if [ "${quality}" -lt 1 ] || [ "${quality}" -gt 100 ]; then die "--quality must be 1-100"; fi
if [ "${scale}" -lt 1 ] || [ "${scale}" -gt 100 ]; then die "--scale must be 1-100"; fi

if [ -n "${out_dir}" ]; then
    ${list_only} && die "--list and --out are mutually exclusive"
    mkdir -p "${out_dir}"
fi

magick_cmd=""
if ${destructive}; then
    ${list_only} && die "--list and --destructive are mutually exclusive"
    if command -v magick >/dev/null 2>&1; then
        magick_cmd="magick"
    elif command -v convert >/dev/null 2>&1; then
        magick_cmd="convert"
    else
        die "--destructive needs ImageMagick (brew install imagemagick / apt install imagemagick)"
    fi
fi

if ${rename} && ${list_only}; then
    die "--list and --rename are mutually exclusive"
fi

# Whether this host can inspect xattrs at all. Without a tool we must say so
# rather than report a reassuring zero: Linux browsers write the download URL to
# user.xdg.origin.url, the direct analogue of macOS kMDItemWhereFroms.
xattr_tool=""
if command -v xattr >/dev/null 2>&1; then
    xattr_tool="macos"
elif command -v getfattr >/dev/null 2>&1 && command -v setfattr >/dev/null 2>&1; then
    xattr_tool="linux"
fi
if [ -z "${xattr_tool}" ] && ! ${keep_xattr} && [ -z "${out_dir}" ]; then
    warn "no xattr/setfattr on this host; extended attributes are NOT being checked or cleared"
fi

# Clear the extended attributes that carry provenance. com.apple.provenance is
# system-managed and cannot be removed - it holds no information about the
# image, so it is excluded from the count and left alone.
clear_xattrs() {
    local f="$1" n=0 a
    case "${xattr_tool}" in
        macos)
            n=$(xattr "${f}" 2>/dev/null | grep -cv '^com\.apple\.provenance$' || true)
            if [ "${n}" -gt 0 ]; then
                xattr -c "${f}" 2>/dev/null || true
            fi
            ;;
        linux)
            while read -r a; do
                [ -n "${a}" ] || continue
                if setfattr -x "${a}" "${f}" 2>/dev/null; then
                    n=$((n + 1))
                fi
            done < <(getfattr --absolute-names -m '^user\.' "${f}" 2>/dev/null |
                         grep '^user\.' || true)
            ;;
    esac
    echo "${n}"
}

count_xattrs() {
    case "${xattr_tool}" in
        macos) xattr "$1" 2>/dev/null | grep -cv '^com\.apple\.provenance$' || true ;;
        linux) getfattr -m '^user\.' "$1" 2>/dev/null | grep -c '^user\.' || true ;;
        *)     echo 0 ;;
    esac
}

strip_one() {
    python3 - "$1" "$2" "$3" "$4" <<'PY'
import os, shutil, struct, sys, tempfile

src, dst, mode, keep_icc = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "true"
listing = mode == "list"

def fail(msg):
    sys.stderr.write("sz-rm-meta: %s: %s\n" % (os.path.basename(src), msg))
    sys.exit(2)

# Follow symlinks: replacing the link node would leave the real file dirty while
# reporting success, and would silently turn the link into a regular file.
src = os.path.realpath(src)
in_place = os.path.realpath(dst) == src
if in_place:
    dst = src

d = open(src, "rb").read()
removed = {}   # label -> [count, bytes], insertion-ordered

def note(label, n):
    slot = removed.setdefault(label, [0, 0])
    slot[0] += 1
    slot[1] += n

def summary():
    parts = []
    for label, (count, size) in removed.items():
        parts.append("%s%s %dB" % (label, " x%d" % count if count > 1 else "", size))
    return ", ".join(parts)

def jpeg_label(marker, body):
    if marker == 0xFE:
        return "comment"
    if body.startswith(b"Exif\x00"):
        return "EXIF"
    if body.startswith(b"http://ns.adobe.com/xap"):
        return "XMP"
    if body.startswith(b"http://ns.adobe.com/xmp/extension"):
        return "XMP-extended"
    if body.startswith(b"Photoshop 3.0"):
        return "IPTC/8BIM"
    if body.startswith(b"ICC_PROFILE"):
        return "ICC"
    if body.startswith(b"Ducky"):
        return "Ducky"
    if body.startswith(b"http://ns.google.com"):
        return "Google-XMP"
    if body.startswith(b"MPF\x00"):
        return "MPF"
    return "APP%d" % (marker - 0xE0)

def strip_jpeg(d):
    out = bytearray(d[:2])
    i, n = 2, len(d)
    saw_eoi = False
    while i < n - 1:
        if d[i] != 0xFF:
            fail("malformed JPEG: expected a marker at offset %d" % i)
        m = d[i + 1]
        if m == 0xFF:                       # fill byte
            out += d[i:i + 1]; i += 1; continue
        if m == 0xD9:                       # EOI
            out += d[i:i + 2]; i += 2; saw_eoi = True; break
        if m == 0x01 or 0xD0 <= m <= 0xD7:  # TEM / RSTn: no payload
            out += d[i:i + 2]; i += 2; continue
        if i + 4 > n:
            fail("truncated JPEG segment header")
        ln = struct.unpack(">H", d[i + 2:i + 4])[0]
        if ln < 2 or i + 2 + ln > n:
            fail("truncated JPEG segment (marker FF%02X)" % m)
        body = d[i + 4:i + 2 + ln]
        keep = True
        if 0xE0 <= m <= 0xEF or m == 0xFE:
            label = jpeg_label(m, body)
            if m == 0xE0 and body.startswith(b"JFIF\x00"):
                # Density header. Its last two fields are thumbnail dimensions;
                # a non-zero pair means an embedded RGB preview of the image
                # rides along, so drop the segment rather than keep the preview.
                keep = not (len(body) >= 14 and body[12] and body[13])
                if not keep:
                    note("JFIF-thumbnail", ln + 2)
            elif m == 0xE0 and body.startswith(b"JFXX\x00"):
                keep = False                # exists only to carry a thumbnail
                note("JFXX-thumbnail", ln + 2)
            elif m == 0xEE and body.startswith(b"Adobe"):
                keep = True                 # colour transform: a codec parameter
            elif label == "ICC" and keep_icc:
                keep = True
            else:
                keep = False
                note(label, ln + 2)
        if keep:
            out += d[i:i + 2 + ln]
        i += 2 + ln
        if m == 0xDA:                       # entropy-coded scan follows
            j = i
            while j < n - 1:
                if d[j] != 0xFF:
                    j += 1; continue
                nx = d[j + 1]
                if nx == 0x00 or 0xD0 <= nx <= 0xD7:
                    j += 2; continue        # byte stuffing / restart marker
                if nx == 0xFF:
                    j += 1; continue        # fill byte
                break
            out += d[i:j]
            i = j
    if not saw_eoi:
        fail("no EOI marker found; refusing to write a truncated file")
    if i < n:
        note("trailer", n - i)              # appended thumbnails, depth maps, junk
    return bytes(out)

# Allowlist, not denylist: anything not named here is dropped. Covers the image
# itself, transparency, colour rendering, and APNG animation. Private chunks
# (mkBF/prVw/cmOD and friends) fall through to the drop path by construction.
PNG_KEEP = {
    b"IHDR", b"PLTE", b"IDAT", b"IEND",             # critical
    b"tRNS", b"gAMA", b"cHRM", b"sRGB", b"sBIT",    # colour / transparency
    b"bKGD", b"pHYs", b"hIST", b"sPLT",             # rendering hints
    b"acTL", b"fcTL", b"fdAT",                      # APNG: dropping these
}                                                   # would flatten the animation

def strip_png(d):
    out = bytearray(d[:8])
    i, n = 8, len(d)
    saw_end = False
    while i + 8 <= n:
        ln = struct.unpack(">I", d[i:i + 4])[0]
        typ = d[i + 4:i + 8]
        end = i + 12 + ln
        if end > n:
            fail("truncated PNG chunk '%s'" % typ.decode("ascii", "replace"))
        keep = typ in PNG_KEEP or (typ == b"iCCP" and keep_icc)
        if keep:
            out += d[i:end]
        else:
            note(typ.decode("ascii", "replace"), ln + 12)
        i = end
        if typ == b"IEND":
            saw_end = True
            break
    if not saw_end:
        fail("no IEND chunk found; refusing to write a truncated file")
    if i < n:
        note("trailer", n - i)
    return bytes(out)

if d[:2] == b"\xff\xd8":
    cleaned = strip_jpeg(d)
elif d[:8] == b"\x89PNG\r\n\x1a\n":
    cleaned = strip_png(d)
else:
    fail("unsupported format (only JPEG and PNG; use exiftool for others)")

links = os.stat(src).st_nlink if in_place else 1

if not listing and (removed or not in_place):
    if in_place and links > 1:
        # Replacing the inode would clean this name and leave every other hard
        # link holding the original metadata. Rewrite the inode instead so all
        # names see the clean data, accepting a non-atomic write.
        with open(dst, "r+b") as fh:
            fh.write(cleaned)
            fh.truncate()
    else:
        # Write beside the destination so the rename is atomic on the same volume.
        dst_dir = os.path.dirname(os.path.abspath(dst)) or "."
        fd, tmp = tempfile.mkstemp(dir=dst_dir, prefix=".sz-rm-meta.")
        try:
            with os.fdopen(fd, "wb") as fh:
                fh.write(cleaned)
            shutil.copymode(src, tmp)
            os.replace(tmp, dst)
        except BaseException:
            if os.path.exists(tmp):
                os.unlink(tmp)
            raise

print("%d\t%d\t%d\t%s" % (len(d), len(cleaned), links, summary()))
PY
}

# Re-encode into a temp file: new quantization tables, new bitstream, slightly
# different dimensions. -strip drops what ImageMagick knows about; the lossless
# pass afterwards catches what it leaves behind (it writes its own tEXt/date
# chunks into PNG, for one).
reencode() {
    local src="$1" tmp="$2"
    "${magick_cmd}" "${src}" -strip -resize "${scale}%" -quality "${quality}" \
        -interlace none "${tmp}" 2>/dev/null
}

random_name() {
    local ext="${1##*.}"
    local stem
    stem=$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 12)
    if [ "${ext}" = "$1" ]; then
        echo "${stem}"
    else
        echo "${stem}.${ext}"
    fi
}

status=0
for f in "${files[@]}"; do
    [ -f "${f}" ] || { >&2 echo "sz-rm-meta: not a file: ${f}"; status=1; continue; }

    orig_size=$(python3 -c 'import os,sys; print(os.path.getsize(sys.argv[1]))' "${f}")

    dst="${f}"
    [ -n "${out_dir}" ] && dst="${out_dir%/}/$(basename "${f}")"

    # Count before rewriting: the rewrite replaces the inode, which drops the
    # xattrs on its own, so counting afterwards would always report zero. In
    # --out mode the source is never touched and the fresh copy starts with no
    # xattrs, so there is nothing to clear or report.
    xa=0
    if ! ${keep_xattr} && [ -z "${out_dir}" ]; then
        xa=$(count_xattrs "${f}")
    fi

    src="${f}"
    tmp_reenc=""
    if ${destructive}; then
        tmp_reenc="$(dirname "${dst}")/.sz-rm-meta-reenc.$$.$(basename "${f}")"
        if ! reencode "${f}" "${tmp_reenc}"; then
            rm -f "${tmp_reenc}"
            >&2 echo "sz-rm-meta: ${f}: re-encode failed"
            status=1; continue
        fi
        src="${tmp_reenc}"
    fi

    mode="strip"; ${list_only} && mode="list"
    if ! result=$(strip_one "${src}" "${dst}" "${mode}" "${keep_icc}"); then
        [ -n "${tmp_reenc}" ] && rm -f "${tmp_reenc}"
        status=1; continue
    fi
    if [ -n "${tmp_reenc}" ]; then
        rm -f "${tmp_reenc}"
    fi

    # Still needed for a file that was already clean, and for the hard-link path
    # where the inode survives the rewrite along with its xattrs.
    if [ "${xa}" -gt 0 ] && ! ${list_only}; then
        clear_xattrs "${dst}" >/dev/null
    fi

    after=$(echo "${result}" | cut -f2)
    links=$(echo "${result}" | cut -f3)
    removed=$(echo "${result}" | cut -f4)

    final="${dst}"
    if ${rename} && ! ${list_only}; then
        final="$(dirname "${dst}")/$(random_name "$(basename "${dst}")")"
        mv -n "${dst}" "${final}" || { >&2 echo "sz-rm-meta: rename failed: ${dst}"; status=1; final="${dst}"; }
    fi

    ${quiet} && continue

    detail="${removed}"
    if [ "${xa}" -gt 0 ]; then
        plural=s; [ "${xa}" -eq 1 ] && plural=
        detail="${detail:+${detail}, }${xa} xattr${plural}"
    fi
    if [ -z "${detail}" ] && ! ${destructive}; then
        echo "${f}: already clean"
    elif ${list_only}; then
        echo "${f}: would remove ${detail} (${orig_size} -> ${after} bytes)"
    else
        if ${destructive}; then
            action="re-encoded q${quality} @${scale}%${detail:+, removed ${detail}}"
        else
            action="removed ${detail}"
        fi
        line="${f}: ${action} (${orig_size} -> ${after} bytes)"
        [ "${links}" -gt 1 ] && line="${line} [${links} hard links, rewritten in place]"
        [ "${final}" != "${dst}" ] && line="${line} -> $(basename "${final}")"
        echo "${line}"
    fi
done

exit "${status}"