1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
#!/bin/bash
# Name: verify-digests.sh
# Title: Gentoo Linux release digest verification
# Author: Robin H Johnson <robbat2@gentoo.org>
# Copyright 2016 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
#
# Description:
# This script exists to help mirrors verify raw digests of release files, to
# detect possible disk and filesystem corruptions. By design, it does NOT check
# GPG signatures.
#
# Usage:
# verify-digests.sh [FILES-OR-DIRECTORIES...]
#
# If passed a digest file:
# - it will be checked.
# If passed a non-digest file:
# - that immediate directory will be checked for all digest files.
# If passed a directory:
# - it and all subdirs will be checked for all digest files.
# If passed no arguments:
# - it will act like the directory '.' was passed.
#
# Return value:
# On success, exits zero.
# On failures, exits non-zero, and writes a file of errors to $TMPDIR.
# Take Gentoo digest files and convert to a plain BSD-format digest file.
# - strip any PGP signing
# - pass existing BSD-format digest
# - convert coreutils-format to BSD-format
transform_digest() {
sed -n -r \
-e '/BEGIN (PGP|GPG) SIGNED MESSAGE/,/^$/d' \
-e '/BEGIN (PGP|GPG) SIGNATURE/,/END (PGP|GPG) SIGNATURE/{d}' \
-e 'p' \
| \
awk \
-e '/^# .* HASH$/{hash=$2}' \
-e '/^[[:xdigit:]]+[[:space:]]+.+/{if(hash != ""){printf "%s (%s) = %s\n",hash,$2,$1}}' \
-e '/^((SHA|MD|RIPEMD)[0-9]+|WHIRLPOOL) \(.*\) = [[:xdigit:]]+/{print $0}' \
-e '/^((SHA|MD|RIPEMD)[0-9]+|WHIRLPOOL) [[:xdigit:]]+ [^[:space:]]+$/{ printf "%s (%s) = %s\n",$1,$3,$2; }'
}
# Pass all directory arguments to find
# Keep all file arguments as-is (so you can pass .asc files directly)
DIGESTS_ARGS=( )
DIGESTS_FIND=( )
if [[ ${#@} -eq 0 ]]; then
DIGESTS_FIND+=( . )
else
for f in "${@}" ; do
if [ -d "$f" ]; then
DIGESTS_FIND+=( "$f" )
else
DIGESTS_ARGS+=( "$f" )
fi
done
fi
# Check if non-dir arguments were digest files or files that you want to get checked
DIGESTS_ARGS2=( )
for f in "${DIGESTS_ARGS[@]}" ; do
if [[ "${f/DIGEST}" != "$f" ]] || grep -sq -m 1 -e '# MD5 HASH' -e '# SHA[0-9]\+ HASH' -e ') = [0-9a-f]\+' $f; then
DIGESTS_ARGS2+=( "$f" )
else
d=$( dirname "$f" )
DIGESTS_FIND2=( )
readarray -t DIGESTS_FIND2 <<< "$(find "$d" -maxdepth 1 ! -type d \( -name '*.DIGESTS' -o -name '*.DIGESTS.asc' \) | fmt -1 |sort | uniq)"
DIGESTS_ARGS2+=( "${DIGESTS_FIND2[@]}" )
DIGESTS_FIND2=( )
fi
done
if [[ "${#DIGESTS_FIND[@]}" -gt 0 ]]; then
readarray -t DIGESTS_FIND <<< "$(find "${DIGESTS_FIND[@]}" ! -type d \( -name '*.DIGESTS' -o -name '*.DIGESTS.asc' \) | fmt -1 | sort | uniq )"
fi
# merge all items
DIGESTS=( "${DIGESTS_ARGS2[@]}" "${DIGESTS_FIND[@]}" )
# Prefer signed digests where possible, but sometimes they were in the original
# .DIGESTS file, and other times there was a seperate .asc file.
DIGESTS2="$(echo "${DIGESTS[@]}" | fmt -1 |sed '/.asc$/s/.asc$//' | sort | uniq)"
DIGESTS=( )
for d in ${DIGESTS2} ; do
if [ -e "${d}" -a -e "${d}.asc" ]; then
DIGESTS+=( "${d}.asc" )
elif [ ! -e "${d}" -a -e "${d}.asc" ]; then
DIGESTS+=( "${d}.asc" )
elif [ -e "${d}" -a ! -e "${d}.asc" ]; then
DIGESTS+=( "${d}" )
fi
done
# Setup storage for digest conversion & results
T=$(date -u +%Y%m%dT%H%M%SZ)
tmp1=$(mktemp --tmpdir)
tmp2=$(mktemp --tmpdir)
failures=$(mktemp --tmpdir gentoo-failures.$T.XXXXXXXXXX)
trap "rm -f $tmp1 $tmp2" SIGINT SIGTERM
# Now check them
failed_digests=()
for d in $(echo "${DIGESTS[@]}" | fmt -1 | sort | uniq); do
sleep 0.01
echo -n "Checking digests from $d: "
transform_digest < "$d" >"$tmp1"
# add leading & trailing space to match
hashes=" $(awk '{print $1}' "$tmp1" | sort | uniq ) "
checked=0
found=0
# order by strength
for h in SHA512 SHA384 SHA256 SHA224 SHA1 MD5 ; do
sleep 0.01
[[ $found -eq 1 ]] && break
if [[ "${hashes/$h}" != "${hashes}" ]]; then
found=1
echo "using $h"
pushd $(dirname $d) >/dev/null
cmd=$(echo ${h}sum | tr '[:upper:]' '[:lower:]')
grep "^$h " $tmp1 | ionice -c 3 --ignore ${cmd} -c - | tee "$tmp2"
rc=${PIPESTATUS[1]}
if [ $rc -ne 0 ]; then
failed_digests+=("$d")
cat "$tmp2" >> "$failures"
fi
checked=1
popd >/dev/null
fi
done
if [[ $checked -eq 0 ]]; then
echo " FAIL - no usable digest"
fi
done
# Handle output of errors
if [[ "${#failed_digests[@]}" -eq 0 ]]; then
exit 0
else
echo "----"
echo "Failures detected in the following DIGESTS:" 1>&2
for f in "${failed_digests[@]}"; do
echo "$f" 1>&2
done
echo "----" 1>&2
echo "Complete output of failed DIGESTS, stored in $failures:" 1>&2
cat "$failures" 1>&2
exit 1
fi
|
GitWeb