# One key per line in $FIRST_IN_HEADERS to get its first value in the headers.
FIRST_VALUE_IN_HEADERS='X-UIDL
Return-Path
Message-ID
Subject
Content analysis details'
# One pair (name, regexp) per line in $DISTINCT_IN_HEADERS_OR_CONTENT to get the distinct matching strings in the headers and the content.
DISTINCT_IN_HEADERS_OR_CONTENT='URL https?://[a-zA-Z0-9./?=_%:-]*
IPv4-addr (([0-9]|[0-9]{2}|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[0-9]{2}|1[0-9]{2}|2[0-4][0-9]|25[0-5])
email-addr [a-zA-Z0-9._]+@[a-zA-Z]+.[a-zA-Z]+
email-addr-with-at [a-zA-Z0-9._]+ *\[at\] *[a-zA-Z]+.[a-zA-Z]+'
group_by_line () {
awk -F : -v i=2 '{
if ($1 - i) {
print str
while ($1 - ++i)
print ""
sub(/[^:]*:/, "")
str = $0 }
else {
sub(/[^:]*:/, "")
if (str)
str = str "," $0
else
str = $0 } }
END {
print str }'
}
TMP=$(mktemp -d)
trap "rm -r $TMP 2>/dev/null" 0
mkdir $TMP/headers $TMP/contents $TMP/fields
files=$(seq 2 $(awk -v RS='\nX-Account-Key: ' -F '\n\n' -v tmp=$TMP/ 'NR - 1 {
printf substr($1, 1, index($1, "\n")) >> tmp "fields/X-Account-Key"
print $1 > tmp "headers/" NR
close(tmp "headers/" NR)
for (i = 2; i <= NF; ++i) {
gsub(/=\n/, "", $i)
print $i >> tmp "contents/" NR }
close(tmp "contents/" NR) }
END {
print NR }' "$@"))
# cd $TMP/contents
# touch $files
cd $TMP/headers
echo "$FIRST_VALUE_IN_HEADERS" | while read -r key
do
mkfifo "../fields/$key"
grep -im 1 "^ *$key:" $files |
sed -e 's/:[^:]*: */:/' -e 's/<//' -e 's/>//' |
group_by_line > "../fields/$key" &
done
for part in headers contents
do
cd ../$part
echo "$DISTINCT_IN_HEADERS_OR_CONTENT" | while read -r name regexp
do
mkfifo "../fields/$name-in-$part"
grep -Eo "$regexp" $files |
sort -ut : -k 1,1n -k 2 |
group_by_line > "../fields/$name-in-$part" &
done
done
cd ../fields/
set *
echo -n '# '
while [ -n "$2" ]
do
i=$(expr $i + 1)
echo -n "$i:$1\t"
shift
done
echo "$(expr $i + 1):$1"
paste *