# an awk script
# an NNTP log summary report generator
#
# NOTE: for systems that are not as yet using the new 4.3 BSD syslog
# (and therefore have nntp messages lumped with everything else), it
# would be best to invoke this script thusly:
#
# egrep nntp syslog.old | awk -f nntp_awk > report_of_the_week
#
# because this script will include in the report all messages in the log
# that it does not recognize (on the assumption that they are errors to
# be dealt with by a human).
#
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
# May 17, 1986 - Norwegian Independence Day
#
# Recognize some new things - February 22, 1987
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
# fix "xmt is not an array" bug - March 11, 1987
# Change Elapsed/CPU fields to break out time values, HH:MM:SS
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
# Add reporting for newnews commands - August 27, 1987
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
# Add nntpxmit connection attempt counting/reporting - December 7, 1987
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
BEGIN{
readers = 0;
transmit = 0;
receive = 0;
polled = 0;
}
### Skip stderr reports from rnews
{
n = split($6, path, "/");
if (path[n] == "rnews:") next;
n = split($7, path, "/");
if (path[n] == "rnews") next;
host = $6;
}
$7 == "group" {
readers = 1;
ng[$8]++;
next;
}
$7 == "ihave" {
receive = 1;
rec[host]++;
if ($9 == "accepted") {
rec_accept[host]++;
if ($10 == "failed") rec_failed[host]++;
} else if ($9 == "rejected") rec_refuse[host]++;
next;
}
# this is from version 1.4 of nntpd
$7 == "ihave_stats" {
receive = 1;
rec[host] += $9 + $11 + $13;
rec_accept[host] += $9;
rec_refuse[host] += $11;
rec_failed[host] += $13;
next;
}
$7 == "connect" {
systems[host]++;
next;
}
# nntpxmit connection errors
# Ooooh! I *wish* awk had N dimensional arrays,
# so I wouldn't have to throw away the error message here!
$7 == "hello:" {
conn[host]++;
if ($8 == "Connection" && $9 == "refused")
rmt_fail[host]++;
else
open_fail[host]++;
next;
}
# we'll get stats from this, don't count conn[]
$7 == "xfer:" {
open_fail[host]++;
# since these are expected to be few in number, we still print
# the exact error (no "next;" statement here).
}
$7 == "greeted" {
conn[host]++;
rmt_fail[host]++;
next;
}
$7 == "host" && $8 == "unknown" {
conn[host]++;
ns_fail[host]++;
next;
}
# nntpd connection abort - all "broken pipe" right now
$7 == "disconnect:" { next }
# syslogd shit
$7 == "repeated" { next }
# inews shit
$11 == "spooled" { next }
$7 == "exit" {
if ($8 > 0) readers = 1;
articles[host] += $8;
groups[host] += $10;
next;
}
$7 == "xmit" {
xmt_cpu[host] += $9 + $11;
xmt_ela[host] += $13;
next;
}
$7 == "times" {
cpu[host] += $9 + $11;
ela[host] += $13;
next;
}
$7 == "stats" {
transmit = 1;
conn[host]++;
xmt[host] += $8;
xmt_accept[host] += $10;
xmt_refuse[host] += $12;
xmt_failed[host] += $14;
next;
}
#
# For the Nth time, I wish awk had two dimensional associative
# arrays. I assume that the last request is the same as all the
# others in this section of logfile.
#
$7 == "newnews" {
polled = 1;
poll[host] ++;
poll_asked[host] = $8;
next;
}
$7 == "newnews_stats" {
poll_offered[host] += $9;
poll_took[host] += $11;
next;
}
$7 == "post" {
readers = 1;
post[host]++;
next;
}
$7 == "timeout" {
timeout[host]++;
timeouts = 1;
next;
}
$7 == "unrecognized" {
unknown[host] = 1;
curious = 1;
}
### Print anything that we don't recognize in the report
{
print;
}
END{
printf("\n");
###############################################################################
### Article Exchange With Peers (other servers) Statistics ###
###############################################################################
if (transmit || receive || polled)
printf("NNTP peer article transfers\n\n");
if (polled) for(s in poll) servers[s]++;
if (receive) for(s in rec) servers[s]++;
if (transmit) for(s in xmt) servers[s]++;
if (receive) {
printf("Article Reception (they contact us)\n");
printf("System Offered Took Toss Fail Toss Elapsed CPU Pct\n");
for(s in rec) {
nrec += rec[s];
nrec_accept += rec_accept[s];
nrec_refuse += rec_refuse[s];
nrec_failed += rec_failed[s];
nrec_cpu += cpu[s];
nrec_ela += ela[s];
they_offered = rec[s];
if (they_offered == 0) they_offered = 1;
we_toss = (rec_refuse[s] / they_offered) * 100 + 0.5;
e_hours = ela[s] / 3600;
e_sec = ela[s] % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = cpu[s] / 3600;
c_sec = cpu[s] % 3600;
c_min = c_sec / 60;
c_sec %= 60;
tmp = ela[s];
if (tmp == 0) tmp = 1;
pct = ((cpu[s] / tmp) * 100.0 + 0.5);
printf("%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n", s, rec[s], rec_accept[s], rec_refuse[s], rec_failed[s], we_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
}
e_hours = nrec_ela / 3600;
e_sec = nrec_ela % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = nrec_cpu / 3600;
c_sec = nrec_cpu % 3600;
c_min = c_sec / 60;
c_sec %= 60;
they_offered = nrec;
if (they_offered == 0) they_offered = 1;
we_toss = (nrec_refuse / they_offered) * 100 + 0.5;
if (nrec_ela == 0) nrec_ela = 1;
pct = ((nrec_cpu / nrec_ela) * 100.0 + 0.5);
printf("\n%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", "TOTALS", nrec, nrec_accept, nrec_refuse, nrec_failed, we_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
}
###############################################################################
if (polled) {
printf("Article Transmission (they poll us)\n");
printf("System Conn Offrd Took Elapsed CPU Pct Groups\n");
npoll = 0;
npoll_offered = 0;
npoll_took = 0;
npoll_cpu = 0;
npoll_ela = 0;
for(s in poll) {
npoll += poll[s];
npoll_offered += poll_offered[s];
npoll_took += poll_took[s];
if (rec[s]) {
printf("%-25s %5d %5d %5d (see Article Reception) %s\n", s, poll[s], poll_offered[s], poll_took[s], poll_asked[s]);
} else {
npoll_ela += ela[s];
npoll_cpu += cpu[s];
e_hours = ela[s] / 3600;
e_sec = ela[s] % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = cpu[s] / 3600;
c_sec = cpu[s] % 3600;
c_min = c_sec / 60;
c_sec %= 60;
tmp = ela[s];
if (tmp == 0) tmp = 1;
pct = ((cpu[s] / tmp) * 100.0 + 0.5);
printf("%-25s %5d %5d %5d %3d:%02d:%02d %3d:%02d:%02d %3d%% %s\n", s, poll[s], poll_offered[s], poll_took[s], e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct, poll_asked[s]);
}
}
printf("\n%-25s %5d %5d %5d", "TOTALS", npoll, npoll_offered, npoll_took);
if (npoll_ela > 0 && npoll_cpu > 0) {
e_hours = npoll_ela / 3600;
e_sec = npoll_ela % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = npoll_cpu / 3600;
c_sec = npoll_cpu % 3600;
c_min = c_sec / 60;
c_sec %= 60;
tmp = npoll_ela;
if (tmp == 0) tmp = 1;
pct = ((npoll_cpu / tmp) * 100.0 + 0.5);
printf(" %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
} else
printf("\n\n");
}
###############################################################################
if (transmit) {
printf("Article Transmission (we contact them)\n");
printf("System Offrd Took Toss Fail Pct Elapsed CPU Pct\n");
for(s in xmt) {
we_offered = xmt[s];
if (we_offered == 0) we_offered = 1;
they_toss = (xmt_refuse[s] / we_offered) * 100 + 0.5;
e_hours = xmt_ela[s] / 3600;
e_sec = xmt_ela[s] % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = xmt_cpu[s] / 3600;
c_sec = xmt_cpu[s] % 3600;
c_min = c_sec / 60;
c_sec %= 60;
elapsed = xmt_ela[s];
if (elapsed == 0) elapsed = 1;
pct = ((xmt_cpu[s] / elapsed) * 100.0 + 0.5);
printf("%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n", s, xmt[s], xmt_accept[s], xmt_refuse[s], xmt_failed[s], they_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
nxmt += xmt[s];
nxmt_accept += xmt_accept[s];
nxmt_refuse += xmt_refuse[s];
nxmt_failed += xmt_failed[s];
nxmt_ela += xmt_ela[s];
nxmt_cpu += xmt_cpu[s];
}
we_offered = nxmt;
if (we_offered == 0) we_offered = 1;
they_toss = (nxmt_refuse / we_offered) * 100 + 0.5;
e_hours = nxmt_ela / 3600;
e_sec = nxmt_ela % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = nxmt_cpu / 3600;
c_sec = nxmt_cpu % 3600;
c_min = c_sec / 60;
c_sec %= 60;
if (nxmt_ela == 0) nxmt_ela = 1;
pct = ((nxmt_cpu / nxmt_ela) * 100.0 + 0.5);
printf("\n%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", "TOTALS", nxmt, nxmt_accept, nxmt_refuse, nxmt_failed, they_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
printf("Transmission Connection Attempts ------errors-------\n");
printf("System Conn OK NS Net Rmt Pct\n");
for(s in xmt) {
tot = conn[s];
if (tot == 0) tot = 1;
errs = rmt_fail[s] + ns_fail[s] + open_fail[s];
ok = (conn[s] - errs);
printf("%-25s %5d %5d %5d %5d %5d %3d%%\n", s, conn[s], ok, ns_fail[s], open_fail[s], rmt_fail[s], (100.0 * errs / tot + 0.5));
ct_tot += conn[s];
ct_ok += ok;
ct_ns += ns_fail[s];
ct_net += open_fail[s];
ct_rmt += rmt_fail[s];
}
tot = ct_tot;
if (tot == 0) tot = 1;
errs = ct_ns + ct_net + ct_rmt;
printf("\n%-25s %5d %5d %5d %5d %5d %3d%%\n\n", "TOTALS", ct_tot, ct_ok, ct_ns, ct_net, ct_rmt, (100.0 * errs / tot + 0.5));
}
###############################################################################
### Article Readership Statistics ###
###############################################################################
if (readers) {
printf("NNTP readership statistics\n");
printf("System Conn Articles Groups Post Elapsed CPU Pct\n");
for(s in systems) {
###
### servers are different animals; they don't belong in this part of the report
###
if (servers[s] > 0 && groups[s] == 0 && articles[s] == 0)
continue;
###
### report the curious server pokers elsewhere
###
if (groups[s] == 0 && articles[s] == 0 && post[s] == 0) {
unknown[s] += systems[s];
curious = 1;
continue;
}
nconn += systems[s];
nart += articles[s];
ngrp += groups[s];
npost += post[s];
ncpu += cpu[s];
nela += ela[s];
e_hours = ela[s] / 3600;
e_sec = ela[s] % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = cpu[s] / 3600;
c_sec = cpu[s] % 3600;
c_min = c_sec / 60;
c_sec %= 60;
elapsed = ela[s];
if (elapsed == 0) elapsed = 1;
pct = ((cpu[s] / elapsed) * 100 + 0.5);
printf("%-25s %5d %8d %6d %4d %3d:%02d:%02d %3d:%02d:%02d %3d%%\n", s, systems[s], articles[s], groups[s], post[s], e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
}
e_hours = nela / 3600;
e_sec = nela % 3600;
e_min = e_sec / 60;
e_sec %= 60;
c_hours = ncpu / 3600;
c_sec = ncpu % 3600;
c_min = c_sec / 60;
c_sec %= 60;
if (nela == 0) nela = 1;
pct = ((ncpu / nela) * 100 + 0.5);
printf("\n%-25s %5d %8d %6d %4d %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", "TOTALS", nconn, nart, ngrp, npost, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
}
###############################################################################
if (curious) {
printf("Unknown NNTP server explorers\nSystem Conn\n");
for(s in unknown) {
printf("%-25s %5d\n", s, unknown[s]);
}
printf("\n");
}
###############################################################################
if (timeouts) {
printf("Server timeouts\n");
for(s in timeout) {
printf("%-25s %5d\n", s, timeout[s]);
}
printf("\n");
}
###############################################################################
if (readers) {
for(g in ng) {
x = length(g);
if (x > max) max = x;
i = index(g, ".");
if (i > 0) top = substr(g, 1, i - 1);
else top = g;
category[top] += ng[g];
}
fmt = sprintf("%%-%ds %%5d\n", max);
printf("Newsgroup Request Counts (by category)\n");
for(g in category) printf(fmt, g, category[g]);
printf("\nNewsgroup Request Counts (by newsgroup)\n");
for(g in ng) printf(fmt, g, ng[g]);
printf("\n");
}
}