This is analog.c in view mode; [Download] [Up]
/*** analog 1.9beta ***/
/* Please read Readme.html, or http://www.statslab.cam.ac.uk/~sret1/analog/ */
/*** analog.c; the main function ***/
#include "analhea2.h"
int main(int argc, char **argv)
{
extern void urltodir(); /* in alias.c */
extern int doaliashost(); /* in alias.c */
extern int doaliasurl(); /* in alias.c */
extern void allaliases(); /* in alias.c */
extern flag included(); /* in alias.c */
extern void hashadd(); /* in hash.c */
extern void domhashadd(); /* in hash.c */
extern void approxhosthashadd(); /* in hash.c */
extern struct genstruct *gensort(); /* in hash.c */
extern int domsort(); /* in hash.c */
extern void subdomsort(); /* in hash.c */
extern void errsort(); /* in hash.c */
extern void datehash(); /* in hash.c */
extern void addref(); /* in hash.c */
extern void addbrowser(); /* in hash.c */
extern void adderr(); /* in hash.c */
extern void initialise(); /* in init.c */
extern void printvbles(); /* in init.c */
extern void output(); /* in output.c */
extern int sscanf_common(); /* in sscanf.c */
extern int sscanf_ncsaold(); /* in sscanf.c */
extern int sscanf_referer(); /* in sscanf.c */
extern long timecode(); /* in utils.c */
extern struct timestruct startofweek(); /* in utils.c */
extern FILE *fopenlog(); /* in utils.c */
extern int fcloselog(); /* in utils.c */
extern char commandname[]; /* all global vars declared in init.c */
extern struct loglist *logfilehead;
extern struct stringlist *cachefilehead, *refloghead, *browloghead;
extern struct stringlist *errloghead;
extern flag vblesonly, byq, browbyq, refbyq;
extern flag filemaskq, hostmaskq, refmaskq, q7, warnq;
extern flag mq, hq, Hq, dq, Dq, Wq, sq, Sq, oq, iq, rq, fq, Bq, bq, cq, eq;
extern int domsortby, reqsortby, dirsortby, hostsortby, refsortby;
extern int browsortby, fullbrowsortby;
extern char monthgraph, daygraph, fulldaygraph, hourgraph, fullhourgraph;
extern char weekgraph;
extern char urlminreqstr[], dirminreqstr[], hostminreqstr[];
extern char refminreqstr[], browminreqstr[], fullbrowminreqstr[];
extern char urlminbytestr[], dirminbytestr[], hostminbytestr[];
extern char refminbytestr[], browminbytestr[], fullbrowminbytestr[];
extern int reqtype;
extern int url_max_reqs, dir_max_reqs, host_max_reqs, ref_max_reqs;
extern int brow_max_reqs, fullbrow_max_reqs;
extern double url_max_bytes, dir_max_bytes, host_max_bytes, ref_max_bytes;
extern double brow_max_bytes, fullbrow_max_bytes;
extern int host_max_length;
extern struct timestruct firsttime, lasttime, fromtime, totime, oldtime;
extern int no_hosts, no_hosts7, no_new_hosts7;
extern int no_urls, no_urls7;
extern int cachereqs, cachereqs7, corrupt_lines, other_lines;
extern double total_bytes, total_bytes7;
extern struct weekly *firstw;
extern struct genstruct *hosthead[], *urlhead[], *dirhead[], *refhead[];
extern struct genstruct *browhead[], *fullbrowhead[];
extern int onumber;
extern struct include *wantfilehead, *wanthosthead;
extern int debug;
extern int status[], status7[], statusnos[];
FILE *lf;
flag ispipe; /* whether the currently open logfile is a pipe */
struct loglist *logfilep;
struct stringlist *otherlogp;
char inputline[MAXLINELENGTH]; /* a particular input line */
int linetype; /* COMMON, NCSAOLD or CORRUPT */
char hostn[MAXSTRINGLENGTH];
int date, year, hr, min, monthno;
long thistimecode;
char fromurl[MAXSTRINGLENGTH];
char browser[MAXSTRINGLENGTH];
char errstr[MAXLINELENGTH];
char filename[MAXSTRINGLENGTH];
int code;
size_t preflength; /* length of filename prefix for this logfile */
flag firstreq = TRUE;
flag datemaskq;
flag fwarn1 = OFF, fwarn2 = OFF; /* have certain warnings been given? */
flag bwarn1 = OFF, bwarn2 = OFF, bwarn3 = OFF;
double bytes; /* long is not big enough; double has more sig. figs,
and copes with overflow automatically. */
char bytestr[16];
struct genstruct *urlsorthead, *dirsorthead, *hostsorthead;
struct genstruct *refsorthead, *browsorthead, *fullbrowsorthead;
int firstdom, errorder[NO_ERRS]; /* heads for sorting */
struct genstruct *hostp, *hostnextp, *urlp, *urlnextp;
int onlist;
flag wantthisone = TRUE; /* whether we want to analyse a particular entry */
flag last7q = OFF; /* are we now in the last 7 days? */
int rc;
char *tempstr, *tempc;
int i, tempint;
flag tempf;
/*** Initialisation ***/
initialise(argc, argv);
datemaskq = (fromtime.code > -INFINITY || totime.code < INFINITY);
if (vblesonly)
printvbles(); /* which also exits */
/*** Now start scanning ***/
for (logfilep = logfilehead; logfilep -> name[0] != '\0';
logfilep = logfilep -> next) { /* for each logfile */
lf = fopenlog(logfilep -> name, "logfile", &ispipe);
if (lf != NULL) {
strcpy(filename, logfilep -> prefix);
preflength = strlen(filename);
while(fgets(inputline, MAXLINELENGTH, lf) != NULL) {
linetype = CORRUPT; /* paranoia :) */
if ((rc = sscanf_common(inputline, hostn, &date, &monthno, &year, &hr,
&min, filename + preflength, fromurl, browser,
&code, bytestr, preflength)) >= 9) {
linetype = COMMON;
bytes = atof(bytestr);
}
else if ((rc = sscanf_ncsaold(inputline, hostn, &monthno, &date, &hr,
&min, &year, filename + preflength,
preflength)) == 7) {
linetype = NCSAOLD;
code = 200;
bytes = 0;
if (byq) {
byq = OFF;
if (domsortby == BYBYTES)
domsortby = BYREQUESTS;
if (reqsortby == BYBYTES)
reqsortby = BYREQUESTS;
if (dirsortby == BYBYTES)
dirsortby = BYREQUESTS;
if (hostsortby == BYBYTES)
hostsortby = BYREQUESTS;
if (monthgraph == 'b' || monthgraph == 'B')
monthgraph = 'r';
if (daygraph == 'b' || daygraph == 'B')
daygraph = 'r';
if (fulldaygraph == 'b' || fulldaygraph == 'B')
fulldaygraph = 'r';
if (hourgraph == 'b' || hourgraph == 'B')
hourgraph = 'r';
if (fullhourgraph == 'b' || fullhourgraph == 'B')
fullhourgraph = 'r';
if (weekgraph == 'b' || weekgraph == 'B')
weekgraph = 'r';
}
}
if (linetype != CORRUPT) {
thistimecode = timecode(date, monthno, year, hr, min);
wantthisone = thistimecode > fromtime.code &&
thistimecode < totime.code;
if (wantthisone && filemaskq) {
doaliasurl(filename);
wantthisone = included(filename, wantfilehead);
}
if (wantthisone && hostmaskq) {
doaliashost(hostn);
wantthisone = included(hostn, wanthosthead);
}
if (!wantthisone)
++other_lines;
else {
/* Are we in the last 7 days? Check this every time in case */
/* logfile is not in chronological order */
if (q7) /* if !q7, last7q stays off (for efficiency) */
last7q = (thistimecode > oldtime.code);
/* add to the right status code total */
tempf = OFF;
for (i = 0; !tempf && i < NO_STATUS; i++) {
if (code <= statusnos[i]) {
status[i]++;
if (last7q)
status7[i]++;
tempf = ON;
}
}
if (code <= 299 || code == 304) { /* successes */
total_bytes += bytes; /* NB only count bytes for successes */
if (last7q)
total_bytes7 += bytes;
if (firstreq) {
firstreq = FALSE;
firsttime.date = date;
firsttime.monthno = monthno;
firsttime.year = year;
firsttime.hr = hr;
firsttime.min = min;
firsttime.code = thistimecode;
if (Wq)
firstw -> start = startofweek(firsttime);
lasttime.date = date;
lasttime.monthno = monthno;
lasttime.year = year;
lasttime.hr = hr;
lasttime.min = min;
lasttime.code = thistimecode;
}
/* date cataloguing */
datehash(year, monthno, date, hr, min, thistimecode, 1, bytes);
/* Now for the request report. We ignore all filename conversions
and aliases until outside the loop; it is more efficient to do
them just once at the end. Also note that we want to construct
a request report even if (iq AND NOT rq) because it is more
efficient to convert filenames into directories once at the
end than every time. */
if (rq || iq) {
hashadd(urlhead, URLHASHSIZE, filename, 1, bytes, last7q,
&no_urls, &no_urls7, &tempint, OFF);
}
/* We leave the directory report until the end; it's easier to do
work once for each filename, not once for each request */
/* Now for the hostname count. Just the same as above. This time,
however, we don't do one if the domain report is on and this
is off, because this takes up a lot of memory. */
if (sq == ON) {
hashadd(hosthead, HOSTHASHSIZE, hostn, 1, bytes, last7q,
&no_hosts, &no_hosts7, &no_new_hosts7, OFF);
}
/* If no exact hostname count, do the domain report now */
else if (oq) {
if (!hostmaskq)
doaliashost(hostn); /* o/wise it's already been done */
domhashadd(hostn, 1, bytes);
}
if (sq == APPROX) {
if (!hostmaskq && !oq)
doaliashost(hostn);
approxhosthashadd(hostn, last7q);
}
if (rc == 11) { /* then do the referer and browser now */
if (fq && fromurl[0] != '\0')
addref(fromurl, filename, bytes, last7q, OFF);
if ((Bq || bq) && browser[0] != '\0')
addbrowser(browser, bytes, last7q);
}
} /* end if code <= 299 || code == 304 */
} /* end if want this one */
} /* end if linetype != CORRUPT */
else { /* line is corrupt */
++corrupt_lines;
if (debug != 0)
fprintf(stderr, "C: %s", inputline);
if (strchr(inputline, '\n') == NULL) {
/* line corrupt by being too long; */
fscanf(lf, "%*[^\n]"); /* read to end of line */
if (debug != 0)
fprintf(stderr, "\n");
}
}
} /* end of reading this logfile */
fcloselog(lf, logfilep -> name, "logfile", ispipe);
}
} /*** End of main loop (for all logfiles) ***/
/*** Now for the other logfiles. First cache files. ***/
/* NB Some of this is shared with main loop and could probably be combined */
min = 30; /* reckon all cache entries at half past the hour */
for (otherlogp = cachefilehead; otherlogp -> name[0] != '\0';
otherlogp = otherlogp -> next) { /* for each referer log */
lf = fopenlog(otherlogp -> name, "cache file", &ispipe);
if (lf != NULL) {
if (fgets(inputline, MAXLINELENGTH, lf) == NULL ||
strncmp(inputline, "CACHE type 1 produced by analog", 31) != 0)
fprintf(stderr, "%s: Warning: %s appears not to be a cache file: ignoring it\n", commandname, otherlogp -> name);
else {
while (fgets(inputline, MAXLINELENGTH, lf) != NULL) {
if ((tempstr = strtok(inputline, ":")) != NULL) {
tempc = tempstr;
if (strlen(tempstr) != 10)
fprintf(stderr, "%s: Warning: ignoring corrupt line in cache file %s starting %s\n", commandname, otherlogp -> name, tempc);
else {
year = 1000 * (*tempstr - '0');
year += 100 * (*(++tempstr) - '0');
year += 10 * (*(++tempstr) - '0');
year += (*(++tempstr) - '0');
monthno = 10 * (*(++tempstr) - '0');
monthno += *(++tempstr) - '0' - 1;
date = 10 * (*(++tempstr) - '0');
date += *(++tempstr) - '0';
hr = 10 * (*(++tempstr) - '0');
hr += *(++tempstr) - '0';
tempf = OFF;
for ( ; hr < 24 && !tempf; hr++) {
if ((tempstr = strtok((char *)NULL, ":")) == NULL) {
fprintf(stderr, "%s: Warning: missing data in cache file %s at line starting %s\n", commandname, otherlogp -> name, tempc);
tempf = ON;
}
else if (tempstr[0] == '*')
tempf = ON;
else {
tempint = atoi(tempstr);
if ((tempstr = strtok((char *)NULL, ":")) == NULL) {
fprintf(stderr, "%s: Warning: missing data in cache file %s at line starting %s\n", commandname, otherlogp -> name, tempc);
tempf = ON;
}
else {
bytes = atof(tempstr);
thistimecode = timecode(date, monthno, year, hr, min);
if (firstreq) {
firstreq = FALSE;
firsttime.date = date;
firsttime.monthno = monthno;
firsttime.year = year;
firsttime.hr = hr;
firsttime.min = min;
firsttime.code = thistimecode;
if (Wq)
firstw -> start = startofweek(firsttime);
lasttime.date = date;
lasttime.monthno = monthno;
lasttime.year = year;
lasttime.hr = hr;
lasttime.min = min;
lasttime.code = thistimecode;
}
if (q7)
last7q = (thistimecode > oldtime.code);
total_bytes += bytes;
cachereqs += tempint;
if (last7q) {
total_bytes7 += bytes;
cachereqs7 += tempint;
}
datehash(year, monthno, date, hr, min, thistimecode,
tempint, bytes);
}
}
}
}
}
}
}
}
}
/*** Now the referer logs ***/
if (fq) {
for (otherlogp = refloghead; otherlogp -> name[0] != '\0';
otherlogp = otherlogp -> next) { /* for each referer log */
lf = fopenlog(otherlogp -> name, "referer log", &ispipe);
if (lf != NULL) {
while(fgets(inputline, MAXLINELENGTH, lf) != NULL) {
if (sscanf_referer(inputline, &date, &monthno, &year, &hr, &min,
fromurl, filename) == 7) {
wantthisone = ON;
last7q = OFF;
if (datemaskq) {
if (date != 0) {
thistimecode = timecode(date, monthno, year, hr, min);
wantthisone = thistimecode > fromtime.code &&
thistimecode < totime.code;
if (q7)
last7q = (thistimecode > oldtime.code);
}
else if (!fwarn1 && warnq) {
fprintf(stderr, "%s: Warning: Referer log contains lines with no date information;\n", commandname);
fprintf(stderr, " cannot filter them by date.\n");
fwarn1 = ON;
}
}
if (!fwarn2 && hostmaskq && wantthisone && warnq) {
fprintf(stderr, "%s: Warning: Referer log contains no host information;\n", commandname);
fprintf(stderr, " cannot filter by host.\n");
fwarn2 = ON;
}
if (wantthisone) {
refbyq = OFF;
if (refsortby == BYBYTES)
refsortby = BYREQUESTS;
addref(fromurl, filename, 0.0, last7q, filemaskq);
}
}
}
fcloselog(lf, otherlogp -> name, "referer log", ispipe);
}
}
}
/* Next the browser logs */
if (bq || Bq) {
for (otherlogp = browloghead; otherlogp -> name[0] != '\0';
otherlogp = otherlogp -> next) {
lf = fopenlog(otherlogp -> name, "browser log", &ispipe);
if (lf != NULL) {
while(fgets(inputline, MAXLINELENGTH, lf) != NULL) {
/* read in the date, if supplied */
if (*(tempstr = inputline) == '[') {
wantthisone = FALSE; /* unless date is valid */
if (sscanf_date(++tempstr, &date, &monthno, &year, &hr, &min) ==
5) {
tempstr += 20;
if (*tempstr == ']') {
if (*(++tempstr) == ' ') {
tempstr++;
wantthisone = TRUE;
}
}
}
}
else {
wantthisone = TRUE;
date = 0; /* as marker */
}
if (wantthisone) {
last7q = OFF;
if (datemaskq) {
if (date != 0) {
thistimecode = timecode(date, monthno, year, hr, min);
wantthisone = thistimecode > fromtime.code &&
thistimecode < totime.code;
if (q7)
last7q = (thistimecode > oldtime.code);
}
else if (!bwarn1 && warnq) {
fprintf(stderr, "%s: Warning: Browser log contains lines with no date information;\n", commandname);
fprintf(stderr, " cannot filter them by date.\n");
bwarn1 = ON;
}
}
if (!bwarn2 && hostmaskq && wantthisone && warnq) {
fprintf(stderr, "%s: Warning: Browser log contains no host information;\n", commandname);
fprintf(stderr, " cannot filter by host.\n");
bwarn2 = ON;
}
if (!bwarn3 && filemaskq && wantthisone && warnq) {
fprintf(stderr, "%s: Warning: Browser log contains no file information;\n", commandname);
fprintf(stderr, " cannot filter by file.\n");
bwarn3 = ON;
}
if (wantthisone) {
if ((tempc = strchr(tempstr, '\n')) != NULL)
*tempc = '\0';
browbyq = OFF;
if (browsortby == BYBYTES)
browsortby = BYREQUESTS;
if (fullbrowsortby == BYBYTES)
fullbrowsortby = BYREQUESTS;
strcpy(browser, tempstr);
addbrowser(browser, 0.0, last7q);
}
}
}
fcloselog(lf, otherlogp -> name, "browser log", ispipe);
}
}
}
/* Finally the error logs */
if (eq) {
for (otherlogp = errloghead; otherlogp -> name[0] != '\0';
otherlogp = otherlogp -> next) { /* for each logfile */
lf = fopenlog(otherlogp -> name, "error log", &ispipe);
if (lf != NULL) {
while(fgets(inputline, MAXLINELENGTH, lf) != NULL) {
tempstr = inputline;
wantthisone = FALSE;
if (*tempstr == '[') { /* others are non-httpd errors */
if (sscanf_olddate(++tempstr, &date, &monthno, &year, &hr, &min)
== 5) {
tempstr += 24;
if (*tempstr == ']') {
if (*(++tempstr) == ' ') {
tempstr++;
wantthisone = TRUE;
}
}
}
}
if (wantthisone) {
if (datemaskq) {
thistimecode = timecode(date, monthno, year, hr, min);
wantthisone = thistimecode > fromtime.code &&
thistimecode < totime.code;
}
if (wantthisone) {
strcpy(errstr, tempstr);
adderr(errstr);
}
}
}
fcloselog(lf, otherlogp -> name, "error log", ispipe);
}
}
}
/* now start final accounting */
tempint = 0;
for (i = 0; statusnos[i] <= 304; i++) {
if (statusnos[i] <= 299 || statusnos[i] == 304)
tempint += status[i];
}
if (tempint == 0) { /* no logfile successes */
if (cachereqs == 0) { /* no cached successes either */
mq = OFF;
dq = OFF;
Dq = OFF;
Wq = OFF;
hq = OFF;
Hq = OFF;
q7 = OFF;
}
oq = OFF;
iq = OFF;
rq = OFF;
Sq = OFF;
cq = OFF;
}
else { /* there are things to report from the main logfile */
tempint = cachereqs7;
if (tempint == 0) {
for (i = 0; i < NO_STATUS; i++)
tempint += status7[i];
}
if (tempint == 0)
q7 = OFF; /* just total_bytes no good in case (!byq) */
/* Next do aliasing of hosts */
if (sq == ON && !hostmaskq) /* if hmq, the aliasing has been done */
allaliases(hosthead, HOSTHASHSIZE, &no_hosts, &no_hosts7,
&no_new_hosts7, 'S');
/* Now the domain report. This is now easy because all the hostnames
are already aliased etc. */
if (oq && sq == ON) {
onlist = 0; /* the list of files we are on */
hostp = hosthead[0]; /* starting at list 0 */
for ( ; onlist < HOSTHASHSIZE; hostp = hostnextp) {
/* run through hosts */
if (hostp -> name == NULL) { /* then finished this list */
hostnextp = hosthead[++onlist]; /* so start the next list */
}
else {
strcpy(hostn, hostp -> name);
domhashadd(hostn, hostp -> reqs, hostp -> bytes);
hostnextp = hostp -> next;
}
}
}
/* Now for aliasing filenames. */
if ((rq || iq) && !filemaskq) /* again, if fmq, aliasing has been done */
allaliases(urlhead, URLHASHSIZE, &no_urls, &no_urls7, &tempint, 'r');
/* Now the directory report. */
if (iq) {
onlist = 0; /* the list of files we are on */
urlp = urlhead[0]; /* starting at list 0 */
for ( ; onlist < URLHASHSIZE; urlp = urlnextp) {
/* run through files */
if (urlp -> name == NULL) { /* then finished this list */
urlnextp = urlhead[++onlist]; /* so start the next list */
}
else {
strcpy(filename, urlp -> name);
urltodir(filename);
hashadd(dirhead, DIRHASHSIZE, filename, urlp -> reqs,
urlp -> bytes, urlp -> last7, &tempint, &tempint,
&tempint, ON);
urlnextp = urlp -> next;
}
}
}
/* Now aliasing referers */
if (fq && !refmaskq) /* if rmq, aliasing has been done in addref() */
allaliases(refhead, REFHASHSIZE, &tempint, &tempint, &tempint, 'f');
/*** now for the checking and sorting ***/
if (rq) {
urlsorthead = gensort(urlhead, URLHASHSIZE, reqsortby, urlminreqstr,
urlminbytestr, reqtype == PAGES, OFF,
&url_max_reqs, &url_max_bytes, &tempint);
if (urlsorthead -> name == NULL)
rq = OFF;
}
if (iq) {
dirsorthead = gensort(dirhead, DIRHASHSIZE, dirsortby, dirminreqstr,
dirminbytestr, OFF, OFF, &dir_max_reqs,
&dir_max_bytes, &tempint);
if (dirsorthead -> name == NULL)
iq = OFF;
}
if (Sq) {
hostsorthead = gensort(hosthead, HOSTHASHSIZE, hostsortby, hostminreqstr,
hostminbytestr, OFF, hostsortby == ALPHABETICAL,
&host_max_reqs, &host_max_bytes,
&host_max_length);
if (hostsorthead -> name == NULL)
Sq = OFF;
}
if (oq) {
firstdom = domsort();
if (onumber == 0)
oq = OFF;
else
subdomsort();
}
} /* end else (there are things to report from the main logfile) */
if (fq) {
refsorthead = gensort(refhead, REFHASHSIZE, refsortby, refminreqstr,
refminbytestr, OFF, OFF, &ref_max_reqs,
&ref_max_bytes, &tempint);
if (refsorthead -> name == NULL)
fq = OFF;
}
if (bq) {
browsorthead = gensort(browhead, BROWHASHSIZE, browsortby, browminreqstr,
browminbytestr, OFF, OFF,
&brow_max_reqs, &brow_max_bytes, &tempint);
if (browsorthead -> name == NULL)
bq = OFF;
}
if (Bq) {
fullbrowsorthead = gensort(fullbrowhead, FULLBROWHASHSIZE,
fullbrowsortby, fullbrowminreqstr,
fullbrowminbytestr, OFF, OFF,
&fullbrow_max_reqs, &fullbrow_max_bytes,
&tempint);
if (fullbrowsorthead -> name == NULL)
Bq = OFF;
}
if (eq)
errsort(errorder);
/*** Finally, do all the output ***/
output(urlsorthead, dirsorthead, hostsorthead, firstdom, refsorthead,
browsorthead, fullbrowsorthead, errorder);
return(OK);
}
These are the contents of the former NiCE NeXT User Group NeXTSTEP/OpenStep software archive, currently hosted by Netfuture.ch.