/************************************************************************************************* * CGI script to search an index * Copyright (C) 2003-2005 Mikio Hirabayashi * This file is part of Estraier, a personal full-text search system. * Estraier is free software; you can redistribute it and/or modify it under the terms of the GNU * General Public License as published by the Free Software Foundation; either version 2 of the * License, or any later version. * Estraier is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU General Public License for more details. * You should have received a copy of the GNU General Public License along with Estraier; * if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, * MA 02111-1307 USA. *************************************************************************************************/ #include "estcommon.h" #define CONFFILE "estsearch.conf" /* name of the configuration file */ #define ENCODING "UTF-8" /* encoding of the page */ #define SAVENAME "estresult.html" /* name of the file to be saved */ #define RDATAMAX 262144 /* max size of data to read */ #define RESROOM 8 /* ratio of hits for max */ #define UNITINC 4 /* unit increasing ratio of search */ #define RELVECNUM 32 /* number of dimension of score vector */ #define URIFLEN 92 /* fixed length of a shown URI */ #define EMCLASSNUM 6 /* number of classes for em elements */ typedef struct { /* type of structure for a cluster */ CBLIST *ids; /* list for ID numbers */ CBMAP *scores; /* map for scores */ int vector[RELVECNUM]; /* vector for calculating similarity */ int point; /* total point */ } CLUSTER; typedef struct { /* type of structure to sort scores */ const char *kbuf; /* pointer to the string */ int ksiz; /* size of the string */ int point; /* total point */ } SCWORD; /* global variables for configurations */ const char *scriptname = NULL; /* name of the script */ const char *indexname = NULL; /* name of the index */ const char *tmplfile = NULL; /* path of the template file */ const char *topfile = NULL; /* path of the top page file */ const char *prefix = NULL; /* prefix of the URI of each document */ const char *suffix = NULL; /* suffix of the URI of each document */ const CBLIST *repllist = NULL; /* list of expressions for URI replacement */ const char *diridx = NULL; /* name of the index file of a directory */ int decuri = FALSE; /* whether the URI of each document is decoded */ int boolunit = 0; /* unit number of boolean search */ int relkeys = 0; /* number of words used with relational search */ int defmax = 0; /* number of shown documents by default */ int reevmax = 0; /* max number of candidates of a regular expression */ int showkeys = 0; /* number of shown keywords of each document */ int sumall = 0; /* number of all words in summary */ int sumtop = 0; /* number of words at the top of summary */ int sumwidth = 0; /* number of words near each keyword in summary */ int clustunit = 0; /* unit number of document clustering */ int clustkeys = 0; /* number of words per shown cluster */ const char *logfile = NULL; /* path of the log file for input search conditions */ const char *ldocs = NULL; /* label shared among all documents */ const char *lperpage = NULL; /* label of the select box for showing number */ const char *lclshow = NULL; /* label of the select box for document clustering */ const char *ldrep = NULL; /* label of the select box for directory rep level */ const char *lsortsc = NULL; /* label of the option of sorting by score */ const char *lsortdt = NULL; /* label of the option of sorting by date */ const char *lreverse = NULL; /* label of the option of reverse sorting */ const char *lexasis = NULL; /* label of the option of as-is expressions */ const char *lexwild = NULL; /* label of the option of expressions with wild cards */ const char *lexregex = NULL; /* label of the option of regular expressions */ const char *lsubmit = NULL; /* label of the submit button to search */ const char *lresult = NULL; /* label of the result */ const char *lclusters = NULL; /* label of the document clusters */ const char *lhits = NULL; /* label of suffix of the hit number */ const char *lmore = NULL; /* label of the link to search thoroughly */ const char *lrelto = NULL; /* label of prefix of the seed document */ const char *ldetail = NULL; /* label of a link to show detail */ const char *lrelated = NULL; /* label of a link to search related documents */ const char *lprev = NULL; /* label of the link to go to the previous page */ const char *lnext = NULL; /* label of the link to go to the next page */ const char *lidnum = NULL; /* label for the attribute of ID number */ const char *luri = NULL; /* label for the attribute of URI */ const char *ltitle = NULL; /* label for the attribute of title */ const char *lauthor = NULL; /* label for the attribute of author */ const char *lrcpt = NULL; /* label for the attribute of recipient */ const char *lmcast = NULL; /* label for the attribute of multicast */ const char *ldate = NULL; /* label for the attribute of date */ const char *ltype = NULL; /* label for the attribute of type */ const char *lenc = NULL; /* label for the attribute of encoding */ const char *lsize = NULL; /* label for the attribute of size */ const char *lkwords = NULL; /* label for the keywords */ const char *ltext = NULL; /* label for the text */ const char *enohit = NULL; /* error message when no document hits */ const char *enoscore = NULL; /* error message when the document has no score */ const char *enoword = NULL; /* error message when no effective word */ const char *altfunc = NULL; /* name of the scripting function for each document */ /* global variables for parameters */ const char *phrase = NULL; /* search phrase */ int max = 0; /* number of shown documents */ int clshow = 0; /* number of shown clusters */ int clcode = 0; /* code of the cluster for narrowing */ int clall = FALSE; /* whether to show all clusters */ int drep = 0; /* level of directory rep */ const char *sort = NULL; /* sorting order */ const char *expr = NULL; /* expression class */ int page = 0; /* numerical order of the page */ int skip = 0; /* number of elements to be skipped */ int unit = 0; /* unit number of current search */ int relid = 0; /* ID number of the seed document */ int detid = 0; /* ID number of the shown document in detail */ int tfidf = TRUE; /* whether search scores are tuned by TF-IDF */ const char *mrglbl = NULL; /* label assigned by mergers */ int showsc = FALSE; /* whether to show score expression */ const char *relsc = NULL; /* score map of seed document */ /* other global variables */ double ustime = 0.0; /* start time of user processing */ double sstime = 0.0; /* start time of system processing */ ODEUM *odeum = NULL; /* handle of the index */ CURIA *scdb = NULL; /* handle of the score database */ DEPOT *dtdb = NULL; /* handle of the date database */ int tabidx = 0; /* counter of tab indexes */ /* function prototypes */ int main(int argc, char **argv); const char *skiplabel(const char *str); CBMAP *getparams(void); void showerror(const char *msg); void xmlprintf(const char *format, ...); void uriprint(const char *path, int mlen); void showform(void); void showrelated(void); void setovec(CBMAP *scores, int *vec); void settvec(CBMAP *osc, CBMAP *tsc, int *vec); void showclusters(ODPAIR *pairs, int *np); void sumupclusters(CLUSTER *clusts, int *np); int scwordcompare(const void *a, const void *b); int clustercompare(const void *a, const void *b); void showdetail(void); void showboolean(void); char *cuturi(const char *uri, int level); void showdoc(const ODDOC *doc, int show, int score, double rel, const ESTWORD *words, int wnum); void showsummary(const ODDOC *doc, const ESTWORD *words, int wnum, int detail); void showpaging(int next, int all); void showinfo(void); void showproctime(void); void showversion(void); void outputlog(void); /* main routine */ int main(int argc, char **argv){ CBLIST *lines, *rlt, *elems; CBMAP *params; const char *tmp; char *esc, *tmpltext, *toptext, path[ESTPATHBUFSIZ], *pname, *pbuf; int i, j, psiz; /* set configurations */ estputenv("LANG", ESTLOCALE); estputenv("LC_ALL", ESTLOCALE); cbstdiobin(); scriptname = argv[0]; if((tmp = getenv("SCRIPT_NAME")) != NULL) scriptname = tmp; if((tmp = strrchr(scriptname, '/')) != NULL) scriptname = tmp + 1; if(!(lines = cbreadlines(CONFFILE))) showerror("the configuration file is missing."); cbglobalgc(lines, (void (*)(void *))cblistclose); rlt = cblistopen(); cbglobalgc(rlt, (void (*)(void *))cblistclose); repllist = rlt; for(i = 0; i < cblistnum(lines); i++){ tmp = cblistval(lines, i, NULL); if(cbstrfwimatch(tmp, "indexname:")){ indexname = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "tmplfile:")){ tmplfile = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "topfile:")){ topfile = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "prefix:")){ prefix = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "suffix:")){ suffix = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "replace:")){ cblistpush(rlt, skiplabel(tmp), -1); } else if(cbstrfwimatch(tmp, "diridx:")){ diridx = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "decuri:")){ if(!cbstricmp(skiplabel(tmp), "true")) decuri = TRUE; } else if(cbstrfwimatch(tmp, "boolunit:")){ boolunit = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "relkeys:")){ relkeys = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "defmax:")){ defmax = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "reevmax:")){ reevmax = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "showkeys:")){ showkeys = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "sumall:")){ sumall = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "sumtop:")){ sumtop = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "sumwidth:")){ sumwidth = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "clustunit:")){ clustunit = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "clustkeys:")){ clustkeys = atoi(skiplabel(tmp)); } else if(cbstrfwimatch(tmp, "logfile:")){ logfile = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldocs:")){ ldocs = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lperpage:")){ lperpage = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lclshow:")){ lclshow = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldrep:")){ ldrep = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsortsc:")){ lsortsc = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsortdt:")){ lsortdt = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lreverse:")){ lreverse = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lexasis:")){ lexasis = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lexwild:")){ lexwild = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lexregex:")){ lexregex = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsubmit:")){ lsubmit = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lresult:")){ lresult = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lclusters:")){ lclusters = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lhits:")){ lhits = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lmore:")){ lmore = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lrelto:")){ lrelto = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldetail:")){ ldetail = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lrelated:")){ lrelated = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lprev:")){ lprev = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lnext:")){ lnext = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lidnum:")){ lidnum = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "luri:")){ luri = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ltitle:")){ ltitle = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lauthor:")){ lauthor = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lrcpt:")){ lrcpt = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lmcast:")){ lmcast = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ldate:")){ ldate = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ltype:")){ ltype = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lsize:")){ lsize = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lkwords:")){ lkwords = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "ltext:")){ ltext = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "lenc:")){ lenc = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "enohit:")){ enohit = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "enoscore:")){ enoscore = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "enoword:")){ enoword = skiplabel(tmp); } else if(cbstrfwimatch(tmp, "altfunc:")){ altfunc = skiplabel(tmp); } } if(!indexname) showerror("indexname is undefined."); if(!tmplfile) showerror("tmplfile is undefined."); if(!topfile) showerror("topfile is undefined."); if(!prefix) prefix = ""; if(!suffix) suffix = ""; if(!diridx) diridx = ""; if(boolunit < 1) showerror("boolunit is undefined."); if(relkeys < 1) showerror("relkeys is undefined."); if(defmax < 1) showerror("defmax is undefined."); if(reevmax < 1) showerror("reevmax is undefined."); if(showkeys < 1) showerror("showkeys is undefined."); if(sumall < 1) showerror("sumall is undefined."); if(sumtop < 1) showerror("sumtop is undefined."); if(sumwidth < 1) showerror("sumwidth is undefined."); if(!logfile) logfile = ""; if(!ldocs) ldocs = ""; if(!lperpage) showerror("lperpage is undefined."); if(!lclshow) showerror("lclshow is undefined."); if(!ldrep) showerror("ldrep is undefined."); if(!lsortsc) showerror("lsortsc is undefined."); if(!lsortdt) showerror("lsortdt is undefined."); if(!lreverse) showerror("lreverse is undefined."); if(!lexasis) showerror("lexasis is undefined."); if(!lexwild) showerror("lexwild is undefined."); if(!lexregex) showerror("lexregex is undefined."); if(!lsubmit) showerror("lsubmit is undefined."); if(!lresult) showerror("lresult is undefined."); if(!lclusters) showerror("lclusters is undefined."); if(!lhits) showerror("lhits is undefined."); if(!lmore) showerror("lmore is undefined."); if(!lrelto) showerror("lrelto is undefined."); if(!ldetail) showerror("ldetail is undefined."); if(!lrelated) showerror("lrelated is undefined."); if(!lprev) showerror("lprev is undefined."); if(!lnext) showerror("lnext is undefined."); if(!lidnum) showerror("lidnum is undefined."); if(!luri) showerror("luri is undefined."); if(!ltitle) showerror("ltitle is undefined."); if(!lauthor) showerror("lauthor is undefined."); if(!lrcpt) showerror("lrcpt is undefined."); if(!lmcast) showerror("lmcast is undefined."); if(!ldate) showerror("ldate is undefined."); if(!ltype) showerror("ltype is undefined."); if(!lenc) showerror("lenc is undefined."); if(!lsize) showerror("lsize is undefined."); if(!lkwords) showerror("lkwords is undefined."); if(!ltext) showerror("ltext is undefined."); if(!enohit) showerror("enohit is undefined."); if(!enoscore) showerror("enoscore is undefined."); if(!enoword) showerror("enoword is undefined."); if(!altfunc) altfunc = ""; /* read other files */ if(!(tmpltext = cbreadfile(tmplfile, NULL))) showerror("the template file is missing."); cbglobalgc(tmpltext, free); if(!(toptext = cbreadfile(topfile, NULL))) showerror("the top page file is missing."); cbglobalgc(toptext, free); /* read parameters */ params = getparams(); cbglobalgc(params, (void (*)(void *))cbmapclose); phrase = cbmapget(params, "phrase", -1, NULL); if((tmp = cbmapget(params, "max", -1, NULL)) != NULL) max = atoi(tmp); if((tmp = cbmapget(params, "clshow", -1, NULL)) != NULL) clshow = atoi(tmp); if((tmp = cbmapget(params, "clcode", -1, NULL)) != NULL) clcode = atoi(tmp); if((tmp = cbmapget(params, "clall", -1, NULL)) != NULL && !strcmp(tmp, "true")) clall = TRUE; if((tmp = cbmapget(params, "drep", -1, NULL)) != NULL) drep = atoi(tmp); sort = cbmapget(params, "sort", -1, NULL); expr = cbmapget(params, "expr", -1, NULL); if((tmp = cbmapget(params, "page", -1, NULL)) != NULL) page = atoi(tmp); if((tmp = cbmapget(params, "skip", -1, NULL)) != NULL) skip = atoi(tmp); if((tmp = cbmapget(params, "unit", -1, NULL)) != NULL) unit = atoi(tmp); if((tmp = cbmapget(params, "relid", -1, NULL)) != NULL) relid = atoi(tmp); if((tmp = cbmapget(params, "detid", -1, NULL)) != NULL) detid = atoi(tmp); if((tmp = cbmapget(params, "tfidf", -1, NULL)) != NULL && !strcmp(tmp, "false")) tfidf = FALSE; mrglbl = cbmapget(params, "mrglbl", -1, NULL); if((tmp = cbmapget(params, "showsc", -1, NULL)) != NULL && !strcmp(tmp, "true")) showsc = TRUE; relsc = cbmapget(params, "relsc", -1, NULL); if(!phrase) phrase = ""; if(max < 1) max = defmax; if(clshow < 0) clshow = 0; if(clcode < 0) clcode = 0; if(drep < 0) drep = 0; if(!sort) sort = "score"; if(!expr) expr = "asis"; if(page < 1) page = 1; if(skip < 1) skip = 0; if(unit < boolunit) unit = boolunit; if(relid < 1) relid = 0; if(detid < 1) detid = 0; if(!mrglbl) mrglbl = ""; if(!relsc) relsc = ""; if(drep > 0) skip = 0; if(relid > 0 || relsc[0] != '\0'){ sort = "score"; expr = "asis"; } if((tmp = cbmapget(params, "enc", -1, NULL)) != NULL){ if((esc = cbiconv(phrase, -1, tmp, ENCODING, NULL, NULL)) != NULL){ cbglobalgc(esc, free); phrase = esc; } if((esc = cbiconv(mrglbl, -1, tmp, ENCODING, NULL, NULL)) != NULL){ cbglobalgc(esc, free); mrglbl = esc; } if((esc = cbiconv(relsc, -1, tmp, ENCODING, NULL, NULL)) != NULL){ cbglobalgc(esc, free); relsc = esc; } } if(cbstrfwimatch(phrase, "[related] ")){ relid = atoi(strchr(phrase, ' ') + 1); phrase = ""; sort = "score"; expr = "asis"; } if(cbstrfwimatch(phrase, "[detail] ")){ detid = atoi(strchr(phrase, ' ') + 1); phrase = ""; } if(cbstrfwimatch(phrase, "[relsc] ")){ esc = cburldecode(strchr(phrase, ' ') + 1, NULL); cbglobalgc(esc, free); relsc = esc; phrase = ""; sort = "score"; expr = "asis"; } /* open the database */ if(!(odeum = odopen(indexname, OD_OREADER))) showerror("the index is missing or broken."); cbglobalgc(odeum, (void (*)(void *))odclose); sprintf(path, "%s%c%s", indexname, ESTPATHCHR, ESTSCDBNAME); if((scdb = cropen(path, CR_OREADER, -1, -1)) != NULL) cbglobalgc(scdb, (void (*)(void *))crclose); sprintf(path, "%s%c%s", indexname, ESTPATHCHR, ESTDTDBNAME); if((dtdb = dpopen(path, DP_OREADER, -1)) != NULL) cbglobalgc(dtdb, (void (*)(void *))dpclose); /* initialize the timer */ cbproctime(&ustime, &sstime); /* send HTTP headers */ printf("Cache-Control: no-cache, must-revalidate, no-transform\r\n"); printf("Pragma: no-cache\r\n"); printf("Content-Disposition: inline; filename=%s\r\n", SAVENAME); printf("Content-Type: text/html; charset=%s\r\n", ENCODING); printf("\r\n"); /* parse the template */ elems = cbxmlbreak(tmpltext, FALSE); cbglobalgc(elems, (void (*)(void *))cblistclose); for(i = 0; i < cblistnum(elems); i++){ tmp = cblistval(elems, i, NULL); if(!strcmp(tmp, "")){ printf("
%@
\n", enoscore); return; } scores = cbmapopenex(RELVECNUM * 2); lines = cbsplit(relsc, -1, "\n"); for(i = 0; i < cblistnum(lines); i++){ word = cblistval(lines, i, &wsiz); if((ep = strchr(word, '\t')) != NULL){ cbmapput(scores, word, ep - word, ep + 1, -1, FALSE); } } cblistclose(lines); } else { if(!(doc = odgetbyid(odeum, relid))){ xmlprintf("%@
\n", enohit); return; } if(!scdb || !(mbuf = crget(scdb, (char *)&relid, sizeof(int), 0, -1, &msiz))){ xmlprintf("%@
\n", enoscore); oddocclose(doc); return; } scores = cbmapload(mbuf, msiz); } /* search for the targets */ mywords = cbmapkeys(scores); myphrase = cbdatumopen("", 0); for(i = 0; i < relkeys && i < cblistnum(mywords); i++){ word = cblistval(mywords, i, &wsiz); if(i > 0) cbdatumcat(myphrase, " [OR] ", -1); cbdatumcat(myphrase, word, wsiz); } words = estsearchwords(cbdatumptr(myphrase), &wnum, FALSE); pairs = estsearch(odeum, words, wnum, unit, tfidf, &pnum, &lnum, FALSE, FALSE, 0); /* get relational order */ setovec(scores, ovec); for(i = 0; i < pnum; i++){ if((tmbuf = crget(scdb, (char *)&(pairs[i].id), sizeof(int), 0, -1, &tmsiz)) != NULL){ tsc = cbmapload(tmbuf, tmsiz); free(tmbuf); } else { tsc = cbmapopenex(1); } settvec(scores, tsc, tvec); cbmapclose(tsc); pairs[i].score = odvectorcosine(ovec, tvec, RELVECNUM) * 10000; if(pairs[i].score >= 9999) pairs[i].score = 10000; } odpairssort(pairs, pnum); for(i = 0; i < pnum; i++){ if(pairs[i].score < 1){ pnum = i; break; } } /* show header of result */ if(doc){ uri = oddocuri(doc); ruri = oddocgetattr(doc, "realuri"); if(ruri){ ubuf = cbmemdup(ruri, -1); } else { if(cbstrfwmatch(uri, "./")) uri += 2; ubuf = decuri ? cburldecode(uri, NULL) : cbmemdup(uri, -1); ulen = strlen(diridx); if(ulen > 0 && cbstrbwmatch(ubuf, diridx)){ ulen = strlen(ubuf) - ulen; if(ulen < 1 || ubuf[ulen-1] == '/') ubuf[ulen] = '\0'; } } } else { ubuf = cbmemdup("", 0); ulen = 0; } xmlprintf("%@
\n", enohit); /* show clusters */ if(scdb && clustunit > 0 && clshow > 0 && clustkeys > 0 && pnum > 0) showclusters(pairs, &pnum); /* show each document */ show = 0; step = 0; dirs = NULL; if(drep > 0) dirs = cbmapopen(); for(i = skip; i < pnum && show < max; i++){ if(!(tdoc = odgetbyid(odeum, pairs[i].id))) continue; if(dirs){ tmp = cuturi(oddocuri(tdoc), drep); if(cbmapget(dirs, tmp, -1, NULL)){ free(tmp); oddocclose(tdoc); continue; } cbmapput(dirs, tmp, -1, "", 0, FALSE); free(tmp); } if(skip < 1 && page > 1 && step < (page - 1) * max){ oddocclose(tdoc); step++; continue; } show++; showdoc(tdoc, show, -1, pairs[i].score / 100.0, NULL, 0); oddocclose(tdoc); } showpaging(i, pnum); /* release resouces */ if(dirs) cbmapclose(dirs); free(ubuf); free(pairs); estfreewords(words, wnum); cbdatumclose(myphrase); cblistclose(mywords); cbmapclose(scores); if(mbuf) free(mbuf); if(doc) oddocclose(doc); } /* set the original score vector */ void setovec(CBMAP *scores, int *vec){ const char *kbuf; int i, ksiz; cbmapiterinit(scores); for(i = 0; i < RELVECNUM; i++){ if((kbuf = cbmapiternext(scores, &ksiz)) != NULL){ vec[i] = atoi(cbmapget(scores, kbuf, ksiz, NULL)); } else { vec[i] = 0; } } } /* set the target score vector */ void settvec(CBMAP *osc, CBMAP *tsc, int *vec){ const char *kbuf, *vbuf; int i, ksiz; cbmapiterinit(osc); for(i = 0; i < RELVECNUM; i++){ if((kbuf = cbmapiternext(osc, &ksiz)) != NULL){ vbuf = cbmapget(tsc, kbuf, ksiz, NULL); vec[i] = vbuf ? atoi(vbuf) : 0; } else { vec[i] = 0; } } } /* show clusters */ void showclusters(ODPAIR *pairs, int *np){ CLUSTER *clusts; CBMAP **tails; ODPAIR *tpairs; const char *kbuf; char *mbuf, numbuf[ESTNUMBUFSIZ]; int i, j, pnum, blnum, clnum, tnum, msiz, ssum, mi, ovec[RELVECNUM], tvec[RELVECNUM]; double msc, sc; pnum = *np; if(pnum > clustunit) pnum = clustunit; blnum = *np; if(blnum > boolunit) blnum = boolunit; clusts = cbmalloc(pnum * sizeof(CLUSTER) + 1); clnum = 0; tails = cbmalloc(blnum * sizeof(CBMAP *) + 1); tpairs = cbmalloc(blnum * sizeof(ODPAIR) + 1); tnum = 0; for(i = 0; i < blnum; i++){ if(i < pnum){ if(!(mbuf = crget(scdb, (char *)&(pairs[i].id), sizeof(int), 0, -1, &msiz))) continue; clusts[clnum].ids = cblistopen(); sprintf(numbuf, "%d:%d", pairs[i].id, pairs[i].score); cblistpush(clusts[clnum].ids, numbuf, -1); clusts[clnum].scores = cbmapload(mbuf, msiz); free(mbuf); setovec(clusts[clnum].scores, clusts[clnum].vector); clusts[clnum].point = pairs[i].score; clnum++; } else { if(!(mbuf = crget(scdb, (char *)&(pairs[i].id), sizeof(int), 0, -1, &msiz))) continue; tails[tnum] = cbmapload(mbuf, msiz); free(mbuf); tpairs[tnum].id = pairs[i].id; tpairs[tnum].score = pairs[i].score; tnum++; } } sumupclusters(clusts, &clnum); for(i = 0; i < tnum; i++){ mi = -1; msc = 0.0; setovec(tails[i], ovec); for(j = 0; j < clnum; j++){ settvec(tails[i], clusts[j].scores, tvec); sc = odvectorcosine(ovec, tvec, RELVECNUM); if(mi < 0 || sc > msc){ mi = j; msc = sc; } } if(!clall && msc < 0.55) continue; if(msc < 0.25) continue; sprintf(numbuf, "%d:%d", tpairs[i].id, tpairs[i].score); cblistpush(clusts[mi].ids, numbuf, -1); } ssum = 0; for(i = 0; i < clnum && (clall || i < clshow); i++){ ssum += cblistnum(clusts[i].ids); } xmlprintf("%@
\n", enohit); estfreewords(words, wnum); return; } id = oddocid(doc); uri = oddocuri(doc); ruri = oddocgetattr(doc, "realuri"); title = oddocgetattr(doc, "title"); author = oddocgetattr(doc, "author"); rcpt = oddocgetattr(doc, "recipient"); mcast = oddocgetattr(doc, "multicast"); date = oddocgetattr(doc, "date"); type = oddocgetattr(doc, "type"); enc = oddocgetattr(doc, "encoding"); size = oddocgetattr(doc, "size"); if(ruri){ ubuf = cbmemdup(ruri, -1); } else { if(cbstrfwmatch(uri, "./")) uri += 2; ubuf = decuri ? cburldecode(uri, NULL) : cbmemdup(uri, -1); ulen = strlen(diridx); if(ulen > 0 && cbstrbwmatch(ubuf, diridx)){ ulen = strlen(ubuf) - ulen; if(ulen < 1 || ubuf[ulen-1] == '/') ubuf[ulen] = '\0'; } } /* show informations */ xmlprintf("%@
\n", enoword); estfreewords(words, wnum); return; } /* get result of search */ pairs = NULL; myunit = unit; if(drep > 0) myunit *= drep + 1; if(strcmp(sort, "score")) myunit = INT_MAX; do { if(pairs){ free(pairs); myunit *= UNITINC; } pairs = estsearch(odeum, words, wnum, myunit, tfidf, &pnum, &lnum, !strcmp(expr, "regex"), !strcmp(expr, "wild"), reevmax); } while(pnum < max * RESROOM && wnum > 1 && lnum > 0); if(!strcmp(sort, "date") || !strcmp(sort, "r-date")){ for(i = 0; i < pnum; i++){ date = 0; if(!dtdb){ if((doc = odgetbyid(odeum, pairs[i].id)) != NULL){ if((dstr = oddocgetattr(doc, "date")) != NULL) date = eststrmktime(dstr); oddocclose(doc); } } else { dpgetwb(dtdb, (char *)&(pairs[i].id), sizeof(int), 0, sizeof(int), (char *)&date); } pairs[i].score = date; } odpairssort(pairs, pnum); } if(cbstrfwmatch(sort, "r-")){ for(i = 0; i < pnum / 2; i++){ pswap = pairs[i]; pairs[i] = pairs[pnum-i-1]; pairs[pnum-i-1] = pswap; } } /* show header of result */ xmlprintf("%@
\n", enohit); /* show clusters */ if(scdb && clustunit > 0 && clshow > 0 && clustkeys > 0 && pnum > 0) showclusters(pairs, &pnum); /* show each document */ show = 0; step = 0; dirs = NULL; if(drep > 0) dirs = cbmapopen(); for(i = skip; i < pnum && show < max; i++){ if(!(doc = odgetbyid(odeum, pairs[i].id))) continue; if(dirs){ tmp = cuturi(oddocuri(doc), drep); if(cbmapget(dirs, tmp, -1, NULL)){ free(tmp); oddocclose(doc); continue; } cbmapput(dirs, tmp, -1, "", 0, FALSE); free(tmp); } if(skip < 1 && page > 1 && step < (page - 1) * max){ oddocclose(doc); step++; continue; } show++; showdoc(doc, show, pairs[i].score, -1, words, wnum); oddocclose(doc); } showpaging(i, pnum); /* release resouces */ if(dirs) cbmapclose(dirs); free(pairs); estfreewords(words, wnum); } /* cut a uri to a directory */ char *cuturi(const char *uri, int level){ char *buf, *pv; int i; buf = cbmemdup(uri, -1); for(i = 0; i < level; i++){ if(!(pv = strrchr(buf, '/'))) break; *pv = '\0'; } return buf; } /* show information of a document */ void showdoc(const ODDOC *doc, int show, int score, double rel, const ESTWORD *words, int wnum){ CBMAP *scores; CBDATUM *scbuf; const char *uri, *ruri, *title, *author, *date, *type, *enc, *size, *label, *key, *val; char *ubuf, numbuf[ESTNUMBUFSIZ], *tmp; int id, ulen, tsiz, ksiz, vsiz, top; id = oddocid(doc); uri = oddocuri(doc); ruri = oddocgetattr(doc, "realuri"); title = oddocgetattr(doc, "title"); author = oddocgetattr(doc, "author"); date = oddocgetattr(doc, "date"); type = oddocgetattr(doc, "type"); enc = oddocgetattr(doc, "encoding"); size = oddocgetattr(doc, "size"); label = ldocs[0] != '\0' ? ldocs : mrglbl; if(ruri){ ubuf = cbmemdup(ruri, -1); } else { if(cbstrfwmatch(uri, "./")) uri += 2; ubuf = decuri ? cburldecode(uri, NULL) : cbmemdup(uri, -1); ulen = strlen(diridx); if(ulen > 0 && cbstrbwmatch(ubuf, diridx)){ ulen = strlen(ubuf) - ulen; if(ulen < 1 || ubuf[ulen-1] == '/') ubuf[ulen] = '\0'; } } if(!title || strlen(title) < 1) title = "(untitled)"; xmlprintf("... ");
/* show phrases around search words */
tb = wnum > 0 && wnum < 4 ? (sumwidth / 2) / wnum : 0;
pv = i;
while(i < lnum){
if(cbmaprnum(kmap) > 0 && cbmaprnum(tmap) < 1){
cbmapclose(tmap);
tmap = cbmapdup(kmap);
}
normal = cblistval(nwords, i, &nwsiz);
asis = cblistval(awords, i, &awsiz);
if(awsiz > 0 && cbmapget(tmap, normal, nwsiz, NULL)){
cbmapout(tmap, normal, nwsiz);
bi = i - (sumwidth + tb) / 2;
bi = bi > pv ? bi : pv;
lshow = 0;
cjk = FALSE;
for(j = bi; j < lnum && j <= bi + sumwidth + tb; j++){
normal = cblistval(nwords, j, &nwsiz);
asis = cblistval(awords, j, &awsiz);
if(awsiz < 1) continue;
if(lshow > 0 && (!cjk || *(unsigned char *)asis < 0xe0)) xmlprintf(" ");
if(nwsiz > 0 && (ibuf = cbmapget(kmap, normal, nwsiz, NULL)) != NULL){
xmlprintf("%@", *(int *)ibuf % EMCLASSNUM, asis);
} else {
xmlprintf("%@", asis);
}
cjk = *(unsigned char *)asis >= 0xe0;
gshow++;
lshow++;
}
xmlprintf(" ... ");
i = j;
pv = i;
} else {
i++;
}
if(gshow > sumall) break;
}
/* infill the left space */
if(pv < lnum - sumwidth && sumall - gshow > sumwidth){
lshow = 0;
cjk = FALSE;
for(i = pv; i < lnum && gshow <= sumall; i++){
asis = cblistval(awords, i, NULL);
if(lshow > 0 && (!cjk || *(unsigned char *)asis < 0xe0)) xmlprintf(" ");
xmlprintf("%@", asis);
cjk = *(unsigned char *)asis >= 0xe0;
gshow++;
lshow++;
}
xmlprintf(" ... ");
}
/* show keywords et cetra */
id = oddocid(doc);
issc = FALSE;
if(scdb && (mbuf = crget(scdb, (char *)&id, sizeof(int), 0, -1, &msiz))){
scores = cbmapload(mbuf, msiz);
cbmapiterinit(scores);
for(i = 0; i < showkeys && (kbuf = cbmapiternext(scores, NULL)) != NULL; i++){
if(i > 0) xmlprintf(" ");
xmlprintf("");
xmlprintf("%@",
scriptname, phrase, phrase[0] != '\0' ? "+" : "",
kbuf, max, drep, sort, expr, unit, clshow, kbuf);
xmlprintf("");
}
xmlprintf(" ... ");
cbmapclose(scores);
free(mbuf);
issc = TRUE;
}
if(detail){
xmlprintf("");
xmlprintf("%@", ldetail);
xmlprintf("");
} else {
xmlprintf("");
xmlprintf("%@",
scriptname, phrase, max, drep, sort, expr, unit, id, clshow, ldetail);
xmlprintf("");
}
if(issc){
xmlprintf(" ");
xmlprintf("");
xmlprintf("%@",
scriptname, max, drep, sort, expr, id, clshow, lrelated);
xmlprintf("");
}
/* release resources */
cbmapclose(tmap);
cbmapclose(kmap);
xmlprintf("