annotate MoinMoin/support/lupy/search/phrase.py @ 0:77665d8e2254

tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0 (automatically generated log message) imported from: moin--main--1.5--base-0
author Thomas Waldmann <tw-public@gmx.de>
date Thu, 22 Sep 2005 15:09:50 +0000
parents
children
rev   line source
0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
1 # This module is part of the Lupy project and is Copyright 2003 Amir
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
2 # Bakhtiar (amir@divmod.org). This is free software; you can redistribute
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
3 # it and/or modify it under the terms of version 2.1 of the GNU Lesser
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
4 # General Public License as published by the Free Software Foundation.
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
5
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
6
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
7 from bisect import insort
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
8 from MoinMoin.support.lupy.search import term, similarity
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
9 import sys
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
10
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
11 class PhraseQuery:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
12 """A query that matches documents containing a particular
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
13 sequence of terms. This may be combined with other terms
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
14 with a L{lupy.search.boolean.BooleanQuery}.
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
15 """
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
16
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
17 def __init__(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
18 """Constructs an empty phrase query."""
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
19
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
20 self.idf = 0.0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
21 self.slop = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
22 self.terms = []
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
23 self.weight = 0.0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
24 self.boost = 1.0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
25
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
26 def add(self, term):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
27 """Adds a term to the end of the query phrase."""
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
28 if len(self.terms) == 0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
29 self.field = term.field()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
30
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
31 elif term.field() != self.field:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
32 raise Exception, 'All phrase terms must be in the same field: ' + str(term)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
33
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
34 self.terms.append(term)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
35
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
36
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
37 def getSlop(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
38 """Returns the slop. See setSlop()."""
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
39 return self.slop
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
40
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
41
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
42 def normalize(self, norm):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
43 # normalize for query
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
44 self.weight *= norm
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
45 # factor from document
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
46 self.weight *= self.idf
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
47
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
48
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
49 def scorer(self, reader):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
50 # optimize zero-term case
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
51 if len(self.terms) == 0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
52 return None
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
53
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
54 # optimize one-term case
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
55 if len(self.terms) == 1:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
56 t = self.terms[0]
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
57 docs = reader.termDocsTerm(t)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
58 if docs is None:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
59 return None
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
60 return term.TermScorer(docs, reader.normsField(t.field()), self.weight)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
61
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
62 tps = []
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
63
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
64 for t in self.terms:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
65 p = reader.termPositionsTerm(t)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
66 if p is None:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
67 # I am not sure how this is ever reached?
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
68 return None
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
69 tps.append(p)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
70
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
71 if self.slop == 0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
72 return ExactPhraseScorer(tps, reader.normsField(self.field),
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
73 self.weight)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
74 else:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
75 return SloppyPhraseScorer(tps, reader.norms(self.field),
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
76 self.weight)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
77
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
78
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
79 def sumOfSquaredWeights(self, searcher):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
80 # sum term IDFs
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
81 for term in self.terms:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
82 self.idf += similarity.idfTerm(term, searcher)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
83
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
84 self.weight = self.idf * self.boost
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
85 # square term weights
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
86 return self.weight * self.weight
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
87
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
88
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
89 def toString(self, f):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
90 """Prints a user-readable version of this query"""
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
91
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
92 buffer = ''
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
93 if not self.field == f :
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
94 buffer += f + ':'
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
95 buffer += '\\'
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
96
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
97 for term in self.terms[:-1]:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
98 buffer += term.text() + ' '
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
99
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
100 buffer += self.terms[-1].text() + '\\'
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
101
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
102 if self.slop != 0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
103 buffer += '~' + str(self.slop)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
104
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
105 if self.boost != 1.0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
106 buffer += '^' + str(self.boost)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
107
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
108 return buffer
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
109
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
110
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
111 class PhraseScorer:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
112
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
113 def __init__(self, tps, n, w):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
114 self.norms = n
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
115 self.weight = w
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
116
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
117 self.pps = [PhrasePositions(tp, i) for i, tp in enumerate(tps)]
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
118 self.pps.sort()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
119
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
120 def phraseQuery(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
121 """Subclass responsibility"""
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
122
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
123 def score(self, end):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
124 # find doc w/ all the terms
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
125 while self.pps[-1].doc < end:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
126 while self.pps[0].doc < self.pps[-1].doc:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
127 self.pps[0].advance()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
128 while self.pps[0].doc < self.pps[-1].doc:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
129 self.pps[0].advance()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
130 self.pps.append(self.pps.pop(0))
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
131 if self.pps[-1].doc >= end:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
132 return
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
133
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
134 # found doc with all terms
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
135 # check for phrase
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
136 freq = self.phraseFreq()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
137
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
138 if freq > 0.0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
139 # compute score
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
140 score = similarity.tf(freq) * self.weight
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
141 # normalize
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
142 score *= similarity.normByte(self.norms[self.pps[0].doc])
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
143 # add to results
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
144 yield (self.pps[0].doc, score)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
145 # resume scanning
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
146 self.pps[-1].advance()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
147
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
148
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
149
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
150
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
151 class ExactPhraseScorer(PhraseScorer):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
152
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
153 def phraseFreq(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
154 for pp in self.pps:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
155 pp.firstPosition()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
156 self.pps.sort()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
157 freq = 0.0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
158
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
159 init = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
160 # the 'init' bits are to simulate a do-while loop :-/
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
161 while init == 0 or self.pps[-1].nextPosition():
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
162 while self.pps[0].position < self.pps[-1].position:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
163 # scan forward in first
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
164 init2 = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
165 while init2 == 0 or self.pps[0].position < self.pps[-1].position:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
166 if not self.pps[0].nextPosition():
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
167 return freq
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
168 init2 = 1
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
169
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
170 self.pps.append(self.pps.pop(0))
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
171 # all equal: a match
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
172 freq += 1
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
173 init = 1
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
174
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
175 return freq
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
176
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
177
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
178 class PhrasePositions(object):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
179
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
180 def __init__(self, t, o):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
181 self.tp = t
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
182 self.offset = o
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
183
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
184 self.position = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
185 self.count = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
186 self.doc = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
187 self.tpiter = iter(t)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
188 self.advance()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
189
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
190
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
191 def firstPosition(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
192 self.count = self.tp.frq
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
193 self.nextPosition()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
194
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
195
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
196 def advance(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
197 """Increments to next doc"""
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
198
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
199 for doc, frq, nextPos in self.tpiter:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
200 self.doc = doc
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
201 self.frq = frq
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
202 self._nextPos = nextPos
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
203 self.position = 0
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
204 return
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
205 else:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
206 # close stream
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
207 self.tp.close()
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
208 # sentinel value
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
209 self.doc = sys.maxint
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
210 return
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
211
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
212
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
213 def nextPosition(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
214 if self.count > 0:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
215 self.count -= 1
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
216 # read subsequent positions
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
217 self.position = self._nextPos.next() - self.offset
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
218 return True
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
219 else:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
220 self.count -= 1
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
221 return False
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
222
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
223
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
224 def __repr__(self):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
225 res = '<pp>d:' + str(self.doc) + ' p:' + str(self.position) + ' o:' + str(self.offset)
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
226 return res
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
227
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
228 def __lt__(this, that):
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
229 if this.doc == that.doc:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
230 return this.position < that.position
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
231 else:
77665d8e2254 tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0
Thomas Waldmann <tw-public@gmx.de>
parents:
diff changeset
232 return this.doc < that.doc