comparison MoinMoin/logfile/logfile.py @ 0:77665d8e2254

tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0 (automatically generated log message) imported from: moin--main--1.5--base-0
author Thomas Waldmann <tw-public@gmx.de>
date Thu, 22 Sep 2005 15:09:50 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:77665d8e2254
1 """
2 MoinMoin basic log stuff
3
4 @license: GNU GPL, see COPYING for details.
5 """
6
7 import os, codecs, errno
8 from MoinMoin import config, wikiutil
9
10 class LogError(Exception):
11 """ Base class for log errors """
12
13 class LogMissing(LogError):
14 """ Raised when the log is missing """
15
16
17 class LineBuffer:
18 """
19 Reads lines from a file
20 self.lines list of lines (Strings)
21 self.offsets list of offset for each line
22 """
23 def __init__(self, file, offset, size, forward=True):
24 """
25 @param file: open file object
26 @param offset: position in file to start from
27 @param size: aproximate number of bytes to read
28 @param forward : read from offset on or from offset-size to offset
29 @type forward: boolean
30 """
31 if forward:
32 file.seek(offset)
33 self.lines = file.readlines(size)
34 self.__calculate_offsets(offset)
35 else:
36 if offset < 2 * size:
37 begin = 0
38 else:
39 begin = offset - size
40 file.seek(begin)
41 self.lines = file.read(offset-begin).splitlines(True)
42 if begin != 0:
43 begin += len(self.lines[0])
44 self.lines = self.lines[1:]
45 # XXX check for min one line read
46 self.__calculate_offsets(begin)
47
48 # Decode lines after offset in file is calculated
49 self.lines = [unicode(line, config.charset) for line in self.lines]
50 self.len = len(self.lines)
51
52 def __calculate_offsets(self, offset):
53 """
54 @param offset: offset of the first line
55 """
56 self.offsets = map(lambda x:len(x), self.lines)
57 self.offsets.append(0)
58 i = 1
59 length = len(self.offsets)
60 tmp = offset
61 while i < length:
62 result = self.offsets[i-1] + tmp
63 tmp = self.offsets[i]
64 self.offsets[i] = result
65 i = i + 1
66 self.offsets[0] = offset
67
68
69 class LogFile:
70 """
71 .filter: function that gets the values from .parser.
72 must return True to keep it or False to remove it
73 Overwrite .parser() and .add() to customize this class to
74 special log files
75 """
76
77 def __init__(self, filename, buffer_size=65536):
78 """
79 @param filename: name of the log file
80 @param buffer_size: approx. size of one buffer in bytes
81 """
82 self.buffer_size = buffer_size
83 self.__filename = filename
84 self.filter = None
85 self.__lineno = 0
86 self.__buffer = None
87 self.__buffer1 = None
88 self.__buffer2 = None
89
90 def __iter__(self):
91 return self
92
93 def reverse(self):
94 """ @rtype: iterator
95 """
96 self.to_end()
97 while 1:
98 try:
99 result = self.previous()
100 except StopIteration:
101 return
102 yield result
103
104 def sanityCheck(self):
105 """ Check for log file write access.
106
107 TODO: os.access should not be used here.
108
109 @rtype: string (error message) or None
110 """
111 if not os.access(self.__filename, os.W_OK):
112 return "The log '%s' is not writable!" % (self.__filename,)
113 return None
114
115 def __getattr__(self, name):
116 """
117 generate some attributes when needed
118 """
119 if name=="_LogFile__rel_index":
120 # starting iteration from begin
121 self.__buffer1 = LineBuffer(self._input, 0, self.buffer_size)
122 self.__buffer2 = LineBuffer(self._input,
123 self.__buffer1.offsets[-1],
124 self.buffer_size)
125 self.__buffer = self.__buffer1
126 self.__rel_index = 0
127 return 0
128 elif name == "_input":
129 try:
130 # Open the file without codecs.open, it break our offset
131 # calculation. We decode it later.
132 # Use binary mode in order to retain \r. Otherwise the offset
133 # calculation would fail
134 self._input = file(self.__filename, "rb",)
135 except IOError:
136 raise StopIteration
137 return self._input
138 elif name == "_output":
139 self._output = codecs.open(self.__filename, 'a', config.charset)
140 try:
141 os.chmod(self.__filename, 0666 & config.umask)
142 except OSError:
143 # TODO: should not ignore errors like this!
144 pass
145 return self._output
146 else:
147 raise AttributeError(name)
148
149 def size(self):
150 """ Return log size in bytes
151
152 Return 0 if the file does not exists. Raises other OSError.
153
154 @return: size of log file in bytes
155 @rtype: Int
156 """
157 try:
158 return os.path.getsize(self.__filename)
159 except OSError, err:
160 if err.errno == errno.ENOENT:
161 return 0
162 raise
163
164 def lines(self):
165 """ Return number of lines in the log file
166
167 Return 0 if the file does not exists. Raises other OSError.
168
169 Expensive for big log files - O(n)
170
171 @return: size of log file in lines
172 @rtype: Int
173 """
174 try:
175 f = codecs.open(self.__filename, 'r')
176 try:
177 count = 0
178 for line in f:
179 count += 1
180 return count
181 finally:
182 f.close()
183 except (OSError, IOError), err:
184 if err.errno == errno.ENOENT:
185 return 0
186 raise
187
188 def date(self):
189 """ Return timestamp of log file in usecs """
190 try:
191 mtime = os.path.getmtime(self.__filename)
192 except OSError, err:
193 if err.errno == errno.ENOENT:
194 # This can happen on fresh wiki when building the index
195 # Usually the first request will create an event log
196 raise LogMissing(str(err))
197 raise
198 return wikiutil.timestamp2version(mtime)
199
200 def peek(self, lines):
201 """ What does this method do?
202
203 @param lines: number of lines, may be negative to move backward
204 moves file position by lines.
205 @return: True if moving more than (WHAT?) to the beginning and moving
206 to the end or beyond
207 @rtype: boolean
208 peek adjusts .__lineno if set
209 This function is not aware of filters!
210 """
211 self.__rel_index = self.__rel_index + lines
212 while self.__rel_index < 0:
213 if self.__buffer == self.__buffer2:
214 # change to buffer 1
215 self.__buffer = self.__buffer1
216 self.__rel_index = self.__rel_index + self.__buffer.len
217 else:
218 if self.__buffer.offsets[0] == 0:
219 # already at the beginning of the file
220 # XXX
221 self.__rel_index = 0
222 self.__lineno = 0
223 return True
224 else:
225 # load previous lines
226 self.__buffer2 = self.__buffer1
227 self.__buffer1 = LineBuffer(self._input,
228 self.__buffer2.offsets[0],
229 self.buffer_size,
230 forward=False)
231 self.__rel_index = (self.__rel_index +
232 self.__buffer1.len)
233 self.__buffer = self.__buffer1
234
235 while self.__rel_index >= self.__buffer.len:
236 if self.__buffer == self.__buffer1:
237 # change to buffer 2
238 self.__rel_index = self.__rel_index - self.__buffer.len
239 self.__buffer = self.__buffer2
240 else:
241 # try to load next buffer
242 tmpbuff = LineBuffer(self._input,
243 self.__buffer1.offsets[-1],
244 self.buffer_size)
245 if tmpbuff.len==0:
246 # end of file
247 if self.__lineno:
248 self.__lineno = (self.__lineno + lines -
249 (self.__rel_index -
250 len(self.__buffer.offsets)))
251 self.__rel_index = len(self.__buffer.offsets)
252 return True
253 # shift buffers
254 self.__buffer1 = self.__buffer2
255 self.__buffer2 = tmpbuff
256 self.__rel_index = self.__rel_index - self.__buffer1.len
257 if self.__lineno: self.__lineno += lines
258 return False
259
260 def __next(self):
261 """get next line already parsed"""
262 if self.peek(0):
263 raise StopIteration
264 result = self.parser(self.__buffer.lines[self.__rel_index])
265 self.peek(1)
266 return result
267
268 def next(self):
269 """
270 @return: next entry
271 raises StopIteration at file end
272 XXX It does not raise anything!
273 """
274 result = None
275 while result == None:
276 while result == None:
277 result = self.__next()
278 if self.filter and not self.filter(result):
279 result = None
280 return result
281
282 def __previous(self):
283 if self.peek(-1): raise StopIteration
284 return self.parser(self.__buffer.lines[self.__rel_index])
285
286 def previous(self):
287 """
288 @return: previous entry and moves file position one line back
289 raises StopIteration at file begin
290 """
291 result = None
292 while result == None:
293 while result == None:
294 result = self.__previous()
295 if self.filter and not self.filter(result):
296 result = None
297 return result
298
299 def to_begin(self):
300 """moves file position to the begin"""
301 if self.__buffer1.offsets[0] != 0:
302 self.__buffer1 = LineBuffer(self._input,
303 0,
304 self.buffer_size)
305 self.__buffer2 = LineBuffer(self._input,
306 self.__buffer1.offsets[-1],
307 self.buffer_size)
308 self.__buffer = self.__buffer1
309 self.__rel_index = 0
310 self.__lineno = 0
311
312 def to_end(self):
313 """moves file position to the end"""
314 self._input.seek(0, 2) # to end of file
315 size = self._input.tell()
316 if (not self.__buffer2) or (size>self.__buffer2.offsets[-1]):
317 self.__buffer2 = LineBuffer(self._input,
318 size,
319 self.buffer_size,
320 forward = False)
321
322 self.__buffer1 = LineBuffer(self._input,
323 self.__buffer2.offsets[0],
324 self.buffer_size,
325 forward = False)
326 self.__buffer = self.__buffer2
327 self.__rel_index = self.__buffer2.len
328 self.__lineno = None
329
330 def position(self):
331 """ Return the current file position
332
333 This can be converted into a String using back-ticks and then
334 be rebuild.
335 For this plain file implementation position is an Integer.
336 """
337 return self.__buffer.offsets[self.__rel_index]
338
339 def seek(self, position, line_no=None):
340 """ moves file position to an value formerly gotten from .position().
341 To enable line counting line_no must be provided.
342 .seek is much more efficient for moving long distances than .peek.
343 raises ValueError if position is invalid
344 """
345 if self.__buffer1.offsets[0] <= position < self.__buffer1.offsets[-1]:
346 # position is in .__buffer1
347 self.__rel_index = self.__buffer1.offsets.index(position)
348 self.__buffer = self.__buffer1
349 elif (self.__buffer2.offsets[0] <= position <
350 self.__buffer2.offsets[-1]):
351 # position is in .__buffer2
352 self.__rel_index = self.__buffer2.offsets.index(position)
353 self.__buffer = self.__buffer2
354 else:
355 # load buffers around position
356 self.__buffer1 = LineBuffer(self._input,
357 position,
358 self.buffer_size,
359 forward = False)
360 self.__buffer2 = LineBuffer(self._input,
361 position,
362 self.buffer_size)
363 self.__buffer = self.__buffer2
364 self.__rel_index = 0
365 # XXX test for valid position
366 self.__lineno = line_no
367
368 def line_no(self):
369 """@return: the current line number or None if line number is unknown"""
370 return self.__lineno
371
372 def calculate_line_no(self):
373 """ Calculate the current line number from buffer offsets
374
375 If line number is unknown it is calculated by parsing the whole file.
376 This may be expensive.
377 """
378 self._input.seek(0, 0)
379 lines = self._input.read(self.__buffer.offsets[self.__rel_index])
380 self.__lineno = len(lines.splitlines())
381 return self.__lineno
382
383 def parser(self, line):
384 """
385 @param line: line as read from file
386 @return: parsed line or None on error
387 Converts the line from file to program representation
388 This implementation uses TAB separated strings.
389 This method should be overwritten by the sub classes.
390 """
391 return line.split("\t")
392
393 def add(self, *data):
394 """
395 add line to log file
396 This implementation save the values as TAB separated strings.
397 This method should be overwritten by the sub classes.
398 """
399 line = "\t".join(data)
400 self._add(line)
401
402 def _add(self, line):
403 """
404 @param line: flat line
405 @type line: String
406 write on entry in the log file
407 """
408 if line != None:
409 if line[-1] != '\n':
410 line += '\n'
411 self._output.write(line)
412