Mercurial > moin > 1.9
changeset 4824:e5ef9d9f9eaa
Groups2009: wikidicts was moved to the datastruct.dicts.backends.wiki_dicts module.
| author | Dmitrijs Milajevs <dimazest@gmail.com> |
|---|---|
| date | Fri, 26 Jun 2009 18:13:35 +0200 |
| parents | 74216148209d |
| children | 8db07c73e972 |
| files | MoinMoin/_tests/test_wikidicts.py MoinMoin/_tests/wikiconfig_groups.py MoinMoin/datastruct/__init__.py MoinMoin/datastruct/dicts/__init__.py MoinMoin/datastruct/dicts/backends/__init__.py MoinMoin/datastruct/dicts/backends/_tests/__init__.py MoinMoin/datastruct/dicts/backends/_tests/test_wiki_dicts.py MoinMoin/datastruct/dicts/backends/wiki_dicts.py MoinMoin/events/wikidictsrescan.py MoinMoin/web/contexts.py MoinMoin/wikidicts.py |
| diffstat | 9 files changed, 254 insertions(+), 235 deletions(-) [+] |
line wrap: on
line diff
--- a/MoinMoin/_tests/test_wikidicts.py Thu Jun 25 15:07:47 2009 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,56 +0,0 @@ -# -*- coding: iso-8859-1 -*- -""" - MoinMoin - MoinMoin.wikidicts tests - - @copyright: 2003-2004 by Juergen Hermann <jh@web.de>, - 2007 by MoinMoin:ThomasWaldmann - @license: GNU GPL, see COPYING for details. -""" - -from MoinMoin import wikidicts -from MoinMoin._tests import become_trusted, create_page, nuke_page - -class TestDictDict: - - def setup_class(self): - request = self.request - become_trusted(request) - - text = ''' -Text ignored - * list items ignored - * Second level list ignored - First:: first item - text with spaces:: second item - -Empty lines ignored, so is this text -Next line has key with empty value - Empty string::\x20 - Last:: last item -''' - - create_page(request, u'SomeTestDict', text) - - def teardown_class(self): - become_trusted(self.request) - nuke_page(self.request, u'SomeTestDict') - - def test_getitem(self): - dicts = self.request.dicts - - some_test_dict = dicts['SomeTestDict'] - assert len(some_test_dict) == 4 - assert some_test_dict['First'] == 'first item' - assert some_test_dict['text with spaces'] == 'second item' - assert some_test_dict['Empty string'] == '' # XXX fails if trailing blank is missing - assert some_test_dict['Last'] == 'last item' - - def test_contains(self): - dicts = self.request.dicts - - assert u'SomeTestDict' in dicts - assert u'SomeNotExistingDict' not in dicts - - -coverage_modules = ['MoinMoin.wikidicts'] -
--- a/MoinMoin/_tests/wikiconfig_groups.py Thu Jun 25 15:07:47 2009 +0200 +++ b/MoinMoin/_tests/wikiconfig_groups.py Fri Jun 26 18:13:35 2009 +0200 @@ -8,7 +8,12 @@ from wikiconfig import LocalConfig from MoinMoin.groups import WikiGroups +from MoinMoin.datastruct.dicts import WikiDicts class Config(LocalConfig): group_manager_init = lambda self, request: WikiGroups(request) + def dict_manager_init(self, request): + dicts = WikiDicts(request) + dicts.load_dicts() + return dicts
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MoinMoin/datastruct/__init__.py Fri Jun 26 18:13:35 2009 +0200 @@ -0,0 +1,8 @@ +# -*- coding: iso-8859-1 -*- +""" +MoinMoin - datastruct (groups, dicts) support. + +@copyright: 2009 MoinMoin:DmitrijsMilajevs +@license: GPL, see COPYING for details +""" +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MoinMoin/datastruct/dicts/__init__.py Fri Jun 26 18:13:35 2009 +0200 @@ -0,0 +1,10 @@ +# -*- coding: iso-8859-1 -*- +""" +MoinMoin - dicts support. + +@copyright: 2009 MoinMoin:DmitrijsMilajevs +@license: GPL, see COPYING for details +""" + +from MoinMoin.datastruct.dicts.backends.wiki_dicts import WikiDicts +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MoinMoin/datastruct/dicts/backends/_tests/test_wiki_dicts.py Fri Jun 26 18:13:35 2009 +0200 @@ -0,0 +1,56 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - MoinMoin.datastruct.dicts.backends.wiki_dicts tests + + @copyright: 2003-2004 by Juergen Hermann <jh@web.de>, + 2007 by MoinMoin:ThomasWaldmann + @license: GNU GPL, see COPYING for details. +""" + +from MoinMoin.datastruct.dicts.backends import wiki_dicts +from MoinMoin._tests import become_trusted, create_page, nuke_page + +class TestDictDict: + + def setup_class(self): + request = self.request + become_trusted(request) + + text = ''' +Text ignored + * list items ignored + * Second level list ignored + First:: first item + text with spaces:: second item + +Empty lines ignored, so is this text +Next line has key with empty value + Empty string::\x20 + Last:: last item +''' + + create_page(request, u'SomeTestDict', text) + + def teardown_class(self): + become_trusted(self.request) + nuke_page(self.request, u'SomeTestDict') + + def test_getitem(self): + dicts = self.request.dicts + + some_test_dict = dicts['SomeTestDict'] + assert len(some_test_dict) == 4 + assert some_test_dict['First'] == 'first item' + assert some_test_dict['text with spaces'] == 'second item' + assert some_test_dict['Empty string'] == '' # XXX fails if trailing blank is missing + assert some_test_dict['Last'] == 'last item' + + def test_contains(self): + dicts = self.request.dicts + + assert u'SomeTestDict' in dicts + assert u'SomeNotExistingDict' not in dicts + + +coverage_modules = ['MoinMoin.datastruct.dicts.backends.wiki_dicts'] +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MoinMoin/datastruct/dicts/backends/wiki_dicts.py Fri Jun 26 18:13:35 2009 +0200 @@ -0,0 +1,172 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - WikiDict functions. + + @copyright: 2003-2007 MoinMoin:ThomasWaldmann, + 2003 by Gustavo Niemeyer + 2009 MoinMoin:DmitrijsMilajevs + @license: GNU GPL, see COPYING for details. +""" +import re, time + +from MoinMoin import caching, Page + +# Version of the internal data structure which is pickled. +# Please increment if you have changed the structure. +DICTS_PICKLE_VERSION = 7 + + +class WikiDict(dict): + """ Mapping of keys to values in a wiki page. + + How a Dict definition page should look like: + + any text ignored + key1:: value1 + * ignored, too + key2:: value2 containing spaces + ... + keyn:: .... + any text ignored + """ + # Key:: Value - ignore all but key:: value pairs, strip whitespace, exactly one space after the :: is required + regex = re.compile(ur'^ (?P<key>.+?):: (?P<val>.*?) *$', re.MULTILINE | re.UNICODE) + + def __init__(self, request=None, pagename=None): + dict.__init__(self) + self.name = None + if request is not None and pagename is not None: + self._loadFromPage(request, pagename) + + def _loadFromPage(self, request, name): + """ load the dict from wiki page <name>'s content """ + self.name = name + text = Page.Page(request, name).get_raw_body() + self._initFromText(text) + + def _initFromText(self, text): + for match in self.regex.finditer(text): + key, val = match.groups() + self[key] = val + + def __repr__(self): + return "<Dict name=%r items=%r>" % (self.name, self.items()) + + +class WikiDicts: + """ a dictionary of Dict objects + + Config: + cfg.page_dict_regex + Default: ".*Dict$" Defs$ Vars$ ??????????????????? + """ + + def __init__(self, request): + self.cfg = request.cfg + self.request = request + + def reset(self): + self.dictdict = {} + self.namespace_timestamp = 0 + self.pageupdate_timestamp = 0 + self.base_timestamp = 0 + self.picklever = DICTS_PICKLE_VERSION + self.disk_cache_id = None + + def values(self, dictname): + """ get values of dict <dictname> """ + try: + d = self.dictdict[dictname] + except KeyError: + return [] + return d.values() + + def __getitem__(self, dictname): + try: + d = self.dictdict[dictname] + except KeyError: + return {} + return d + + def _adddict(self, request, dictname): + """ add a new dict (will be read from the wiki page) """ + self.dictdict[dictname] = WikiDict(request, dictname) + + def __contains__(self, dictname): + return self.dictdict.has_key(dictname) + + def load_dicts(self): + """ load the dict from the cache """ + request = self.request + rescan = False + arena = 'wikidicts' + key = 'dicts' + cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True) + current_disk_cache_id = cache.uid() + try: + self.__dict__.update(self.cfg.cache.DICTS_DATA) + if (current_disk_cache_id is None or + current_disk_cache_id != self.disk_cache_id): + self.reset() + raise AttributeError # not fresh, force load from disk + else: + return + except AttributeError: + try: + data = cache.content() + self.__dict__.update(data) + self.disk_cache_id = current_disk_cache_id + + # invalidate the cache if the pickle version changed + if self.picklever != DICTS_PICKLE_VERSION: + raise # force rescan + except: + self.reset() + rescan = True + + if rescan: + self.scan_dicts() + self.load_dicts() # try again + return + + data = { + "disk_cache_id": self.disk_cache_id, + "dictdict": self.dictdict, + "picklever": self.picklever + } + + # remember it (persistent environments) + self.cfg.cache.DICTS_DATA = data + + def scan_dicts(self): + """ scan all pages matching the dict regex and cache the + results on disk + """ + request = self.request + self.reset() + + # XXX get cache write lock here + scan_begin_time = time.time() + + # Get all pages in the wiki - without user filtering using filter + # function - this makes the page list about 10 times faster. + isdict = self.cfg.cache.page_dict_regexact.search + dictpages = request.rootpage.getPageList(user='', filter=isdict) + for pagename in dictpages: + self._adddict(request, pagename) + + scan_end_time = time.time() + + arena = 'wikidicts' + key = 'dicts' + cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True) + data = { + "scan_begin_time": scan_begin_time, + "scan_end_time": scan_end_time, + "dictdict": self.dictdict, + "picklever": self.picklever + } + cache.update(data) + # XXX release cache write lock here + +
--- a/MoinMoin/events/wikidictsrescan.py Thu Jun 25 15:07:47 2009 +0200 +++ b/MoinMoin/events/wikidictsrescan.py Fri Jun 26 18:13:35 2009 +0200 @@ -12,7 +12,6 @@ logging = log.getLogger(__name__) from MoinMoin import events as ev -from MoinMoin import wikidicts def handle(event): # "changed" includes creation, deletion, renamed and copied @@ -34,8 +33,7 @@ page = event.page logging.debug("groupsdicts changed: %r, scan_dicts started", page.page_name) - del request.dicts - gd = wikidicts.DictDict(request) - gd.scan_dicts() + request.dicts = request.cfg.dict_manager_init(request) + request.dicts.scan_dicts() logging.debug("groupsdicts changed: scan_dicts finished")
--- a/MoinMoin/web/contexts.py Thu Jun 25 15:07:47 2009 +0200 +++ b/MoinMoin/web/contexts.py Fri Jun 26 18:13:35 2009 +0200 @@ -371,9 +371,7 @@ def dicts(self): """ Lazy initialize the dicts on the first access """ - from MoinMoin import wikidicts - dicts = wikidicts.DictDict(self) - dicts.load_dicts() + dicts = self.cfg.dict_manager_init(self) return dicts dicts = EnvironProxy(dicts)
--- a/MoinMoin/wikidicts.py Thu Jun 25 15:07:47 2009 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,172 +0,0 @@ -# -*- coding: iso-8859-1 -*- -""" - MoinMoin - WikiDict functions. - - @copyright: 2003-2007 MoinMoin:ThomasWaldmann, - 2003 by Gustavo Niemeyer - 2009 MoinMoin:DmitrijsMilajevs - @license: GNU GPL, see COPYING for details. -""" -import re, time - -from MoinMoin import caching, Page - -# Version of the internal data structure which is pickled. -# Please increment if you have changed the structure. -DICTS_PICKLE_VERSION = 7 - - -class Dict(dict): - """ Mapping of keys to values in a wiki page. - - How a Dict definition page should look like: - - any text ignored - key1:: value1 - * ignored, too - key2:: value2 containing spaces - ... - keyn:: .... - any text ignored - """ - # Key:: Value - ignore all but key:: value pairs, strip whitespace, exactly one space after the :: is required - regex = re.compile(ur'^ (?P<key>.+?):: (?P<val>.*?) *$', re.MULTILINE | re.UNICODE) - - def __init__(self, request=None, pagename=None): - dict.__init__(self) - self.name = None - if request is not None and pagename is not None: - self._loadFromPage(request, pagename) - - def _loadFromPage(self, request, name): - """ load the dict from wiki page <name>'s content """ - self.name = name - text = Page.Page(request, name).get_raw_body() - self._initFromText(text) - - def _initFromText(self, text): - for match in self.regex.finditer(text): - key, val = match.groups() - self[key] = val - - def __repr__(self): - return "<Dict name=%r items=%r>" % (self.name, self.items()) - - -class DictDict: - """ a dictionary of Dict objects - - Config: - cfg.page_dict_regex - Default: ".*Dict$" Defs$ Vars$ ??????????????????? - """ - - def __init__(self, request): - self.cfg = request.cfg - self.request = request - - def reset(self): - self.dictdict = {} - self.namespace_timestamp = 0 - self.pageupdate_timestamp = 0 - self.base_timestamp = 0 - self.picklever = DICTS_PICKLE_VERSION - self.disk_cache_id = None - - def values(self, dictname): - """ get values of dict <dictname> """ - try: - d = self.dictdict[dictname] - except KeyError: - return [] - return d.values() - - def __getitem__(self, dictname): - try: - d = self.dictdict[dictname] - except KeyError: - return {} - return d - - def _adddict(self, request, dictname): - """ add a new dict (will be read from the wiki page) """ - self.dictdict[dictname] = Dict(request, dictname) - - def __contains__(self, dictname): - return self.dictdict.has_key(dictname) - - def load_dicts(self): - """ load the dict from the cache """ - request = self.request - rescan = False - arena = 'wikidicts' - key = 'dicts' - cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True) - current_disk_cache_id = cache.uid() - try: - self.__dict__.update(self.cfg.cache.DICTS_DATA) - if (current_disk_cache_id is None or - current_disk_cache_id != self.disk_cache_id): - self.reset() - raise AttributeError # not fresh, force load from disk - else: - return - except AttributeError: - try: - data = cache.content() - self.__dict__.update(data) - self.disk_cache_id = current_disk_cache_id - - # invalidate the cache if the pickle version changed - if self.picklever != DICTS_PICKLE_VERSION: - raise # force rescan - except: - self.reset() - rescan = True - - if rescan: - self.scan_dicts() - self.load_dicts() # try again - return - - data = { - "disk_cache_id": self.disk_cache_id, - "dictdict": self.dictdict, - "picklever": self.picklever - } - - # remember it (persistent environments) - self.cfg.cache.DICTS_DATA = data - - def scan_dicts(self): - """ scan all pages matching the dict regex and cache the - results on disk - """ - request = self.request - self.reset() - - # XXX get cache write lock here - scan_begin_time = time.time() - - # Get all pages in the wiki - without user filtering using filter - # function - this makes the page list about 10 times faster. - isdict = self.cfg.cache.page_dict_regexact.search - dictpages = request.rootpage.getPageList(user='', filter=isdict) - for pagename in dictpages: - self._adddict(request, pagename) - - scan_end_time = time.time() - - arena = 'wikidicts' - key = 'dicts' - cache = caching.CacheEntry(request, arena, key, scope='wiki', use_pickle=True) - data = { - "scan_begin_time": scan_begin_time, - "scan_end_time": scan_end_time, - "dictdict": self.dictdict, - "picklever": self.picklever - } - cache.update(data) - # XXX release cache write lock here - -