view m.py @ 2311:c4abbe125226

rewrite Makefile as python script for windows and unix
author RogerHaase
date Tue, 03 Dec 2013 11:46:31 -0700
parents
children
line wrap: on
line source
#!/usr/bin/python
# Copyright: 2013 MoinMoin:RogerHaase
# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.

"""
m.py

Provides a menu of functions frequently used by moin2 developers and desktop wiki users.
    - duplicates some common moin commands, do "moin --help" for all alternatives
    - adds default file names for selected moin commands (backup, restore, ...)
    - creates log files for functions with large output, extracts success/failure messages
    - displays error messages when user tries to run commands out of sequence
"""

import os
import sys
import subprocess
import glob
import shutil
import fnmatch
from collections import Counter

import MoinMoin  # validate python version


# text files created by commands with high volume output
QUICKINSTALL = 'm-quickinstall.txt'
PYTEST = 'm-pytest.txt'
PEP8 = 'm-pep8.txt'
CODING_STD = 'm-coding-std.txt'
DOCS = 'm-docs.txt'
NEWWIKI = 'm-new-wiki.txt'
DELWIKI = 'm-delete-wiki.txt'
BACKUPWIKI = 'm-backup-wiki.txt'
EXTRAS = 'm-extras.txt'
DIST = 'm-create-dist.txt'
# default files used for backup and restore
BACKUP_FILENAME = 'wiki/backup.moin'
JUST_IN_CASE_BACKUP = 'wiki/deleted-backup.moin'


if os.name == 'nt':
    M = 'm'  # customize help to local OS
    WINDOWS_OS = True
else:
    M = './m'
    WINDOWS_OS = False


# commands that create log files; "tests" creates 2 log files - pytest + pep8
CMD_LOGS = {
    'quickinstall': QUICKINSTALL,
    'pytest': PYTEST,
    'pep8': PEP8,
    # 'coding-std': CODING_STD,  # not logged due to small output
    'docs': DOCS,
    'new-wiki': NEWWIKI,
    'del-wiki': DELWIKI,
    'backup': BACKUPWIKI,
    'extras': EXTRAS,
    'dist': DIST,
}


help = r"""

usage: "%s <target>" where <target> is:

quickinstall    create or update virtual environment with required packages
docs            create local Sphinx html documentation
extras          install OpenID, Pillow, pymongo, sqlalchemy, ldap, upload.py
interwiki       refresh contrib\interwiki\intermap.txt (hg version control)
log <target>    view detailed log generated by <target>, omit to see list

new-wiki        create empty wiki
sample          create wiki and load sample data
restore *       create wiki and restore wiki\backup.moin *option, specify file
import <dir>    import a moin 1.9 wiki/data instance from <dir>

run             run built-in wiki server with local OS and logging options
backup *        roll 3 prior backups and create new backup *option, specify file

css             run Stylus to update CSS files
tests           run tests, output goes to pytest.txt and pytestpep8.txt
coding-std      correct scripts that taint the repository with trailing spaces..
api             update Sphinx API docs (files are under hg version control)
dist            delete wiki data, then create distribution archive in /dist

del-all         same as running the 4 del-* commands below
del-orig        delete all files matching *.orig
del-pyc         delete all files matching *.pyc
del-rej         delete all files matching *.rej
del-wiki        create a backup, then delete all wiki data
""" % M


def search_for_phrase(filename):
    """Search a text file for key phrases and print the lines of interest or print a count by phrase."""
    files = {
        # filename: (list of phrases)
        QUICKINSTALL: ('could not find', 'error', 'fail', 'timeout', 'traceback', 'success', 'cache location', 'must be deactivated', ),
        NEWWIKI: ('error', 'fail', 'timeout', 'traceback', 'success', ),
        BACKUPWIKI: ('error', 'fail', 'timeout', 'traceback', 'success', ),
        # 'error ' to avoid catching .../Modules/errors.o....
        EXTRAS: ('error ', 'error:', 'error.', 'error,', 'fail', 'timeout', 'traceback', 'success', 'already satisfied', 'active version', 'installed', 'finished', ),
        PYTEST: ('seconds =', ),
        PEP8: ('seconds =', ),
        CODING_STD: ('remove trailing blanks', 'dos line endings', 'unix line endings', 'remove empty lines', ),
        DIST: ('creating', 'copying', 'adding', 'hard linking', ),
        DOCS: ('build finished', 'build succeeded', 'traceback', 'failed', 'error', 'usage', 'importerror', 'Exception occurred', )
    }
    # for these file names, display a count of occurrances rather than each found line
    print_counts = (CODING_STD, DIST, )

    with open(filename, "r") as f:
        lines = f.readlines()
    name = os.path.split(filename)[1]
    phrases = files[name]
    counts = Counter()
    for idx, line in enumerate(lines):
        line = line.lower()
        for phrase in phrases:
            if phrase in line:
                if filename in print_counts:
                    counts[phrase] += 1
                else:
                    print idx + 1, line.rstrip()
                    break
    for key in counts:
        print 'The phrase "%s" was found %s times.' % (key, counts[key])


def wiki_exists():
    """Return truthy if a wiki exists."""
    return glob.glob('wiki/index/_all_revs_*.toc')


def make_wiki(command):
    """Process command to create a new wiki."""
    if wiki_exists():
        print 'Error: a wiki exists, delete it and try again.'
    else:
        print 'Output messages redirected to %s' % NEWWIKI
        with open(NEWWIKI, 'w') as messages:
            result = subprocess.call(command, shell=True, stderr=messages, stdout=messages)
        if result == 0:
            print '\nSuccess: a new wiki has been created.'
        else:
            print 'Important messages from %s are shown below:' % NEWWIKI
            search_for_phrase(NEWWIKI)
            print '\nError: attempt to create wiki failed. Do "%s log new-wiki" to see complete log.' % M


def delete_files(pattern):
    """Recursively delete all files matching pattern."""
    matches = []
    for root, dirnames, filenames in os.walk(os.path.abspath(os.path.dirname(__file__))):
        for filename in fnmatch.filter(filenames, pattern):
            matches.append(os.path.join(root, filename))
    for match in matches:
        os.remove(match)
    print 'Deleted %s files matching "%s".' % (len(matches), pattern)


class Menu(object):
    """Each cmd_ method processes an option on the menu."""
    def __init__(self):
        pass

    def cmd_quickinstall(self, *args):
        """create or update a virtual environment with the required packages"""
        command = '%s quickinstall.py %s' % (sys.executable, ' '.join(args))
        print 'Running quickinstall.py... output messages redirected to %s' % QUICKINSTALL
        with open(QUICKINSTALL, 'w') as messages:
            result = subprocess.call(command, shell=True, stderr=messages, stdout=messages)
        if result != 0:
            open(QUICKINSTALL, 'a').write('Error: quickinstall passed non-zero return code: %s' % result)
        print 'Searching %s, important messages are shown below... Do "%s log quickinstall" to see complete log.\n' % (QUICKINSTALL, M)
        search_for_phrase(QUICKINSTALL)

    def cmd_docs(self, *args):
        """create local Sphinx html documentation"""
        if WINDOWS_OS:
            command = 'activate.bat & cd docs & make.bat html'  # windows separates commands with "&"
        else:
            # in terminal "source activate" works, but shell requires "source ./activate"
            command = 'source ./activate; cd docs; make html'  # unix separates commands with ";"
        print 'Creating HTML docs... output messages written to %s.' % DOCS
        with open(DOCS, 'w') as messages:
            result = subprocess.call(command, shell=True, stderr=messages, stdout=messages)
        print 'Searching %s, important messages are shown below...\n' % DOCS
        search_for_phrase(DOCS)
        if result == 0:
            print 'HTML docs successfully created in docs/_build/html.'
        else:
            print 'Error: creation of HTML docs failed with return code "%s". Do "%s log docs" to see complete log.' % (result, M)

    def cmd_extras(self, *args):
        """install optional packages: OpenID, Pillow, pymongo, sqlalchemy, ldap; and upload.py"""
        upload = '%s MoinMoin/script/win/wget.py https://codereview.appspot.com/static/upload.py upload.py' % sys.executable
        if WINDOWS_OS:
            print 'Installing OpenId, Pillow, pymongo, sqlalchemy, upload.py... output messages written to %s.' % EXTRAS
            # easy_install is used for windows because it installs binary packages, pip does not
            command = 'activate.bat & easy_install python-openid & easy_install pillow & easy_install pymongo & easy_install sqlalchemy' + ' & ' + upload
            # TODO: easy_install python-ldap fails on windows
            # try google: installing python-ldap in a virtualenv on windows
            # or, download from http://www.lfd.uci.edu/~gohlke/pythonlibs/#python-ldap
            #   activate.bat
            #   easy_install <path to downloaded .exe file>
            #   deactivate.bat
        else:
            print 'Installing OpenId, Pillow, pymongo, sqlalchemy, ldap, upload.py... output messages written to %s.' % EXTRAS
            command = 'source ./activate; pip install python-openid; pip install pillow; pip install pymongo; pip install sqlalchemy; pip install python-ldap' + '; ' + upload
        with open(EXTRAS, 'w') as messages:
            subprocess.call(command, shell=True, stderr=messages, stdout=messages)
        print 'Important messages from %s are shown below. Do "%s log extras" to see complete log.' % (EXTRAS, M)
        search_for_phrase(EXTRAS)

    def cmd_interwiki(self, *args):
        """refresh contrib\interwiki\intermap.txt"""
        print 'Refreshing contrib\interwiki\intermap.txt...'
        command = '%s MoinMoin/script/win/wget.py http://master19.moinmo.in/InterWikiMap?action=raw contrib/interwiki/intermap.txt' % sys.executable
        subprocess.call(command, shell=True)

    def cmd_log(self, *args):
        """View a log file with the default text editor"""

        def log_help(logs):
            """Print list of available logs to view."""
            print "usage: %s log <target> where <target> is:\n\n" % M
            choices = '{0: <16}- {1}'
            for log in sorted(logs):
                if os.path.isfile(CMD_LOGS[log]):
                    print choices.format(log, CMD_LOGS[log])
                else:
                    print choices.format(log, '* file does not exist')

        logs = set(CMD_LOGS.keys())
        if args and args[0] in logs and os.path.isfile(CMD_LOGS[args[0]]):
            if WINDOWS_OS:
                command = 'start %s' % CMD_LOGS[args[0]]
            else:
                command = '${VISUAL:-${FCEDIT:-${EDITOR:-less}}} %s' % CMD_LOGS[args[0]]
            subprocess.call(command, shell=True)
        else:
            log_help(logs)

    def cmd_new_wiki(self, *args):
        """create empty wiki"""
        if WINDOWS_OS:
            command = 'moin.bat index-create -s -i'
        else:
            command = './moin index-create -s -i'
        print 'Creating a new empty wiki...'
        make_wiki(command)  # share code with loading sample data or restoring backups

    def cmd_sample(self, *args):
        """create wiki and load sample data"""
        if WINDOWS_OS:
            command = 'moin.bat index-create -s -i & moin.bat load --file contrib/serialized/items.moin & moin.bat index-build'
        else:
            command = './moin index-create -s -i; ./moin load --file contrib/serialized/items.moin; ./moin index-build'
        print 'Creating a new wiki populated with sample data...'
        make_wiki(command)

    def cmd_restore(self, *args):
        """create wiki and load data from wiki/backup.moin or user specified path"""
        if WINDOWS_OS:
            command = 'moin.bat index-create -s -i & moin.bat load --file %s & moin.bat index-build'
        else:
            command = './moin index-create -s -i; ./moin load --file %s; ./moin index-build'
        filename = BACKUP_FILENAME
        if args:
            filename = args[0]
        if os.path.isfile(filename):
            command = command % filename
            print 'Creating a new wiki and loading it with data from %s...' % filename
            make_wiki(command)
        else:
            print 'Error: cannot create wiki because %s does not exist.' % filename

    def cmd_import(self, *args):
        """import a moin 1.9 wiki directory named dir"""
        if WINDOWS_OS:
            command = 'moin.bat import19 -s -i --data_dir %s'
        else:
            command = './moin import19 -s -i --data_dir %s'
        if args:
            dirname = args[0]
            if os.path.isdir(dirname):
                command = command % dirname
                print 'Creating a new wiki populated with data from %s...' % dirname
                make_wiki(command)
            else:
                print 'Error: cannot create wiki because %s does not exist.' % dirname
        else:
            print 'Error: a path to the Moin 1.9 wiki/data data directory is required.'

    def cmd_run(self, *args):
        """run built-in wiki server with local options"""
        if wiki_exists():
            if os.path.isfile('logging.conf'):
                if WINDOWS_OS:
                    logfile = 'set MOINLOGGINGCONF=logging.conf & '
                else:
                    logfile = 'MOINLOGGINGCONF=logging.conf; export MOINLOGGINGCONF; '
            else:
                logfile = ''
            if WINDOWS_OS:
                command = '%smoin.bat moin %s --threaded' % (logfile, ' '.join(args))
            else:
                command = '%s./moin moin %s' % (logfile, ' '.join(args))
            try:
                subprocess.call(command, shell=True)
            except KeyboardInterrupt:
                pass  # on windows pass eliminates traceback but "Terminate batch job..." message is displayed twice
        else:
            print 'Error: a wiki must be created before running the built-in server.'

    def cmd_backup(self, *args):
        """roll 3 prior backups and create new wiki/backup.moin or backup to user specified file"""
        if wiki_exists():
            filename = BACKUP_FILENAME
            if args:
                filename = args[0]
                print 'Creating a wiki backup to %s...' % filename
            else:
                print 'Creating a wiki backup to %s after rolling 3 prior backups...'
                b3 = BACKUP_FILENAME.replace('.', '3.')
                b2 = BACKUP_FILENAME.replace('.', '2.')
                b1 = BACKUP_FILENAME.replace('.', '1.')
                if os.path.exists(b3):
                    os.remove(b3)
                for src, dst in ((b2, b3), (b1, b2), (BACKUP_FILENAME, b1)):
                    if os.path.exists(src):
                        os.rename(src, dst)

            if WINDOWS_OS:
                command = 'moin.bat save --all-backends --file %s' % filename
            else:
                command = './moin save --all-backends --file %s' % filename
            with open(BACKUPWIKI, 'w') as messages:
                result = subprocess.call(command, shell=True, stderr=messages, stdout=messages)
            if result == 0:
                print 'Success: wiki was backed up to %s' % filename
            else:
                print 'Important messages from %s are shown below. Do "%s log backup" to see complete log.' % (BACKUPWIKI, M)
                search_for_phrase(BACKUPWIKI)
                print '\nError: attempt to backup wiki failed.'
        else:
            print 'Error: cannot backup wiki because it has not been created.'

    def cmd_css(self, *args):
        """run Stylus to update CSS files"""
        print 'Running Stylus to update CSS files...'
        if WINDOWS_OS:
            command = r'cd MoinMoin\themes\modernized\static\css\stylus & stylus --include-css --compress < main.styl > ../common.css'
        else:
            command = 'cd MoinMoin/themes/modernized/static/css/stylus; stylus --include-css --compress < main.styl > ../common.css'
        result = subprocess.call(command, shell=True)

        if WINDOWS_OS:
            command = r'cd MoinMoin\themes\foobar\static\css\stylus & stylus --include-css --compress < main.styl > ../common.css'
        else:
            command = 'cd MoinMoin/themes/foobar/static/css/stylus; stylus --include-css --compress < main.styl > ../common.css'
        result2 = subprocess.call(command, shell=True)

        if result == 0 and result2 == 0:
            print 'Success: CSS files updated.'
        else:
            print 'Error: stylus failed to update css files, see error messages above.'

    def cmd_tests(self, *args):
        """run tests, output goes to pytest.txt and pytestpep8.txt"""
        print 'Running tests... output written to %s and %s.' % (PYTEST, PEP8)
        if WINDOWS_OS:
            command = 'activate.bat & py.test.exe > %s 2>&1 & py.test.exe --pep8 -k pep8 --clearcache > %s 2>&1' % (PYTEST, PEP8)
        else:
            command = 'source ./activate; py.test > %s 2>&1; py.test --pep8 -k pep8 --clearcache > %s 2>&1' % (PYTEST, PEP8)
        result = subprocess.call(command, shell=True)
        print 'Summary message from %s is shown below. Do "%s log pytest" to see complete log.' % (PYTEST, M)
        search_for_phrase(PYTEST)
        print 'Summary message from %s is shown below. Do "%s log pep8" to see complete log.' % (PEP8, M)
        search_for_phrase(PEP8)

    def cmd_coding_std(self, *args):
        """correct scripts that taint the HG repository and clutter subsequent code reviews"""
        print 'Checking for trailing blanks, DOS line endings, Unix line endings, empty lines at eof...'
        command = '%s contrib/pep8/coding_std.py' % sys.executable
        subprocess.call(command, shell=True)

    def cmd_api(self, *args):
        """update Sphinx API docs, these docs are under hg version control"""
        print 'Refreshing api docs...'
        if WINDOWS_OS:
            command = 'activate.bat & sphinx-apidoc -f -o docs/devel/api MoinMoin & %s MoinMoin/script/win/dos2unix.py docs/devel/api' % sys.executable
        else:
            command = 'source ./activate; sphinx-apidoc -f -o docs/devel/api MoinMoin'
        result = subprocess.call(command, shell=True)

    def cmd_dist(self, *args):
        """create distribution archive in dist/"""
        print 'Deleting wiki data, then creating distribution archive in /dist, output written to %s.' % DIST
        self.cmd_del_wiki(*args)
        command = '%s setup.py sdist' % sys.executable
        with open(DIST, 'w') as messages:
            result = subprocess.call(command, shell=True, stderr=messages, stdout=messages)
        print 'Summary message from %s is shown below:' % DIST
        search_for_phrase(DIST)
        if result == 0:
            print 'Success: a distribution archive was created in /dist.'
        else:
            print 'Error: create dist failed with return code = %s. Do "%s log dist" to see complete log.' % (result, M)

    def cmd_del_all(self, *args):
        """same as running the 4 del-* commands below"""
        self.cmd_del_orig(*args)
        self.cmd_del_pyc(*args)
        self.cmd_del_rej(*args)
        self.cmd_del_wiki(*args)

    def cmd_del_orig(self, *args):
        """delete all files matching *.orig"""
        delete_files('*.orig')

    def cmd_del_pyc(self, *args):
        """delete all files matching *.pyc"""
        delete_files('*.pyc')

    def cmd_del_rej(self, *args):
        """delete all files matching *.rej"""
        delete_files('*.rej')

    def cmd_del_wiki(self, *args):
        """create a just-in-case backup, then delete all wiki data"""
        if WINDOWS_OS:
            command = 'moin.bat save --all-backends --file %s' % JUST_IN_CASE_BACKUP
        else:
            command = './moin save --all-backends --file %s' % JUST_IN_CASE_BACKUP
        if wiki_exists():
            print 'Creating a backup named %s; then deleting all wiki data and indexes...' % JUST_IN_CASE_BACKUP
            with open(DELWIKI, 'w') as messages:
                result = subprocess.call(command, shell=True, stderr=messages, stdout=messages)
            if result != 0:
                print 'Error: backup failed with return code = %s. Complete log is in %s.' % (result, DELWIKI)
        # destroy wiki even though files are damaged and backup may have failed
        if os.path.isdir('wiki/data') or os.path.isdir('wiki/index'):
            shutil.rmtree('wiki/data')
            shutil.rmtree('wiki/index')
            print 'Wiki data successfully deleted.'
        else:
            print 'Wiki data not deleted because it does not exist.'


if __name__ == '__main__':
    # create a set of valid menu choices
    menu = Menu()
    choices = set()
    names = dir(menu)
    for name in names:
        if name.startswith('cmd_'):
            choices.add(name)

    if len(sys.argv) == 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help':
        print help
    else:
        if sys.argv[1] != 'quickinstall' and not (os.path.isfile('activate') or os.path.isfile('activate.bat')):
            print 'Error: files created by quickinstall are missing, run "%s quickinstall" and try again.' % M
        else:
            choice = 'cmd_%s' % sys.argv[1]
            choice = choice.replace('-', '_')
            if choice in choices:
                choice = getattr(menu, choice)
                choice(*sys.argv[2:])
            else:
                print help
                print 'Error: unknown menu selection "%s"' % sys.argv[1]