COMP/WMCORE/setup.py

#!/usr/bin/env python
from distutils.core import setup, Command
from unittest import TextTestRunner, TestLoader, TestSuite
from glob import glob
from os.path import splitext, basename, join as pjoin, walk
from ConfigParser import ConfigParser, NoOptionError
import os, sys, os.path
import logging
import unittest
#PyLinter and coverage aren't standard, but aren't strictly necessary
can_lint = False
can_coverage = False
can_nose = False
try:
    from pylint.lint import Run 
    from pylint.lint import preprocess_options, cb_init_hook
    from pylint import checkers
    can_lint = True
except:
    pass
try:
    import coverage
    can_coverage = True
except:
    pass

try:
    import nose
    can_nose = True
except:
    pass

if can_nose:
    class TestCommand(Command):
        """Runs our test suite"""
        
        user_options = [ ]

        def initialize_options(self):
            pass
        
        def finalize_options(self):
            pass
    
        def run(self):
            print "committed at abour 14:11"
            sys.stdout.flush()
            retval =  nose.run(argv=[__file__,'--all-modules','-v','test/python'])
            if retval:
                sys.exit( 0 ) 
            else:
                sys.exit( 1 )
else:
    class TestCommand(Command):
        user_options = [ ]
        def run(self):
            print "Nose isn't installed. You must install the nose package to run tests (easy_install nose might do it)"
            pass
        
        def initialize_options(self):
            pass
        
        def finalize_options(self):
            pass
        pass

if can_lint:
    class LinterRun(Run):
        def __init__(self, args, reporter=None):
            self._rcfile = None
            self._plugins = []
            preprocess_options(args, {
                # option: (callback, takearg)
                'rcfile':       (self.cb_set_rcfile, True),
                'load-plugins': (self.cb_add_plugins, True),
                })
            self.linter = linter = self.LinterClass((
                ('rcfile',
                 {'action' : 'callback', 'callback' : lambda *args: 1,
                  'type': 'string', 'metavar': '<file>',
                  'help' : 'Specify a configuration file.'}),
    
                ('init-hook',
                 {'action' : 'callback', 'type' : 'string', 'metavar': '<code>',
                  'callback' : cb_init_hook,
                  'help' : 'Python code to execute, usually for sys.path \
    manipulation such as pygtk.require().'}),
    
                ('help-msg',
                 {'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>',
                  'callback' : self.cb_help_message,
                  'group': 'Commands',
                  'help' : '''Display a help message for the given message id and \
    exit. The value may be a comma separated list of message ids.'''}),
    
                ('list-msgs',
                 {'action' : 'callback', 'metavar': '<msg-id>',
                  'callback' : self.cb_list_messages,
                  'group': 'Commands',
                  'help' : "Generate pylint's full documentation."}),
    
                ('generate-rcfile',
                 {'action' : 'callback', 'callback' : self.cb_generate_config,
                  'group': 'Commands',
                  'help' : '''Generate a sample configuration file according to \
    the current configuration. You can put other options before this one to get \
    them in the generated configuration.'''}),
    
                ('generate-man',
                 {'action' : 'callback', 'callback' : self.cb_generate_manpage,
                  'group': 'Commands',
                  'help' : "Generate pylint's man page.",'hide': 'True'}),
    
                ('errors-only',
                 {'action' : 'callback', 'callback' : self.cb_error_mode,
                  'short': 'e',
                  'help' : '''In error mode, checkers without error messages are \
    disabled and for others, only the ERROR messages are displayed, and no reports \
    are done by default'''}),
    
                ('profile',
                 {'type' : 'yn', 'metavar' : '<y_or_n>',
                  'default': False,
                  'help' : 'Profiled execution.'}),
    
                ), option_groups=self.option_groups,
                   reporter=reporter, pylintrc=self._rcfile)
            # register standard checkers
            checkers.initialize(linter)
            # load command line plugins
            linter.load_plugin_modules(self._plugins)
            # read configuration
            linter.disable_message('W0704')
            linter.read_config_file()
            # is there some additional plugins in the file configuration, in
            config_parser = linter._config_parser
            if config_parser.has_option('MASTER', 'load-plugins'):
                plugins = splitstrip(config_parser.get('MASTER', 'load-plugins'))
                linter.load_plugin_modules(plugins)
            # now we can load file config and command line, plugins (which can
            # provide options) have been registered
            linter.load_config_file()
            if reporter:
                # if a custom reporter is provided as argument, it may be overriden
                # by file parameters, so re-set it here, but before command line
                # parsing so it's still overrideable by command line option
                linter.set_reporter(reporter)
            args = linter.load_command_line_configuration(args)
            # insert current working directory to the python path to have a correct
            # behaviour
            sys.path.insert(0, os.getcwd())
            if self.linter.config.profile:
                print >> sys.stderr, '** profiled run'
                from hotshot import Profile, stats
                prof = Profile('stones.prof')
                prof.runcall(linter.check, args)
                prof.close()
                data = stats.load('stones.prof')
                data.strip_dirs()
                data.sort_stats('time', 'calls')
                data.print_stats(30)
            sys.path.pop(0)
        
        def cb_set_rcfile(self, name, value):
            """callback for option preprocessing (ie before optik parsing)"""
            self._rcfile = value
    
        def cb_add_plugins(self, name, value):
            """callback for option preprocessing (ie before optik parsing)"""
            self._plugins.extend(splitstrip(value))
    
        def cb_error_mode(self, *args, **kwargs):
            """error mode:
            * checkers without error messages are disabled
            * for others, only the ERROR messages are displayed
            * disable reports
            * do not save execution information
            """
            self.linter.disable_noerror_checkers()
            self.linter.set_option('disable-msg-cat', 'WCRFI')
            self.linter.set_option('reports', False)
            self.linter.set_option('persistent', False)
    
        def cb_generate_config(self, *args, **kwargs):
            """optik callback for sample config file generation"""
            self.linter.generate_config(skipsections=('COMMANDS',))
            
        def cb_generate_manpage(self, *args, **kwargs):
            """optik callback for sample config file generation"""
            from pylint import __pkginfo__
            self.linter.generate_manpage(__pkginfo__)
            
        def cb_help_message(self, option, opt_name, value, parser):
            """optik callback for printing some help about a particular message"""
            self.linter.help_message(splitstrip(value))
    
        def cb_list_messages(self, option, opt_name, value, parser):
            """optik callback for printing available messages"""
            self.linter.list_messages()
else:
    class LinterRun:
        def __init__(self):
            pass

"""
Build, clean and test the WMCore package.
"""

def generate_filelist(basepath=None, recurse=True, ignore=False):
    if basepath:
        walkpath = os.path.join(get_relative_path(), 'src/python', basepath)
    else:
        walkpath = os.path.join(get_relative_path(), 'src/python')
    
    files = []
    
    if walkpath.endswith('.py'):
        if ignore and walkpath.endswith(ignore):
            files.append(walkpath)
    else:
        for dirpath, dirnames, filenames in os.walk(walkpath):
            # skipping CVS directories and their contents
            pathelements = dirpath.split('/')
            result = []
            if not 'CVS' in pathelements:
                # to build up a list of file names which contain tests
                for file in filenames:
                    if file.endswith('.py'):
                        filepath = '/'.join([dirpath, file]) 
                        files.append(filepath)

    if len(files) == 0 and recurse:
        files = generate_filelist(basepath + '.py', not recurse)

    return files

def lint_score(stats, evaluation):
    return eval(evaluation, {}, stats)

def lint_files(files, reports=False):
    """
    lint a (list of) file(s) and return the results as a dictionary containing
    filename : result_dict
    """
    
    rcfile=os.path.join(get_relative_path(),'standards/.pylintrc')
    
    arguements = ['--rcfile=%s' % rcfile, '--ignore=DefaultConfig.py']
    
    if not reports:
        arguements.append('-rn')
    
    arguements.extend(files)
    
    lntr = LinterRun(arguements)
    
    results = {}
    for file in files:
        lntr.linter.check(file)
        results[file] = {'stats': lntr.linter.stats,
                         'score': lint_score(lntr.linter.stats, 
                                             lntr.linter.config.evaluation)
                         }
        if reports:
            print '----------------------------------'
            print 'Your code has been rated at %.2f/10' % \
                    lint_score(lntr.linter.stats, lntr.linter.config.evaluation)

    return results, lntr.linter.config.evaluation

class CleanCommand(Command):
    description = "Clean up (delete) compiled files"
    user_options = [ ]

    def initialize_options(self):
        self._clean_me = [ ]
        for root, dirs, files in os.walk('.'):
            for f in files:
                if f.endswith('.pyc'):
                    self._clean_me.append(pjoin(root, f))

    def finalize_options(self):
        pass

    def run(self):
        for clean_me in self._clean_me:
            try:
                os.unlink(clean_me)
            except:
                pass
            
class LintCommand(Command):
    description = "Lint all files in the src tree"
    """
    TODO: better format the test results, get some global result, make output 
    more buildbot friendly.    
    """
    
    user_options = [ ('package=', 'p', 'package to lint, default to None'),
                ('report', 'r', 'return a detailed lint report, default False')]
    
    def initialize_options(self):
        self._dir = get_relative_path()
        self.package = None
        self.report = False
        
    def finalize_options(self):
        if self.report:
            self.report = True

    def run(self):
        '''
        Find the code and run lint on it
        '''
        if can_lint:
            srcpypath = os.path.join(self._dir, 'src/python/')
            
            sys.path.append(srcpypath) 
            
            files_to_lint = []
            
            if self.package:
                if self.package.endswith('.py'):
                    cnt = self.package.count('.') - 1
                    files_to_lint = generate_filelist(self.package.replace('.', '/', cnt), 'DeafultConfig.py')
                else:
                    files_to_lint = generate_filelist(self.package.replace('.', '/'), 'DeafultConfig.py')
            else:
                files_to_lint = generate_filelist(ignore='DeafultConfig.py')
            
            results, evaluation = lint_files(files_to_lint, self.report)
            ln = len(results)
            scr = 0
            print
            for k, v in results.items():
                print "%s: %.2f/10" % (k.replace('src/python/', ''), v['score'])
                scr += v['score']
            if ln > 1:
                print '--------------------------------------------------------'
                print 'Average pylint score for %s is: %.2f/10' % (self.package,
                                                                  scr/ln)
                
        else:
            print 'You need to install pylint before using the lint command'
            
class ReportCommand(Command):
    description = "Generate a simple html report for ease of viewing in buildbot"
    """
    To contain:
        average lint score
        % code coverage
        list of classes missing tests
        etc.
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        """
        run all the tests needed to generate the report and make an
        html table
        """
        files = generate_filelist()
        
        error = 0 
        warning = 0
        refactor = 0
        convention = 0 
        statement = 0
        
        srcpypath = '/'.join([get_relative_path(), 'src/python/'])
        sys.path.append(srcpypath)

        cfg = ConfigParser()
        cfg.read('standards/.pylintrc')
        
        # Supress stdout/stderr
        sys.stderr = open('/dev/null', 'w')
        sys.stdout = open('/dev/null', 'w')
        # wrap it in an exception handler, otherwise we can't see why it fails
        try:
            # lint the code
            for stats in lint_files(files):
                error += stats['error']
                warning += stats['warning']
                refactor += stats['refactor']
                convention += stats['convention'] 
                statement += stats['statement']
        except Exception,e:
            # and restore the stdout/stderr
            sys.stderr = sys.__stderr__
            sys.stdout = sys.__stderr__
            raise e
        
        # and restore the stdout/stderr
        sys.stderr = sys.__stderr__
        sys.stdout = sys.__stderr__ 
        
        stats = {'error': error,
            'warning': warning,
            'refactor': refactor,
            'convention': convention, 
            'statement': statement}
        
        lint_score = eval(cfg.get('MASTER', 'evaluation'), {}, stats)
        coverage = 0 # TODO: calculate this
        testless_classes = [] # TODO: generate this
        
        print "<table>"
        print "<tr>"
        print "<td colspan=2><h1>WMCore test report</h1></td>"
        print "</tr>"
        print "<tr>"
        print "<td>Average lint score</td>"
        print "<td>%.2f</td>" % lint_score
        print "</tr>"
        print "<tr>"
        print "<td>% code coverage</td>"
        print "<td>%s</td>" % coverage
        print "</tr>"
        print "<tr>"
        print "<td>Classes missing tests</td>"
        print "<td>"
        if len(testless_classes) == 0:
            print "None"
        else:
            print "<ul>"
            for c in testless_classes:
                print "<li>%c</li>" % c
            print "</ul>"
        print "</td>"
        print "</tr>"
        print "</table>"
            
class CoverageCommand(Command):
    description = "Run code coverage tests"
    """
    To do this, we need to run all the unittests within the coverage
    framework to record all the lines(and branches) executed
    unfortunately, we have multiple code paths per database schema, so
    we need to find a way to merge them.
      
    TODO: modify the test command to have a flag to record code coverage
          the file thats used can then be used here, saving us from running
          our tests twice    
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        """
        Determine the code's test coverage and return that as a float
        
        http://nedbatchelder.com/code/coverage/
        """
        if can_coverage:
            files = generate_filelist()
            dataFile = None
            cov = None
            
            # attempt to load previously cached coverage information if it exists
            try:
                dataFile = open("wmcore-coverage.dat","r")
                cov = coverage.coverage(branch = True, data_file='wmcore-coverage.dat')
                cov.load()
            except:  
                cov = coverage.coverage(branch = True, )
                cov.start()
                runUnitTests()
                cov.stop()
                cov.save()
                
            # we have our coverage information, now let's do something with it
            # get a list of modules
            cov.report(morfs = files, file=sys.stdout)
            return 0
        else:
            print 'You need the coverage module installed before running the' +\
                            ' coverage command'
    
class DumbCoverageCommand(Command):
    description = "Run a simple coverage test - find classes that don't have a unit test"
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        """
        Determine the code's test coverage in a dumb way and return that as a 
        float.
        """
        print "This determines test coverage in a very crude manner. If your"
        print "test file is incorrectly named it will not be counted, and" 
        print "result in a lower coverage score." 
        print '----------------------------------------------------------------'
        filelist = generate_filelist()
        tests = 0
        files = 0
        pkgcnt = 0
        dir = get_relative_path()
        pkg = {'name': '', 'files': 0, 'tests': 0}
        for f in filelist:
            testpath = '/'.join([dir, f])
            pth = testpath.split('./src/python/')
            pth.append(pth[1].replace('/', '_t/').replace('.', '_t.'))
            if pkg['name'] == pth[2].rsplit('/', 1)[0].replace('_t/', '/'):
                # pkg hasn't changed, increment counts
                pkg['files'] += 1
            else:
                # new package, print stats for old package
                pkgcnt += 1
                if pkg['name'] != '' and pkg['files'] > 0:
                    print 'Package %s has coverage %.1f percent' % (pkg['name'], 
                                (float(pkg['tests'])/float(pkg['files']) * 100))
                # and start over for the new package
                pkg['name'] = pth[2].rsplit('/', 1)[0].replace('_t/', '/')
                # do global book keeping
                files += pkg['files']
                tests += pkg['tests'] 
                pkg['files'] = 0 
                pkg['tests'] = 0
            pth[1] = 'test/python'
            testpath = '/'.join(pth)
            try: 
                os.stat(testpath)
                pkg['tests'] += 1
            except:
                pass
            
        coverage = (float(tests) / float(files)) * 100
        print '----------------------------------------------------------------'
        print 'Code coverage (%s packages) is %.2f percent' % (pkgcnt, coverage)
        return coverage

class EnvCommand(Command):
    description = "Configure the PYTHONPATH, DATABASE and PATH variables to" +\
    "some sensible defaults, if not already set. Call with -q when eval-ing," +\
    """ e.g.:
        eval `python setup.py -q env`
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        if not os.getenv('DATABASE', False):
            # Use an in memory sqlite one if none is configured. 
             print 'export DATABASE=sqlite://'
            
        here = get_relative_path()
        
        tests = here + '/test/python'
        source = here + '/src/python'
        webpth = source + '/WMCore/WebTools'
        
        pypath=os.getenv('PYTHONPATH', '').strip(':').split(':')
         
        for pth in [tests, source]:
            if pth not in pypath:
                pypath.append(pth)
        
        # We might want to add other executables to PATH
        expath=os.getenv('PATH', '').split(':')
        for pth in [webpth]:
            if pth not in expath:
                expath.append(pth)
              
        print 'export PYTHONPATH=%s' % ':'.join(pypath)
        print 'export PATH=%s' % ':'.join(expath)
        
        #We want the WMCORE root set, too
        print 'export WMCOREBASE=%s' % get_relative_path()


def getPackages(package_dirs = []):
    packages = []
    for dir in package_dirs:
        for dirpath, dirnames, filenames in os.walk('./%s' % dir):
            # Exclude things here
            if dirpath not in ['./src/python/', './src/python/IMProv']: 
                pathelements = dirpath.split('/')
                if not 'CVS' in pathelements:
                    path = pathelements[3:]
                    packages.append('.'.join(path))
    return packages

package_dir = {'WMCore': 'src/python/WMCore',
               'WMComponent' : 'src/python/WMComponent',
               'WMQuality' : 'src/python/WMQuality'}

setup (name = 'wmcore',
       version = '1.0',
       maintainer_email = 'hn-cms-wmDevelopment@cern.ch',
       cmdclass = {'clean': CleanCommand, 
                   'lint': LintCommand,
                   'report': ReportCommand,
                   'coverage': CoverageCommand ,
                   'missing': DumbCoverageCommand,
                   'env': EnvCommand,
                   'test' : TestCommand },
       package_dir = package_dir,
       packages = getPackages(package_dir.values()),)
Revision:	1.108
Committed:	Thu Feb 4 20:50:34 2010 UTC (15 years, 2 months ago) by meloam
Content type:	text/x-python
Branch:	MAIN
Changes since 1.107:	+1 -0 lines
Log Message:	trying to find out where my debug statements go
#	User	Rev	Content
1	fvlingen	1.1	#!/usr/bin/env python
2	metson	1.21	from distutils.core import setup, Command
3			from unittest import TextTestRunner, TestLoader, TestSuite
4			from glob import glob
5			from os.path import splitext, basename, join as pjoin, walk
6	metson	1.85	from ConfigParser import ConfigParser, NoOptionError
7	meloam	1.53	import os, sys, os.path
8	metson	1.85	import logging
9			import unittest
10	metson	1.51	#PyLinter and coverage aren't standard, but aren't strictly necessary
11			can_lint = False
12			can_coverage = False
13	meloam	1.88	can_nose = False
14	sfoulkes	1.29	try:
15	metson	1.87	from pylint.lint import Run
16	metson	1.85	from pylint.lint import preprocess_options, cb_init_hook
17			from pylint import checkers
18	metson	1.51	can_lint = True
19			except:
20			pass
21			try:
22	meloam	1.47	import coverage
23	metson	1.51	can_coverage = True
24	sfoulkes	1.29	except:
25			pass
26	fvlingen	1.1
27	meloam	1.88	try:
28			import nose
29			can_nose = True
30			except:
31			pass
32
33			if can_nose:
34	meloam	1.96	class TestCommand(Command):
35	meloam	1.100	"""Runs our test suite"""
36
37	meloam	1.88	user_options = [ ]
38
39			def initialize_options(self):
40			pass
41
42			def finalize_options(self):
43			pass
44
45			def run(self):
46	meloam	1.107	print "committed at abour 14:11"
47	meloam	1.108	sys.stdout.flush()
48	meloam	1.90	retval = nose.run(argv=[__file__,'--all-modules','-v','test/python'])
49	meloam	1.94	if retval:
50	meloam	1.107	sys.exit( 0 )
51	meloam	1.94	else:
52	meloam	1.107	sys.exit( 1 )
53	meloam	1.88	else:
54	meloam	1.101	class TestCommand(Command):
55	meloam	1.88	user_options = [ ]
56			def run(self):
57	meloam	1.97	print "Nose isn't installed. You must install the nose package to run tests (easy_install nose might do it)"
58	meloam	1.88	pass
59
60			def initialize_options(self):
61			pass
62
63			def finalize_options(self):
64			pass
65			pass
66
67	metson	1.87	if can_lint:
68			class LinterRun(Run):
69			def __init__(self, args, reporter=None):
70			self._rcfile = None
71			self._plugins = []
72			preprocess_options(args, {
73			# option: (callback, takearg)
74			'rcfile': (self.cb_set_rcfile, True),
75			'load-plugins': (self.cb_add_plugins, True),
76			})
77			self.linter = linter = self.LinterClass((
78			('rcfile',
79			{'action' : 'callback', 'callback' : lambda *args: 1,
80			'type': 'string', 'metavar': '<file>',
81			'help' : 'Specify a configuration file.'}),
82
83			('init-hook',
84			{'action' : 'callback', 'type' : 'string', 'metavar': '<code>',
85			'callback' : cb_init_hook,
86			'help' : 'Python code to execute, usually for sys.path \
87			manipulation such as pygtk.require().'}),
88
89			('help-msg',
90			{'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>',
91			'callback' : self.cb_help_message,
92			'group': 'Commands',
93			'help' : '''Display a help message for the given message id and \
94			exit. The value may be a comma separated list of message ids.'''}),
95
96			('list-msgs',
97			{'action' : 'callback', 'metavar': '<msg-id>',
98			'callback' : self.cb_list_messages,
99			'group': 'Commands',
100			'help' : "Generate pylint's full documentation."}),
101
102			('generate-rcfile',
103			{'action' : 'callback', 'callback' : self.cb_generate_config,
104			'group': 'Commands',
105			'help' : '''Generate a sample configuration file according to \
106			the current configuration. You can put other options before this one to get \
107			them in the generated configuration.'''}),
108
109			('generate-man',
110			{'action' : 'callback', 'callback' : self.cb_generate_manpage,
111			'group': 'Commands',
112			'help' : "Generate pylint's man page.",'hide': 'True'}),
113
114			('errors-only',
115			{'action' : 'callback', 'callback' : self.cb_error_mode,
116			'short': 'e',
117			'help' : '''In error mode, checkers without error messages are \
118			disabled and for others, only the ERROR messages are displayed, and no reports \
119			are done by default'''}),
120
121			('profile',
122			{'type' : 'yn', 'metavar' : '<y_or_n>',
123			'default': False,
124			'help' : 'Profiled execution.'}),
125
126			), option_groups=self.option_groups,
127			reporter=reporter, pylintrc=self._rcfile)
128			# register standard checkers
129			checkers.initialize(linter)
130			# load command line plugins
131			linter.load_plugin_modules(self._plugins)
132			# read configuration
133			linter.disable_message('W0704')
134			linter.read_config_file()
135			# is there some additional plugins in the file configuration, in
136			config_parser = linter._config_parser
137			if config_parser.has_option('MASTER', 'load-plugins'):
138			plugins = splitstrip(config_parser.get('MASTER', 'load-plugins'))
139			linter.load_plugin_modules(plugins)
140			# now we can load file config and command line, plugins (which can
141			# provide options) have been registered
142			linter.load_config_file()
143			if reporter:
144			# if a custom reporter is provided as argument, it may be overriden
145			# by file parameters, so re-set it here, but before command line
146			# parsing so it's still overrideable by command line option
147			linter.set_reporter(reporter)
148			args = linter.load_command_line_configuration(args)
149			# insert current working directory to the python path to have a correct
150			# behaviour
151			sys.path.insert(0, os.getcwd())
152			if self.linter.config.profile:
153			print >> sys.stderr, '** profiled run'
154			from hotshot import Profile, stats
155			prof = Profile('stones.prof')
156			prof.runcall(linter.check, args)
157			prof.close()
158			data = stats.load('stones.prof')
159			data.strip_dirs()
160			data.sort_stats('time', 'calls')
161			data.print_stats(30)
162			sys.path.pop(0)
163
164			def cb_set_rcfile(self, name, value):
165			"""callback for option preprocessing (ie before optik parsing)"""
166			self._rcfile = value
167
168			def cb_add_plugins(self, name, value):
169			"""callback for option preprocessing (ie before optik parsing)"""
170			self._plugins.extend(splitstrip(value))
171
172			def cb_error_mode(self, args, *kwargs):
173			"""error mode:
174			* checkers without error messages are disabled
175			* for others, only the ERROR messages are displayed
176			* disable reports
177			* do not save execution information
178			"""
179			self.linter.disable_noerror_checkers()
180			self.linter.set_option('disable-msg-cat', 'WCRFI')
181			self.linter.set_option('reports', False)
182			self.linter.set_option('persistent', False)
183
184			def cb_generate_config(self, args, *kwargs):
185			"""optik callback for sample config file generation"""
186			self.linter.generate_config(skipsections=('COMMANDS',))
187
188			def cb_generate_manpage(self, args, *kwargs):
189			"""optik callback for sample config file generation"""
190			from pylint import __pkginfo__
191			self.linter.generate_manpage(__pkginfo__)
192
193			def cb_help_message(self, option, opt_name, value, parser):
194			"""optik callback for printing some help about a particular message"""
195			self.linter.help_message(splitstrip(value))
196
197			def cb_list_messages(self, option, opt_name, value, parser):
198			"""optik callback for printing available messages"""
199			self.linter.list_messages()
200			else:
201			class LinterRun:
202			def __init__(self):
203			pass
204	metson	1.85
205	metson	1.22	"""
206			Build, clean and test the WMCore package.
207			"""
208
209	metson	1.92	def generate_filelist(basepath=None, recurse=True, ignore=False):
210	metson	1.85	if basepath:
211			walkpath = os.path.join(get_relative_path(), 'src/python', basepath)
212			else:
213			walkpath = os.path.join(get_relative_path(), 'src/python')
214
215	metson	1.36	files = []
216	metson	1.85
217			if walkpath.endswith('.py'):
218	metson	1.92	if ignore and walkpath.endswith(ignore):
219			files.append(walkpath)
220	metson	1.85	else:
221			for dirpath, dirnames, filenames in os.walk(walkpath):
222			# skipping CVS directories and their contents
223			pathelements = dirpath.split('/')
224			result = []
225			if not 'CVS' in pathelements:
226			# to build up a list of file names which contain tests
227			for file in filenames:
228			if file.endswith('.py'):
229			filepath = '/'.join([dirpath, file])
230			files.append(filepath)
231
232			if len(files) == 0 and recurse:
233			files = generate_filelist(basepath + '.py', not recurse)
234
235	metson	1.36	return files
236
237	metson	1.85	def lint_score(stats, evaluation):
238			return eval(evaluation, {}, stats)
239
240			def lint_files(files, reports=False):
241	metson	1.36	"""
242	metson	1.85	lint a (list of) file(s) and return the results as a dictionary containing
243			filename : result_dict
244	metson	1.36	"""
245	metson	1.85
246			rcfile=os.path.join(get_relative_path(),'standards/.pylintrc')
247
248	metson	1.92	arguements = ['--rcfile=%s' % rcfile, '--ignore=DefaultConfig.py']
249	metson	1.86
250			if not reports:
251			arguements.append('-rn')
252
253	metson	1.85	arguements.extend(files)
254
255			lntr = LinterRun(arguements)
256
257			results = {}
258			for file in files:
259			lntr.linter.check(file)
260			results[file] = {'stats': lntr.linter.stats,
261			'score': lint_score(lntr.linter.stats,
262			lntr.linter.config.evaluation)
263			}
264			if reports:
265			print '----------------------------------'
266			print 'Your code has been rated at %.2f/10' % \
267			lint_score(lntr.linter.stats, lntr.linter.config.evaluation)
268	metson	1.36
269	metson	1.85	return results, lntr.linter.config.evaluation
270	meloam	1.69
271	metson	1.21	class CleanCommand(Command):
272	metson	1.49	description = "Clean up (delete) compiled files"
273	metson	1.21	user_options = [ ]
274
275			def initialize_options(self):
276			self._clean_me = [ ]
277			for root, dirs, files in os.walk('.'):
278			for f in files:
279			if f.endswith('.pyc'):
280			self._clean_me.append(pjoin(root, f))
281
282			def finalize_options(self):
283			pass
284
285			def run(self):
286			for clean_me in self._clean_me:
287			try:
288			os.unlink(clean_me)
289			except:
290			pass
291
292	metson	1.23	class LintCommand(Command):
293	metson	1.49	description = "Lint all files in the src tree"
294	metson	1.24	"""
295			TODO: better format the test results, get some global result, make output
296			more buildbot friendly.
297			"""
298
299	metson	1.86	user_options = [ ('package=', 'p', 'package to lint, default to None'),
300			('report', 'r', 'return a detailed lint report, default False')]
301	metson	1.23
302			def initialize_options(self):
303	metson	1.85	self._dir = get_relative_path()
304			self.package = None
305	metson	1.86	self.report = False
306	metson	1.85
307	metson	1.23	def finalize_options(self):
308	metson	1.86	if self.report:
309			self.report = True
310
311	metson	1.23	def run(self):
312			'''
313			Find the code and run lint on it
314			'''
315	metson	1.51	if can_lint:
316	metson	1.85	srcpypath = os.path.join(self._dir, 'src/python/')
317
318	metson	1.51	sys.path.append(srcpypath)
319
320	metson	1.85	files_to_lint = []
321
322			if self.package:
323			if self.package.endswith('.py'):
324			cnt = self.package.count('.') - 1
325	metson	1.92	files_to_lint = generate_filelist(self.package.replace('.', '/', cnt), 'DeafultConfig.py')
326	metson	1.85	else:
327	metson	1.92	files_to_lint = generate_filelist(self.package.replace('.', '/'), 'DeafultConfig.py')
328	metson	1.85	else:
329	metson	1.92	files_to_lint = generate_filelist(ignore='DeafultConfig.py')
330	metson	1.86
331			results, evaluation = lint_files(files_to_lint, self.report)
332	metson	1.85	ln = len(results)
333			scr = 0
334			print
335			for k, v in results.items():
336			print "%s: %.2f/10" % (k.replace('src/python/', ''), v['score'])
337			scr += v['score']
338			if ln > 1:
339			print '--------------------------------------------------------'
340			print 'Average pylint score for %s is: %.2f/10' % (self.package,
341			scr/ln)
342
343	metson	1.51	else:
344			print 'You need to install pylint before using the lint command'
345	metson	1.36
346			class ReportCommand(Command):
347	metson	1.49	description = "Generate a simple html report for ease of viewing in buildbot"
348	metson	1.36	"""
349			To contain:
350			average lint score
351			% code coverage
352			list of classes missing tests
353			etc.
354			"""
355
356			user_options = [ ]
357
358			def initialize_options(self):
359			pass
360
361			def finalize_options(self):
362			pass
363
364			def run(self):
365			"""
366			run all the tests needed to generate the report and make an
367			html table
368			"""
369			files = generate_filelist()
370
371			error = 0
372			warning = 0
373			refactor = 0
374			convention = 0
375			statement = 0
376
377	metson	1.85	srcpypath = '/'.join([get_relative_path(), 'src/python/'])
378	metson	1.36	sys.path.append(srcpypath)
379	metson	1.37
380			cfg = ConfigParser()
381			cfg.read('standards/.pylintrc')
382
383	metson	1.40	# Supress stdout/stderr
384	metson	1.38	sys.stderr = open('/dev/null', 'w')
385	metson	1.39	sys.stdout = open('/dev/null', 'w')
386	meloam	1.47	# wrap it in an exception handler, otherwise we can't see why it fails
387			try:
388			# lint the code
389			for stats in lint_files(files):
390			error += stats['error']
391			warning += stats['warning']
392			refactor += stats['refactor']
393			convention += stats['convention']
394			statement += stats['statement']
395			except Exception,e:
396			# and restore the stdout/stderr
397			sys.stderr = sys.__stderr__
398			sys.stdout = sys.__stderr__
399			raise e
400	metson	1.37
401	metson	1.40	# and restore the stdout/stderr
402	meloam	1.47	sys.stderr = sys.__stderr__
403			sys.stdout = sys.__stderr__
404	metson	1.38
405	metson	1.37	stats = {'error': error,
406			'warning': warning,
407			'refactor': refactor,
408			'convention': convention,
409			'statement': statement}
410
411			lint_score = eval(cfg.get('MASTER', 'evaluation'), {}, stats)
412	metson	1.36	coverage = 0 # TODO: calculate this
413			testless_classes = [] # TODO: generate this
414
415			print "<table>"
416			print "<tr>"
417			print "<td colspan=2><h1>WMCore test report</h1></td>"
418			print "</tr>"
419			print "<tr>"
420			print "<td>Average lint score</td>"
421			print "<td>%.2f</td>" % lint_score
422			print "</tr>"
423			print "<tr>"
424			print "<td>% code coverage</td>"
425			print "<td>%s</td>" % coverage
426			print "</tr>"
427			print "<tr>"
428			print "<td>Classes missing tests</td>"
429			print "<td>"
430			if len(testless_classes) == 0:
431			print "None"
432			else:
433			print "<ul>"
434			for c in testless_classes:
435			print "<li>%c</li>" % c
436			print "</ul>"
437			print "</td>"
438			print "</tr>"
439			print "</table>"
440
441			class CoverageCommand(Command):
442	metson	1.49	description = "Run code coverage tests"
443	metson	1.36	"""
444	metson	1.49	To do this, we need to run all the unittests within the coverage
445			framework to record all the lines(and branches) executed
446	meloam	1.47	unfortunately, we have multiple code paths per database schema, so
447			we need to find a way to merge them.
448
449			TODO: modify the test command to have a flag to record code coverage
450			the file thats used can then be used here, saving us from running
451			our tests twice
452	metson	1.36	"""
453
454			user_options = [ ]
455
456			def initialize_options(self):
457			pass
458
459			def finalize_options(self):
460			pass
461
462			def run(self):
463			"""
464			Determine the code's test coverage and return that as a float
465
466			http://nedbatchelder.com/code/coverage/
467			"""
468	metson	1.51	if can_coverage:
469			files = generate_filelist()
470			dataFile = None
471			cov = None
472	meloam	1.48
473	metson	1.51	# attempt to load previously cached coverage information if it exists
474			try:
475			dataFile = open("wmcore-coverage.dat","r")
476			cov = coverage.coverage(branch = True, data_file='wmcore-coverage.dat')
477			cov.load()
478			except:
479			cov = coverage.coverage(branch = True, )
480			cov.start()
481			runUnitTests()
482			cov.stop()
483			cov.save()
484
485			# we have our coverage information, now let's do something with it
486			# get a list of modules
487			cov.report(morfs = files, file=sys.stdout)
488			return 0
489			else:
490			print 'You need the coverage module installed before running the' +\
491			' coverage command'
492	metson	1.36
493	metson	1.41	class DumbCoverageCommand(Command):
494	metson	1.49	description = "Run a simple coverage test - find classes that don't have a unit test"
495	metson	1.41
496			user_options = [ ]
497
498			def initialize_options(self):
499			pass
500
501			def finalize_options(self):
502			pass
503
504			def run(self):
505			"""
506			Determine the code's test coverage in a dumb way and return that as a
507			float.
508			"""
509	metson	1.42	print "This determines test coverage in a very crude manner. If your"
510			print "test file is incorrectly named it will not be counted, and"
511			print "result in a lower coverage score."
512	metson	1.43	print '----------------------------------------------------------------'
513	metson	1.42	filelist = generate_filelist()
514			tests = 0
515			files = 0
516			pkgcnt = 0
517	metson	1.85	dir = get_relative_path()
518	metson	1.42	pkg = {'name': '', 'files': 0, 'tests': 0}
519			for f in filelist:
520	metson	1.41	testpath = '/'.join([dir, f])
521			pth = testpath.split('./src/python/')
522			pth.append(pth[1].replace('/', '_t/').replace('.', '_t.'))
523	metson	1.42	if pkg['name'] == pth[2].rsplit('/', 1)[0].replace('_t/', '/'):
524			# pkg hasn't changed, increment counts
525			pkg['files'] += 1
526			else:
527			# new package, print stats for old package
528			pkgcnt += 1
529			if pkg['name'] != '' and pkg['files'] > 0:
530			print 'Package %s has coverage %.1f percent' % (pkg['name'],
531			(float(pkg['tests'])/float(pkg['files']) * 100))
532			# and start over for the new package
533			pkg['name'] = pth[2].rsplit('/', 1)[0].replace('_t/', '/')
534			# do global book keeping
535			files += pkg['files']
536			tests += pkg['tests']
537			pkg['files'] = 0
538			pkg['tests'] = 0
539	metson	1.41	pth[1] = 'test/python'
540			testpath = '/'.join(pth)
541	metson	1.42	try:
542			os.stat(testpath)
543			pkg['tests'] += 1
544	metson	1.41	except:
545	metson	1.42	pass
546
547			coverage = (float(tests) / float(files)) * 100
548	metson	1.43	print '----------------------------------------------------------------'
549	metson	1.42	print 'Code coverage (%s packages) is %.2f percent' % (pkgcnt, coverage)
550	metson	1.41	return coverage
551	metson	1.49
552			class EnvCommand(Command):
553			description = "Configure the PYTHONPATH, DATABASE and PATH variables to" +\
554			"some sensible defaults, if not already set. Call with -q when eval-ing," +\
555	metson	1.50	""" e.g.:
556	metson	1.49	eval `python setup.py -q env`
557			"""
558
559			user_options = [ ]
560
561			def initialize_options(self):
562			pass
563
564			def finalize_options(self):
565			pass
566	metson	1.41
567	metson	1.49	def run(self):
568			if not os.getenv('DATABASE', False):
569			# Use an in memory sqlite one if none is configured.
570			print 'export DATABASE=sqlite://'
571
572	metson	1.93	here = get_relative_path()
573
574	metson	1.49	tests = here + '/test/python'
575			source = here + '/src/python'
576			webpth = source + '/WMCore/WebTools'
577	metson	1.93
578			pypath=os.getenv('PYTHONPATH', '').strip(':').split(':')
579	metson	1.49
580			for pth in [tests, source]:
581			if pth not in pypath:
582			pypath.append(pth)
583	metson	1.50
584			# We might want to add other executables to PATH
585	metson	1.49	expath=os.getenv('PATH', '').split(':')
586			for pth in [webpth]:
587			if pth not in expath:
588			expath.append(pth)
589
590			print 'export PYTHONPATH=%s' % ':'.join(pypath)
591			print 'export PATH=%s' % ':'.join(expath)
592	metson	1.85
593			#We want the WMCORE root set, too
594	meloam	1.89	print 'export WMCOREBASE=%s' % get_relative_path()
595	metson	1.85
596	metson	1.49
597	metson	1.22	def getPackages(package_dirs = []):
598			packages = []
599			for dir in package_dirs:
600			for dirpath, dirnames, filenames in os.walk('./%s' % dir):
601			# Exclude things here
602			if dirpath not in ['./src/python/', './src/python/IMProv']:
603			pathelements = dirpath.split('/')
604			if not 'CVS' in pathelements:
605			path = pathelements[3:]
606			packages.append('.'.join(path))
607			return packages
608
609			package_dir = {'WMCore': 'src/python/WMCore',
610			'WMComponent' : 'src/python/WMComponent',
611			'WMQuality' : 'src/python/WMQuality'}
612
613			setup (name = 'wmcore',
614			version = '1.0',
615	metson	1.36	maintainer_email = 'hn-cms-wmDevelopment@cern.ch',
616	meloam	1.96	cmdclass = {'clean': CleanCommand,
617	metson	1.36	'lint': LintCommand,
618			'report': ReportCommand,
619	metson	1.41	'coverage': CoverageCommand ,
620	metson	1.49	'missing': DumbCoverageCommand,
621	meloam	1.88	'env': EnvCommand,
622	meloam	1.96	'test' : TestCommand },
623	metson	1.22	package_dir = package_dir,
624			packages = getPackages(package_dir.values()),)