COMP/WMCORE/setup.py

#!/usr/bin/env python
from distutils.core import setup, Command
from unittest import TextTestRunner, TestLoader, TestSuite
from glob import glob
from os.path import splitext, basename, join as pjoin, walk
from ConfigParser import ConfigParser, NoOptionError
import os, sys, os.path
import logging
import unittest
#PyLinter and coverage aren't standard, but aren't strictly necessary
can_lint = False
can_coverage = False
can_nose = False
try:
    from pylint.lint import Run 
    from pylint.lint import preprocess_options, cb_init_hook
    from pylint import checkers
    can_lint = True
except:
    pass
try:
    import coverage
    can_coverage = True
except:
    pass

try:
    import nose
    can_nose = True
except:
    pass

if can_nose:
    class TestCommand(Command):
        """Runs our test suite"""
        
        user_options = [ ]

        def initialize_options(self):
            pass
        
        def finalize_options(self):
            pass
    
        def run(self):
            print "committed at abour 11:23 AM"
            retval =  nose.run(argv=[__file__,'--all-modules','-v','test/python'])
            if retval:
                sys.exit( 1 ) 
            else:
                sys.exit( 0 )
else:
    class TestCommand(Command):
        user_options = [ ]
        def run(self):
            print "Nose isn't installed. You must install the nose package to run tests (easy_install nose might do it)"
            pass
        
        def initialize_options(self):
            pass
        
        def finalize_options(self):
            pass
        pass

if can_lint:
    class LinterRun(Run):
        def __init__(self, args, reporter=None):
            self._rcfile = None
            self._plugins = []
            preprocess_options(args, {
                # option: (callback, takearg)
                'rcfile':       (self.cb_set_rcfile, True),
                'load-plugins': (self.cb_add_plugins, True),
                })
            self.linter = linter = self.LinterClass((
                ('rcfile',
                 {'action' : 'callback', 'callback' : lambda *args: 1,
                  'type': 'string', 'metavar': '<file>',
                  'help' : 'Specify a configuration file.'}),
    
                ('init-hook',
                 {'action' : 'callback', 'type' : 'string', 'metavar': '<code>',
                  'callback' : cb_init_hook,
                  'help' : 'Python code to execute, usually for sys.path \
    manipulation such as pygtk.require().'}),
    
                ('help-msg',
                 {'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>',
                  'callback' : self.cb_help_message,
                  'group': 'Commands',
                  'help' : '''Display a help message for the given message id and \
    exit. The value may be a comma separated list of message ids.'''}),
    
                ('list-msgs',
                 {'action' : 'callback', 'metavar': '<msg-id>',
                  'callback' : self.cb_list_messages,
                  'group': 'Commands',
                  'help' : "Generate pylint's full documentation."}),
    
                ('generate-rcfile',
                 {'action' : 'callback', 'callback' : self.cb_generate_config,
                  'group': 'Commands',
                  'help' : '''Generate a sample configuration file according to \
    the current configuration. You can put other options before this one to get \
    them in the generated configuration.'''}),
    
                ('generate-man',
                 {'action' : 'callback', 'callback' : self.cb_generate_manpage,
                  'group': 'Commands',
                  'help' : "Generate pylint's man page.",'hide': 'True'}),
    
                ('errors-only',
                 {'action' : 'callback', 'callback' : self.cb_error_mode,
                  'short': 'e',
                  'help' : '''In error mode, checkers without error messages are \
    disabled and for others, only the ERROR messages are displayed, and no reports \
    are done by default'''}),
    
                ('profile',
                 {'type' : 'yn', 'metavar' : '<y_or_n>',
                  'default': False,
                  'help' : 'Profiled execution.'}),
    
                ), option_groups=self.option_groups,
                   reporter=reporter, pylintrc=self._rcfile)
            # register standard checkers
            checkers.initialize(linter)
            # load command line plugins
            linter.load_plugin_modules(self._plugins)
            # read configuration
            linter.disable_message('W0704')
            linter.read_config_file()
            # is there some additional plugins in the file configuration, in
            config_parser = linter._config_parser
            if config_parser.has_option('MASTER', 'load-plugins'):
                plugins = splitstrip(config_parser.get('MASTER', 'load-plugins'))
                linter.load_plugin_modules(plugins)
            # now we can load file config and command line, plugins (which can
            # provide options) have been registered
            linter.load_config_file()
            if reporter:
                # if a custom reporter is provided as argument, it may be overriden
                # by file parameters, so re-set it here, but before command line
                # parsing so it's still overrideable by command line option
                linter.set_reporter(reporter)
            args = linter.load_command_line_configuration(args)
            # insert current working directory to the python path to have a correct
            # behaviour
            sys.path.insert(0, os.getcwd())
            if self.linter.config.profile:
                print >> sys.stderr, '** profiled run'
                from hotshot import Profile, stats
                prof = Profile('stones.prof')
                prof.runcall(linter.check, args)
                prof.close()
                data = stats.load('stones.prof')
                data.strip_dirs()
                data.sort_stats('time', 'calls')
                data.print_stats(30)
            sys.path.pop(0)
        
        def cb_set_rcfile(self, name, value):
            """callback for option preprocessing (ie before optik parsing)"""
            self._rcfile = value
    
        def cb_add_plugins(self, name, value):
            """callback for option preprocessing (ie before optik parsing)"""
            self._plugins.extend(splitstrip(value))
    
        def cb_error_mode(self, *args, **kwargs):
            """error mode:
            * checkers without error messages are disabled
            * for others, only the ERROR messages are displayed
            * disable reports
            * do not save execution information
            """
            self.linter.disable_noerror_checkers()
            self.linter.set_option('disable-msg-cat', 'WCRFI')
            self.linter.set_option('reports', False)
            self.linter.set_option('persistent', False)
    
        def cb_generate_config(self, *args, **kwargs):
            """optik callback for sample config file generation"""
            self.linter.generate_config(skipsections=('COMMANDS',))
            
        def cb_generate_manpage(self, *args, **kwargs):
            """optik callback for sample config file generation"""
            from pylint import __pkginfo__
            self.linter.generate_manpage(__pkginfo__)
            
        def cb_help_message(self, option, opt_name, value, parser):
            """optik callback for printing some help about a particular message"""
            self.linter.help_message(splitstrip(value))
    
        def cb_list_messages(self, option, opt_name, value, parser):
            """optik callback for printing available messages"""
            self.linter.list_messages()
else:
    class LinterRun:
        def __init__(self):
            pass

"""
Build, clean and test the WMCore package.
"""

def generate_filelist(basepath=None, recurse=True, ignore=False):
    if basepath:
        walkpath = os.path.join(get_relative_path(), 'src/python', basepath)
    else:
        walkpath = os.path.join(get_relative_path(), 'src/python')
    
    files = []
    
    if walkpath.endswith('.py'):
        if ignore and walkpath.endswith(ignore):
            files.append(walkpath)
    else:
        for dirpath, dirnames, filenames in os.walk(walkpath):
            # skipping CVS directories and their contents
            pathelements = dirpath.split('/')
            result = []
            if not 'CVS' in pathelements:
                # to build up a list of file names which contain tests
                for file in filenames:
                    if file.endswith('.py'):
                        filepath = '/'.join([dirpath, file]) 
                        files.append(filepath)

    if len(files) == 0 and recurse:
        files = generate_filelist(basepath + '.py', not recurse)

    return files

def lint_score(stats, evaluation):
    return eval(evaluation, {}, stats)

def lint_files(files, reports=False):
    """
    lint a (list of) file(s) and return the results as a dictionary containing
    filename : result_dict
    """
    
    rcfile=os.path.join(get_relative_path(),'standards/.pylintrc')
    
    arguements = ['--rcfile=%s' % rcfile, '--ignore=DefaultConfig.py']
    
    if not reports:
        arguements.append('-rn')
    
    arguements.extend(files)
    
    lntr = LinterRun(arguements)
    
    results = {}
    for file in files:
        lntr.linter.check(file)
        results[file] = {'stats': lntr.linter.stats,
                         'score': lint_score(lntr.linter.stats, 
                                             lntr.linter.config.evaluation)
                         }
        if reports:
            print '----------------------------------'
            print 'Your code has been rated at %.2f/10' % \
                    lint_score(lntr.linter.stats, lntr.linter.config.evaluation)

    return results, lntr.linter.config.evaluation

class CleanCommand(Command):
    description = "Clean up (delete) compiled files"
    user_options = [ ]

    def initialize_options(self):
        self._clean_me = [ ]
        for root, dirs, files in os.walk('.'):
            for f in files:
                if f.endswith('.pyc'):
                    self._clean_me.append(pjoin(root, f))

    def finalize_options(self):
        pass

    def run(self):
        for clean_me in self._clean_me:
            try:
                os.unlink(clean_me)
            except:
                pass
            
class LintCommand(Command):
    description = "Lint all files in the src tree"
    """
    TODO: better format the test results, get some global result, make output 
    more buildbot friendly.    
    """
    
    user_options = [ ('package=', 'p', 'package to lint, default to None'),
                ('report', 'r', 'return a detailed lint report, default False')]
    
    def initialize_options(self):
        self._dir = get_relative_path()
        self.package = None
        self.report = False
        
    def finalize_options(self):
        if self.report:
            self.report = True

    def run(self):
        '''
        Find the code and run lint on it
        '''
        if can_lint:
            srcpypath = os.path.join(self._dir, 'src/python/')
            
            sys.path.append(srcpypath) 
            
            files_to_lint = []
            
            if self.package:
                if self.package.endswith('.py'):
                    cnt = self.package.count('.') - 1
                    files_to_lint = generate_filelist(self.package.replace('.', '/', cnt), 'DeafultConfig.py')
                else:
                    files_to_lint = generate_filelist(self.package.replace('.', '/'), 'DeafultConfig.py')
            else:
                files_to_lint = generate_filelist(ignore='DeafultConfig.py')
            
            results, evaluation = lint_files(files_to_lint, self.report)
            ln = len(results)
            scr = 0
            print
            for k, v in results.items():
                print "%s: %.2f/10" % (k.replace('src/python/', ''), v['score'])
                scr += v['score']
            if ln > 1:
                print '--------------------------------------------------------'
                print 'Average pylint score for %s is: %.2f/10' % (self.package,
                                                                  scr/ln)
                
        else:
            print 'You need to install pylint before using the lint command'
            
class ReportCommand(Command):
    description = "Generate a simple html report for ease of viewing in buildbot"
    """
    To contain:
        average lint score
        % code coverage
        list of classes missing tests
        etc.
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        """
        run all the tests needed to generate the report and make an
        html table
        """
        files = generate_filelist()
        
        error = 0 
        warning = 0
        refactor = 0
        convention = 0 
        statement = 0
        
        srcpypath = '/'.join([get_relative_path(), 'src/python/'])
        sys.path.append(srcpypath)

        cfg = ConfigParser()
        cfg.read('standards/.pylintrc')
        
        # Supress stdout/stderr
        sys.stderr = open('/dev/null', 'w')
        sys.stdout = open('/dev/null', 'w')
        # wrap it in an exception handler, otherwise we can't see why it fails
        try:
            # lint the code
            for stats in lint_files(files):
                error += stats['error']
                warning += stats['warning']
                refactor += stats['refactor']
                convention += stats['convention'] 
                statement += stats['statement']
        except Exception,e:
            # and restore the stdout/stderr
            sys.stderr = sys.__stderr__
            sys.stdout = sys.__stderr__
            raise e
        
        # and restore the stdout/stderr
        sys.stderr = sys.__stderr__
        sys.stdout = sys.__stderr__ 
        
        stats = {'error': error,
            'warning': warning,
            'refactor': refactor,
            'convention': convention, 
            'statement': statement}
        
        lint_score = eval(cfg.get('MASTER', 'evaluation'), {}, stats)
        coverage = 0 # TODO: calculate this
        testless_classes = [] # TODO: generate this
        
        print "<table>"
        print "<tr>"
        print "<td colspan=2><h1>WMCore test report</h1></td>"
        print "</tr>"
        print "<tr>"
        print "<td>Average lint score</td>"
        print "<td>%.2f</td>" % lint_score
        print "</tr>"
        print "<tr>"
        print "<td>% code coverage</td>"
        print "<td>%s</td>" % coverage
        print "</tr>"
        print "<tr>"
        print "<td>Classes missing tests</td>"
        print "<td>"
        if len(testless_classes) == 0:
            print "None"
        else:
            print "<ul>"
            for c in testless_classes:
                print "<li>%c</li>" % c
            print "</ul>"
        print "</td>"
        print "</tr>"
        print "</table>"
            
class CoverageCommand(Command):
    description = "Run code coverage tests"
    """
    To do this, we need to run all the unittests within the coverage
    framework to record all the lines(and branches) executed
    unfortunately, we have multiple code paths per database schema, so
    we need to find a way to merge them.
      
    TODO: modify the test command to have a flag to record code coverage
          the file thats used can then be used here, saving us from running
          our tests twice    
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        """
        Determine the code's test coverage and return that as a float
        
        http://nedbatchelder.com/code/coverage/
        """
        if can_coverage:
            files = generate_filelist()
            dataFile = None
            cov = None
            
            # attempt to load previously cached coverage information if it exists
            try:
                dataFile = open("wmcore-coverage.dat","r")
                cov = coverage.coverage(branch = True, data_file='wmcore-coverage.dat')
                cov.load()
            except:  
                cov = coverage.coverage(branch = True, )
                cov.start()
                runUnitTests()
                cov.stop()
                cov.save()
                
            # we have our coverage information, now let's do something with it
            # get a list of modules
            cov.report(morfs = files, file=sys.stdout)
            return 0
        else:
            print 'You need the coverage module installed before running the' +\
                            ' coverage command'
    
class DumbCoverageCommand(Command):
    description = "Run a simple coverage test - find classes that don't have a unit test"
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        """
        Determine the code's test coverage in a dumb way and return that as a 
        float.
        """
        print "This determines test coverage in a very crude manner. If your"
        print "test file is incorrectly named it will not be counted, and" 
        print "result in a lower coverage score." 
        print '----------------------------------------------------------------'
        filelist = generate_filelist()
        tests = 0
        files = 0
        pkgcnt = 0
        dir = get_relative_path()
        pkg = {'name': '', 'files': 0, 'tests': 0}
        for f in filelist:
            testpath = '/'.join([dir, f])
            pth = testpath.split('./src/python/')
            pth.append(pth[1].replace('/', '_t/').replace('.', '_t.'))
            if pkg['name'] == pth[2].rsplit('/', 1)[0].replace('_t/', '/'):
                # pkg hasn't changed, increment counts
                pkg['files'] += 1
            else:
                # new package, print stats for old package
                pkgcnt += 1
                if pkg['name'] != '' and pkg['files'] > 0:
                    print 'Package %s has coverage %.1f percent' % (pkg['name'], 
                                (float(pkg['tests'])/float(pkg['files']) * 100))
                # and start over for the new package
                pkg['name'] = pth[2].rsplit('/', 1)[0].replace('_t/', '/')
                # do global book keeping
                files += pkg['files']
                tests += pkg['tests'] 
                pkg['files'] = 0 
                pkg['tests'] = 0
            pth[1] = 'test/python'
            testpath = '/'.join(pth)
            try: 
                os.stat(testpath)
                pkg['tests'] += 1
            except:
                pass
            
        coverage = (float(tests) / float(files)) * 100
        print '----------------------------------------------------------------'
        print 'Code coverage (%s packages) is %.2f percent' % (pkgcnt, coverage)
        return coverage

class EnvCommand(Command):
    description = "Configure the PYTHONPATH, DATABASE and PATH variables to" +\
    "some sensible defaults, if not already set. Call with -q when eval-ing," +\
    """ e.g.:
        eval `python setup.py -q env`
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        if not os.getenv('DATABASE', False):
            # Use an in memory sqlite one if none is configured. 
             print 'export DATABASE=sqlite://'
            
        here = get_relative_path()
        
        tests = here + '/test/python'
        source = here + '/src/python'
        webpth = source + '/WMCore/WebTools'
        
        pypath=os.getenv('PYTHONPATH', '').strip(':').split(':')
         
        for pth in [tests, source]:
            if pth not in pypath:
                pypath.append(pth)
        
        # We might want to add other executables to PATH
        expath=os.getenv('PATH', '').split(':')
        for pth in [webpth]:
            if pth not in expath:
                expath.append(pth)
              
        print 'export PYTHONPATH=%s' % ':'.join(pypath)
        print 'export PATH=%s' % ':'.join(expath)
        
        #We want the WMCORE root set, too
        print 'export WMCOREBASE=%s' % get_relative_path()


def getPackages(package_dirs = []):
    packages = []
    for dir in package_dirs:
        for dirpath, dirnames, filenames in os.walk('./%s' % dir):
            # Exclude things here
            if dirpath not in ['./src/python/', './src/python/IMProv']: 
                pathelements = dirpath.split('/')
                if not 'CVS' in pathelements:
                    path = pathelements[3:]
                    packages.append('.'.join(path))
    return packages

package_dir = {'WMCore': 'src/python/WMCore',
               'WMComponent' : 'src/python/WMComponent',
               'WMQuality' : 'src/python/WMQuality'}

setup (name = 'wmcore',
       version = '1.0',
       maintainer_email = 'hn-cms-wmDevelopment@cern.ch',
       cmdclass = {'clean': CleanCommand, 
                   'lint': LintCommand,
                   'report': ReportCommand,
                   'coverage': CoverageCommand ,
                   'missing': DumbCoverageCommand,
                   'env': EnvCommand,
                   'test' : TestCommand },
       package_dir = package_dir,
       packages = getPackages(package_dir.values()),)
Revision:	1.106
Committed:	Thu Feb 4 17:21:39 2010 UTC (15 years, 2 months ago) by meloam
Content type:	text/x-python
Branch:	MAIN
Changes since 1.105:	+3 -3 lines
Log Message:	fixing failingness
#	User	Rev	Content
1	fvlingen	1.1	#!/usr/bin/env python
2	metson	1.21	from distutils.core import setup, Command
3			from unittest import TextTestRunner, TestLoader, TestSuite
4			from glob import glob
5			from os.path import splitext, basename, join as pjoin, walk
6	metson	1.85	from ConfigParser import ConfigParser, NoOptionError
7	meloam	1.53	import os, sys, os.path
8	metson	1.85	import logging
9			import unittest
10	metson	1.51	#PyLinter and coverage aren't standard, but aren't strictly necessary
11			can_lint = False
12			can_coverage = False
13	meloam	1.88	can_nose = False
14	sfoulkes	1.29	try:
15	metson	1.87	from pylint.lint import Run
16	metson	1.85	from pylint.lint import preprocess_options, cb_init_hook
17			from pylint import checkers
18	metson	1.51	can_lint = True
19			except:
20			pass
21			try:
22	meloam	1.47	import coverage
23	metson	1.51	can_coverage = True
24	sfoulkes	1.29	except:
25			pass
26	fvlingen	1.1
27	meloam	1.88	try:
28			import nose
29			can_nose = True
30			except:
31			pass
32
33			if can_nose:
34	meloam	1.96	class TestCommand(Command):
35	meloam	1.100	"""Runs our test suite"""
36
37	meloam	1.88	user_options = [ ]
38
39			def initialize_options(self):
40			pass
41
42			def finalize_options(self):
43			pass
44
45			def run(self):
46	meloam	1.106	print "committed at abour 11:23 AM"
47	meloam	1.90	retval = nose.run(argv=[__file__,'--all-modules','-v','test/python'])
48	meloam	1.94	if retval:
49	meloam	1.106	sys.exit( 1 )
50	meloam	1.94	else:
51	meloam	1.106	sys.exit( 0 )
52	meloam	1.88	else:
53	meloam	1.101	class TestCommand(Command):
54	meloam	1.88	user_options = [ ]
55			def run(self):
56	meloam	1.97	print "Nose isn't installed. You must install the nose package to run tests (easy_install nose might do it)"
57	meloam	1.88	pass
58
59			def initialize_options(self):
60			pass
61
62			def finalize_options(self):
63			pass
64			pass
65
66	metson	1.87	if can_lint:
67			class LinterRun(Run):
68			def __init__(self, args, reporter=None):
69			self._rcfile = None
70			self._plugins = []
71			preprocess_options(args, {
72			# option: (callback, takearg)
73			'rcfile': (self.cb_set_rcfile, True),
74			'load-plugins': (self.cb_add_plugins, True),
75			})
76			self.linter = linter = self.LinterClass((
77			('rcfile',
78			{'action' : 'callback', 'callback' : lambda *args: 1,
79			'type': 'string', 'metavar': '<file>',
80			'help' : 'Specify a configuration file.'}),
81
82			('init-hook',
83			{'action' : 'callback', 'type' : 'string', 'metavar': '<code>',
84			'callback' : cb_init_hook,
85			'help' : 'Python code to execute, usually for sys.path \
86			manipulation such as pygtk.require().'}),
87
88			('help-msg',
89			{'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>',
90			'callback' : self.cb_help_message,
91			'group': 'Commands',
92			'help' : '''Display a help message for the given message id and \
93			exit. The value may be a comma separated list of message ids.'''}),
94
95			('list-msgs',
96			{'action' : 'callback', 'metavar': '<msg-id>',
97			'callback' : self.cb_list_messages,
98			'group': 'Commands',
99			'help' : "Generate pylint's full documentation."}),
100
101			('generate-rcfile',
102			{'action' : 'callback', 'callback' : self.cb_generate_config,
103			'group': 'Commands',
104			'help' : '''Generate a sample configuration file according to \
105			the current configuration. You can put other options before this one to get \
106			them in the generated configuration.'''}),
107
108			('generate-man',
109			{'action' : 'callback', 'callback' : self.cb_generate_manpage,
110			'group': 'Commands',
111			'help' : "Generate pylint's man page.",'hide': 'True'}),
112
113			('errors-only',
114			{'action' : 'callback', 'callback' : self.cb_error_mode,
115			'short': 'e',
116			'help' : '''In error mode, checkers without error messages are \
117			disabled and for others, only the ERROR messages are displayed, and no reports \
118			are done by default'''}),
119
120			('profile',
121			{'type' : 'yn', 'metavar' : '<y_or_n>',
122			'default': False,
123			'help' : 'Profiled execution.'}),
124
125			), option_groups=self.option_groups,
126			reporter=reporter, pylintrc=self._rcfile)
127			# register standard checkers
128			checkers.initialize(linter)
129			# load command line plugins
130			linter.load_plugin_modules(self._plugins)
131			# read configuration
132			linter.disable_message('W0704')
133			linter.read_config_file()
134			# is there some additional plugins in the file configuration, in
135			config_parser = linter._config_parser
136			if config_parser.has_option('MASTER', 'load-plugins'):
137			plugins = splitstrip(config_parser.get('MASTER', 'load-plugins'))
138			linter.load_plugin_modules(plugins)
139			# now we can load file config and command line, plugins (which can
140			# provide options) have been registered
141			linter.load_config_file()
142			if reporter:
143			# if a custom reporter is provided as argument, it may be overriden
144			# by file parameters, so re-set it here, but before command line
145			# parsing so it's still overrideable by command line option
146			linter.set_reporter(reporter)
147			args = linter.load_command_line_configuration(args)
148			# insert current working directory to the python path to have a correct
149			# behaviour
150			sys.path.insert(0, os.getcwd())
151			if self.linter.config.profile:
152			print >> sys.stderr, '** profiled run'
153			from hotshot import Profile, stats
154			prof = Profile('stones.prof')
155			prof.runcall(linter.check, args)
156			prof.close()
157			data = stats.load('stones.prof')
158			data.strip_dirs()
159			data.sort_stats('time', 'calls')
160			data.print_stats(30)
161			sys.path.pop(0)
162
163			def cb_set_rcfile(self, name, value):
164			"""callback for option preprocessing (ie before optik parsing)"""
165			self._rcfile = value
166
167			def cb_add_plugins(self, name, value):
168			"""callback for option preprocessing (ie before optik parsing)"""
169			self._plugins.extend(splitstrip(value))
170
171			def cb_error_mode(self, args, *kwargs):
172			"""error mode:
173			* checkers without error messages are disabled
174			* for others, only the ERROR messages are displayed
175			* disable reports
176			* do not save execution information
177			"""
178			self.linter.disable_noerror_checkers()
179			self.linter.set_option('disable-msg-cat', 'WCRFI')
180			self.linter.set_option('reports', False)
181			self.linter.set_option('persistent', False)
182
183			def cb_generate_config(self, args, *kwargs):
184			"""optik callback for sample config file generation"""
185			self.linter.generate_config(skipsections=('COMMANDS',))
186
187			def cb_generate_manpage(self, args, *kwargs):
188			"""optik callback for sample config file generation"""
189			from pylint import __pkginfo__
190			self.linter.generate_manpage(__pkginfo__)
191
192			def cb_help_message(self, option, opt_name, value, parser):
193			"""optik callback for printing some help about a particular message"""
194			self.linter.help_message(splitstrip(value))
195
196			def cb_list_messages(self, option, opt_name, value, parser):
197			"""optik callback for printing available messages"""
198			self.linter.list_messages()
199			else:
200			class LinterRun:
201			def __init__(self):
202			pass
203	metson	1.85
204	metson	1.22	"""
205			Build, clean and test the WMCore package.
206			"""
207
208	metson	1.92	def generate_filelist(basepath=None, recurse=True, ignore=False):
209	metson	1.85	if basepath:
210			walkpath = os.path.join(get_relative_path(), 'src/python', basepath)
211			else:
212			walkpath = os.path.join(get_relative_path(), 'src/python')
213
214	metson	1.36	files = []
215	metson	1.85
216			if walkpath.endswith('.py'):
217	metson	1.92	if ignore and walkpath.endswith(ignore):
218			files.append(walkpath)
219	metson	1.85	else:
220			for dirpath, dirnames, filenames in os.walk(walkpath):
221			# skipping CVS directories and their contents
222			pathelements = dirpath.split('/')
223			result = []
224			if not 'CVS' in pathelements:
225			# to build up a list of file names which contain tests
226			for file in filenames:
227			if file.endswith('.py'):
228			filepath = '/'.join([dirpath, file])
229			files.append(filepath)
230
231			if len(files) == 0 and recurse:
232			files = generate_filelist(basepath + '.py', not recurse)
233
234	metson	1.36	return files
235
236	metson	1.85	def lint_score(stats, evaluation):
237			return eval(evaluation, {}, stats)
238
239			def lint_files(files, reports=False):
240	metson	1.36	"""
241	metson	1.85	lint a (list of) file(s) and return the results as a dictionary containing
242			filename : result_dict
243	metson	1.36	"""
244	metson	1.85
245			rcfile=os.path.join(get_relative_path(),'standards/.pylintrc')
246
247	metson	1.92	arguements = ['--rcfile=%s' % rcfile, '--ignore=DefaultConfig.py']
248	metson	1.86
249			if not reports:
250			arguements.append('-rn')
251
252	metson	1.85	arguements.extend(files)
253
254			lntr = LinterRun(arguements)
255
256			results = {}
257			for file in files:
258			lntr.linter.check(file)
259			results[file] = {'stats': lntr.linter.stats,
260			'score': lint_score(lntr.linter.stats,
261			lntr.linter.config.evaluation)
262			}
263			if reports:
264			print '----------------------------------'
265			print 'Your code has been rated at %.2f/10' % \
266			lint_score(lntr.linter.stats, lntr.linter.config.evaluation)
267	metson	1.36
268	metson	1.85	return results, lntr.linter.config.evaluation
269	meloam	1.69
270	metson	1.21	class CleanCommand(Command):
271	metson	1.49	description = "Clean up (delete) compiled files"
272	metson	1.21	user_options = [ ]
273
274			def initialize_options(self):
275			self._clean_me = [ ]
276			for root, dirs, files in os.walk('.'):
277			for f in files:
278			if f.endswith('.pyc'):
279			self._clean_me.append(pjoin(root, f))
280
281			def finalize_options(self):
282			pass
283
284			def run(self):
285			for clean_me in self._clean_me:
286			try:
287			os.unlink(clean_me)
288			except:
289			pass
290
291	metson	1.23	class LintCommand(Command):
292	metson	1.49	description = "Lint all files in the src tree"
293	metson	1.24	"""
294			TODO: better format the test results, get some global result, make output
295			more buildbot friendly.
296			"""
297
298	metson	1.86	user_options = [ ('package=', 'p', 'package to lint, default to None'),
299			('report', 'r', 'return a detailed lint report, default False')]
300	metson	1.23
301			def initialize_options(self):
302	metson	1.85	self._dir = get_relative_path()
303			self.package = None
304	metson	1.86	self.report = False
305	metson	1.85
306	metson	1.23	def finalize_options(self):
307	metson	1.86	if self.report:
308			self.report = True
309
310	metson	1.23	def run(self):
311			'''
312			Find the code and run lint on it
313			'''
314	metson	1.51	if can_lint:
315	metson	1.85	srcpypath = os.path.join(self._dir, 'src/python/')
316
317	metson	1.51	sys.path.append(srcpypath)
318
319	metson	1.85	files_to_lint = []
320
321			if self.package:
322			if self.package.endswith('.py'):
323			cnt = self.package.count('.') - 1
324	metson	1.92	files_to_lint = generate_filelist(self.package.replace('.', '/', cnt), 'DeafultConfig.py')
325	metson	1.85	else:
326	metson	1.92	files_to_lint = generate_filelist(self.package.replace('.', '/'), 'DeafultConfig.py')
327	metson	1.85	else:
328	metson	1.92	files_to_lint = generate_filelist(ignore='DeafultConfig.py')
329	metson	1.86
330			results, evaluation = lint_files(files_to_lint, self.report)
331	metson	1.85	ln = len(results)
332			scr = 0
333			print
334			for k, v in results.items():
335			print "%s: %.2f/10" % (k.replace('src/python/', ''), v['score'])
336			scr += v['score']
337			if ln > 1:
338			print '--------------------------------------------------------'
339			print 'Average pylint score for %s is: %.2f/10' % (self.package,
340			scr/ln)
341
342	metson	1.51	else:
343			print 'You need to install pylint before using the lint command'
344	metson	1.36
345			class ReportCommand(Command):
346	metson	1.49	description = "Generate a simple html report for ease of viewing in buildbot"
347	metson	1.36	"""
348			To contain:
349			average lint score
350			% code coverage
351			list of classes missing tests
352			etc.
353			"""
354
355			user_options = [ ]
356
357			def initialize_options(self):
358			pass
359
360			def finalize_options(self):
361			pass
362
363			def run(self):
364			"""
365			run all the tests needed to generate the report and make an
366			html table
367			"""
368			files = generate_filelist()
369
370			error = 0
371			warning = 0
372			refactor = 0
373			convention = 0
374			statement = 0
375
376	metson	1.85	srcpypath = '/'.join([get_relative_path(), 'src/python/'])
377	metson	1.36	sys.path.append(srcpypath)
378	metson	1.37
379			cfg = ConfigParser()
380			cfg.read('standards/.pylintrc')
381
382	metson	1.40	# Supress stdout/stderr
383	metson	1.38	sys.stderr = open('/dev/null', 'w')
384	metson	1.39	sys.stdout = open('/dev/null', 'w')
385	meloam	1.47	# wrap it in an exception handler, otherwise we can't see why it fails
386			try:
387			# lint the code
388			for stats in lint_files(files):
389			error += stats['error']
390			warning += stats['warning']
391			refactor += stats['refactor']
392			convention += stats['convention']
393			statement += stats['statement']
394			except Exception,e:
395			# and restore the stdout/stderr
396			sys.stderr = sys.__stderr__
397			sys.stdout = sys.__stderr__
398			raise e
399	metson	1.37
400	metson	1.40	# and restore the stdout/stderr
401	meloam	1.47	sys.stderr = sys.__stderr__
402			sys.stdout = sys.__stderr__
403	metson	1.38
404	metson	1.37	stats = {'error': error,
405			'warning': warning,
406			'refactor': refactor,
407			'convention': convention,
408			'statement': statement}
409
410			lint_score = eval(cfg.get('MASTER', 'evaluation'), {}, stats)
411	metson	1.36	coverage = 0 # TODO: calculate this
412			testless_classes = [] # TODO: generate this
413
414			print "<table>"
415			print "<tr>"
416			print "<td colspan=2><h1>WMCore test report</h1></td>"
417			print "</tr>"
418			print "<tr>"
419			print "<td>Average lint score</td>"
420			print "<td>%.2f</td>" % lint_score
421			print "</tr>"
422			print "<tr>"
423			print "<td>% code coverage</td>"
424			print "<td>%s</td>" % coverage
425			print "</tr>"
426			print "<tr>"
427			print "<td>Classes missing tests</td>"
428			print "<td>"
429			if len(testless_classes) == 0:
430			print "None"
431			else:
432			print "<ul>"
433			for c in testless_classes:
434			print "<li>%c</li>" % c
435			print "</ul>"
436			print "</td>"
437			print "</tr>"
438			print "</table>"
439
440			class CoverageCommand(Command):
441	metson	1.49	description = "Run code coverage tests"
442	metson	1.36	"""
443	metson	1.49	To do this, we need to run all the unittests within the coverage
444			framework to record all the lines(and branches) executed
445	meloam	1.47	unfortunately, we have multiple code paths per database schema, so
446			we need to find a way to merge them.
447
448			TODO: modify the test command to have a flag to record code coverage
449			the file thats used can then be used here, saving us from running
450			our tests twice
451	metson	1.36	"""
452
453			user_options = [ ]
454
455			def initialize_options(self):
456			pass
457
458			def finalize_options(self):
459			pass
460
461			def run(self):
462			"""
463			Determine the code's test coverage and return that as a float
464
465			http://nedbatchelder.com/code/coverage/
466			"""
467	metson	1.51	if can_coverage:
468			files = generate_filelist()
469			dataFile = None
470			cov = None
471	meloam	1.48
472	metson	1.51	# attempt to load previously cached coverage information if it exists
473			try:
474			dataFile = open("wmcore-coverage.dat","r")
475			cov = coverage.coverage(branch = True, data_file='wmcore-coverage.dat')
476			cov.load()
477			except:
478			cov = coverage.coverage(branch = True, )
479			cov.start()
480			runUnitTests()
481			cov.stop()
482			cov.save()
483
484			# we have our coverage information, now let's do something with it
485			# get a list of modules
486			cov.report(morfs = files, file=sys.stdout)
487			return 0
488			else:
489			print 'You need the coverage module installed before running the' +\
490			' coverage command'
491	metson	1.36
492	metson	1.41	class DumbCoverageCommand(Command):
493	metson	1.49	description = "Run a simple coverage test - find classes that don't have a unit test"
494	metson	1.41
495			user_options = [ ]
496
497			def initialize_options(self):
498			pass
499
500			def finalize_options(self):
501			pass
502
503			def run(self):
504			"""
505			Determine the code's test coverage in a dumb way and return that as a
506			float.
507			"""
508	metson	1.42	print "This determines test coverage in a very crude manner. If your"
509			print "test file is incorrectly named it will not be counted, and"
510			print "result in a lower coverage score."
511	metson	1.43	print '----------------------------------------------------------------'
512	metson	1.42	filelist = generate_filelist()
513			tests = 0
514			files = 0
515			pkgcnt = 0
516	metson	1.85	dir = get_relative_path()
517	metson	1.42	pkg = {'name': '', 'files': 0, 'tests': 0}
518			for f in filelist:
519	metson	1.41	testpath = '/'.join([dir, f])
520			pth = testpath.split('./src/python/')
521			pth.append(pth[1].replace('/', '_t/').replace('.', '_t.'))
522	metson	1.42	if pkg['name'] == pth[2].rsplit('/', 1)[0].replace('_t/', '/'):
523			# pkg hasn't changed, increment counts
524			pkg['files'] += 1
525			else:
526			# new package, print stats for old package
527			pkgcnt += 1
528			if pkg['name'] != '' and pkg['files'] > 0:
529			print 'Package %s has coverage %.1f percent' % (pkg['name'],
530			(float(pkg['tests'])/float(pkg['files']) * 100))
531			# and start over for the new package
532			pkg['name'] = pth[2].rsplit('/', 1)[0].replace('_t/', '/')
533			# do global book keeping
534			files += pkg['files']
535			tests += pkg['tests']
536			pkg['files'] = 0
537			pkg['tests'] = 0
538	metson	1.41	pth[1] = 'test/python'
539			testpath = '/'.join(pth)
540	metson	1.42	try:
541			os.stat(testpath)
542			pkg['tests'] += 1
543	metson	1.41	except:
544	metson	1.42	pass
545
546			coverage = (float(tests) / float(files)) * 100
547	metson	1.43	print '----------------------------------------------------------------'
548	metson	1.42	print 'Code coverage (%s packages) is %.2f percent' % (pkgcnt, coverage)
549	metson	1.41	return coverage
550	metson	1.49
551			class EnvCommand(Command):
552			description = "Configure the PYTHONPATH, DATABASE and PATH variables to" +\
553			"some sensible defaults, if not already set. Call with -q when eval-ing," +\
554	metson	1.50	""" e.g.:
555	metson	1.49	eval `python setup.py -q env`
556			"""
557
558			user_options = [ ]
559
560			def initialize_options(self):
561			pass
562
563			def finalize_options(self):
564			pass
565	metson	1.41
566	metson	1.49	def run(self):
567			if not os.getenv('DATABASE', False):
568			# Use an in memory sqlite one if none is configured.
569			print 'export DATABASE=sqlite://'
570
571	metson	1.93	here = get_relative_path()
572
573	metson	1.49	tests = here + '/test/python'
574			source = here + '/src/python'
575			webpth = source + '/WMCore/WebTools'
576	metson	1.93
577			pypath=os.getenv('PYTHONPATH', '').strip(':').split(':')
578	metson	1.49
579			for pth in [tests, source]:
580			if pth not in pypath:
581			pypath.append(pth)
582	metson	1.50
583			# We might want to add other executables to PATH
584	metson	1.49	expath=os.getenv('PATH', '').split(':')
585			for pth in [webpth]:
586			if pth not in expath:
587			expath.append(pth)
588
589			print 'export PYTHONPATH=%s' % ':'.join(pypath)
590			print 'export PATH=%s' % ':'.join(expath)
591	metson	1.85
592			#We want the WMCORE root set, too
593	meloam	1.89	print 'export WMCOREBASE=%s' % get_relative_path()
594	metson	1.85
595	metson	1.49
596	metson	1.22	def getPackages(package_dirs = []):
597			packages = []
598			for dir in package_dirs:
599			for dirpath, dirnames, filenames in os.walk('./%s' % dir):
600			# Exclude things here
601			if dirpath not in ['./src/python/', './src/python/IMProv']:
602			pathelements = dirpath.split('/')
603			if not 'CVS' in pathelements:
604			path = pathelements[3:]
605			packages.append('.'.join(path))
606			return packages
607
608			package_dir = {'WMCore': 'src/python/WMCore',
609			'WMComponent' : 'src/python/WMComponent',
610			'WMQuality' : 'src/python/WMQuality'}
611
612			setup (name = 'wmcore',
613			version = '1.0',
614	metson	1.36	maintainer_email = 'hn-cms-wmDevelopment@cern.ch',
615	meloam	1.96	cmdclass = {'clean': CleanCommand,
616	metson	1.36	'lint': LintCommand,
617			'report': ReportCommand,
618	metson	1.41	'coverage': CoverageCommand ,
619	metson	1.49	'missing': DumbCoverageCommand,
620	meloam	1.88	'env': EnvCommand,
621	meloam	1.96	'test' : TestCommand },
622	metson	1.22	package_dir = package_dir,
623			packages = getPackages(package_dir.values()),)