COMP/WMCORE/setup.py

#!/usr/bin/env python
from distutils.core import setup, Command
from unittest import TextTestRunner, TestLoader, TestSuite
from glob import glob
from os.path import splitext, basename, join as pjoin, walk
from ConfigParser import ConfigParser, NoOptionError
import os, sys, os.path
import logging
import unittest
#PyLinter and coverage aren't standard, but aren't strictly necessary
can_lint = False
can_coverage = False
can_nose = False
try:
    from pylint.lint import Run 
    from pylint.lint import preprocess_options, cb_init_hook
    from pylint import checkers
    can_lint = True
except:
    pass
try:
    import coverage
    can_coverage = True
except:
    pass

try:
    import nose
    can_nose = True
except:
    pass

if can_nose:
    class TestCommand(Command):
        """Runs our test suite"""
        
        user_options = [ ]

        def initialize_options(self):
            pass
        
        def finalize_options(self):
            pass
    
        def run(self):
            retval =  nose.run(argv=[__file__,'--all-modules','-v','test/python'])
            if retval:
                return 0 
            else:
                return 1
else:
    class TestCommand(Command):
        user_options = [ ]
        def run(self):
            print "Nose isn't installed. You must install the nose package to run tests (easy_install nose might do it)"
            pass
        
        def initialize_options(self):
            pass
        
        def finalize_options(self):
            pass
        pass

if can_lint:
    class LinterRun(Run):
        def __init__(self, args, reporter=None):
            self._rcfile = None
            self._plugins = []
            preprocess_options(args, {
                # option: (callback, takearg)
                'rcfile':       (self.cb_set_rcfile, True),
                'load-plugins': (self.cb_add_plugins, True),
                })
            self.linter = linter = self.LinterClass((
                ('rcfile',
                 {'action' : 'callback', 'callback' : lambda *args: 1,
                  'type': 'string', 'metavar': '<file>',
                  'help' : 'Specify a configuration file.'}),
    
                ('init-hook',
                 {'action' : 'callback', 'type' : 'string', 'metavar': '<code>',
                  'callback' : cb_init_hook,
                  'help' : 'Python code to execute, usually for sys.path \
    manipulation such as pygtk.require().'}),
    
                ('help-msg',
                 {'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>',
                  'callback' : self.cb_help_message,
                  'group': 'Commands',
                  'help' : '''Display a help message for the given message id and \
    exit. The value may be a comma separated list of message ids.'''}),
    
                ('list-msgs',
                 {'action' : 'callback', 'metavar': '<msg-id>',
                  'callback' : self.cb_list_messages,
                  'group': 'Commands',
                  'help' : "Generate pylint's full documentation."}),
    
                ('generate-rcfile',
                 {'action' : 'callback', 'callback' : self.cb_generate_config,
                  'group': 'Commands',
                  'help' : '''Generate a sample configuration file according to \
    the current configuration. You can put other options before this one to get \
    them in the generated configuration.'''}),
    
                ('generate-man',
                 {'action' : 'callback', 'callback' : self.cb_generate_manpage,
                  'group': 'Commands',
                  'help' : "Generate pylint's man page.",'hide': 'True'}),
    
                ('errors-only',
                 {'action' : 'callback', 'callback' : self.cb_error_mode,
                  'short': 'e',
                  'help' : '''In error mode, checkers without error messages are \
    disabled and for others, only the ERROR messages are displayed, and no reports \
    are done by default'''}),
    
                ('profile',
                 {'type' : 'yn', 'metavar' : '<y_or_n>',
                  'default': False,
                  'help' : 'Profiled execution.'}),
    
                ), option_groups=self.option_groups,
                   reporter=reporter, pylintrc=self._rcfile)
            # register standard checkers
            checkers.initialize(linter)
            # load command line plugins
            linter.load_plugin_modules(self._plugins)
            # read configuration
            linter.disable_message('W0704')
            linter.read_config_file()
            # is there some additional plugins in the file configuration, in
            config_parser = linter._config_parser
            if config_parser.has_option('MASTER', 'load-plugins'):
                plugins = splitstrip(config_parser.get('MASTER', 'load-plugins'))
                linter.load_plugin_modules(plugins)
            # now we can load file config and command line, plugins (which can
            # provide options) have been registered
            linter.load_config_file()
            if reporter:
                # if a custom reporter is provided as argument, it may be overriden
                # by file parameters, so re-set it here, but before command line
                # parsing so it's still overrideable by command line option
                linter.set_reporter(reporter)
            args = linter.load_command_line_configuration(args)
            # insert current working directory to the python path to have a correct
            # behaviour
            sys.path.insert(0, os.getcwd())
            if self.linter.config.profile:
                print >> sys.stderr, '** profiled run'
                from hotshot import Profile, stats
                prof = Profile('stones.prof')
                prof.runcall(linter.check, args)
                prof.close()
                data = stats.load('stones.prof')
                data.strip_dirs()
                data.sort_stats('time', 'calls')
                data.print_stats(30)
            sys.path.pop(0)
        
        def cb_set_rcfile(self, name, value):
            """callback for option preprocessing (ie before optik parsing)"""
            self._rcfile = value
    
        def cb_add_plugins(self, name, value):
            """callback for option preprocessing (ie before optik parsing)"""
            self._plugins.extend(splitstrip(value))
    
        def cb_error_mode(self, *args, **kwargs):
            """error mode:
            * checkers without error messages are disabled
            * for others, only the ERROR messages are displayed
            * disable reports
            * do not save execution information
            """
            self.linter.disable_noerror_checkers()
            self.linter.set_option('disable-msg-cat', 'WCRFI')
            self.linter.set_option('reports', False)
            self.linter.set_option('persistent', False)
    
        def cb_generate_config(self, *args, **kwargs):
            """optik callback for sample config file generation"""
            self.linter.generate_config(skipsections=('COMMANDS',))
            
        def cb_generate_manpage(self, *args, **kwargs):
            """optik callback for sample config file generation"""
            from pylint import __pkginfo__
            self.linter.generate_manpage(__pkginfo__)
            
        def cb_help_message(self, option, opt_name, value, parser):
            """optik callback for printing some help about a particular message"""
            self.linter.help_message(splitstrip(value))
    
        def cb_list_messages(self, option, opt_name, value, parser):
            """optik callback for printing available messages"""
            self.linter.list_messages()
else:
    class LinterRun:
        def __init__(self):
            pass

"""
Build, clean and test the WMCore package.
"""

def generate_filelist(basepath=None, recurse=True, ignore=False):
    if basepath:
        walkpath = os.path.join(get_relative_path(), 'src/python', basepath)
    else:
        walkpath = os.path.join(get_relative_path(), 'src/python')
    
    files = []
    
    if walkpath.endswith('.py'):
        if ignore and walkpath.endswith(ignore):
            files.append(walkpath)
    else:
        for dirpath, dirnames, filenames in os.walk(walkpath):
            # skipping CVS directories and their contents
            pathelements = dirpath.split('/')
            result = []
            if not 'CVS' in pathelements:
                # to build up a list of file names which contain tests
                for file in filenames:
                    if file.endswith('.py'):
                        filepath = '/'.join([dirpath, file]) 
                        files.append(filepath)

    if len(files) == 0 and recurse:
        files = generate_filelist(basepath + '.py', not recurse)

    return files

def lint_score(stats, evaluation):
    return eval(evaluation, {}, stats)

def lint_files(files, reports=False):
    """
    lint a (list of) file(s) and return the results as a dictionary containing
    filename : result_dict
    """
    
    rcfile=os.path.join(get_relative_path(),'standards/.pylintrc')
    
    arguements = ['--rcfile=%s' % rcfile, '--ignore=DefaultConfig.py']
    
    if not reports:
        arguements.append('-rn')
    
    arguements.extend(files)
    
    lntr = LinterRun(arguements)
    
    results = {}
    for file in files:
        lntr.linter.check(file)
        results[file] = {'stats': lntr.linter.stats,
                         'score': lint_score(lntr.linter.stats, 
                                             lntr.linter.config.evaluation)
                         }
        if reports:
            print '----------------------------------'
            print 'Your code has been rated at %.2f/10' % \
                    lint_score(lntr.linter.stats, lntr.linter.config.evaluation)

    return results, lntr.linter.config.evaluation

class CleanCommand(Command):
    description = "Clean up (delete) compiled files"
    user_options = [ ]

    def initialize_options(self):
        self._clean_me = [ ]
        for root, dirs, files in os.walk('.'):
            for f in files:
                if f.endswith('.pyc'):
                    self._clean_me.append(pjoin(root, f))

    def finalize_options(self):
        pass

    def run(self):
        for clean_me in self._clean_me:
            try:
                os.unlink(clean_me)
            except:
                pass
            
class LintCommand(Command):
    description = "Lint all files in the src tree"
    """
    TODO: better format the test results, get some global result, make output 
    more buildbot friendly.    
    """
    
    user_options = [ ('package=', 'p', 'package to lint, default to None'),
                ('report', 'r', 'return a detailed lint report, default False')]
    
    def initialize_options(self):
        self._dir = get_relative_path()
        self.package = None
        self.report = False
        
    def finalize_options(self):
        if self.report:
            self.report = True

    def run(self):
        '''
        Find the code and run lint on it
        '''
        if can_lint:
            srcpypath = os.path.join(self._dir, 'src/python/')
            
            sys.path.append(srcpypath) 
            
            files_to_lint = []
            
            if self.package:
                if self.package.endswith('.py'):
                    cnt = self.package.count('.') - 1
                    files_to_lint = generate_filelist(self.package.replace('.', '/', cnt), 'DeafultConfig.py')
                else:
                    files_to_lint = generate_filelist(self.package.replace('.', '/'), 'DeafultConfig.py')
            else:
                files_to_lint = generate_filelist(ignore='DeafultConfig.py')
            
            results, evaluation = lint_files(files_to_lint, self.report)
            ln = len(results)
            scr = 0
            print
            for k, v in results.items():
                print "%s: %.2f/10" % (k.replace('src/python/', ''), v['score'])
                scr += v['score']
            if ln > 1:
                print '--------------------------------------------------------'
                print 'Average pylint score for %s is: %.2f/10' % (self.package,
                                                                  scr/ln)
                
        else:
            print 'You need to install pylint before using the lint command'
            
class ReportCommand(Command):
    description = "Generate a simple html report for ease of viewing in buildbot"
    """
    To contain:
        average lint score
        % code coverage
        list of classes missing tests
        etc.
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        """
        run all the tests needed to generate the report and make an
        html table
        """
        files = generate_filelist()
        
        error = 0 
        warning = 0
        refactor = 0
        convention = 0 
        statement = 0
        
        srcpypath = '/'.join([get_relative_path(), 'src/python/'])
        sys.path.append(srcpypath)

        cfg = ConfigParser()
        cfg.read('standards/.pylintrc')
        
        # Supress stdout/stderr
        sys.stderr = open('/dev/null', 'w')
        sys.stdout = open('/dev/null', 'w')
        # wrap it in an exception handler, otherwise we can't see why it fails
        try:
            # lint the code
            for stats in lint_files(files):
                error += stats['error']
                warning += stats['warning']
                refactor += stats['refactor']
                convention += stats['convention'] 
                statement += stats['statement']
        except Exception,e:
            # and restore the stdout/stderr
            sys.stderr = sys.__stderr__
            sys.stdout = sys.__stderr__
            raise e
        
        # and restore the stdout/stderr
        sys.stderr = sys.__stderr__
        sys.stdout = sys.__stderr__ 
        
        stats = {'error': error,
            'warning': warning,
            'refactor': refactor,
            'convention': convention, 
            'statement': statement}
        
        lint_score = eval(cfg.get('MASTER', 'evaluation'), {}, stats)
        coverage = 0 # TODO: calculate this
        testless_classes = [] # TODO: generate this
        
        print "<table>"
        print "<tr>"
        print "<td colspan=2><h1>WMCore test report</h1></td>"
        print "</tr>"
        print "<tr>"
        print "<td>Average lint score</td>"
        print "<td>%.2f</td>" % lint_score
        print "</tr>"
        print "<tr>"
        print "<td>% code coverage</td>"
        print "<td>%s</td>" % coverage
        print "</tr>"
        print "<tr>"
        print "<td>Classes missing tests</td>"
        print "<td>"
        if len(testless_classes) == 0:
            print "None"
        else:
            print "<ul>"
            for c in testless_classes:
                print "<li>%c</li>" % c
            print "</ul>"
        print "</td>"
        print "</tr>"
        print "</table>"
            
class CoverageCommand(Command):
    description = "Run code coverage tests"
    """
    To do this, we need to run all the unittests within the coverage
    framework to record all the lines(and branches) executed
    unfortunately, we have multiple code paths per database schema, so
    we need to find a way to merge them.
      
    TODO: modify the test command to have a flag to record code coverage
          the file thats used can then be used here, saving us from running
          our tests twice    
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        """
        Determine the code's test coverage and return that as a float
        
        http://nedbatchelder.com/code/coverage/
        """
        if can_coverage:
            files = generate_filelist()
            dataFile = None
            cov = None
            
            # attempt to load previously cached coverage information if it exists
            try:
                dataFile = open("wmcore-coverage.dat","r")
                cov = coverage.coverage(branch = True, data_file='wmcore-coverage.dat')
                cov.load()
            except:  
                cov = coverage.coverage(branch = True, )
                cov.start()
                runUnitTests()
                cov.stop()
                cov.save()
                
            # we have our coverage information, now let's do something with it
            # get a list of modules
            cov.report(morfs = files, file=sys.stdout)
            return 0
        else:
            print 'You need the coverage module installed before running the' +\
                            ' coverage command'
    
class DumbCoverageCommand(Command):
    description = "Run a simple coverage test - find classes that don't have a unit test"
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        """
        Determine the code's test coverage in a dumb way and return that as a 
        float.
        """
        print "This determines test coverage in a very crude manner. If your"
        print "test file is incorrectly named it will not be counted, and" 
        print "result in a lower coverage score." 
        print '----------------------------------------------------------------'
        filelist = generate_filelist()
        tests = 0
        files = 0
        pkgcnt = 0
        dir = get_relative_path()
        pkg = {'name': '', 'files': 0, 'tests': 0}
        for f in filelist:
            testpath = '/'.join([dir, f])
            pth = testpath.split('./src/python/')
            pth.append(pth[1].replace('/', '_t/').replace('.', '_t.'))
            if pkg['name'] == pth[2].rsplit('/', 1)[0].replace('_t/', '/'):
                # pkg hasn't changed, increment counts
                pkg['files'] += 1
            else:
                # new package, print stats for old package
                pkgcnt += 1
                if pkg['name'] != '' and pkg['files'] > 0:
                    print 'Package %s has coverage %.1f percent' % (pkg['name'], 
                                (float(pkg['tests'])/float(pkg['files']) * 100))
                # and start over for the new package
                pkg['name'] = pth[2].rsplit('/', 1)[0].replace('_t/', '/')
                # do global book keeping
                files += pkg['files']
                tests += pkg['tests'] 
                pkg['files'] = 0 
                pkg['tests'] = 0
            pth[1] = 'test/python'
            testpath = '/'.join(pth)
            try: 
                os.stat(testpath)
                pkg['tests'] += 1
            except:
                pass
            
        coverage = (float(tests) / float(files)) * 100
        print '----------------------------------------------------------------'
        print 'Code coverage (%s packages) is %.2f percent' % (pkgcnt, coverage)
        return coverage

class EnvCommand(Command):
    description = "Configure the PYTHONPATH, DATABASE and PATH variables to" +\
    "some sensible defaults, if not already set. Call with -q when eval-ing," +\
    """ e.g.:
        eval `python setup.py -q env`
    """
    
    user_options = [ ]
    
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
    
    def run(self):
        if not os.getenv('DATABASE', False):
            # Use an in memory sqlite one if none is configured. 
             print 'export DATABASE=sqlite://'
            
        here = get_relative_path()
        
        tests = here + '/test/python'
        source = here + '/src/python'
        webpth = source + '/WMCore/WebTools'
        
        pypath=os.getenv('PYTHONPATH', '').strip(':').split(':')
         
        for pth in [tests, source]:
            if pth not in pypath:
                pypath.append(pth)
        
        # We might want to add other executables to PATH
        expath=os.getenv('PATH', '').split(':')
        for pth in [webpth]:
            if pth not in expath:
                expath.append(pth)
              
        print 'export PYTHONPATH=%s' % ':'.join(pypath)
        print 'export PATH=%s' % ':'.join(expath)
        
        #We want the WMCORE root set, too
        print 'export WMCOREBASE=%s' % get_relative_path()


def getPackages(package_dirs = []):
    packages = []
    for dir in package_dirs:
        for dirpath, dirnames, filenames in os.walk('./%s' % dir):
            # Exclude things here
            if dirpath not in ['./src/python/', './src/python/IMProv']: 
                pathelements = dirpath.split('/')
                if not 'CVS' in pathelements:
                    path = pathelements[3:]
                    packages.append('.'.join(path))
    return packages

package_dir = {'WMCore': 'src/python/WMCore',
               'WMComponent' : 'src/python/WMComponent',
               'WMQuality' : 'src/python/WMQuality'}

setup (name = 'wmcore',
       version = '1.0',
       maintainer_email = 'hn-cms-wmDevelopment@cern.ch',
       cmdclass = {'clean': CleanCommand, 
                   'lint': LintCommand,
                   'report': ReportCommand,
                   'coverage': CoverageCommand ,
                   'missing': DumbCoverageCommand,
                   'env': EnvCommand,
                   'test' : TestCommand },
       package_dir = package_dir,
       packages = getPackages(package_dir.values()),)
Revision:	1.103
Committed:	Wed Feb 3 22:30:39 2010 UTC (15 years, 2 months ago) by meloam
Content type:	text/x-python
Branch:	MAIN
Changes since 1.102:	+2 -2 lines
Log Message:	backing out a change. buildbot was using central, not UTC time, so the CVS checkouts weren't working properly. undoing what I just did to see if it actually worked
#	User	Rev	Content
1	fvlingen	1.1	#!/usr/bin/env python
2	metson	1.21	from distutils.core import setup, Command
3			from unittest import TextTestRunner, TestLoader, TestSuite
4			from glob import glob
5			from os.path import splitext, basename, join as pjoin, walk
6	metson	1.85	from ConfigParser import ConfigParser, NoOptionError
7	meloam	1.53	import os, sys, os.path
8	metson	1.85	import logging
9			import unittest
10	metson	1.51	#PyLinter and coverage aren't standard, but aren't strictly necessary
11			can_lint = False
12			can_coverage = False
13	meloam	1.88	can_nose = False
14	sfoulkes	1.29	try:
15	metson	1.87	from pylint.lint import Run
16	metson	1.85	from pylint.lint import preprocess_options, cb_init_hook
17			from pylint import checkers
18	metson	1.51	can_lint = True
19			except:
20			pass
21			try:
22	meloam	1.47	import coverage
23	metson	1.51	can_coverage = True
24	sfoulkes	1.29	except:
25			pass
26	fvlingen	1.1
27	meloam	1.88	try:
28			import nose
29			can_nose = True
30			except:
31			pass
32
33			if can_nose:
34	meloam	1.96	class TestCommand(Command):
35	meloam	1.100	"""Runs our test suite"""
36
37	meloam	1.88	user_options = [ ]
38
39			def initialize_options(self):
40			pass
41
42			def finalize_options(self):
43			pass
44
45			def run(self):
46	meloam	1.90	retval = nose.run(argv=[__file__,'--all-modules','-v','test/python'])
47	meloam	1.94	if retval:
48	meloam	1.103	return 0
49	meloam	1.94	else:
50	meloam	1.103	return 1
51	meloam	1.88	else:
52	meloam	1.101	class TestCommand(Command):
53	meloam	1.88	user_options = [ ]
54			def run(self):
55	meloam	1.97	print "Nose isn't installed. You must install the nose package to run tests (easy_install nose might do it)"
56	meloam	1.88	pass
57
58			def initialize_options(self):
59			pass
60
61			def finalize_options(self):
62			pass
63			pass
64
65	metson	1.87	if can_lint:
66			class LinterRun(Run):
67			def __init__(self, args, reporter=None):
68			self._rcfile = None
69			self._plugins = []
70			preprocess_options(args, {
71			# option: (callback, takearg)
72			'rcfile': (self.cb_set_rcfile, True),
73			'load-plugins': (self.cb_add_plugins, True),
74			})
75			self.linter = linter = self.LinterClass((
76			('rcfile',
77			{'action' : 'callback', 'callback' : lambda *args: 1,
78			'type': 'string', 'metavar': '<file>',
79			'help' : 'Specify a configuration file.'}),
80
81			('init-hook',
82			{'action' : 'callback', 'type' : 'string', 'metavar': '<code>',
83			'callback' : cb_init_hook,
84			'help' : 'Python code to execute, usually for sys.path \
85			manipulation such as pygtk.require().'}),
86
87			('help-msg',
88			{'action' : 'callback', 'type' : 'string', 'metavar': '<msg-id>',
89			'callback' : self.cb_help_message,
90			'group': 'Commands',
91			'help' : '''Display a help message for the given message id and \
92			exit. The value may be a comma separated list of message ids.'''}),
93
94			('list-msgs',
95			{'action' : 'callback', 'metavar': '<msg-id>',
96			'callback' : self.cb_list_messages,
97			'group': 'Commands',
98			'help' : "Generate pylint's full documentation."}),
99
100			('generate-rcfile',
101			{'action' : 'callback', 'callback' : self.cb_generate_config,
102			'group': 'Commands',
103			'help' : '''Generate a sample configuration file according to \
104			the current configuration. You can put other options before this one to get \
105			them in the generated configuration.'''}),
106
107			('generate-man',
108			{'action' : 'callback', 'callback' : self.cb_generate_manpage,
109			'group': 'Commands',
110			'help' : "Generate pylint's man page.",'hide': 'True'}),
111
112			('errors-only',
113			{'action' : 'callback', 'callback' : self.cb_error_mode,
114			'short': 'e',
115			'help' : '''In error mode, checkers without error messages are \
116			disabled and for others, only the ERROR messages are displayed, and no reports \
117			are done by default'''}),
118
119			('profile',
120			{'type' : 'yn', 'metavar' : '<y_or_n>',
121			'default': False,
122			'help' : 'Profiled execution.'}),
123
124			), option_groups=self.option_groups,
125			reporter=reporter, pylintrc=self._rcfile)
126			# register standard checkers
127			checkers.initialize(linter)
128			# load command line plugins
129			linter.load_plugin_modules(self._plugins)
130			# read configuration
131			linter.disable_message('W0704')
132			linter.read_config_file()
133			# is there some additional plugins in the file configuration, in
134			config_parser = linter._config_parser
135			if config_parser.has_option('MASTER', 'load-plugins'):
136			plugins = splitstrip(config_parser.get('MASTER', 'load-plugins'))
137			linter.load_plugin_modules(plugins)
138			# now we can load file config and command line, plugins (which can
139			# provide options) have been registered
140			linter.load_config_file()
141			if reporter:
142			# if a custom reporter is provided as argument, it may be overriden
143			# by file parameters, so re-set it here, but before command line
144			# parsing so it's still overrideable by command line option
145			linter.set_reporter(reporter)
146			args = linter.load_command_line_configuration(args)
147			# insert current working directory to the python path to have a correct
148			# behaviour
149			sys.path.insert(0, os.getcwd())
150			if self.linter.config.profile:
151			print >> sys.stderr, '** profiled run'
152			from hotshot import Profile, stats
153			prof = Profile('stones.prof')
154			prof.runcall(linter.check, args)
155			prof.close()
156			data = stats.load('stones.prof')
157			data.strip_dirs()
158			data.sort_stats('time', 'calls')
159			data.print_stats(30)
160			sys.path.pop(0)
161
162			def cb_set_rcfile(self, name, value):
163			"""callback for option preprocessing (ie before optik parsing)"""
164			self._rcfile = value
165
166			def cb_add_plugins(self, name, value):
167			"""callback for option preprocessing (ie before optik parsing)"""
168			self._plugins.extend(splitstrip(value))
169
170			def cb_error_mode(self, args, *kwargs):
171			"""error mode:
172			* checkers without error messages are disabled
173			* for others, only the ERROR messages are displayed
174			* disable reports
175			* do not save execution information
176			"""
177			self.linter.disable_noerror_checkers()
178			self.linter.set_option('disable-msg-cat', 'WCRFI')
179			self.linter.set_option('reports', False)
180			self.linter.set_option('persistent', False)
181
182			def cb_generate_config(self, args, *kwargs):
183			"""optik callback for sample config file generation"""
184			self.linter.generate_config(skipsections=('COMMANDS',))
185
186			def cb_generate_manpage(self, args, *kwargs):
187			"""optik callback for sample config file generation"""
188			from pylint import __pkginfo__
189			self.linter.generate_manpage(__pkginfo__)
190
191			def cb_help_message(self, option, opt_name, value, parser):
192			"""optik callback for printing some help about a particular message"""
193			self.linter.help_message(splitstrip(value))
194
195			def cb_list_messages(self, option, opt_name, value, parser):
196			"""optik callback for printing available messages"""
197			self.linter.list_messages()
198			else:
199			class LinterRun:
200			def __init__(self):
201			pass
202	metson	1.85
203	metson	1.22	"""
204			Build, clean and test the WMCore package.
205			"""
206
207	metson	1.92	def generate_filelist(basepath=None, recurse=True, ignore=False):
208	metson	1.85	if basepath:
209			walkpath = os.path.join(get_relative_path(), 'src/python', basepath)
210			else:
211			walkpath = os.path.join(get_relative_path(), 'src/python')
212
213	metson	1.36	files = []
214	metson	1.85
215			if walkpath.endswith('.py'):
216	metson	1.92	if ignore and walkpath.endswith(ignore):
217			files.append(walkpath)
218	metson	1.85	else:
219			for dirpath, dirnames, filenames in os.walk(walkpath):
220			# skipping CVS directories and their contents
221			pathelements = dirpath.split('/')
222			result = []
223			if not 'CVS' in pathelements:
224			# to build up a list of file names which contain tests
225			for file in filenames:
226			if file.endswith('.py'):
227			filepath = '/'.join([dirpath, file])
228			files.append(filepath)
229
230			if len(files) == 0 and recurse:
231			files = generate_filelist(basepath + '.py', not recurse)
232
233	metson	1.36	return files
234
235	metson	1.85	def lint_score(stats, evaluation):
236			return eval(evaluation, {}, stats)
237
238			def lint_files(files, reports=False):
239	metson	1.36	"""
240	metson	1.85	lint a (list of) file(s) and return the results as a dictionary containing
241			filename : result_dict
242	metson	1.36	"""
243	metson	1.85
244			rcfile=os.path.join(get_relative_path(),'standards/.pylintrc')
245
246	metson	1.92	arguements = ['--rcfile=%s' % rcfile, '--ignore=DefaultConfig.py']
247	metson	1.86
248			if not reports:
249			arguements.append('-rn')
250
251	metson	1.85	arguements.extend(files)
252
253			lntr = LinterRun(arguements)
254
255			results = {}
256			for file in files:
257			lntr.linter.check(file)
258			results[file] = {'stats': lntr.linter.stats,
259			'score': lint_score(lntr.linter.stats,
260			lntr.linter.config.evaluation)
261			}
262			if reports:
263			print '----------------------------------'
264			print 'Your code has been rated at %.2f/10' % \
265			lint_score(lntr.linter.stats, lntr.linter.config.evaluation)
266	metson	1.36
267	metson	1.85	return results, lntr.linter.config.evaluation
268	meloam	1.69
269	metson	1.21	class CleanCommand(Command):
270	metson	1.49	description = "Clean up (delete) compiled files"
271	metson	1.21	user_options = [ ]
272
273			def initialize_options(self):
274			self._clean_me = [ ]
275			for root, dirs, files in os.walk('.'):
276			for f in files:
277			if f.endswith('.pyc'):
278			self._clean_me.append(pjoin(root, f))
279
280			def finalize_options(self):
281			pass
282
283			def run(self):
284			for clean_me in self._clean_me:
285			try:
286			os.unlink(clean_me)
287			except:
288			pass
289
290	metson	1.23	class LintCommand(Command):
291	metson	1.49	description = "Lint all files in the src tree"
292	metson	1.24	"""
293			TODO: better format the test results, get some global result, make output
294			more buildbot friendly.
295			"""
296
297	metson	1.86	user_options = [ ('package=', 'p', 'package to lint, default to None'),
298			('report', 'r', 'return a detailed lint report, default False')]
299	metson	1.23
300			def initialize_options(self):
301	metson	1.85	self._dir = get_relative_path()
302			self.package = None
303	metson	1.86	self.report = False
304	metson	1.85
305	metson	1.23	def finalize_options(self):
306	metson	1.86	if self.report:
307			self.report = True
308
309	metson	1.23	def run(self):
310			'''
311			Find the code and run lint on it
312			'''
313	metson	1.51	if can_lint:
314	metson	1.85	srcpypath = os.path.join(self._dir, 'src/python/')
315
316	metson	1.51	sys.path.append(srcpypath)
317
318	metson	1.85	files_to_lint = []
319
320			if self.package:
321			if self.package.endswith('.py'):
322			cnt = self.package.count('.') - 1
323	metson	1.92	files_to_lint = generate_filelist(self.package.replace('.', '/', cnt), 'DeafultConfig.py')
324	metson	1.85	else:
325	metson	1.92	files_to_lint = generate_filelist(self.package.replace('.', '/'), 'DeafultConfig.py')
326	metson	1.85	else:
327	metson	1.92	files_to_lint = generate_filelist(ignore='DeafultConfig.py')
328	metson	1.86
329			results, evaluation = lint_files(files_to_lint, self.report)
330	metson	1.85	ln = len(results)
331			scr = 0
332			print
333			for k, v in results.items():
334			print "%s: %.2f/10" % (k.replace('src/python/', ''), v['score'])
335			scr += v['score']
336			if ln > 1:
337			print '--------------------------------------------------------'
338			print 'Average pylint score for %s is: %.2f/10' % (self.package,
339			scr/ln)
340
341	metson	1.51	else:
342			print 'You need to install pylint before using the lint command'
343	metson	1.36
344			class ReportCommand(Command):
345	metson	1.49	description = "Generate a simple html report for ease of viewing in buildbot"
346	metson	1.36	"""
347			To contain:
348			average lint score
349			% code coverage
350			list of classes missing tests
351			etc.
352			"""
353
354			user_options = [ ]
355
356			def initialize_options(self):
357			pass
358
359			def finalize_options(self):
360			pass
361
362			def run(self):
363			"""
364			run all the tests needed to generate the report and make an
365			html table
366			"""
367			files = generate_filelist()
368
369			error = 0
370			warning = 0
371			refactor = 0
372			convention = 0
373			statement = 0
374
375	metson	1.85	srcpypath = '/'.join([get_relative_path(), 'src/python/'])
376	metson	1.36	sys.path.append(srcpypath)
377	metson	1.37
378			cfg = ConfigParser()
379			cfg.read('standards/.pylintrc')
380
381	metson	1.40	# Supress stdout/stderr
382	metson	1.38	sys.stderr = open('/dev/null', 'w')
383	metson	1.39	sys.stdout = open('/dev/null', 'w')
384	meloam	1.47	# wrap it in an exception handler, otherwise we can't see why it fails
385			try:
386			# lint the code
387			for stats in lint_files(files):
388			error += stats['error']
389			warning += stats['warning']
390			refactor += stats['refactor']
391			convention += stats['convention']
392			statement += stats['statement']
393			except Exception,e:
394			# and restore the stdout/stderr
395			sys.stderr = sys.__stderr__
396			sys.stdout = sys.__stderr__
397			raise e
398	metson	1.37
399	metson	1.40	# and restore the stdout/stderr
400	meloam	1.47	sys.stderr = sys.__stderr__
401			sys.stdout = sys.__stderr__
402	metson	1.38
403	metson	1.37	stats = {'error': error,
404			'warning': warning,
405			'refactor': refactor,
406			'convention': convention,
407			'statement': statement}
408
409			lint_score = eval(cfg.get('MASTER', 'evaluation'), {}, stats)
410	metson	1.36	coverage = 0 # TODO: calculate this
411			testless_classes = [] # TODO: generate this
412
413			print "<table>"
414			print "<tr>"
415			print "<td colspan=2><h1>WMCore test report</h1></td>"
416			print "</tr>"
417			print "<tr>"
418			print "<td>Average lint score</td>"
419			print "<td>%.2f</td>" % lint_score
420			print "</tr>"
421			print "<tr>"
422			print "<td>% code coverage</td>"
423			print "<td>%s</td>" % coverage
424			print "</tr>"
425			print "<tr>"
426			print "<td>Classes missing tests</td>"
427			print "<td>"
428			if len(testless_classes) == 0:
429			print "None"
430			else:
431			print "<ul>"
432			for c in testless_classes:
433			print "<li>%c</li>" % c
434			print "</ul>"
435			print "</td>"
436			print "</tr>"
437			print "</table>"
438
439			class CoverageCommand(Command):
440	metson	1.49	description = "Run code coverage tests"
441	metson	1.36	"""
442	metson	1.49	To do this, we need to run all the unittests within the coverage
443			framework to record all the lines(and branches) executed
444	meloam	1.47	unfortunately, we have multiple code paths per database schema, so
445			we need to find a way to merge them.
446
447			TODO: modify the test command to have a flag to record code coverage
448			the file thats used can then be used here, saving us from running
449			our tests twice
450	metson	1.36	"""
451
452			user_options = [ ]
453
454			def initialize_options(self):
455			pass
456
457			def finalize_options(self):
458			pass
459
460			def run(self):
461			"""
462			Determine the code's test coverage and return that as a float
463
464			http://nedbatchelder.com/code/coverage/
465			"""
466	metson	1.51	if can_coverage:
467			files = generate_filelist()
468			dataFile = None
469			cov = None
470	meloam	1.48
471	metson	1.51	# attempt to load previously cached coverage information if it exists
472			try:
473			dataFile = open("wmcore-coverage.dat","r")
474			cov = coverage.coverage(branch = True, data_file='wmcore-coverage.dat')
475			cov.load()
476			except:
477			cov = coverage.coverage(branch = True, )
478			cov.start()
479			runUnitTests()
480			cov.stop()
481			cov.save()
482
483			# we have our coverage information, now let's do something with it
484			# get a list of modules
485			cov.report(morfs = files, file=sys.stdout)
486			return 0
487			else:
488			print 'You need the coverage module installed before running the' +\
489			' coverage command'
490	metson	1.36
491	metson	1.41	class DumbCoverageCommand(Command):
492	metson	1.49	description = "Run a simple coverage test - find classes that don't have a unit test"
493	metson	1.41
494			user_options = [ ]
495
496			def initialize_options(self):
497			pass
498
499			def finalize_options(self):
500			pass
501
502			def run(self):
503			"""
504			Determine the code's test coverage in a dumb way and return that as a
505			float.
506			"""
507	metson	1.42	print "This determines test coverage in a very crude manner. If your"
508			print "test file is incorrectly named it will not be counted, and"
509			print "result in a lower coverage score."
510	metson	1.43	print '----------------------------------------------------------------'
511	metson	1.42	filelist = generate_filelist()
512			tests = 0
513			files = 0
514			pkgcnt = 0
515	metson	1.85	dir = get_relative_path()
516	metson	1.42	pkg = {'name': '', 'files': 0, 'tests': 0}
517			for f in filelist:
518	metson	1.41	testpath = '/'.join([dir, f])
519			pth = testpath.split('./src/python/')
520			pth.append(pth[1].replace('/', '_t/').replace('.', '_t.'))
521	metson	1.42	if pkg['name'] == pth[2].rsplit('/', 1)[0].replace('_t/', '/'):
522			# pkg hasn't changed, increment counts
523			pkg['files'] += 1
524			else:
525			# new package, print stats for old package
526			pkgcnt += 1
527			if pkg['name'] != '' and pkg['files'] > 0:
528			print 'Package %s has coverage %.1f percent' % (pkg['name'],
529			(float(pkg['tests'])/float(pkg['files']) * 100))
530			# and start over for the new package
531			pkg['name'] = pth[2].rsplit('/', 1)[0].replace('_t/', '/')
532			# do global book keeping
533			files += pkg['files']
534			tests += pkg['tests']
535			pkg['files'] = 0
536			pkg['tests'] = 0
537	metson	1.41	pth[1] = 'test/python'
538			testpath = '/'.join(pth)
539	metson	1.42	try:
540			os.stat(testpath)
541			pkg['tests'] += 1
542	metson	1.41	except:
543	metson	1.42	pass
544
545			coverage = (float(tests) / float(files)) * 100
546	metson	1.43	print '----------------------------------------------------------------'
547	metson	1.42	print 'Code coverage (%s packages) is %.2f percent' % (pkgcnt, coverage)
548	metson	1.41	return coverage
549	metson	1.49
550			class EnvCommand(Command):
551			description = "Configure the PYTHONPATH, DATABASE and PATH variables to" +\
552			"some sensible defaults, if not already set. Call with -q when eval-ing," +\
553	metson	1.50	""" e.g.:
554	metson	1.49	eval `python setup.py -q env`
555			"""
556
557			user_options = [ ]
558
559			def initialize_options(self):
560			pass
561
562			def finalize_options(self):
563			pass
564	metson	1.41
565	metson	1.49	def run(self):
566			if not os.getenv('DATABASE', False):
567			# Use an in memory sqlite one if none is configured.
568			print 'export DATABASE=sqlite://'
569
570	metson	1.93	here = get_relative_path()
571
572	metson	1.49	tests = here + '/test/python'
573			source = here + '/src/python'
574			webpth = source + '/WMCore/WebTools'
575	metson	1.93
576			pypath=os.getenv('PYTHONPATH', '').strip(':').split(':')
577	metson	1.49
578			for pth in [tests, source]:
579			if pth not in pypath:
580			pypath.append(pth)
581	metson	1.50
582			# We might want to add other executables to PATH
583	metson	1.49	expath=os.getenv('PATH', '').split(':')
584			for pth in [webpth]:
585			if pth not in expath:
586			expath.append(pth)
587
588			print 'export PYTHONPATH=%s' % ':'.join(pypath)
589			print 'export PATH=%s' % ':'.join(expath)
590	metson	1.85
591			#We want the WMCORE root set, too
592	meloam	1.89	print 'export WMCOREBASE=%s' % get_relative_path()
593	metson	1.85
594	metson	1.49
595	metson	1.22	def getPackages(package_dirs = []):
596			packages = []
597			for dir in package_dirs:
598			for dirpath, dirnames, filenames in os.walk('./%s' % dir):
599			# Exclude things here
600			if dirpath not in ['./src/python/', './src/python/IMProv']:
601			pathelements = dirpath.split('/')
602			if not 'CVS' in pathelements:
603			path = pathelements[3:]
604			packages.append('.'.join(path))
605			return packages
606
607			package_dir = {'WMCore': 'src/python/WMCore',
608			'WMComponent' : 'src/python/WMComponent',
609			'WMQuality' : 'src/python/WMQuality'}
610
611			setup (name = 'wmcore',
612			version = '1.0',
613	metson	1.36	maintainer_email = 'hn-cms-wmDevelopment@cern.ch',
614	meloam	1.96	cmdclass = {'clean': CleanCommand,
615	metson	1.36	'lint': LintCommand,
616			'report': ReportCommand,
617	metson	1.41	'coverage': CoverageCommand ,
618	metson	1.49	'missing': DumbCoverageCommand,
619	meloam	1.88	'env': EnvCommand,
620	meloam	1.96	'test' : TestCommand },
621	metson	1.22	package_dir = package_dir,
622			packages = getPackages(package_dir.values()),)