| 1 | #!/usr/bin/python -u
|
|---|
| 2 | import glob, os, string, sys, thread, time
|
|---|
| 3 | # import difflib
|
|---|
| 4 | import libxml2
|
|---|
| 5 |
|
|---|
| 6 | ###
|
|---|
| 7 | #
|
|---|
| 8 | # This is a "Work in Progress" attempt at a python script to run the
|
|---|
| 9 | # various regression tests. The rationale for this is that it should be
|
|---|
| 10 | # possible to run this on most major platforms, including those (such as
|
|---|
| 11 | # Windows) which don't support gnu Make.
|
|---|
| 12 | #
|
|---|
| 13 | # The script is driven by a parameter file which defines the various tests
|
|---|
| 14 | # to be run, together with the unique settings for each of these tests. A
|
|---|
| 15 | # script for Linux is included (regressions.xml), with comments indicating
|
|---|
| 16 | # the significance of the various parameters. To run the tests under Windows,
|
|---|
| 17 | # edit regressions.xml and remove the comment around the default parameter
|
|---|
| 18 | # "<execpath>" (i.e. make it point to the location of the binary executables).
|
|---|
| 19 | #
|
|---|
| 20 | # Note that this current version requires the Python bindings for libxml2 to
|
|---|
| 21 | # have been previously installed and accessible
|
|---|
| 22 | #
|
|---|
| 23 | # See Copyright for the status of this software.
|
|---|
| 24 | # William Brack ([email protected])
|
|---|
| 25 | #
|
|---|
| 26 | ###
|
|---|
| 27 | defaultParams = {} # will be used as a dictionary to hold the parsed params
|
|---|
| 28 |
|
|---|
| 29 | # This routine is used for comparing the expected stdout / stdin with the results.
|
|---|
| 30 | # The expected data has already been read in; the result is a file descriptor.
|
|---|
| 31 | # Within the two sets of data, lines may begin with a path string. If so, the
|
|---|
| 32 | # code "relativises" it by removing the path component. The first argument is a
|
|---|
| 33 | # list already read in by a separate thread; the second is a file descriptor.
|
|---|
| 34 | # The two 'base' arguments are to let me "relativise" the results files, allowing
|
|---|
| 35 | # the script to be run from any directory.
|
|---|
| 36 | def compFiles(res, expected, base1, base2):
|
|---|
| 37 | l1 = len(base1)
|
|---|
| 38 | exp = expected.readlines()
|
|---|
| 39 | expected.close()
|
|---|
| 40 | # the "relativisation" is done here
|
|---|
| 41 | for i in range(len(res)):
|
|---|
| 42 | j = string.find(res[i],base1)
|
|---|
| 43 | if (j == 0) or ((j == 2) and (res[i][0:2] == './')):
|
|---|
| 44 | col = string.find(res[i],':')
|
|---|
| 45 | if col > 0:
|
|---|
| 46 | start = string.rfind(res[i][:col], '/')
|
|---|
| 47 | if start > 0:
|
|---|
| 48 | res[i] = res[i][start+1:]
|
|---|
| 49 |
|
|---|
| 50 | for i in range(len(exp)):
|
|---|
| 51 | j = string.find(exp[i],base2)
|
|---|
| 52 | if (j == 0) or ((j == 2) and (exp[i][0:2] == './')):
|
|---|
| 53 | col = string.find(exp[i],':')
|
|---|
| 54 | if col > 0:
|
|---|
| 55 | start = string.rfind(exp[i][:col], '/')
|
|---|
| 56 | if start > 0:
|
|---|
| 57 | exp[i] = exp[i][start+1:]
|
|---|
| 58 |
|
|---|
| 59 | ret = 0
|
|---|
| 60 | # ideally we would like to use difflib functions here to do a
|
|---|
| 61 | # nice comparison of the two sets. Unfortunately, during testing
|
|---|
| 62 | # (using python 2.3.3 and 2.3.4) the following code went into
|
|---|
| 63 | # a dead loop under windows. I'll pursue this later.
|
|---|
| 64 | # diff = difflib.ndiff(res, exp)
|
|---|
| 65 | # diff = list(diff)
|
|---|
| 66 | # for line in diff:
|
|---|
| 67 | # if line[:2] != ' ':
|
|---|
| 68 | # print string.strip(line)
|
|---|
| 69 | # ret = -1
|
|---|
| 70 |
|
|---|
| 71 | # the following simple compare is fine for when the two data sets
|
|---|
| 72 | # (actual result vs. expected result) are equal, which should be true for
|
|---|
| 73 | # us. Unfortunately, if the test fails it's not nice at all.
|
|---|
| 74 | rl = len(res)
|
|---|
| 75 | el = len(exp)
|
|---|
| 76 | if el != rl:
|
|---|
| 77 | print 'Length of expected is %d, result is %d' % (el, rl)
|
|---|
| 78 | ret = -1
|
|---|
| 79 | for i in range(min(el, rl)):
|
|---|
| 80 | if string.strip(res[i]) != string.strip(exp[i]):
|
|---|
| 81 | print '+:%s-:%s' % (res[i], exp[i])
|
|---|
| 82 | ret = -1
|
|---|
| 83 | if el > rl:
|
|---|
| 84 | for i in range(rl, el):
|
|---|
| 85 | print '-:%s' % exp[i]
|
|---|
| 86 | ret = -1
|
|---|
| 87 | elif rl > el:
|
|---|
| 88 | for i in range (el, rl):
|
|---|
| 89 | print '+:%s' % res[i]
|
|---|
| 90 | ret = -1
|
|---|
| 91 | return ret
|
|---|
| 92 |
|
|---|
| 93 | # Separate threads to handle stdout and stderr are created to run this function
|
|---|
| 94 | def readPfile(file, list, flag):
|
|---|
| 95 | data = file.readlines() # no call by reference, so I cheat
|
|---|
| 96 | for l in data:
|
|---|
| 97 | list.append(l)
|
|---|
| 98 | file.close()
|
|---|
| 99 | flag.append('ok')
|
|---|
| 100 |
|
|---|
| 101 | # This routine runs the test program (e.g. xmllint)
|
|---|
| 102 | def runOneTest(testDescription, filename, inbase, errbase):
|
|---|
| 103 | if 'execpath' in testDescription:
|
|---|
| 104 | dir = testDescription['execpath'] + '/'
|
|---|
| 105 | else:
|
|---|
| 106 | dir = ''
|
|---|
| 107 | cmd = os.path.abspath(dir + testDescription['testprog'])
|
|---|
| 108 | if 'flag' in testDescription:
|
|---|
| 109 | for f in string.split(testDescription['flag']):
|
|---|
| 110 | cmd += ' ' + f
|
|---|
| 111 | if 'stdin' not in testDescription:
|
|---|
| 112 | cmd += ' ' + inbase + filename
|
|---|
| 113 | if 'extarg' in testDescription:
|
|---|
| 114 | cmd += ' ' + testDescription['extarg']
|
|---|
| 115 |
|
|---|
| 116 | noResult = 0
|
|---|
| 117 | expout = None
|
|---|
| 118 | if 'resext' in testDescription:
|
|---|
| 119 | if testDescription['resext'] == 'None':
|
|---|
| 120 | noResult = 1
|
|---|
| 121 | else:
|
|---|
| 122 | ext = '.' + testDescription['resext']
|
|---|
| 123 | else:
|
|---|
| 124 | ext = ''
|
|---|
| 125 | if not noResult:
|
|---|
| 126 | try:
|
|---|
| 127 | fname = errbase + filename + ext
|
|---|
| 128 | expout = open(fname, 'rt')
|
|---|
| 129 | except:
|
|---|
| 130 | print "Can't open result file %s - bypassing test" % fname
|
|---|
| 131 | return
|
|---|
| 132 |
|
|---|
| 133 | noErrors = 0
|
|---|
| 134 | if 'reserrext' in testDescription:
|
|---|
| 135 | if testDescription['reserrext'] == 'None':
|
|---|
| 136 | noErrors = 1
|
|---|
| 137 | else:
|
|---|
| 138 | if len(testDescription['reserrext'])>0:
|
|---|
| 139 | ext = '.' + testDescription['reserrext']
|
|---|
| 140 | else:
|
|---|
| 141 | ext = ''
|
|---|
| 142 | else:
|
|---|
| 143 | ext = ''
|
|---|
| 144 | if not noErrors:
|
|---|
| 145 | try:
|
|---|
| 146 | fname = errbase + filename + ext
|
|---|
| 147 | experr = open(fname, 'rt')
|
|---|
| 148 | except:
|
|---|
| 149 | experr = None
|
|---|
| 150 | else:
|
|---|
| 151 | experr = None
|
|---|
| 152 |
|
|---|
| 153 | pin, pout, perr = os.popen3(cmd)
|
|---|
| 154 | if 'stdin' in testDescription:
|
|---|
| 155 | infile = open(inbase + filename, 'rt')
|
|---|
| 156 | pin.writelines(infile.readlines())
|
|---|
| 157 | infile.close()
|
|---|
| 158 | pin.close()
|
|---|
| 159 |
|
|---|
| 160 | # popen is great fun, but can lead to the old "deadly embrace", because
|
|---|
| 161 | # synchronizing the writing (by the task being run) of stdout and stderr
|
|---|
| 162 | # with respect to the reading (by this task) is basically impossible. I
|
|---|
| 163 | # tried several ways to cheat, but the only way I have found which works
|
|---|
| 164 | # is to do a *very* elementary multi-threading approach. We can only hope
|
|---|
| 165 | # that Python threads are implemented on the target system (it's okay for
|
|---|
| 166 | # Linux and Windows)
|
|---|
| 167 |
|
|---|
| 168 | th1Flag = [] # flags to show when threads finish
|
|---|
| 169 | th2Flag = []
|
|---|
| 170 | outfile = [] # lists to contain the pipe data
|
|---|
| 171 | errfile = []
|
|---|
| 172 | th1 = thread.start_new_thread(readPfile, (pout, outfile, th1Flag))
|
|---|
| 173 | th2 = thread.start_new_thread(readPfile, (perr, errfile, th2Flag))
|
|---|
| 174 | while (len(th1Flag)==0) or (len(th2Flag)==0):
|
|---|
| 175 | time.sleep(0.001)
|
|---|
| 176 | if not noResult:
|
|---|
| 177 | ret = compFiles(outfile, expout, inbase, 'test/')
|
|---|
| 178 | if ret != 0:
|
|---|
| 179 | print 'trouble with %s' % cmd
|
|---|
| 180 | else:
|
|---|
| 181 | if len(outfile) != 0:
|
|---|
| 182 | for l in outfile:
|
|---|
| 183 | print l
|
|---|
| 184 | print 'trouble with %s' % cmd
|
|---|
| 185 | if experr != None:
|
|---|
| 186 | ret = compFiles(errfile, experr, inbase, 'test/')
|
|---|
| 187 | if ret != 0:
|
|---|
| 188 | print 'trouble with %s' % cmd
|
|---|
| 189 | else:
|
|---|
| 190 | if not noErrors:
|
|---|
| 191 | if len(errfile) != 0:
|
|---|
| 192 | for l in errfile:
|
|---|
| 193 | print l
|
|---|
| 194 | print 'trouble with %s' % cmd
|
|---|
| 195 |
|
|---|
| 196 | if 'stdin' not in testDescription:
|
|---|
| 197 | pin.close()
|
|---|
| 198 |
|
|---|
| 199 | # This routine is called by the parameter decoding routine whenever the end of a
|
|---|
| 200 | # 'test' section is encountered. Depending upon file globbing, a large number of
|
|---|
| 201 | # individual tests may be run.
|
|---|
| 202 | def runTest(description):
|
|---|
| 203 | testDescription = defaultParams.copy() # set defaults
|
|---|
| 204 | testDescription.update(description) # override with current ent
|
|---|
| 205 | if 'testname' in testDescription:
|
|---|
| 206 | print "## %s" % testDescription['testname']
|
|---|
| 207 | if not 'file' in testDescription:
|
|---|
| 208 | print "No file specified - can't run this test!"
|
|---|
| 209 | return
|
|---|
| 210 | # Set up the source and results directory paths from the decoded params
|
|---|
| 211 | dir = ''
|
|---|
| 212 | if 'srcdir' in testDescription:
|
|---|
| 213 | dir += testDescription['srcdir'] + '/'
|
|---|
| 214 | if 'srcsub' in testDescription:
|
|---|
| 215 | dir += testDescription['srcsub'] + '/'
|
|---|
| 216 |
|
|---|
| 217 | rdir = ''
|
|---|
| 218 | if 'resdir' in testDescription:
|
|---|
| 219 | rdir += testDescription['resdir'] + '/'
|
|---|
| 220 | if 'ressub' in testDescription:
|
|---|
| 221 | rdir += testDescription['ressub'] + '/'
|
|---|
| 222 |
|
|---|
| 223 | testFiles = glob.glob(os.path.abspath(dir + testDescription['file']))
|
|---|
| 224 | if testFiles == []:
|
|---|
| 225 | print "No files result from '%s'" % testDescription['file']
|
|---|
| 226 | return
|
|---|
| 227 |
|
|---|
| 228 | # Some test programs just don't work (yet). For now we exclude them.
|
|---|
| 229 | count = 0
|
|---|
| 230 | excl = []
|
|---|
| 231 | if 'exclfile' in testDescription:
|
|---|
| 232 | for f in string.split(testDescription['exclfile']):
|
|---|
| 233 | glb = glob.glob(dir + f)
|
|---|
| 234 | for g in glb:
|
|---|
| 235 | excl.append(os.path.abspath(g))
|
|---|
| 236 |
|
|---|
| 237 | # Run the specified test program
|
|---|
| 238 | for f in testFiles:
|
|---|
| 239 | if not os.path.isdir(f):
|
|---|
| 240 | if f not in excl:
|
|---|
| 241 | count = count + 1
|
|---|
| 242 | runOneTest(testDescription, os.path.basename(f), dir, rdir)
|
|---|
| 243 |
|
|---|
| 244 | #
|
|---|
| 245 | # The following classes are used with the xmlreader interface to interpret the
|
|---|
| 246 | # parameter file. Once a test section has been identified, runTest is called
|
|---|
| 247 | # with a dictionary containing the parsed results of the interpretation.
|
|---|
| 248 | #
|
|---|
| 249 |
|
|---|
| 250 | class testDefaults:
|
|---|
| 251 | curText = '' # accumulates text content of parameter
|
|---|
| 252 |
|
|---|
| 253 | def addToDict(self, key):
|
|---|
| 254 | txt = string.strip(self.curText)
|
|---|
| 255 | # if txt == '':
|
|---|
| 256 | # return
|
|---|
| 257 | if key not in defaultParams:
|
|---|
| 258 | defaultParams[key] = txt
|
|---|
| 259 | else:
|
|---|
| 260 | defaultParams[key] += ' ' + txt
|
|---|
| 261 |
|
|---|
| 262 | def processNode(self, reader, curClass):
|
|---|
| 263 | if reader.Depth() == 2:
|
|---|
| 264 | if reader.NodeType() == 1:
|
|---|
| 265 | self.curText = '' # clear the working variable
|
|---|
| 266 | elif reader.NodeType() == 15:
|
|---|
| 267 | if (reader.Name() != '#text') and (reader.Name() != '#comment'):
|
|---|
| 268 | self.addToDict(reader.Name())
|
|---|
| 269 | elif reader.Depth() == 3:
|
|---|
| 270 | if reader.Name() == '#text':
|
|---|
| 271 | self.curText += reader.Value()
|
|---|
| 272 |
|
|---|
| 273 | elif reader.NodeType() == 15: # end of element
|
|---|
| 274 | print "Defaults have been set to:"
|
|---|
| 275 | for k in defaultParams.keys():
|
|---|
| 276 | print " %s : '%s'" % (k, defaultParams[k])
|
|---|
| 277 | curClass = rootClass()
|
|---|
| 278 | return curClass
|
|---|
| 279 |
|
|---|
| 280 |
|
|---|
| 281 | class testClass:
|
|---|
| 282 | def __init__(self):
|
|---|
| 283 | self.testParams = {} # start with an empty set of params
|
|---|
| 284 | self.curText = '' # and empty text
|
|---|
| 285 |
|
|---|
| 286 | def addToDict(self, key):
|
|---|
| 287 | data = string.strip(self.curText)
|
|---|
| 288 | if key not in self.testParams:
|
|---|
| 289 | self.testParams[key] = data
|
|---|
| 290 | else:
|
|---|
| 291 | if self.testParams[key] != '':
|
|---|
| 292 | data = ' ' + data
|
|---|
| 293 | self.testParams[key] += data
|
|---|
| 294 |
|
|---|
| 295 | def processNode(self, reader, curClass):
|
|---|
| 296 | if reader.Depth() == 2:
|
|---|
| 297 | if reader.NodeType() == 1:
|
|---|
| 298 | self.curText = '' # clear the working variable
|
|---|
| 299 | if reader.Name() not in self.testParams:
|
|---|
| 300 | self.testParams[reader.Name()] = ''
|
|---|
| 301 | elif reader.NodeType() == 15:
|
|---|
| 302 | if (reader.Name() != '#text') and (reader.Name() != '#comment'):
|
|---|
| 303 | self.addToDict(reader.Name())
|
|---|
| 304 | elif reader.Depth() == 3:
|
|---|
| 305 | if reader.Name() == '#text':
|
|---|
| 306 | self.curText += reader.Value()
|
|---|
| 307 |
|
|---|
| 308 | elif reader.NodeType() == 15: # end of element
|
|---|
| 309 | runTest(self.testParams)
|
|---|
| 310 | curClass = rootClass()
|
|---|
| 311 | return curClass
|
|---|
| 312 |
|
|---|
| 313 |
|
|---|
| 314 | class rootClass:
|
|---|
| 315 | def processNode(self, reader, curClass):
|
|---|
| 316 | if reader.Depth() == 0:
|
|---|
| 317 | return curClass
|
|---|
| 318 | if reader.Depth() != 1:
|
|---|
| 319 | print "Unexpected junk: Level %d, type %d, name %s" % (
|
|---|
| 320 | reader.Depth(), reader.NodeType(), reader.Name())
|
|---|
| 321 | return curClass
|
|---|
| 322 | if reader.Name() == 'test':
|
|---|
| 323 | curClass = testClass()
|
|---|
| 324 | curClass.testParams = {}
|
|---|
| 325 | elif reader.Name() == 'defaults':
|
|---|
| 326 | curClass = testDefaults()
|
|---|
| 327 | return curClass
|
|---|
| 328 |
|
|---|
| 329 | def streamFile(filename):
|
|---|
| 330 | try:
|
|---|
| 331 | reader = libxml2.newTextReaderFilename(filename)
|
|---|
| 332 | except:
|
|---|
| 333 | print "unable to open %s" % (filename)
|
|---|
| 334 | return
|
|---|
| 335 |
|
|---|
| 336 | curClass = rootClass()
|
|---|
| 337 | ret = reader.Read()
|
|---|
| 338 | while ret == 1:
|
|---|
| 339 | curClass = curClass.processNode(reader, curClass)
|
|---|
| 340 | ret = reader.Read()
|
|---|
| 341 |
|
|---|
| 342 | if ret != 0:
|
|---|
| 343 | print "%s : failed to parse" % (filename)
|
|---|
| 344 |
|
|---|
| 345 | # OK, we're finished with all the routines. Now for the main program:-
|
|---|
| 346 | if len(sys.argv) != 2:
|
|---|
| 347 | print "Usage: maketest {filename}"
|
|---|
| 348 | sys.exit(-1)
|
|---|
| 349 |
|
|---|
| 350 | streamFile(sys.argv[1])
|
|---|
