Preprocessor.py
No OneTemporary
Actions

Size

15 KB

Subscribers

None

Preprocessor.py
View Options

	"""
	This is a very primitive line based preprocessor, for times when using
	a C preprocessor isn't an option.
	"""

	# This Source Code Form is subject to the terms of the Mozilla Public
	# License, v. 2.0. If a copy of the MPL was not distributed with this
	# file, You can obtain one at http://mozilla.org/MPL/2.0/.

	import sys
	import os
	import os.path
	import re
	from optparse import OptionParser
	import errno

	# hack around win32 mangling our line endings
	# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
	if sys.platform == "win32":
	import msvcrt
	msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
	os.linesep = '\n'

	import Expression

	__all__ = ['Preprocessor', 'preprocess']


	class Preprocessor:
	"""
	Class for preprocessing text files.
	"""
	class Error(RuntimeError):
	def __init__(self, cpp, MSG, context):
	self.file = cpp.context['FILE']
	self.line = cpp.context['LINE']
	self.key = MSG
	RuntimeError.__init__(self, (self.file, self.line, self.key, context))
	def __init__(self):
	self.context = Expression.Context()
	for k,v in {'FILE': '',
	'LINE': 0,
	'DIRECTORY': os.path.abspath('.')}.iteritems():
	self.context[k] = v
	self.actionLevel = 0
	self.disableLevel = 0
	# ifStates can be
	# 0: hadTrue
	# 1: wantsTrue
	# 2: #else found
	self.ifStates = []
	self.checkLineNumbers = False
	self.writtenLines = 0
	self.filters = []
	self.cmds = {}
	for cmd, level in {'define': 0,
	'undef': 0,
	'if': sys.maxint,
	'ifdef': sys.maxint,
	'ifndef': sys.maxint,
	'else': 1,
	'elif': 1,
	'elifdef': 1,
	'elifndef': 1,
	'endif': sys.maxint,
	'expand': 0,
	'literal': 0,
	'filter': 0,
	'unfilter': 0,
	'include': 0,
	'includesubst': 0,
	'error': 0}.iteritems():
	self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
	self.out = sys.stdout
	self.setMarker('#')
	self.LE = '\n'
	self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)

	def warnUnused(self, file):
	if self.actionLevel == 0:
	sys.stderr.write('{0}: WARNING: no preprocessor directives found\n'.format(file))
	elif self.actionLevel == 1:
	sys.stderr.write('{0}: WARNING: no useful preprocessor directives found\n'.format(file))
	pass

	def setLineEndings(self, aLE):
	"""
	Set the line endings to be used for output.
	"""
	self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]

	def setMarker(self, aMarker):
	"""
	Set the marker to be used for processing directives.
	Used for handling CSS files, with pp.setMarker('%'), for example.
	The given marker may be None, in which case no markers are processed.
	"""
	self.marker = aMarker
	if aMarker:
	self.instruction = re.compile('{0}(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'
	.format(aMarker),
	re.U)
	self.comment = re.compile(aMarker, re.U)
	else:
	class NoMatch(object):
	def match(self, *args):
	return False
	self.instruction = self.comment = NoMatch()

	def clone(self):
	"""
	Create a clone of the current processor, including line ending
	settings, marker, variable definitions, output stream.
	"""
	rv = Preprocessor()
	rv.context.update(self.context)
	rv.setMarker(self.marker)
	rv.LE = self.LE
	rv.out = self.out
	return rv

	def applyFilters(self, aLine):
	for f in self.filters:
	aLine = f[1](aLine)
	return aLine

	def write(self, aLine):
	"""
	Internal method for handling output.
	"""
	if self.checkLineNumbers:
	self.writtenLines += 1
	ln = self.context['LINE']
	if self.writtenLines != ln:
	self.out.write('//@line {line} "{file}"{le}'.format(line=ln,
	file=self.context['FILE'],
	le=self.LE))
	self.writtenLines = ln
	filteredLine = self.applyFilters(aLine)
	if filteredLine != aLine:
	self.actionLevel = 2
	# ensure our line ending. Only need to handle \n, as we're reading
	# with universal line ending support, at least for files.
	filteredLine = re.sub('\n', self.LE, filteredLine)
	self.out.write(filteredLine)

	def handleCommandLine(self, args, defaultToStdin = False):
	"""
	Parse a commandline into this parser.
	Uses OptionParser internally, no args mean sys.argv[1:].
	"""
	p = self.getCommandLineParser()
	(options, args) = p.parse_args(args=args)
	includes = options.I
	if options.output:
	dir = os.path.dirname(options.output)
	if dir and not os.path.exists(dir):
	try:
	os.makedirs(dir)
	except OSError as error:
	if error.errno != errno.EEXIST:
	raise
	self.out = open(options.output, 'w')
	if defaultToStdin and len(args) == 0:
	args = [sys.stdin]
	includes.extend(args)
	if includes:
	for f in includes:
	self.do_include(f, False)
	self.warnUnused(f)
	pass

	def getCommandLineParser(self, unescapeDefines = False):
	escapedValue = re.compile('".*"$')
	numberValue = re.compile('\d+$')
	def handleE(option, opt, value, parser):
	for k,v in os.environ.iteritems():
	self.context[k] = v
	def handleD(option, opt, value, parser):
	vals = value.split('=', 1)
	if len(vals) == 1:
	vals.append(1)
	elif unescapeDefines and escapedValue.match(vals[1]):
	# strip escaped string values
	vals[1] = vals[1][1:-1]
	elif numberValue.match(vals[1]):
	vals[1] = int(vals[1])
	self.context[vals[0]] = vals[1]
	def handleU(option, opt, value, parser):
	del self.context[value]
	def handleF(option, opt, value, parser):
	self.do_filter(value)
	def handleLE(option, opt, value, parser):
	self.setLineEndings(value)
	def handleMarker(option, opt, value, parser):
	self.setMarker(value)
	p = OptionParser()
	p.add_option('-I', action='append', type="string", default = [],
	metavar="FILENAME", help='Include file')
	p.add_option('-E', action='callback', callback=handleE,
	help='Import the environment into the defined variables')
	p.add_option('-D', action='callback', callback=handleD, type="string",
	metavar="VAR[=VAL]", help='Define a variable')
	p.add_option('-U', action='callback', callback=handleU, type="string",
	metavar="VAR", help='Undefine a variable')
	p.add_option('-F', action='callback', callback=handleF, type="string",
	metavar="FILTER", help='Enable the specified filter')
	p.add_option('-o', '--output', type="string", default=None,
	metavar="FILENAME", help='Output to the specified file '+
	'instead of stdout')
	p.add_option('--line-endings', action='callback', callback=handleLE,
	type="string", metavar="[cr\|lr\|crlf]",
	help='Use the specified line endings [Default: OS dependent]')
	p.add_option('--marker', action='callback', callback=handleMarker,
	type="string",
	help='Use the specified marker instead of #')
	return p

	def handleLine(self, aLine):
	"""
	Handle a single line of input (internal).
	"""
	if self.actionLevel == 0 and self.comment.match(aLine):
	self.actionLevel = 1
	m = self.instruction.match(aLine)
	if m:
	args = None
	cmd = m.group('cmd')
	try:
	args = m.group('args')
	except IndexError:
	pass
	if cmd not in self.cmds:
	raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
	level, cmd = self.cmds[cmd]
	if (level >= self.disableLevel):
	cmd(args)
	if cmd != 'literal':
	self.actionLevel = 2
	elif self.disableLevel == 0 and not self.comment.match(aLine):
	self.write(aLine)
	pass

	# Instruction handlers
	# These are named do_'instruction name' and take one argument

	# Variables
	def do_define(self, args):
	m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
	if not m:
	raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
	val = 1
	if m.group('value'):
	val = self.applyFilters(m.group('value'))
	try:
	val = int(val)
	except:
	pass
	self.context[m.group('name')] = val
	def do_undef(self, args):
	m = re.match('(?P<name>\w+)$', args, re.U)
	if not m:
	raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
	if args in self.context:
	del self.context[args]
	# Logic
	def ensure_not_else(self):
	if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
	sys.stderr.write('WARNING: bad nesting of #else\n')
	def do_if(self, args, replace=False):
	if self.disableLevel and not replace:
	self.disableLevel += 1
	return
	val = None
	try:
	e = Expression.Expression(args)
	val = e.evaluate(self.context)
	except Exception:
	# XXX do real error reporting
	raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
	if type(val) == str:
	# we're looking for a number value, strings are false
	val = False
	if not val:
	self.disableLevel = 1
	if replace:
	if val:
	self.disableLevel = 0
	self.ifStates[-1] = self.disableLevel
	else:
	self.ifStates.append(self.disableLevel)
	pass
	def do_ifdef(self, args, replace=False):
	if self.disableLevel and not replace:
	self.disableLevel += 1
	return
	if re.match('\W', args, re.U):
	raise Preprocessor.Error(self, 'INVALID_VAR', args)
	if args not in self.context:
	self.disableLevel = 1
	if replace:
	if args in self.context:
	self.disableLevel = 0
	self.ifStates[-1] = self.disableLevel
	else:
	self.ifStates.append(self.disableLevel)
	pass
	def do_ifndef(self, args, replace=False):
	if self.disableLevel and not replace:
	self.disableLevel += 1
	return
	if re.match('\W', args, re.U):
	raise Preprocessor.Error(self, 'INVALID_VAR', args)
	if args in self.context:
	self.disableLevel = 1
	if replace:
	if args not in self.context:
	self.disableLevel = 0
	self.ifStates[-1] = self.disableLevel
	else:
	self.ifStates.append(self.disableLevel)
	pass
	def do_else(self, args, ifState = 2):
	self.ensure_not_else()
	hadTrue = self.ifStates[-1] == 0
	self.ifStates[-1] = ifState # in-else
	if hadTrue:
	self.disableLevel = 1
	return
	self.disableLevel = 0
	def do_elif(self, args):
	if self.disableLevel == 1:
	if self.ifStates[-1] == 1:
	self.do_if(args, replace=True)
	else:
	self.do_else(None, self.ifStates[-1])
	def do_elifdef(self, args):
	if self.disableLevel == 1:
	if self.ifStates[-1] == 1:
	self.do_ifdef(args, replace=True)
	else:
	self.do_else(None, self.ifStates[-1])
	def do_elifndef(self, args):
	if self.disableLevel == 1:
	if self.ifStates[-1] == 1:
	self.do_ifndef(args, replace=True)
	else:
	self.do_else(None, self.ifStates[-1])
	def do_endif(self, args):
	if self.disableLevel > 0:
	self.disableLevel -= 1
	if self.disableLevel == 0:
	self.ifStates.pop()
	# output processing
	def do_expand(self, args):
	lst = re.split('__(\w+)__', args, re.U)
	do_replace = False
	def vsubst(v):
	if v in self.context:
	return str(self.context[v])
	return ''
	for i in range(1, len(lst), 2):
	lst[i] = vsubst(lst[i])
	lst.append('\n') # add back the newline
	self.write(reduce(lambda x, y: x+y, lst, ''))
	def do_literal(self, args):
	self.write(args + self.LE)
	def do_filter(self, args):
	filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
	if len(filters) == 0:
	return
	current = dict(self.filters)
	for f in filters:
	current[f] = getattr(self, 'filter_' + f)
	filterNames = current.keys()
	filterNames.sort()
	self.filters = [(fn, current[fn]) for fn in filterNames]
	return
	def do_unfilter(self, args):
	filters = args.split(' ')
	current = dict(self.filters)
	for f in filters:
	if f in current:
	del current[f]
	filterNames = current.keys()
	filterNames.sort()
	self.filters = [(fn, current[fn]) for fn in filterNames]
	return
	# Filters
	#
	# emptyLines
	# Strips blank lines from the output.
	def filter_emptyLines(self, aLine):
	if aLine == '\n':
	return ''
	return aLine
	# slashslash
	# Strips everything after //
	def filter_slashslash(self, aLine):
	if (aLine.find('//') == -1):
	return aLine
	[aLine, rest] = aLine.split('//', 1)
	if rest:
	aLine += '\n'
	return aLine
	# spaces
	# Collapses sequences of spaces into a single space
	def filter_spaces(self, aLine):
	return re.sub(' +', ' ', aLine).strip(' ')
	# substition
	# helper to be used by both substition and attemptSubstitution
	def filter_substitution(self, aLine, fatal=True):
	def repl(matchobj):
	varname = matchobj.group('VAR')
	if varname in self.context:
	return str(self.context[varname])
	if fatal:
	raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
	return matchobj.group(0)
	return self.varsubst.sub(repl, aLine)
	def filter_attemptSubstitution(self, aLine):
	return self.filter_substitution(aLine, fatal=False)
	# File ops
	def do_include(self, args, filters=True):
	"""
	Preprocess a given file.
	args can either be a file name, or a file-like object.
	Files should be opened, and will be closed after processing.
	"""
	isName = type(args) == str or type(args) == unicode
	oldWrittenLines = self.writtenLines
	oldCheckLineNumbers = self.checkLineNumbers
	self.checkLineNumbers = False
	if isName:
	try:
	args = str(args)
	if filters:
	args = self.applyFilters(args)
	if not os.path.isabs(args):
	args = os.path.join(self.context['DIRECTORY'], args)
	args = open(args, 'rU')
	except Preprocessor.Error:
	raise
	except:
	raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
	self.checkLineNumbers = bool(re.search('\.(js\|jsm\|java)(?:\.in)?$', args.name))
	oldFile = self.context['FILE']
	oldLine = self.context['LINE']
	oldDir = self.context['DIRECTORY']
	if args.isatty():
	# we're stdin, use '-' and '' for file and dir
	self.context['FILE'] = '-'
	self.context['DIRECTORY'] = ''
	else:
	abspath = os.path.abspath(args.name)
	self.context['FILE'] = abspath
	self.context['DIRECTORY'] = os.path.dirname(abspath)
	self.context['LINE'] = 0
	self.writtenLines = 0
	for l in args:
	self.context['LINE'] += 1
	self.handleLine(l)
	args.close()
	self.context['FILE'] = oldFile
	self.checkLineNumbers = oldCheckLineNumbers
	self.writtenLines = oldWrittenLines
	self.context['LINE'] = oldLine
	self.context['DIRECTORY'] = oldDir
	def do_includesubst(self, args):
	args = self.filter_substitution(args)
	self.do_include(args)
	def do_error(self, args):
	raise Preprocessor.Error(self, 'Error: ', str(args))

	def main():
	pp = Preprocessor()
	pp.handleCommandLine(None, True)
	return

	def preprocess(includes=[sys.stdin], defines={},
	output = sys.stdout,
	line_endings='\n', marker='#'):
	pp = Preprocessor()
	pp.context.update(defines)
	pp.setLineEndings(line_endings)
	pp.setMarker(marker)
	pp.out = output
	for f in includes:
	pp.do_include(f, False)

	if __name__ == "__main__":
	main()

File Metadata

Mime Type: text/plain
Expires: Sat, Jan 31, 11:09 PM (1 d, 17 h)
Storage Engine: local-disk
Storage Format: Raw Data
Storage Handle: 75/f4/2c2988b494072fbd324ef2b24742

Preprocessor.pyNo OneTemporaryActions

Preprocessor.pyView Options

File Metadata

Event Timeline

Preprocessor.py
No OneTemporary
Actions

Preprocessor.py
View Options