#################################################
# SPYCE - Python-based HTML Scripting
# Copyright (c) 2002 Rimon Barr.
#
# Refer to spyce.py
##################################################

#rimtodo:
# - fix compaction (it assumed newlines parsed independently)

import re  # otherwise apache 2.0 pcre library conflicts
           # we just can't win! either stack limits (sre), or 
           # library conflicts (pre)! :)

from cStringIO import StringIO
import sys, string, token, tokenize, os, md5, base64
import spyceTag, spyceUtil, spyce
from spyceException import spyceSyntaxError

__doc__ = '''Compile Spyce files into Python code.'''

##################################################
# Special method names
#

# external interface of generated code
SPYCE_CLASS = 'spyceImpl'
SPYCE_PROCESS_FUNC = 'spyceProcess'
SPYCE_LIBNAME = 'spyceTagcollection'
SPYCE_WRAPPER = '_spyceWrapper' # for raising spyceRuntimeException

# codepoints
GLOBAL_CODEPATH = ['global']
CLASS_CODEPATH = [SPYCE_CLASS]
SPYCEPROCESS_CODEPATH = [SPYCE_PROCESS_FUNC]
# MODULES_CODEPATH is different depending on whether we are compiling a
# tagcollection, and will be created later
TAGLIB_CODEPATH = [SPYCE_PROCESS_FUNC, 'taglibs']
LOGIN_CODEPATH = [SPYCE_PROCESS_FUNC, 'login']
HANDLER_CODEPATH = [SPYCE_PROCESS_FUNC, 'handler']

##################################################
# Dos-to-Unix linebreaks
#

# split a buffer into lines (regardless of terminators)
def splitLines(buf):
  lines=[]
  f=StringIO(buf)
  l=f.readline()
  while l:
    while l and l[-1] in ['\r', '\n']:
      l=l[:-1]
    lines.append(l)
    l=f.readline()
  return lines

# encode document with LF
def CRLF2LF(s):
  return string.join(splitLines(s), '\n')+'\n'

# encode document with CRLF
def LF2CRLF(s):
  return string.join(splitLines(s), '\r\n')+'\r\n'


##################################################
# Tokens
#

T_ESC      = -2
T_EOF      = -1
T_TEXT     = 0
T_EVAL     = 1
T_STMT     = 2
T_CHUNK    = 3
T_CHUNKC   = 4
T_DIRECT   = 5
T_LAMBDA   = 6
T_END      = 7
T_CMNT     = 8
T_END_CMNT = 9

TOKENS = (
  # in the order that they should be tested
  # (i.e. usually longest first)
  (T_ESC,      r'\\\[\[', r'\\<%', r'\\\]\]', r'\\%>'),  # escapes
  (T_CHUNKC,   r'\[\[!', r'<%!'),                     # open class chunk
  (T_CHUNK,    r'\[\[\\', r'<%\\'),                      # open chunk
  (T_EVAL,     r'\[\[=', r'<%='),                        # open eval
  (T_DIRECT,   r'\[\[\.', r'<%\.', r'<%@'),              # open directive
  (T_LAMBDA,   r'\[\[spy', r'<%spy'),                    # open lambda
  (T_CMNT,     r'\[\[--', r'<%--'),                      # open comment
  (T_END_CMNT, r'--\]\]', r'--%>'),                      # close comment
  (T_STMT,     r'\[\[', r'<%'),                          # open statement
  (T_END,      r'\]\]', r'%>'),                          # close
)

def genTokensRE(tokens):
  regexp = []
  typelookup = [None,]
  for group in tokens:
    type, matchstrings = group[0], group[1:]
    for s in matchstrings:
      regexp.append('(%s)' % s)
      typelookup.append(type)
  regexp = string.join(regexp, '|')
  return re.compile(regexp, re.M), typelookup

RE_TOKENS = None
TOKEN_TYPES = None
if not RE_TOKENS:
  RE_TOKENS, TOKEN_TYPES = genTokensRE(TOKENS)

def spyceTokenize(buf):
  # scan using regexp
  tokens = []
  buflen = len(buf)
  pos = 0
  brow = bcol = erow = ecol = 0
  while pos < buflen:
    m = RE_TOKENS.search(buf, pos)
    try:
      mstart, mend = m.start(), m.end()
      other, token = buf[pos:mstart], buf[mstart:mend]
      if other:
        tokens.append((T_TEXT, other, pos, mstart))
      try:
        type = TOKEN_TYPES[m.lastindex]
      except AttributeError, e: 
        # Python 1.5 does not support lastindex
        lastindex = 1
        for x in m.groups():
          if x: break
          lastindex = lastindex + 1
        type = TOKEN_TYPES[lastindex]
      if type==T_ESC:
        token = token[1:]
        type = T_TEXT
      tokens.append((type, token, mstart, mend))
      pos = mend
    except AttributeError, e:
      # handle text before EOF...
      other = buf[pos:]
      if other:
        tokens.append((T_TEXT, other, pos, buflen))
      pos = buflen
  # compute row, col
  brow, bcol = 1, 0
  tokens2 = []
  for type, text, begin, end in tokens:
    lines = string.split(text[:-1], '\n')
    numlines = len(lines)
    erow = brow + numlines - 1
    ecol = bcol
    if numlines>1: ecol = 0
    ecol = ecol + len(lines[-1])
    tokens2.append((type, text, (brow, bcol), (erow, ecol)))
    if text[-1]=='\n':
      brow = erow + 1
      bcol = 0
    else:
      brow = erow
      bcol = ecol + 1
  return tokens2


def spyceTokenize4Parse(buf):
  # add eof and reverse (so that you can pop() tokens)
  tokens = spyceTokenize(buf)
  try:
    _, _, _, end = tokens[-1]
  except:
    end = 0;
  tokens.append((T_EOF, '<EOF>', end, end))
  tokens.reverse()
  return tokens

def processMagic(buf):
  if buf[:2]=='#!':
    buf = string.join(string.split(buf, '\n')[1:], '\n')
  return buf

##################################################
# Directives / Active Tags / Multi-line quotes
#

DIRECTIVE_NAME = re.compile('[a-zA-Z][-a-zA-Z0-9_:]*')
DIRECTIVE_ATTR = re.compile(
    r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
    r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./:;+*%?!&$\(\)_#=~]*))?')
def parseDirective(text):
  "Parse a Spyce directive into name and an attribute list."
  attrs = {}
  match = DIRECTIVE_NAME.match(text)
  if not match: return None, {}
  name = string.lower(text[:match.end()])
  text = string.strip(text[match.end()+1:])
  while text:
    match = DIRECTIVE_ATTR.match(text)
    if not match: break
    attrname, rest, attrvalue = match.group(1, 2, 3)
    if not rest: attrvalue = None
    elif attrvalue[:1] == "'" == attrvalue[-1:] or \
        attrvalue[:1] == '"' == attrvalue[-1:]:
      attrvalue = attrvalue[1:-1]
    attrs[string.lower(attrname)] = attrvalue
    text = text[match.end()+1:]
  return name, attrs

RE_LIB_TAG = re.compile(r'''<          # beginning of tag
  (?P<end>/?)                          # ending tag
  (?P<lib>[a-zA-Z][-.a-zA-Z0-9_]*):    # lib name
  (?P<name>[a-zA-Z][-.a-zA-Z0-9_]*)    # tag name
  (?P<attrs>(?:\s+                     # attributes
    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*       # attribute name
      (?:\s*=\s*                       # value indicator
        (?:'[^']*'                     # LITA-enclosed value
          |"[^"]*"                     # LIT-enclosed value
          |[^'">\s]+                   # bare value
        )
      )?
    )
  )*)
  \s*                                  # trailing whitespace
  (?P<single>/?)                       # single / unpaired tag
  >''', re.VERBOSE)                    # end of tag
def calcEndPos(begin, str):
  if not str: raise 'empty string'
  beginrow, begincol = begin
  eol = 0
  if str[-1]=='\n': 
    str = str[:-1]+' '
    eol = 1
  lines = string.split(str, '\n')
  endrow = beginrow + len(lines)-1
  if endrow!=beginrow:
    begincol = 0
  endcol = begincol + len(lines[-1]) - 1
  beginrow, begincol = endrow, endcol + 1
  if eol:
    begincol = 0
    beginrow = beginrow + 1
  return (endrow, endcol), (beginrow, begincol)

def removeMultiLineQuotes(s):
  def findMultiLineQuote(s):
    quotelist = []
    def eatToken(type, string, begin, end, _, quotelist=quotelist):
      if type == token.STRING:
        if string.startswith('r"""') or string.startswith("r'''") \
        or string.startswith('"""') or string.startswith("'''"):
          quotelist.append((string, begin, end))
    tokenize.tokenize(StringIO(s).readline, eatToken)
    return quotelist
  def replaceRegionWithLine(s, begin, end, s2):
    (beginrow, begincol), (endrow, endcol) = begin, end
    beginrow, endrow = beginrow-1, endrow-1
    s = string.split(s, '\n')
    s1, s3 = s[:beginrow], s[endrow+1:]
    s2 = s[beginrow][:begincol] + s2 + s[endrow][endcol:]
    return string.join(s1 + [s2] + s3, '\n')
  match = findMultiLineQuote(s)
  offsets = {}
  for _, (obr, _), (oer, _) in match:
    offsets[obr] = oer - obr
  while match:
    s2, begin, end = match[0]
    s = replaceRegionWithLine(s, begin, end, `eval(s2)`)
    match = findMultiLineQuote(s)
  return s, offsets

##################################################
# Pre-Python AST
#

# ast node types
AST_PY      = 0
AST_PYEVAL  = 1
AST_TEXT    = 2
AST_COMPACT = 3

# compacting modes
COMPACT_OFF   = 0
COMPACT_LINE  = 1
COMPACT_SPACE = 2
COMPACT_FULL  = 3

class Codepoint:
  def __init__(self, path):
    self.elements = {}
    self.fragments = []
    self.path = path
  def add(self, code):
    self.fragments.append(code)
    if isinstance(code, Codepoint):
      self.elements[code.path[-1]] = code
  def __str__(self, depth=0):
    L = []
    indent = '\t' * depth
    L.append(indent + 'path: ' + str(self.path))
    for code in self.fragments:
      if isinstance(code, Codepoint):
        L.append(code.__str__(depth + 1))
      else:
        L.append(indent + str(code))
    return '\n'.join(L)

class Leaf:
  def __init__(self, type, code, ref):
    self.type = type
    self.code = code
    self.ref = ref
  def __str__(self):
    return str((self.type, self.code, self.ref))

class ppyAST:
  "Generate a pre-Python AST"
  def __init__(self):
    "Initialise parser data structures, AST, token handlers, ..."
    # set up ast
    self._root = Codepoint([])
    self._activepoint = self._root
    self._mods = []
    self._taglibs = {}
  # routines to navigate AST
  def mergeCode(self, codepath):
    restore_point = self._activepoint
    self.selectCodepath(codepath)
    restore_point.fragments += self._activepoint.fragments
    self._activepoint.fragments = []
    for key in self._activepoint.elements:
      restore_point.elements[key] = self._activepoint.elements[key]
    self._activepoint.elements = {}
    self._activepoint = restore_point
  def selectCodepath(self, codepath):
    """
    codepath is a list of which key in the elements dict to follow.
    selectCodepath starts at the top of the AST and follows this path down.
    """
    self._activepoint = self._root
    for point in codepath:
      self._descendCodepath(point)
  def getCodepath(self):
    return self._activepoint.path
  def _descendCodepath(self, codepoint):
    try:
      self._activepoint = self._activepoint.elements[codepoint]
    except KeyError:
      raise 'codepoint %s not found in elements %s of ast %s' % (codepoint, self._activepoint.elements, self._root)
  # routines that modify the ast
  def appendCodepoint(self, pointname, ref=None, descend=True):
    new_path = list(self._activepoint.path)
    new_path.append(pointname)
    self._activepoint.add(Codepoint(new_path))
    if descend:
      self._descendCodepath(pointname)
  def addCode(self, code, ref=None, path=None):
    if path is not None:
      restore_point = self.getCodepath()
      self.selectCodepath(path)
    self._activepoint.add(Leaf(AST_PY, code, ref))
    if path is not None:
      self.selectCodepath(restore_point)
  def addEval(self, eval, ref=None):
    self._activepoint.add(Leaf(AST_PYEVAL, eval, ref))
  def addCodeIndented(self, code, ref, classcode=0):
    code, replacelist = removeMultiLineQuotes(code)
    # funky hack: put in NULLs to preserve indentation
    #   NULLs don't appear in code, and the BraceConverter will
    #   turn them back into spaces. If we leave them as spaces,
    #   BraceConverter is just going to ignore them and pay attention
    #   only to the braces. (not the best compile-time performance!)
    code = string.split(code, '\n')
    code = map(lambda l: (len(l)-len(string.lstrip(l)), l), code)
    code = map(lambda (indent, l): chr(0)*indent + l, code)
    code.append('')
    # split code lines
    (brow, bcol), (erow, ecol), text, file = ref
    row = brow
    for l in code:
      cbcol = 0
      cecol = len(l)
      if row==brow: cbcol = bcol
      if row==erow: cecol = ecol
      try: row2 = row + replacelist[row-brow+1]
      except: row2 = row
      ref = (row, cbcol), (row2, cecol), l, file
      if classcode: self.addCode(l, ref, CLASS_CODEPATH)
      else: self.addCode(l, ref)
      row = row2 + 1
  def addText(self, text, ref=None):
    self._activepoint.add(Leaf(AST_TEXT, text, ref))
  def addCompact(self, compact, ref):
    self._activepoint.add(Leaf(AST_COMPACT, compact, ref))
  def addModule(self, modname, modfrom, modas):
    self._mods.append((modname, modfrom, modas))

##################################################
# Parse
#

class spyceParse:
  def initStandard(self, sig):
    self._ast.appendCodepoint(GLOBAL_CODEPATH[0])
    self._ast.addCode('from spyceException import spyceDone, spyceRedirect, spyceRuntimeException, HandlerError')
    self._ast.selectCodepath([])

    # class codepoint, for class chunk when we start compiling
    # todo: allow this in compiled libraries?
    self._ast.appendCodepoint(SPYCE_CLASS)
    self._ast.addCode('class %s: {' % (SPYCE_CLASS))
    # define spyceProcess
    self._ast.selectCodepath([])
    self._ast.appendCodepoint(SPYCE_PROCESS_FUNC)
    if sig:
      sig = 'self, ' + sig
    else:
      sig = 'self'
    self._ast.addCode('def %s(%s): {' % (SPYCE_PROCESS_FUNC, sig))
    self._ast.addCode('try:{')

    global MODULES_CODEPATH
    MODULES_CODEPATH = [SPYCE_PROCESS_FUNC, 'spymod']
    self._ast.appendCodepoint(MODULES_CODEPATH[-1], descend=False)

    self._ast.appendCodepoint(LOGIN_CODEPATH[-1], descend=False)

    self._ast.appendCodepoint(HANDLER_CODEPATH[-1])
    self._ast.addCode("_handlers = {}")
    self._ast.addCode("_forms_by_handler = {}")
    self._ast.selectCodepath(SPYCEPROCESS_CODEPATH)

    # set up a point to add taglib load calls later
    self._ast.appendCodepoint(TAGLIB_CODEPATH[-1], descend=False)

  def finishStandard(self):
    # login stuff, must allow login_required to terminate execution before running handlers!
    self._ast.selectCodepath(LOGIN_CODEPATH)
    # munge login_required tag invocation to the end of this codepath, 
    # after the request.login_id stuff
    old_login_fragments = self._ast._activepoint.fragments
    self._ast._activepoint.fragments = []
 
    if self.filename:
      # (if no filename, it's a spylamba, and we don't need to re-munge request obj b/c
      # it's passed in from a real page request)
      self._ast.addCode("import _coreutil")
      # login_id() is a callable so that there is no overhead if it's not needed.
      # (w/o explicitly binding request in lambda you will get "unknown global 'request'" errors)
      self._ast.addCode("request.login_id = lambda request=request, _coreutil=_coreutil: _coreutil.login_from_cookie(request)")
      if self._login_possible:
        self._ast.addCode("if _coreutil.login_pending(request):{")
        self._ast.addCode("  if self._login_validator:{")
        self._ast.addCode("    from spyceCompile import _evalWithImport")
        self._ast.addCode("    _validator = _evalWithImport(self._login_validator)")
        self._ast.addCode("  }else:{")
        self._ast.addCode("    import spyceConfig")
        self._ast.addCode("    _validator = spyceConfig.login_defaultvalidator")
        self._ast.addCode("  }")
        self._ast.addCode("  _login_id = _coreutil.login_perform(request, _validator)")
        self._ast.addCode("  request.login_id = lambda _login_id=_login_id: _login_id")
        self._ast.addCode("}")
      if self._login_required:
        self._ast.addCode("if not request.login_id():{")
        # TODO fix hack of special-casing core taglib here (general case done by addLoadTaglibs)
        self._ast.addCode("taglib.load('core','core.py','spy')") 
        self._ast._activepoint.fragments.extend(old_login_fragments)
        self._ast.addCode("}")

    # handlers
    if self._call_handlers:
      self._ast.selectCodepath(HANDLER_CODEPATH)
  
      # call handlers on postback
      self._ast.addCode("_validation_error = {}")
      self._ast.addCode("for _key in request.getpost():{")
      self._ast.addCode("  if _key.startswith('_submit'):{")
      self._ast.addCode("    _handlerid = _key[7:]")
      # don't assume handler is there -- tag in question could have
      # been output by included file or parent, which will run
      # its own copy of this code
      self._ast.addCode("    if _handlerid in _handlers:{")
      self._ast.addCode("      from spyceCompile import _evalWithImport")
      self._ast.addCode("      for _handler in _handlers[_handlerid]:{")
      # imports done in tag scope won't be visible here; make up for that w/autoimport here
      self._ast.addCode("        _f = _evalWithImport(_handler, locals())")
      self._ast.addCode("        from spyceCompile import _marshallArgs")
      self._ast.addCode("        _args, _kwargs = _marshallArgs(request, _f)")
      self._ast.addCode("        try:{")
      self._ast.addCode("          _f(*_args, **_kwargs)")
      self._ast.addCode("        } except HandlerError, _e:{")
      self._ast.addCode("          _form_id = _forms_by_handler[_handlerid]")
      self._ast.addCode("          _validation_error[_form_id] = _e")
      self._ast.addCode("        }")
      self._ast.addCode("      break")
      self._ast.addCode("}}}}")
      if spyce.DEBUG_ERROR:
        self._ast.addCode("if '_handlerid' not in locals():{")
        self._ast.addCode("  import spyce; spyce.DEBUG('(no active handlers to run for this form)')")
        self._ast.addCode("}")
    self._ast.selectCodepath(SPYCEPROCESS_CODEPATH)
  
    # spyceProcess post
    self._ast.addCode('} except spyceDone: pass')
    self._ast.addCode('except spyceRedirect: raise')
    self._ast.addCode('except KeyboardInterrupt: raise')
    self._ast.addCode('except:{ raise spyceRuntimeException(%s) }'%SPYCE_WRAPPER)
    self._ast.addCode('}}') # matches spyceProcess/SPYCE_CLASS

    self._ast.addCode('_has_parent = %r' % self._has_parent, path=CLASS_CODEPATH)
    self._ast.addCode('_login_validator = %r' % self._login_validator, path=CLASS_CODEPATH)

    if self._taglibs_used:
      self._ast.addModule('taglib', None, None)
      self._ast.selectCodepath(TAGLIB_CODEPATH)
      self.addLoadTaglibs()
    if self._load_spylambda: self._ast.addModule('spylambda', None, None)

  def initTaglib(self):
    global MODULES_CODEPATH
    MODULES_CODEPATH = ['spymod']
    self._ast.appendCodepoint(MODULES_CODEPATH[-1], descend=False)
    pass

  def finishTaglib(self):
    self._ast.addCode('''
class %s(spyceTagLibrary):
  tags = [%s]
''' % (SPYCE_LIBNAME, ','.join(self._tagsdefined)))

  def next_tagid(self):
    self._tagcount += 1
    return self._tagidprefix + str(self._tagcount)

  def addLoadTaglibs(self):
    for taglib in self._taglibs_used:
      libname, libfrom = self._ast._taglibs[taglib]
      self._tag_dependencies.append(libfrom)
      self._ast.addCode('taglib.load(%s, %s, %s)'%(repr(libname), repr(libfrom), repr(taglib)))

  def __init__(self, server, buf, filename, gentaglib, sig):
    try:
      # initialization
      self._current_form_id = None
      self._brace_stack = []
      self._tagcount = 0
      self._server = server
      self._tagChecker = spyceTag.spyceTagChecker(server)
      self._load_spylambda = False
      self._has_parent = False
      self._login_required = False
      self._login_possible = False
      self._login_validator = False
      self._call_handlers = False
      self._taglibs_used = {}
      self._tag_dependencies = []
      self._tagsdefined = []
      self._definingtag = spyceUtil.attrdict()
      self._gentaglib = gentaglib
      self._curdir, self._curfile = os.getcwd(), '<string>'
      self._tagidprefix = base64.encodestring(md5.md5(buf).digest())[:-1]
      self.filename = filename # so request-munger can tell if this is a spylambda (and req is already munged)
      if filename:
        self._curdir, self._curfile = os.path.split(filename)
      if not self._curdir:
        self._curdir = os.getcwd() # TODO this is sorta broken; we don't chdir for each request
      self._path = os.path.join(self._curdir, self._curfile)
      # prime ast
      self._ast = ppyAST()
      for tag_tuple in server.config.globaltags:
        path, name = server._findModule(tag_tuple[0], tag_tuple[1], None)
        # a 2.0-style active tag may not call other tags in the same collection
        if path == self._path:
          continue
        args = list(tag_tuple) + [None, None]
        self.addTaglib(*args)

      if gentaglib:
        self.initTaglib()
      else:
        self.initStandard(sig)

      # spyceProcess body
      self._tokens = spyceTokenize4Parse(processMagic(buf))
      self._tokenType = None
      self.popToken()
      self.processSpyce()

      if self._brace_stack:
        ref = self._brace_stack[-1]
        raise spyceSyntaxError("unclosed opening brace '{'", ref)

      if gentaglib:
        self.finishTaglib()
      else:
        self.finishStandard()
      
      # post processing
      self._tagChecker.finish()
    except spyceSyntaxError, e:
      raise
      if e.info:
        begin, end, text, _ = e.info
        e.info = begin, end, text, self._curfile
      raise e
  def addTaglib(self, libname, libfrom, libas, fullfile, ref):
    if not libas: libas=libname
    self._tagChecker.loadLib(libname, libfrom, libas, fullfile, ref)
    self._ast._taglibs[libas] = libname, libfrom
  def info(self):
    return self._ast._root, self._ast._mods, self._tag_dependencies
  def popToken(self):
    if self._tokenType!=T_EOF:
      self._tokenType, self._tokenText, self._tokenBegin, self._tokenEnd = self._tokens.pop()

  def processSpyce(self):
    while self._tokenType!=T_EOF:
      [
        self.processText,        # T_TEXT
        self.processEval,        # T_EVAL
        self.processStmt,        # T_STMT
        self.processChunk,       # T_CHUNK
        self.processClassChunk,  # T_CHUNKC
        self.processDirective,   # T_DIRECT
        self.processUnexpected,  # T_LAMBDA
        self.processUnexpected,  # T_END
        self.processComment,     # T_CMNT
        self.processUnexpected,  # T_END_CMNT
      ][self._tokenType]()
      self.popToken()
  def processComment(self):
    # collect comment
    self.popToken()
    while self._tokenType not in [T_END_CMNT, T_EOF]:
      self.popToken()
    if self._tokenType==T_EOF:
      self.processUnexpected()
  def addText(self, text, ref):
    if self._gentaglib and not self._definingtag:
      s = re.sub(r'''\s''', '', text)
      if s:
        raise spyceSyntaxError('text found outside of tag definition in tagcollection: "%s"' % s, ref)
      return # whitespace is OK, but we don't want to add it to the AST
    self._ast.addText(text, ref)
  def processText(self):
    "Process HTML (possibly with some active tags)"
    html, begin, end = self._tokenText, self._tokenBegin, self._tokenEnd
    while True:
      # TODO refactor this so can share code with include.spycecode
      # something like a second-stage parse that separates text into
      # PLAIN and TAG sections...
      m = RE_LIB_TAG.search(html)
      if not m:
        break
      spyce.DEBUG('active tag hit: %s' % html[m.start():m.end()])
      # emit text literal before tag start, if any
      plain = html[:m.start()]
      if plain:
        plain_end, tag_begin = calcEndPos(begin, plain)
        self.addText(plain, (begin, plain_end, '<html string>', self._curfile))
      else: tag_begin = begin

      tag = m.group(0)
      tag_end, begin = calcEndPos(tag_begin, tag)
      self.processActiveTag(tag, 
        not not m.group('end'), m.group('lib'), m.group('name'), 
        m.group('attrs'), not m.group('single'),
        tag_begin, tag_end)
      html = html[m.end():]
    self.addText(html, (begin, end, '<html string>', self._curfile))
  def processActiveTag(self, tag, tagend, taglib, tagname, tagattrs, tagpair, begin, end):
    """
    Process HTML and Spyce tags
    tagend: true if tag starts with </
    taglib: tag library alias, e.g. 'spy' or 'f' (as in spy:for, f:checkbox)
    tagname: duh
    tagattrs: dict of attributes
    tagpair: true if tag doesn't end with />
    """
    ref = (begin, end, tag, self._curfile)
    if self._gentaglib and not self._definingtag:
      raise spyceSyntaxError("active tag used outside of tag definition in tagcollection", ref)
    # make sure prefix belongs to loaded taglibrary
    if not self._ast._taglibs.has_key(taglib):
      self.addText(tag, (begin, end, '<html string>', self._curfile))
      return
    # parse process tag attributes
    _, tagattrs = parseDirective('x '+tagattrs)
    # get tag class
    tagclass = self._tagChecker.getTagClass(self._ast._taglibs[taglib], tagname, ref)
    codepath = None
    if tagclass.__name__ == 'tag_parent':
      self._has_parent = True
    elif tagclass.__name__ == 'tag_login_required':
      self._login_required = True
      codepath = LOGIN_CODEPATH
    elif tagclass.__name__ == 'form_form':
      if not self._gentaglib:
        self._current_form_id = self.next_tagid()
        self._ast.addCode("_current_form_id = %r" % self._current_form_id, ref)
    if tagclass.__name__ in ['tag_login', 'tag_login_required']:
      self._login_possible = True
      if 'validator' in tagattrs:
        self._login_validator = tagattrs['validator']
    if self._gentaglib and self._login_possible:
      raise spyceSyntaxError("login tags may not currently be used in user-defined Active Tags")
    # syntax check
    if not tagend: # start tag
      self._tagChecker.startTag(self._ast._taglibs[taglib], 
        tagname, tagattrs, tagpair, ref)
    else: # end tag
      self._tagChecker.endTag(self._ast._taglibs[taglib], tagname, ref)
    if tag in self._taglibs_used.setdefault(taglib, {}):
      firstuse = False
    else:
      firstuse = True
      self._taglibs_used[taglib][tag] = True
    if not tagend or not tagpair: # open or singleton tag
      if tagclass.handlers is None:
        tagid = None
        if 'handler' in tagattrs:
          raise spyceSyntaxError('''handler cannot apply to this "%s:%s" tag; handlers may only be assigned to tags that have a 'handlers' list, such as the form library's submit tag''' % (taglib, tagname), ref)
      else:
        tagid = self.next_tagid()
        allhandlers = {}
        for (subid, subhandlers) in tagclass.handlers.items():
          fullid = tagid + subid
          allhandlers[fullid] = subhandlers
        if 'handler' in tagattrs:
          if tagattrs['handler'].startswith('='):
            raise spyceSyntaxError("handler attribute may not be evaluated at runtime (i.e., may not start with '=')", ref)
          L = tagattrs['handler'].split(',')
          if not allhandlers:
            allhandlers[tagid] = []
          for (fullid, subhandlers) in allhandlers.items():
            allhandlers[fullid] = subhandlers + L
          # tag doesn't have to do anything further to set up handler,
          # but we'll pass the attr along so it can perform any validation it wants
          # (e.g, don't allow form action + handler)
        elif not allhandlers:
          allhandlers[tagid] = []
        for (fullid, L) in allhandlers.items():
          if not L:
            continue
          if self._gentaglib:
            self._definingtag.handlers[fullid] = L
          else:
            self._call_handlers = True
            self._ast.addCode("_handlers[%r] = %r" % (fullid, L), path=HANDLER_CODEPATH)
            self._ast.addCode("_forms_by_handler[%r] = %r" % (fullid, self._current_form_id), path=HANDLER_CODEPATH)
      # end handler code

      if firstuse:
        if tagclass.classcode:
          cref = tagclass.classcode
          self.addChunk(cref[2], cref, True)

      self._ast.addCode('taglib.tagPush(%s, %s, %s, locals(), %s, %s)' % (
          repr(taglib), repr(tagname), repr(tagid), repr(tagattrs), repr(tagpair)), ref, codepath)
      self._ast.addCode('try: {', ref, codepath)
      if tagclass.catches:
        self._ast.addCode('try: {', ref, codepath)
      if tagclass.conditional:
        self._ast.addCode('if taglib.tagBegin(): {', ref, codepath)
      else:
        self._ast.addCode('taglib.tagBegin()', ref, codepath)
      if tagclass.mustend:
        self._ast.addCode('try: {', ref, codepath)
      if tagclass.loops:
        self._ast.addCode('while 1: {', ref, codepath)
      # handle exports
      if tagclass.exports:
        # use _foo instead of __foo to avoid problems with name mangling
        # -- since we're in a class, the assignment gets mangled, but the
        # code in exec does not, yeilding a NameError.  Doh!
        self._ast.addCode('_tagexports = taglib.tagExport()', ref, codepath)
        self._ast.addCode('for _tagkey in _tagexports: {'
                          + """exec("%s = _tagexports['%s']" % (_tagkey, _tagkey))"""
                          + '}',
                          ref, codepath)
    if tagend or not tagpair: # close or singleton tag
        if tagclass.loops:
          self._ast.addCode('if not taglib.tagBody(): break }', ref, codepath)
        else:
          self._ast.addCode('taglib.tagBody()', ref, codepath)
        if tagclass.mustend:
          self._ast.addCode('} finally: taglib.tagEnd()', ref, codepath)
        else:
          self._ast.addCode('taglib.tagEnd()', ref, codepath)
        if tagclass.conditional:
          self._ast.addCode('}', ref, codepath)
        if tagclass.catches:
          self._ast.addCode('} except: taglib.tagCatch()', ref, codepath)
        self._ast.addCode('} finally: taglib.tagPop()', ref, codepath)
  def processEval(self):
    # collect expression
    begin = self._tokenBegin
    self.popToken()
    expr = ''
    while self._tokenType not in [T_END, T_EOF]:
      if self._tokenType==T_TEXT:
        expr = expr + self._tokenText
      elif self._tokenType==T_LAMBDA:
        expr = expr + self.processLambda()
      else: self.processUnexpected()
      self.popToken()
    expr = string.strip(expr)
    if not expr: self.processUnexpected()
    # add expression to ast
    self._ast.addEval(expr, (begin, self._tokenEnd, '='+expr, self._curfile))
  def processStmt(self):
    # collect statement
    self.popToken()
    beginrow, begincol = self._tokenBegin
    stmt = ''
    while self._tokenType not in [T_END, T_EOF]:
      if self._tokenType==T_TEXT:
        stmt = stmt + self._tokenText
      elif self._tokenType==T_LAMBDA:
        stmt = stmt + self.processLambda()
      else: self.processUnexpected()
      endrow, endcol = self._tokenEnd
      self.popToken()
    if not string.strip(stmt): self.processUnexpected()
    # add statement to ast, row-by-row
    currow = beginrow
    lines = string.split(stmt, '\n')
    for l in lines:
      if currow==beginrow: curcolbegin = begincol
      else: curcolbegin = 0
      if currow==endrow: curcolend = endcol
      else: curcolend = len(l)
      l = string.strip(l)
      if l:
        ref = ((currow, curcolbegin), (currow, curcolend), l, self._curfile)
        def braceTokenEater(type, string, begin, end, line):
          if type==token.OP:
            if string == '{':
              self._brace_stack.append(ref)
            elif string == '}':
              if not self._brace_stack:
                raise spyceSyntaxError("extra close brace '}'", ref)
              self._brace_stack.pop()
        try:
          tokenize.tokenize(StringIO(l).readline, braceTokenEater)
        except tokenize.TokenError:
          # eof before close brace found; this is expected
          pass
        self._ast.addCode(l, ref)
      currow = currow + 1
  def processChunk(self, classChunk=0):
    # collect chunk
    self.popToken()
    begin = self._tokenBegin
    chunk = ''
    while self._tokenType not in [T_END, T_EOF]:
      if self._tokenType==T_TEXT:
        chunk = chunk + self._tokenText
      elif self._tokenType==T_LAMBDA:
        chunk = chunk + self.processLambda()
      else: self.processUnexpected()
      end = self._tokenEnd
      self.popToken()
    ref = (begin, end, chunk.strip(), self._curfile)
    if self._gentaglib:
      if not self._definingtag:
        raise spyceSyntaxError('tagcollection code chunks may only appear inside tag definitions', ref )
      if classChunk:
        self._definingtag.classcode = ref
        return
    # add chunk block at ast
    if chunk:
      self.addChunk(chunk, ref, classChunk)

  def addChunk(self, chunk, ref, classChunk):
    (begin, end, _, curfile) = ref
    chunk = string.split(chunk, '\n')
    # eliminate initial blank lines
    brow, bcol = begin
    while chunk and not string.strip(chunk[0]):
      chunk = chunk[1:]
      brow = brow + 1
      bcol = 0
    begin = brow, bcol
    if not chunk: self.processUnexpected()
    # outdent chunk based on first line
    # note: modifies multi-line strings having more spaces than first line outdent
    #    by removing outdent number of spaces at the beginning of each line.
    #    -- difficult to deal with efficiently (without parsing python) so just 
    #    don't do this!
    outdent = len(chunk[0]) - len(string.lstrip(chunk[0]))
    for i in range(len(chunk)):
      if string.strip(chunk[i][:outdent]):
        chunk[i] = ' '*outdent + chunk[i]
    chunk = map(lambda l, outdent=outdent: l[outdent:], chunk)
    chunk = string.join(chunk, '\n')
    ref = (begin, end, chunk, curfile)
    try:
      self._ast.addCodeIndented(chunk, ref, classChunk)
    except tokenize.TokenError, e:
      # removeMultiLineQuotes raised
      msg, _ = e
      raise spyceSyntaxError(msg, ref)

  def processClassChunk(self):
    self.processChunk(1)

  def processDirective(self):
    # collect directive
    begin = self._tokenBegin
    self.popToken()
    directive = ''
    while self._tokenType not in [T_END, T_EOF]:
      if self._tokenType==T_TEXT:
        directive = directive + self._tokenText
      else: self.processUnexpected()
      end = self._tokenEnd
      self.popToken()
    directive = string.strip(directive)
    if not directive: self.processUnexpected()
    ref = (begin, end, directive, self._curfile)
    # process directives
    name, attrs = parseDirective(directive)
    if name=='compact':
      compact_mode = COMPACT_FULL
      if attrs.has_key('mode'):
        mode = string.lower(attrs['mode'])
        if mode=='off':
          compact_mode = COMPACT_OFF
        elif mode=='line':
          compact_mode = COMPACT_LINE
        elif mode=='space':
          compact_mode = COMPACT_SPACE
        elif mode=='full':
          compact_mode = COMPACT_FULL
        else:
          raise spyceSyntaxError('invalid compacting mode "%s" specified'%mode, ref)
      self._ast.addCompact(compact_mode, (begin, end, '<spyce compact directive>', self._curfile))
    elif name in ('module', 'import'):
      if not attrs.has_key('name') and not attrs.has_key('names'):
        raise spyceSyntaxError('name or names attribute required', ref)
      if attrs.has_key('names'):
        mod_names = filter(None, map(string.strip, string.split(attrs['names'],',')))
        for mod_name in mod_names:
          self._ast.addModule(mod_name, None, None)
          self._ast.addCode('%s.init()'%mod_name, ref)
      else:
        mod_name = attrs['name']
        mod_from = attrs.get('from')
        mod_as = attrs.get('as')
        mod_args = attrs.get('args', '')
        if mod_as: theName=mod_as
        else: theName=mod_name
        self._ast.addModule(mod_name, mod_from, mod_as)
        self._ast.addCode('%s.init(%s)' % (theName, mod_args), ref, MODULES_CODEPATH)
    elif name in ('taglib',):
      if not attrs.has_key('name') and not attrs.has_key('from'):
        raise spyceSyntaxError('at least one of {name, from} attributes required', ref)
      taglib_name = attrs.get('name', SPYCE_LIBNAME) # ignored if a tagcollection
      taglib_from = attrs.get('from')
      taglib_as = attrs.get('as')
      path, name = self._server._findModule(taglib_name, taglib_from, self._path)
      if path == self._path:
        raise spyceSyntaxError('Compiled active tag may not reference other tags in the same library', ref)
      self.addTaglib(taglib_name, taglib_from, taglib_as, self._path, ref)
    elif name == 'tagcollection':
      if not self._gentaglib:
        raise spyceSyntaxError('tagcollection directive may only be used in a dedicated tag collection file', ref)
    elif name == 'begin':
      if not self._gentaglib:
        raise spyceSyntaxError('begin directive may only be used in a dedicated tag collection file', ref)
      if self._definingtag:
        raise spyceSyntaxError('cannot nest begin directives; expected end for "%s" first' % self._definingtag.name, ref)
      if not attrs.has_key('name'):
        raise spyceSyntaxError('name attribute required', ref)
      name = attrs['name']
      self._definingtag.name = name
      self._definingtag.attrs = []
      self._definingtag.handlers = {}
      self._definingtag.exports = []
      self._definingtag.classcode = None
      self._definingtag.buffer = 'buffers' in attrs and eval(attrs['buffers'])
      self._tagsdefined.append(name)
      self._ast.addCode('class %s(spyceTagPlus):{' % name, ref)
      self._ast.addCode("name = '%s'" % name, ref)
      self._ast.addCode('buffer = %s' % self._definingtag.buffer, ref)
      if 'singleton' in attrs and eval(attrs['singleton']):
        if self._definingtag.buffer:
          raise spyceSyntaxError('Buffer option is exclusive with singleton option', ref)
        self._ast.addCode('def syntax(self):{', ref)
        self._ast.addCode('self.syntaxSingleOnly()}', ref)
      else:
        self._ast.addCode('def syntax(self):{', ref)
        self._ast.addCode('self.syntaxPairOnly()}', ref)
      self._definingtag.kwattrs = 'kwattrs' in attrs and eval(attrs['kwattrs'])
      
      self._ast.appendCodepoint(name + 'Begin', descend=False)
    elif name == 'attr':
      if not self._definingtag:
        raise spyceSyntaxError('attr found, but no corresponding begin', ref)
      if not attrs.has_key('name'):
        raise spyceSyntaxError('name attribute required', ref)
      name = attrs['name']
      if 'default' in attrs:
        self._definingtag.attrs.append((name, attrs['default']))
      else:
        self._definingtag.attrs.append(name)
    elif name == 'export':
      if not attrs.has_key('var'):
        raise spyceSyntaxError('var attribute required', ref)
      if self._definingtag.buffer:
        raise spyceSyntaxError('buffering tags may not export variables', ref)
      if 'as' in attrs:
        exportas = attrs['as']
      else:
        exportas = attrs['var']
      self._definingtag.exports.append((attrs['var'], exportas))
    elif name == 'end':
      if not self._definingtag:
        raise spyceSyntaxError('end found, but no corresponding begin', ref)
      if self._load_spylambda:
        self._ast.addModule('spylambda', None, None)
        self._load_spylambda = False
      def attrsort(a, b):
        return cmp(isinstance(a, tuple), isinstance(b, tuple))
      self._definingtag.attrs.sort(attrsort)
      L = []
      for attr in self._definingtag.attrs:
        if attr[0][0] != attr[0]: # not a string, must be a tuple
          L.append("%s='%s'" % attr)
        else:
          L.append(attr)
      if self._definingtag.kwattrs:
        L.append('**kwargs')
      s = ','.join(L)
      if s:
        s = ',' + s
      self._ast.selectCodepath([self._definingtag.name + 'Begin'])
      # always emit begin so tagchecker can veryify attrs
      self._ast.addCode('def begin(self%s):{' % s, ref)
      if self._definingtag.buffer:
        self._ast.addCode('pass}', ref)
        self._ast.addCode('def body(self, _content):{', ref)
        for attr in self._definingtag.attrs:
          if attr[0][0] != attr[0]: # not a string, must be a tuple
            attr, default = attr
            self._ast.addCode("try:{", ref)
            self._ast.addCode("  %s = self._attrs['%s']" % (attr, attr), ref)
            self._ast.addCode("} except KeyError: {", ref)
            self._ast.addCode("  %s = '%s'" % (attr, default), ref)
            self._ast.addCode("}", ref)
          else:
            self._ast.addCode("%s = self._attrs['%s']" % (attr, attr), ref)
      for modname in spyce.DEFAULT_MODULES:
        self._ast.addCode("%s = self._api.getModule('%s')" % (modname, modname))
      for modname, modfrom, modas in self._ast._mods:
        self._ast.addCode('%s = self._api._startModule(%s, %s, %s)' % (
          modname, repr(modname), repr(modfrom), repr(modas)), ref)
      self._ast.mergeCode(MODULES_CODEPATH)
      self._ast._mods = []
      if self._taglibs_used:
        self._ast.addCode("taglib = self._api.getModules()['taglib']", ref)
        self.addLoadTaglibs()
        self._taglibs_used = {}
      if self._definingtag.handlers:
        self._ast.addCode("if not self.getParent('form'):{", ref)
        self._ast.addCode("  raise 'active handlers may not be used without a parent form active tag, i.e., f:form'}", ref)

      self._ast.selectCodepath([])
      # record exported values for export() to access
      for (var, exportas) in self._definingtag.exports:
        self._ast.addCode("self.%s = %s" % (var, var), ref)
      self._ast.addCode('}', ref)
      if self._definingtag.exports:
        self._ast.addCode("exports = 1", ref)
        self._ast.addCode("def export(self):{", ref)
        L = []
        for (var, exportas) in self._definingtag.exports:
          L.append("'%s': self.%s" % (exportas, var))
        self._ast.addCode("return {%s}" % ','.join(L))
        self._ast.addCode('}', ref)
      # class code chunk, if any
      if self._definingtag.classcode:
        self._ast.addCode("classcode = %s" % repr(self._definingtag.classcode), ref)
      # handlers
      self._ast.addCode("handlers = %s" % repr(self._definingtag.handlers), ref)
      self._ast.addCode('}', ref)
      self._definingtag = spyceUtil.attrdict()
    elif name=='include':
      # deprecated (undocumented) post-1.3
      if not attrs.has_key('file'):
        raise spyceSyntaxError('file attribute missing', ref)
      filename = spyceUtil.url2file(attrs['file'], os.path.join(self._curdir, self._curfile))
      f = None
      try:
        try:
          f = open(filename)
          buf = f.read()
        finally:
          if f: f.close()
      except KeyboardInterrupt: raise
      except:
        raise spyceSyntaxError('unable to open included file: %s'%filename, ref)
      prev = (self._curdir, self._curfile, self._tokens,
        self._tokenType, self._tokenText, self._tokenBegin, self._tokenEnd)
      self._curdir, self._curfile = os.path.dirname(filename), filename
      self._tokens = spyceTokenize4Parse(processMagic(buf))
      self.popToken()
      self.processSpyce()
      (self._curdir, self._curfile, self._tokens,
        self._tokenType, self._tokenText, self._tokenBegin, self._tokenEnd) = prev
    else:
      raise spyceSyntaxError('invalid spyce directive', ref)
  def processLambda(self):
    # collect lambda
    self.popToken()
    begin = self._tokenBegin
    lamb = ''
    depth = 1
    while self._tokenType!=T_EOF:
      if self._tokenType in [T_END,]:
        depth = depth - 1
        if not depth: break
        lamb = lamb + self._tokenText
      elif self._tokenType in [T_EVAL, T_STMT, T_CHUNK, T_CHUNKC, T_DIRECT, T_LAMBDA]:
        depth = depth + 1
        lamb = lamb + self._tokenText
      elif self._tokenType==T_CMNT:
        self.processComment()
      else:
        lamb = lamb + self._tokenText
      end = self._tokenEnd
      self.popToken()
    # process lambda
    lamb = string.split(lamb, ':')
    try:
      params = lamb[0]
      memoize = 0
      if params and params[0]=='!':
        params = params[1:]
        memoize = 1
      lamb = string.join(lamb[1:],':')
    except:
      raise spyceSyntaxError('invalid spyce lambda', (begin, end, lamb, self._curfile))
    self._load_spylambda = True
    lamb = 'spylambda.define(%s,%s,%d)' % (`string.strip(params)`, `lamb`, memoize)
    return lamb
  def processUnexpected(self):
    raise spyceSyntaxError('unexpected token: "%s"'%self._tokenText, 
      (self._tokenBegin, self._tokenEnd, self._tokenText, self._curfile))

##################################################
# Peep-hole optimizer
#
class spyceOptimize:
  def __init__(self, ast):
    self.compaction(ast)
    self.sideBySideWrites(ast)
    #self.splitCodeLines(ast)
  def splitCodeLines(self, ast):
    nodes = ast.fragments
    i = 0
    while i < len(nodes):
      row = 1
      if isinstance(nodes[i], Codepoint):
        self.splitCodeLines(nodes[i])
      elif nodes[i].type == AST_PY and nodes[i].ref:
        code = nodes[i].code
        (brow, bcol), (erow, ecol), code, file = nodes[i].ref
        lines = string.split(code, '\n')
        if code==text and len(lines)>1:
          del nodes[i]
          row = brow
          for l in lines:
            cbcol = 0
            cecol = len(l)
            if row==brow: cbcol = bcol
            if row==erow: becol = ecol
            nodes.insert(i+(brow-row), Leaf(AST_PY, l, ((row, cbcol), (row, cecol), l, file)))
            row = row + 1
      i = i + row

  def sideBySideWrites(self, ast):
    nodes = ast.fragments
    i = 0
    while i < len(nodes):
      if isinstance(nodes[i], Codepoint):
        self.sideBySideWrites(nodes[i])
      elif i + 1 < len(nodes) and isinstance(nodes[i + 1], Leaf):
        type1, text1, ref1 = (nodes[i].type, nodes[i].code, nodes[i].ref)
        type2, text2, ref2 = (nodes[i + 1].type, nodes[i + 1].code, nodes[i + 1].ref)
        file1 = None
        file2 = None
        if ref1:
          _, _, _, file1 = ref1
        if ref2:
          _, _, _, file2 = ref2
        if type1==AST_TEXT and type2==AST_TEXT and file1==file2:
          text = text1 + text2
          begin, _, orig, _ = ref1
          _, end, _, _ = ref2
          nodes[i] = Leaf(AST_TEXT, text, (begin, end, orig, file1))
          del nodes[i+1]
          i -= 1
      i += 1
  
  def compaction(self, ast):
    nodes = ast.fragments
    compact = COMPACT_LINE
    i = 0
    while i < len(nodes):
      if isinstance(nodes[i], Codepoint):
        self.compaction(nodes[i])
      else:
        type, text, ref = (nodes[i].type, nodes[i].code, nodes[i].ref)
        if type==AST_COMPACT:
          compact = text
        elif type==AST_TEXT:
          # line compaction
          if compact==COMPACT_LINE or compact==COMPACT_FULL:
            # remove any trailing whitespace
            text = string.split(text, '\n')
            for j in range(len(text)-1):
              text[j] = string.rstrip(text[j])
            text = string.join(text, '\n')
            # gobble the end of the line
            ((row, _), _, _, file) = ref
            rowtext = string.split(text, '\n')
            if rowtext: rowtext = string.strip(rowtext[0])
            crow = row ; cfile = file
            j = i - 1
            while j > 0 and not rowtext and isinstance(nodes[j], Leaf):
              type2, text2, ref2 = (nodes[j].type, nodes[j].code, nodes[j].ref)
              if ref2: (_, (crow, _), _, cfile) = ref2
              if crow != row or file != cfile: break
              if type2 == AST_TEXT:
                text2 = string.split(text2, '\n')
                if text2: text2 = text2[-1]
                rowtext += string.strip(text2)
              elif type2 == AST_PYEVAL:
                rowtext = 'x'
              j -= 1
            if not rowtext:
              text = string.split(text, '\n')
              if text and not string.strip(text[0]):
                text = text[1:]
              text = string.join(text, '\n')
            # gobble beginning of the line
            (_, (row, _), _, file) = ref
            rowtext = string.split(text, '\n')
            if rowtext: rowtext = string.strip(rowtext[-1])
            crow = row ; cfile = file
            j = i + 1
            while j < len(nodes) and not rowtext and isinstance(nodes[j], Leaf):
              type2, text2, ref2 = (nodes[j].type, nodes[j].code, nodes[j].ref)
              if ref2: ((crow, _), _, _, cfile) = ref2
              if crow != row or file != cfile: break
              if type2 == AST_TEXT:
                text2 = string.split(text2, '\n')
                if text2: text2 = text2[0]
                rowtext += string.strip(text2)
              elif type2 == AST_PYEVAL:
                rowtext = 'x'
              j += 1
            if not rowtext:
              text = string.split(text, '\n')
              if text: text[-1] = string.strip(text[-1])
              text = string.join(text, '\n')
          # space compaction
          if compact==COMPACT_SPACE or compact==COMPACT_FULL:
            text = spyceUtil.spaceCompact(text)
          # update text, if any
          if text: nodes[i] = Leaf(type, text, ref)
          else: 
            del nodes[i]
            i -= 1
        elif type in [AST_PY, AST_PYEVAL, None]:
          pass
        else:
          raise 'error: unknown AST node type'
      i = i + 1

##################################################
# Output classes
#

class LineWriter:
  "Output class that counts lines written."
  def __init__(self, f, initialLine = 1):
    self.f = f
    self.lineno = initialLine
  def write(self, s):
    self.f.write(s)
    self.lineno = self.lineno + len(string.split(s,'\n'))-1
  def writeln(self, s):
    self.f.write(s+'\n')
  def close(self):
    self.f.close()

class IndentingWriter:
  "Output class that helps with indentation of code."
  # Note: this writer is line-oriented.
  def __init__(self, f, indentSize=2):
    self._f = f
    self._indentSize = indentSize
    self._indent = 0
    self._indentString = ' '*(self._indent*self._indentSize)
    self._currentLine = ''
  def close(self):
    if self._indent > 0:
      raise 'unmatched open brace'
    self._f.close()
  def indent(self):
    self._indent = self._indent + 1
    self._indentString = ' '*(self._indent*self._indentSize)
  def outdent(self):
    self._indent = self._indent - 1
    if self._indent<0: 
      raise 'unmatched close brace'
    self._indentString = ' '*(self._indent*self._indentSize)
  def dumpLine(self, s):
    self._f.write(self._indentString+s+'\n')
  def write(self, s):
    self._currentLine = self._currentLine + s
    lines = string.split(self._currentLine, '\n')
    for l in lines[:-1]:
      self.dumpLine(l)
    self._currentLine=lines[-1]
  def writeln(self, s=''):
    self.write(s+'\n')
  # remaining methods are defined in terms of writeln(), indent(), outdent()
  def pln(self, s=''):
    self.writeln(s)
  def pIln(self, s=''):
    self.indent(); self.pln(s)
  def plnI(self, s=''):
    self.pln(s); self.indent()
  def pOln(self, s=''):
    self.outdent(); self.pln(s)
  def plnO(self, s=''):
    self.pln(s); self.outdent()
  def pOlnI(self, s=''):
    self.outdent(); self.pln(s); self.indent()
  def pIlnO(self, s=''):
    self.indent(); self.pln(s); self.outdent()

##################################################
# Print out Braced Python
#

class emitBracedPython:
  def __init__(self, out, ast, gentaglib):
    self._gentaglib = gentaglib
    out = LineWriter(out)
    self._spyceRefs = {}
    # text compaction
    self.compact = COMPACT_LINE
    self._gobblelineNumber = 1
    self._gobblelineText = ''
    # do the deed!
    self.emitSpyceRec(out, ast)
  def getSpyceRefs(self):
    return self._spyceRefs
  def emitSpyceRec(self, out, ast):
    nodes = ast.fragments
    if self._gentaglib:
      outstr = 'self._out'
    else:
      outstr = 'response'
    for code in nodes:
      if isinstance(code, Codepoint):
        self.emitSpyceRec(out, code)
        continue
      type, text, ref = (code.type, code.code, code.ref)
      line1 = out.lineno
      if type==AST_TEXT:
        out.write('%s.writeStatic(%s)\n' % (outstr, `text`))
      elif type==AST_PY:
        out.write(text+'\n')
      elif type==AST_PYEVAL:
        out.write('%s.writeExpr(%s)\n' % (outstr, text))
      elif type==AST_COMPACT:
        self.compact = text
      else:
        raise 'error: unknown AST node type'
      line2 = out.lineno
      if ref:
        for l in range(line1, line2):
          self._spyceRefs[l] = ref
    if not nodes and not ast.elements:
      out.write('pass\n')

##################################################
# Print out regular Python
#

class BraceConverter:
  "Convert Python with braces into indented (normal) Python code."
  def __init__(self, out):
    self.out = IndentingWriter(out)
    self.prevname = 0
    self.prevstring = 0
    self.dictlevel = 0
  def emitToken(self, type, string):
    if type==token.NAME:
      if self.prevname: self.out.write(' ')
      if self.prevstring: self.out.write(' ')
      self.out.write(string)
    elif type==token.STRING:
      if self.prevname: self.out.write(' ')
      string = `eval(string)`  # get rid of multi-line strings
      self.out.write(string)
    elif type==token.NUMBER:
      if self.prevname: self.out.write(' ')
      self.out.write(string)
    elif type==token.OP:
      if string=='{': 
        if self.prevcolon and not self.dictlevel:
          self.out.plnI()
        else:
          self.dictlevel = self.dictlevel + 1
          self.out.write(string)
      elif string=='}':
        if not self.dictlevel:
          self.out.plnO()
        else:
          self.dictlevel = self.dictlevel - 1
          self.out.write(string)
      else:
        self.out.write(string)
    elif type==token.ERRORTOKEN and string==chr(0):
      self.out.write(' ')
    else:
      self.out.write(string)
    self.prevname = type==token.NAME
    self.prevstring = type==token.STRING
    self.prevcolon = type==token.OP and string==':'

def emitPython(out, bracedPythonCode, spyceRefs):
  out = LineWriter(out)
  spyceRefs2 = {}
  braceConv = BraceConverter(out)
  def eatToken(type, string, begin, end, _, out=out, braceConv=braceConv, spyceRefs=spyceRefs, spyceRefs2=spyceRefs2):
    try:
      beginrow, _ = begin
      line1 = out.lineno
      try:
        braceConv.emitToken(type, string)
      except:
        raise spyceSyntaxError('emitToken %s: %s' % (string, spyceUtil.exceptionString()))
      line2 = out.lineno
      if spyceRefs.has_key(beginrow):
        for l in range(line1, line2):
          spyceRefs2[l] = spyceRefs[beginrow]
    except:
      raise spyceSyntaxError('eatToken: %s' % spyceUtil.exceptionString())
  try:
    tokenize.tokenize(StringIO(bracedPythonCode).readline, eatToken)
  except tokenize.TokenError, e:
    msg, (row, col) = e
    raise spyceSyntaxError('tokenization error "%s" at (%d, %d) in\n%s'
                            % (msg, row, col, bracedPythonCode))
  return spyceRefs2

def calcRowCol(str, pos):
  lines = string.split(str, '\n')
  row = 1
  while pos > len(lines[0]):
    pos = pos - len(lines[0]) - 1
    del lines[0]
    row = row + 1
  return row, pos

_bool_values = {
  'true': True,
  'false': False,
  't': True,
  'f': False,
  'yes': True,
  'no': False,
  'on': True,
  'off': False,
  }
def _convertArg(v, convert_to):
  if convert_to == 'int':
    try:
      v = int(v)
    except ValueError:
      raise ValueError('invalid %s: %s' % (convert_to, v))
  elif convert_to == 'float':
    try:
      v = float(v)
    except ValueError:
      raise ValueError('invalid %s: %s' % (convert_to, v))
  elif convert_to == 'bool':
    if v:
      try:
        v = int(v)
      except ValueError:
        # (using eval would be a security hole)
        try:
          v = _bool_values[v.lower()]
        except KeyError:
          raise ValueError('invalid %s: %s' % (convert_to, v))
      else:
        # int succeeded, now make bool
        v = bool(v)
    else:
      # empty string is always False
      v = False
  return v

def _evalWithImport(expr, env=None):
  result = None
  L = expr.split('.')
  if len(L) > 1:
    prefix = L[0]
    try:
      eval(prefix, env)
    except NameError:
      try:
        code = "%s = __import__('%s')" % (prefix, prefix)
        d = {}
        # an "unqualified exec" causes strange errors in user code;
        # see http://spyced.blogspot.com/2005/04/how-well-do-you-know-python-part-4.html
        exec code in d
        result = eval(expr, d)
      except ImportError:
        import spyceConfig, spyceUtil
        msg = '''Unable to import %s while trying to execute %s.
        You probably need to add its location to sys.path in your spyce config file
        (%s)
        
        Raw error message was: %s''' % (prefix, expr, spyceConfig.__file__, spyceUtil.exceptionString())
  if not result:
    result = eval(expr, env)
  return result
  
# called by generated code
def _marshallArgs(req, callable):
  import inspect
  (desired_args, _, _, defaults) = inspect.getargspec(callable)
  if defaults:
    args_with_defaults = dict(zip(desired_args[-len(defaults):], defaults))
  else:
    args_with_defaults = {}
  # ignore 'self'
  if desired_args[0] == 'self':
    desired_args = desired_args[1:]
  if not desired_args:
    return []

  # pre-process request input
  input = {}
  for rawname, values in req.getpost().iteritems():
    L = rawname.split(':')
    name, modifiers = L[0], L[1:]
    op = 'assign'
    convert = ''
    for m in modifiers:
      if m == 'list':
        op = 'append'
      elif m == 'int':
        convert = 'int'
      elif m == 'float':
        convert = 'float'
      elif m == 'bool':
        convert = 'bool'
      else:
        raise 'invalid argument transformation %s' % m
    if op == 'assign':
      input[name] = _convertArg(values[0], convert)
    else:
      input[name] = []
      for v in values:
        input[name].append(_convertArg(v, convert))
  spyce.DEBUG('marshalled input is %s' % input)

  # first non-self arg is modulefinder; others we try to look up in request
  from spyceModule import moduleFinder
  args = [moduleFinder(req._api)]
  kwargs = {}
  for argname in desired_args[1:]:
    try:
      v = input[argname]
    except KeyError:
      if argname in args_with_defaults:
        continue
      raise 'Required parameter %s not present in request' % argname
    if argname in args_with_defaults:
      kwargs[argname] = v
    else:
      args.append(v)
  return args, kwargs

##############################################
# Compile spyce files
#

# (sig is usually '', but spylambda sticks its arguments there; spy:parent puts child there too)
def spyceCompile(buf, filename, sig, server, gentaglib=False):
  # parse
  ast, libs, tags = spyceParse(server, CRLF2LF(buf), filename, gentaglib, sig).info()
  # optimize the ast
  spyceOptimize(ast)
  # generate braced code
  out = StringIO()
  refs = emitBracedPython(out, ast, gentaglib).getSpyceRefs()
  # then, generate regular python code
  bracedPython = out.getvalue()

  out = StringIO()
  refs = emitPython(out, bracedPython, refs)
  return out.getvalue(), refs, libs, tags

def test():
  import spyce
  f = open(sys.argv[1])
  gentaglib = spyceUtil.isTagCollection(f)
  spycecode = f.read()
  f.close()
  tokens = spyceTokenize(processMagic(CRLF2LF(spycecode)))
  print 'TOKENS:'
  for type, text, begin, end in tokens:
    print '%s (%s, %s): %s' % (type, begin, end, `text`)
  pythoncode, refs, libs, tags = spyceCompile(spycecode, sys.argv[1], '', spyce.getServer(), gentaglib)
  L = pythoncode.split('\n')
  print 'CODE:'
  for i in range(len(L)):
    print '%s %s' % (str(i + 1).rjust(3), L[i])
  print 'REFS:'
  for line, ref in refs.items():
    print '%s %s' % (str(line).rjust(3), ref)
  print 'REFERENCED MODULES: %s' % libs

if __name__ == '__main__':
  test()