/* 
  mxte -- A table driven tagging engine for Python (Version 0.9)

  Copyright (c) 2000, Marc-Andre Lemburg; mailto:mal@lemburg.com
  Copyright (c) 2000-2001, eGenix.com Software GmbH; mailto:info@egenix.com
*/

/* Debugging switches */
/*#define MAL_DEBUG*/
/*#define MAL_REF_DEBUG*/

/* Logging file used by debugging facility */
#ifndef MAL_DEBUG_OUTPUTFILE
# define MAL_DEBUG_OUTPUTFILE "mxTagEngine.log"
#endif

#include "mx.h"
#include "mxstdlib.h"
#include "mxTextTools.h"

/* --- Internal definitions ----------------------------------------------- */

#if 0

/* The beginnings of a Tag Table compiler... */

typedef struct {
    PyObject *tagobj;			/* Tag object to assign or NULL */
    int cmd;				/* Command (encoded as integer) */
    PyObject *match;			/* Matching argument */
    int jne, je;			/* Jump offsets */
} TagTableEntry;

#endif

/* --- Tagging Enigne ----------------------------------------------------- */

/* Forwards */
static int match_append(int flags, PyObject *pytext, PyObject *taglist,
			PyObject *tagobj, int l, int r, PyObject *subtags);

/* fast_tag: a table driven parser engine
   
   - return codes: rc = 2: match ok; rc = 1: match failed; rc = 0: error
   - doesn't check type of passed arguments !
   - doesn't increment reference counts of passed objects !
*/

int fast_tag(PyObject *pytext,  	/* must be a Python string */
	     char *text,		/* text of that string */
	     int len_text,		/* "length" of the Python string */
	     PyObject *table,		/* must be a tuple (this is
					   *not* checked) */
	     int start_text,		/* start position in text */
	     PyObject *taglist,		/* must be a Python list or None */
	     int *next)			/* output: next position in text */
{
    register int x;		/* current (head) position in text */
    int i; 			/* index of current table entry */
    int start = start_text;	/* start position for current tag */
    int len_table = PyTuple_GET_SIZE(table); /* table length */
    int rc = -1;		/* return code: -1 not set, 0 error, 1
				   not ok, 2 ok */
    int loopcount = -1; 	/* loop counter */
    int loopstart = start;	/* loop start position */
    int flags;			/* flags set in command */
    int cmd;			/* command */
    int jne;			/* rel. jump distance on 'not matched' */
    int je;			/* dito on 'matched' */
    PyObject *tagobj,		/* tag object */
	     *match;		/* matching parameter */

    /* Main loop */
    for (x = start, i = 0, je = 0;;) {

    next_entry:
	/* Get next entry */
	i += je;
	if (i >= len_table || i < 0)
	    /* Out of bounds: we're finished */
	    goto finished;

	je = 1;
	jne = 0;
		
	/* Parse arguments - inlined for speed */
	{
	    register PyObject *entry;
	    register PyObject *w;

	    /* Parse entry */
	    entry = PyTuple_GET_ITEM(table,i);
	    Py_AssertWithArg(
		      PyTuple_Check(entry) && PyTuple_GET_SIZE(entry) >= 3,
		      PyExc_TypeError,
		      "tag table entry %i: expected a tuple of the form "
		      "(tagobj,command,arg[,jne[,je]])",i);

	    /* Get tagobj */
	    tagobj = PyTuple_GET_ITEM(entry,0);

	    /* Get cmd */
	    w = PyTuple_GET_ITEM(entry,1);
	    Py_AssertWithArg(PyInt_Check(w),
			     PyExc_TypeError,
			     "tag table entry %i: "
			     "command must be an integer",i);
	    cmd = PyInt_AS_LONG(w);

	    /* Get match object */
	    match = PyTuple_GET_ITEM(entry,2);

	    /* Optional parameters*/
	    if (PyTuple_GET_SIZE(entry) > 3) {
		/* get jne */
		w = PyTuple_GET_ITEM(entry,3);
		Py_AssertWithArg(PyInt_Check(w),
				 PyExc_TypeError,
				 "tag table entry %i: "
				 "jne must be an integer",i);
		jne = PyInt_AS_LONG(w);

		if (PyTuple_GET_SIZE(entry) > 4) {
		    /* get je */
		    w = PyTuple_GET_ITEM(entry,4);
		    Py_AssertWithArg(PyInt_Check(w),
				     PyExc_TypeError,
				     "tag table entry %i: "
				     "je must be an integer",i);
		    je = PyInt_AS_LONG(w);
		}
	    }
	}
	
	/* Decode flags and cmd */
	flags = (cmd >> 8) << 8;
	cmd = cmd & 0xff;

	IF_DEBUGGING {
	    PyObject *v;
	    
	    mxDebugPrintf("\n# Tag Table entry %i: ",i);
	    v = PyObject_Repr(tagobj);
	    if (!v) {
		v = tagobj;
		Py_INCREF(v);
	    }
	    if (PyString_Check(v))
		mxDebugPrintf("tagobj=%s cmd=%i flags=%i position=%i\n",
			      PyString_AsString(v),cmd,flags,x);
	    else
		mxDebugPrintf("tagobj at 0x%lx cmd=%i flags=%i position=%i\n",
			      (long)v,cmd,flags,x);
	    Py_DECREF(v);
	}
	
	/* Low-level matching commands */

	if (cmd > MATCH_MAX_SPECIALS && cmd < MATCH_MAX_LOWLEVEL) {
	    char *m;
	    
	    Py_AssertWithArg(PyString_Check(match),
			     PyExc_TypeError,
			     "Tag Table entry %i: arg must be a string",i);
	    m = PyString_AS_STRING(match);

	    /* save starting position */
	    start = x;

	    switch (cmd) {

	    case MATCH_ALLIN:

		{
		    register int ml = PyString_GET_SIZE(match);
		    register char *tx = text + x;

		    DPRINTF("\nAllIn :\n"
			    " looking for   = '%.40s'\n"
			    " in string     = '%.40s'\n",m,tx);
	    
		    if (ml > 1)
			for (; x < len_text; tx++, x++) {
			    register int j;
			    register char *mj = m;
			    register char ctx = *tx;
			    for (j=0; j < ml && ctx != *mj; mj++, j++) ;
			    if (j == ml) break;
			}
		    else if (ml == 1)
			/* one char only: use faster variant: */
			for (; x < len_text && *tx == *m; tx++, x++) ;

		    break;
		}

	    case MATCH_ALLNOTIN:

		{
		    register int ml = PyString_GET_SIZE(match);
		    register char *tx = text + x;

		    DPRINTF("\nAllNotIn :\n"
			    " looking for   = '%.40s'\n"
			    " not in string = '%.40s'\n",m,tx);
	    
		    if (ml != 1)
			for (; x < len_text; tx++, x++) {
			    register int j;
			    register char *mj = m;
			    register char ctx = *tx;
			    for (j=0; j < ml && ctx != *mj; mj++, j++) ;
			    if (j != ml) break;
			}
		    else
			/* one char only: use faster variant: */
			for (; x < len_text && *tx != *m; tx++, x++) ;
		    break;
		}

	    case MATCH_IS: 
		    
		{
		    DPRINTF("\nIs :\n"
			    " looking for   = '%.40s'\n"
			    " in string     = '%.40s'\n",m,text+x);
	    
		    if (*(text + x) == *m && x < len_text)
			x++;
		    break;
		}

	    case MATCH_ISIN:

		{
		    register int ml = PyString_GET_SIZE(match);
		    register char ctx = *(text + x);

		    DPRINTF("\nIsIn :\n"
			    " looking for   = '%.40s'\n"
			    " in string     = '%.40s'\n",m,text+x);
	    
		    if (ml > 0 && x < len_text) {
			register int j;
			register char *mj = m;
			for (j=0; j < ml && ctx != *mj; mj++, j++) ;
			if (j != ml) x++;
		    }

		    break;
		}

	    case MATCH_ISNOTIN:

		{
		    register int ml = PyString_GET_SIZE(match);
		    register char ctx = *(text + x);

		    DPRINTF("\nIsNotIn :\n"
			    " looking for   = '%.40s'\n"
			    " not in string = '%.40s'\n",m,text+x);
	    
		    if (ml > 0 && x < len_text) {
			register int j;
			register char *mj = m;
			for (j=0; j < ml && ctx != *mj; mj++, j++) ;
			if (j == ml) x++;
		    }
		    else
			x++;

		    break;
		}

	    case MATCH_WORD:

		{
		    int ml1 = PyString_GET_SIZE(match) - 1;
		    register char *tx = text + x + ml1;
		    register int j = ml1;
		    register char *mj = m + j;

		    DPRINTF("\nWord :\n"
			    " looking for   = '%.40s'\n"
			    " in string     = '%.40s'\n",m,tx);
	    
		    if (x+ml1 >= len_text) break;
		    
		    /* compare from right to left */
		    for (; j >= 0 && *tx == *mj;
			 tx--, mj--, j--) ;

		    if (j >= 0) /* not matched */
			x = start; /* reset */
		    else
			x += ml1 + 1;
		    break;
		}

	    case MATCH_WORDSTART:
	    case MATCH_WORDEND:

		{
		    int ml1 = PyString_GET_SIZE(match) - 1;

		    if (ml1 >= 0) {
			register char *tx = text + x;
			    
			DPRINTF("\nWordStart/End :\n"
				" looking for   = '%.40s'\n"
				" in string     = '%.40s'\n",m,tx);
	    
			/* Brute-force method; from right to left */
			for (;;) {
			    register int j = ml1;
			    register char *mj = m + j;

			    if (x+j >= len_text) {
				/* reached eof: no match, rewind */
				x = start;
				break;
			    }

			    /* scan from right to left */
			    for (tx += j; j >= 0 && *tx == *mj; 
				 tx--, mj--, j--) ;
			    /*
			    DPRINTF("match text[%i+%i]: %c == %c\n",
			            x,j,*tx,*mj);
			    */

			    if (j < 0) {
				/* found */
				if (cmd == MATCH_WORDEND) x += ml1 + 1;
				break;
			    }
			    /* not found: rewind and advance one char */
			    tx -= j - 1;
			    x++;
			}
		    }

		    break;
		}

	    case MATCH_ALLINSET:

		{
		    register char *tx = text + x;

		    Py_AssertWithArg(PyString_GET_SIZE(match) == 32,
				     PyExc_TypeError,
				     "Tag Table entry %i: "
				     "sets must have 32 chars "
				     "(cmd=AllInSet)",i);

		    DPRINTF("\nAllInSet :\n"
			    " looking for   = set at 0x%lx\n"
			    " in string     = '%.40s'\n",(long)match,tx);
	    
		    for (;x < len_text &&
			  ((unsigned char) m[(unsigned char)*tx >> 3] & 
			   (1 << (*tx & 7))) > 0;
			 tx++, x++) ;
		
		    break;
		}

	    case MATCH_ISINSET:

		{
		    register char *tx = text + x;

		    Py_AssertWithArg(PyString_GET_SIZE(match) == 32,
				     PyExc_TypeError,
				     "Tag Table entry %i: "
				     "sets must have 32 chars "
				     "(cmd=IsInSet)",i);

		    DPRINTF("\nIsInSet :\n"
			    " looking for   = set at 0x%lx\n"
			    " in string     = '%.40s'\n",(long)match,tx);
	    
		    if (x < len_text &&
			((unsigned char) m[(unsigned char)*tx >> 3] & 
			 (1 << (*tx & 7))) > 0) x++;
		
		    break;
		}

	    default:

		Py_ErrorWithArg(PyExc_TypeError,
				"Tag Table entry %i: "
				"unknown command",i);

	    } /* switch */
	    
	    /* Not matched */
	    if (start == x) { 
		DPRINTF(" (no success)\n");
		if (jne == 0) { 
		    /* failed */
		    rc = 1; 
		    goto finished; 
		}
		else 
		    je = jne;
		goto next_entry;
	    }

	    /* Matched */
	    if (tagobj != Py_None) { 
		if (match_append(flags,pytext,taglist,tagobj,
				 start,x,NULL) < 0)
		    goto onError;
		DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
			start,x);
	    }
	    else
		DPRINTF(" [%i:%i] (matched but not saved)\n",start,x);

	    if (flags & MATCH_LOOKAHEAD) {
		x = start;
		DPRINTF(" LOOKAHEAD flag set: reseting position to %i\n",
			x);
	    }

	    goto next_entry;
	}
	
	/* Jumps & special commands */

	if (cmd < MATCH_MAX_SPECIALS) {
	
	    switch (cmd) {

	    case MATCH_FAIL: /* == MATCH_JUMP */

		if (jne == 0) { /* match failed */
		    rc = 1;
		    goto finished; 
		}
		else 
		    je = jne;
		goto next_entry;
	    
	    case MATCH_SKIP:

		if (PyInt_Check(match)) {

		    DPRINTF("\nSkip %li characters\n"
			    " in string    = '%.40s'\n",
			    PyInt_AS_LONG(match),text+x);

		    start = x;
		    x += PyInt_AS_LONG(match);
		    
		    break;
		}
		else
		    Py_ErrorWithArg(PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected an integer (cmd=Skip)",i);

	    case MATCH_MOVE:

		if (PyInt_Check(match)) {

		    start = x;
		    x = PyInt_AS_LONG(match);
		    if (x < 0)
			/* Relative to end of the slice */
			x += len_text + 1;
		    else
			/* Relative to beginning of the slice */
			x += start_text;

		    DPRINTF("\nMove to position %i \n"
			    " string       = '%.40s'\n",
			    x,text+x);
		    break;
		}
		else
		    Py_ErrorWithArg(PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected an integer (cmd=Move)",i);
		
	    case MATCH_EOF:

		DPRINTF("\nEOF at position %i ? \n"
			" string       = '%.40s'\n",
			x,text+x);

		if (len_text > x) { /* not matched */
		    DPRINTF(" (no success)\n");
		    if (jne == 0) { /* match failed */
			rc = 1;
			goto finished; 
		    }
		    else 
			je = jne;
		    goto next_entry;
		}

		/* Matched & finished */
		if (tagobj != Py_None) {
		    x = len_text;
		    if (match_append(flags,pytext,taglist,tagobj,
				     x,x,NULL) < 0)
			goto onError;
		    DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
			    x,x);
		}
		else
		    DPRINTF(" [%i:%i] (matched but not saved)\n",
			    x,x);
		rc = 2;
		goto finished;

	    default:

		Py_ErrorWithArg(PyExc_TypeError,
				"Tag Table entry %i: "
				"unknown command",i);
	    }

	    /* Matched */
	    if (x < 0)
		Py_ErrorWithArg(PyExc_TypeError,
				"Tag Table entry %i: "
				"moved/skipped beyond start of text",i);
	    
	    if (tagobj != Py_None) {
		if (match_append(flags,pytext,taglist,tagobj,
				 start,x,NULL) < 0)
		    goto onError;
		DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
			start,x);
	    }
	    else
		DPRINTF(" [%i:%i] (matched but not saved)\n",start,x);

	    if (flags & MATCH_LOOKAHEAD) {
		x = start;
		DPRINTF(" LOOKAHEAD flag set: reseting position to %i\n",
			x);
	    }

	    goto next_entry;
	}

	/* Higher level matching commands */

	switch (cmd) {

	case MATCH_SWORDSTART:
	case MATCH_SWORDEND:
	case MATCH_SFINDWORD:
	    {
		int len_match;

		DPRINTF("\nsWordStart/End/sFindWord :\n"
			" in string   = '%.40s'\n",text+x);

		/* Boyer Moore search */
		if (_mxBMS_Check(match)) { 
		    mxBMSObject *so = (mxBMSObject *)match;

		    len_match = so->c->len_match;
		    start = x;

		    if (so->tr) {
			/* search with translate table */
			x = bm_tr_search(so->c,
					 text,
					 start,
					 len_text,
					 PyString_AS_STRING(so->tr));
		    }
		    else {
			/* exact search */
			x = bm_search(so->c,
				      text,
				      start,
				      len_text);
		    }
		}
#ifdef MXFASTSEARCH		
		/* Fast Search */
		else if (_mxFS_Check(match)) { 
		    mxFSObject *so = (mxFSObject *)match;

		    len_match = so->c->len_match;
		    start = x;

		    if (so->tr) {
			/* search with translate table */
			x = fs_tr_search(so->c,
					 text,
					 start,
					 len_text,
					 PyString_AS_STRING(so->tr));
		    }
		    else {
			/* exact search */
			x = fs_search(so->c,
				      text,
				      start,
				      len_text);
		    }
		}
#endif
		else {
		    Py_ErrorWithArg(PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected a search object "
				    "(cmd=WordStart/End)",i);
		}
		
		if (start == x) { 
		    /* not matched */
		    DPRINTF(" (no success)\n");
		    if (jne == 0) { 
			/* match failed */
			rc = 1; 
			goto finished; 
		    }
		    else 
			je = jne;
		} 
		else { 
		    /* matched */
		    /* Apply correction of x for SWORDSTART and start
                       for SFINDWORD */
		    if (cmd == MATCH_SWORDSTART)
			x -= len_match;
		    else if (cmd == MATCH_SFINDWORD)
			start = x - len_match;

		    if (tagobj != Py_None) {
			if (match_append(flags,pytext,taglist,tagobj,
					 start,x,NULL) < 0)
			    goto onError;
			DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
				start,x);
		    }
		    else
			DPRINTF(" [%i:%i] (matched but not saved)\n",start,x);

		    if (flags & MATCH_LOOKAHEAD) {
			x = start;
			DPRINTF(" LOOKAHEAD option set: reseting position to %i\n",
				x);
		    }
		}
		
		goto next_entry;
	    }
	    
	case MATCH_TABLE:
	case MATCH_SUBTABLE:

	    if (PyInt_Check(match) && 
		PyInt_AS_LONG(match) == MATCH_THISTABLE)
		match = table;

	    if (PyTuple_Check(match)) { 
		PyObject *subtags;
		int y = x;
		int newrc = 0;
		int len_taglist;

		if (taglist != Py_None && cmd != MATCH_SUBTABLE) {
		    /* Create a new list for use as subtaglist */
		    subtags = PyList_New(0);
		    if (subtags == NULL) 
			goto onError;
		    len_taglist = 0;
		}
		else {
		    /* Use taglist as subtaglist */
		    subtags = taglist;
		    Py_INCREF(subtags);
		    len_taglist = PyList_Size(taglist);
		    if (len_taglist < 0)
			goto onError;
		}

		DPRINTF("\n[Sub]Table : using table at 0x%lx\n",(long)match);

		start = x;

		/* match other table */
		newrc = fast_tag(pytext,text,len_text,match,start,subtags,&y);
		if (newrc == 0) {
		    Py_DECREF(subtags);
		    goto onError;
		}

		if (newrc == 1) {
		    /* not matched */
		    DPRINTF(" (no success)\n");
		    /* Undo changes to taglist in case of SUBTABLE match */
		    if (cmd == MATCH_SUBTABLE && taglist != Py_None) {
			DPRINTF("  undoing changes: del taglist[%i:%i]\n",
				len_taglist, PyList_Size(taglist));
			if (PyList_SetSlice(taglist, 
					    len_taglist, 
					    PyList_Size(taglist), 
					    NULL))
			    goto onError;
		    }
		    if (jne == 0) {
			/* match failed */
			rc = 1; 
			Py_DECREF(subtags);
			goto finished;
		    }
		    else 
			je = jne;
		} 
		else { 
		    /* matched */

		    /* move x to new position */
		    x = y;

		    if (tagobj != Py_None) {
			if (match_append(flags,pytext,taglist,tagobj,
					 start,x,subtags) < 0) {
			    /* append failed */
			    Py_DECREF(subtags);
			    goto onError;
			    DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
				    start,x);
			}
		    }
		    else
			DPRINTF(" [%i:%i] (matched but not saved)\n",start,x);

		    if (flags & MATCH_LOOKAHEAD) {
			x = start;
			DPRINTF(" LOOKAHEAD option set: reseting position to %i\n",
				x);
		    }
		}
		Py_DECREF(subtags);
		goto next_entry;
	    }
	    else
		Py_ErrorWithArg(PyExc_TypeError,
				"Tag Table entry %i: "
				"expected a table (cmd=[Sub]Table)",i);

	    goto next_entry;
	    
	case MATCH_TABLEINLIST:
	case MATCH_SUBTABLEINLIST:
	    
	    if (PyTuple_Check(match)) {
		PyObject *subtags;
		int y = x;
		int newrc = 0;
		int len = PyTuple_GET_SIZE(match);
		int len_taglist;

		if (len == 2) {
		    PyObject *w;
		    PyObject *tables;
		    int index;

		    /* get tables (a list of tuples) */
		    tables = PyTuple_GET_ITEM(match,0);
		    Py_AssertWithArg(tables != NULL && PyList_Check(tables),
				     PyExc_TypeError,
				     "Tag Table entry %i: "
				     "first entry in arg tuple "
				     "must be a list",i);

		    /* get index (integer) */
		    w = PyTuple_GET_ITEM(match,1);
		    Py_AssertWithArg(w != NULL && PyInt_Check(w),
				     PyExc_TypeError,
				     "Tag Table entry %i: "
				     "second entry in arg tuple "
				     "must be an integer",i);
		    index = PyInt_AS_LONG(w);

		    /* get matching table and incr ref count for it */
		    match = PyList_GetItem(tables,index);
		    if (match != NULL) {
			Py_INCREF(match);
			if (!PyTuple_Check(match)) {
			    Py_DECREF(match);
			    Py_ErrorWithArg(PyExc_TypeError,
					    "Tag Table entry %i: "
					    "Tag Tables must be a tuples",i);
			}
		    }
		    else /* not found */
			Py_ErrorWithArg(PyExc_TypeError,
					"Tag Table entry %i: "
					"matching table not found "
					"in list of tables",i);
		}
		else /* incorrect length */
		    Py_ErrorWithArg(PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected (list,index) as arg",i);
		
		if (taglist != Py_None && cmd != MATCH_SUBTABLEINLIST) {
		    /* Create a new list for use as subtaglist */
		    subtags = PyList_New(0);
		    if (subtags == NULL) {
			Py_DECREF(match);
			goto onError;
		    }
		    len_taglist = 0;
		}
		else {
		    /* Use taglist as subtaglist */
		    subtags = taglist;
		    Py_INCREF(subtags);
		    len_taglist = PyList_Size(taglist);
		    if (len_taglist < 0)
			goto onError;
		}

		DPRINTF("\n[Sub]TableInList : using table at 0x%lx\n",
			(long)match);
	    
		start = x;

		/* match other table */
		newrc = fast_tag(pytext,text,len_text,match,start,subtags,&y);
		if (newrc == 0) {
		    Py_DECREF(subtags);
		    Py_DECREF(match);
		    goto onError;
		}

		if (newrc == 1) {
		    /* not matched */
		    DPRINTF(" (no success)\n");
		    /* Undo changes to taglist in case of SUBTABLE match */
		    if (cmd == MATCH_SUBTABLE && taglist != Py_None) {
			DPRINTF("  undoing changes: del taglist[%i:%i]\n",
				len_taglist, PyList_Size(taglist));
			if (PyList_SetSlice(taglist, 
					    len_taglist, 
					    PyList_Size(taglist), 
					    NULL))
			    goto onError;
		    }
		    if (jne == 0) {
			/* match failed */
			rc = 1;
			Py_DECREF(subtags);
			Py_DECREF(match);
			goto finished;
		    }
		    else 
			je = jne;
		} 
		else { 
		    /* matched */

		    /* move x to new position */
		    x = y;

		    if (tagobj != Py_None) {
			if (match_append(flags,pytext,taglist,tagobj,
					 start,x,subtags) < 0) {
			    /* append failed */
			    Py_DECREF(subtags);
			    Py_DECREF(match);
			    goto onError;
			}
			DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
				start,x);
		    }
		    else {
			DPRINTF(" [%i:%i] (matched but not saved)\n",start,x);
		    }

		    if (flags & MATCH_LOOKAHEAD) {
			x = start;
			DPRINTF(" LOOKAHEAD option set: reseting position to %i\n",
				x);
		    }
		}
		Py_DECREF(subtags);
		Py_DECREF(match);
		goto next_entry;
	    }
	    else
		Py_ErrorWithArg(PyExc_TypeError,
				"Tag Table entry %i: "
				"expected a tuple (tables,index) "
				"(cmd=TableInList)",i);

	    goto next_entry;
	    
	case MATCH_LOOP:

	    DPRINTF("\nLoop: pre loop counter = %i\n",loopcount);
	    
	    if (loopcount > 0)
		/* we are inside a loop */
		loopcount--;

	    else if (loopcount < 0) {
		/* starting a new loop */
		if (PyInt_Check(match)) {
		    loopcount = PyInt_AS_LONG(match);
		    loopstart = x;
		}
		else
		    Py_ErrorWithArg(PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected an integer (cmd=Loop)",i);
	    }

	    if (loopcount == 0) {
		/* finished loop */
		loopcount = -1;
		if (loopstart == x) {
		    /* not matched */

		    DPRINTF(" (no success)\n");
	    
		}
		else if (tagobj != Py_None) {
		    /* matched */
		    if (match_append(flags,pytext,taglist,tagobj,
				     loopstart,x,NULL) < 0)
			goto onError;
		    DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
			    loopstart,x);
		}
		else {
		    DPRINTF(" [%i:%i] (matched but not saved)\n",start,x);
		}

		if (flags & MATCH_LOOKAHEAD) {
		    x = start;
		    DPRINTF(" LOOKAHEAD option set: reseting position to %i\n",
			    x);
		}

		/* skip loop body */
		je = jne; 
	    }

	    DPRINTF("\nloop: post loop counter = %i\n",loopcount);
	    
	    goto next_entry;

	case MATCH_LOOPCONTROL:

	    if (PyInt_Check(match)) {

		DPRINTF("\nLoopControl: loop counter = %i, "
			"setting it to = %li\n",
			loopcount,PyInt_AS_LONG(match));

		loopcount = PyInt_AS_LONG(match);
		goto next_entry;
	    }
	    else
		Py_ErrorWithArg(PyExc_TypeError,
				"Tag Table entry %i: "
				"expected an integer (cmd=LoopControl)",i);

	    goto next_entry;

	case MATCH_CALL:
	case MATCH_CALLARG:
	    {
		PyObject *fct;
		int argc;
	    
		if (!PyTuple_Check(match)) {
		    argc = 0;
		    fct = match;
		}
		else {
		    argc = PyTuple_GET_SIZE(match) - 1;
		    Py_AssertWithArg(argc >= 0,
				     PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected a tuple (fct,arg0,arg1,...)"
				    "(cmd=CallArg)",i);
		    fct = PyTuple_GET_ITEM(match,0);
		}
	    
		if (PyCallable_Check(fct)) {
		    PyObject *args;
		    register PyObject *w;
		    register int i;

		    DPRINTF("\nCall[Arg] :\n");
	    
		    start = x;

		    /* Build args = (pytext,start,len_text[,arg0,arg1,...]) */
		    args = PyTuple_New(3 + argc);
		    if (!args)
			goto onError;
		    Py_INCREF(pytext);
		    PyTuple_SET_ITEM(args,0,pytext);
		    w = PyInt_FromLong(start);
		    if (!w)
			goto onError;
		    PyTuple_SET_ITEM(args,1,w);
		    w = PyInt_FromLong(len_text);
		    if (!w)
			goto onError;
		    PyTuple_SET_ITEM(args,2,w);
		    for (i = 0; i < argc; i++) {
			w = PyTuple_GET_ITEM(match,i + 1);
			Py_INCREF(w);
			PyTuple_SET_ITEM(args,3 + i,w);
		    }

		    w = PyEval_CallObject(fct,args);
		    Py_DECREF(args);
		    if (w == NULL) 
			goto onError;

		    Py_AssertWithArg(PyInt_Check(w),
				     PyExc_TypeError,
				     "Tag Table entry %i: "
				     "matching fct has to return an integer",i);
		    x = PyInt_AS_LONG(w);
		    Py_DECREF(w);

		    if (start == x) { 
			/* not matched */
			DPRINTF(" (no success)\n");
			if (jne == 0) { 
			    /* match failed */
			    rc = 1; 
			    goto finished; 
			}
			else 
			    je = jne;
		    } 
		    else if (tagobj != Py_None) { 
			/* matched */
			if (match_append(flags,pytext,taglist,tagobj,
					 start,x,NULL) < 0)
			    goto onError;
			DPRINTF(" [%i:%i] (matched and remembered this slice)\n",
				start,x);
		    }
		    else {
			DPRINTF(" [%i:%i] (matched but not saved)\n",
				start,x);
		    }

		    if (flags & MATCH_LOOKAHEAD) {
			x = start;
			DPRINTF(" LOOKAHEAD option set: reseting position to %i\n",
				x);
		    }

		    goto next_entry;
		}
		else
		    Py_ErrorWithArg(PyExc_TypeError,
				    "Tag Table entry %i: "
				    "expected a callable object "
				    "(cmd=Call[Arg])",i);
	    }
	    goto next_entry;

	default:

	    Py_ErrorWithArg(PyExc_TypeError,
			    "Tag Table entry %i: unknown command",i);

	} /* switch */

    } /* for-loop */

 finished:

    /* In case no specific return code was set, check if we have
       matched successfully (pointer beyond the end of the table) or
       failed to match (pointer negative) */
    if (rc < 0) {
	if (i >= len_table)
	    rc = 2;
	else if (i < 0)
	    rc = 1;
	else
	    Py_ErrorWith2Args(PyExc_SystemError,
			      "Internal Error: "
			      "tagging engine finished with no proper result"
			      "at position %i in table 0x%0x",
			      i,(int)table);
    }

    DPRINTF("\nTag Engine finished: %s; Tag Table entry %i; position %i\n",
	    rc==1?"failed":"success",i,x);

    /* Record the current head position */
    *next = x;

    return rc;

 onError:
    rc = 0;
    return rc;
}

/*
  What to do with a successful match depends on the value of flags:

  flags mod x == 1:
  -----------------
  MATCH_CALLTAG: 
  	call the tagobj with (taglist,pytext,l,r,subtags) and
	let it decide what to do
  MATCH_APPENDTAG:
  	do a tagobj.append((None,l,r,subtags))
  default:
  	do a taglist.append((tagobj,l,r,subtags)) 
 
 - subtags is made to reference Py_None, if subtags == NULL
 - returns -1 if not successful, 0 on success
 - refcounts: all reference counts are incremented upon success only

*/
static
int match_append(int flags,
		 PyObject *pytext,
		 PyObject *taglist,
		 PyObject *tagobj,
		 int l,
		 int r,
		 PyObject *subtags)
{
    register PyObject *w;

    if (subtags == NULL)
	subtags = Py_None;

    /* Default mechanism: */

    if (flags == 0) {
	/* append result to taglist */

	if (taglist == Py_None) 
	    return 0; /* nothing to be done */

	/* Build w = (tagobj,l,r,subtags) */
	w = PyTuple_New(4);
	if (!w)
	    goto onError;

	Py_INCREF(tagobj);
	PyTuple_SET_ITEM(w,0,tagobj);
	PyTuple_SET_ITEM(w,1,PyInt_FromLong(l));
	PyTuple_SET_ITEM(w,2,PyInt_FromLong(r));
	Py_INCREF(subtags);
	PyTuple_SET_ITEM(w,3,subtags);

	if (PyList_Append(taglist,w))
	    goto onError;
	Py_DECREF(w);
	return 0;
    }

    /* Flags are set: */
    
    if (flags & MATCH_APPENDTAGOBJ) {
	/* append the tagobj to the taglist */
	if (taglist == Py_None) 
	    return 0; /* nothing to be done */
	return PyList_Append(taglist,tagobj);
    }

    if (flags & MATCH_APPENDMATCH) {
	/* append the match obj to the taglist */
	register PyObject *v;
	
	if (taglist == Py_None) 
	    return 0; /* nothing to be done */
	v = PyString_FromStringAndSize(PyString_AS_STRING(pytext) + l, r-l);
	if (!v)
	    goto onError;
	if (PyList_Append(taglist,v))
	    goto onError;
	Py_DECREF(v);
	return 0;
    }

    if (flags & MATCH_CALLTAG) { 
	/* call tagobj */
	register PyObject *args;
	
	/* Build args = (taglist,pytext,l,r,subtags) */
	args = PyTuple_New(5);
	if (!args)
	    goto onError;

	Py_INCREF(taglist);
	PyTuple_SET_ITEM(args,0,taglist);
	Py_INCREF(pytext);
	PyTuple_SET_ITEM(args,1,pytext);
	PyTuple_SET_ITEM(args,2,PyInt_FromLong(l));
	PyTuple_SET_ITEM(args,3,PyInt_FromLong(r));
	Py_INCREF(subtags);
	PyTuple_SET_ITEM(args,4,subtags);

	w = PyEval_CallObject(tagobj,args);
	Py_DECREF(args);
	if (w == NULL)
	    goto onError;
	Py_DECREF(w);
	return 0;
    }

    if (flags & MATCH_APPENDTAG) { 
	/* append to tagobj */
	Py_Assert(PyList_Check(tagobj),
		  PyExc_TypeError,
		  "Tag Table: used AppendToTag, but tagobj is not a list");

	/* Build w = (None,l,r,subtags) */
	w = PyTuple_New(4);
	if (!w)
	    goto onError;

	Py_INCREF(Py_None);
	PyTuple_SET_ITEM(w,0,Py_None);
	PyTuple_SET_ITEM(w,1,PyInt_FromLong(l));
	PyTuple_SET_ITEM(w,2,PyInt_FromLong(r));
	Py_INCREF(subtags);
	PyTuple_SET_ITEM(w,3,subtags);

	if (PyList_Append(tagobj,w))
	    goto onError;
	Py_DECREF(w);
	return 0;
    }
    Py_Error(PyExc_TypeError,
	     "Tag Table: unknown flag in command");
 onError:
    return -1;
}