# $Header: /home/orodruin/h/bair/phelps/spine/tkman/RCS/texi.tcl 1.55 1997/12/31 00:53:28 phelps Exp phelps $

# protocol: show directory, show file.  works for GNU info, Tcl proc browser, Java class browser

#
# GNU info *source* reader
# 19 January 1997
#
# 1997
# 19 Jan	basically works
# 20		nested table, itemize, enumerate
# 21		index builder
# 28		dynamic index building
# 31		support indexes and crossreferences
#			back to the disk database, sigh, but at least orders of magnitude smaller than Texinfo's online versions

# to do:
# random tags (r that overrides)
# if no match on hyperlink try all lowercase (recover from author's sloppiness)
# if everything collapsed (as under Title Page), then expand it
# update for Texinfo 3.11 (image, deftypemethod, ifnothtml, kbdinputstyle, ...)


# find <l>...@end <l> in text widget, 
# delete that text, return range of bracked text
proc texiRegionFind {t l start end} {
#puts "texiRegionFind $start $end"
	set ws "\[ \t]"
	set open [$t search -regexp -count openlen -- "^@$l$ws*" $start $end]
	if {$open==""} return
	set close [$t search -regexp -- "^@end $l$ws*\$" "$open+1l" $end]
	if {$close==""} return

	# delete labels
	set tag [string trim [$t get $open+1c "$open+${openlen}c"]]; # unknown amount of whitespace
	set arg [$t get $open+${openlen}c "$open lineend"]
	$t delete $close "$close+1l"
	$t delete $open "$open+1l"

	# return range of bracked text
	return [list $tag $arg [$t index $open] [$t index $close-2l+1l]]
}

proc texiRegionFindAll {t l start end} {
	set all {}
	while {[set tuple [texiRegionFind $t $l $start $end]]!=""} {
		lappend all [lindex $tuple 0] [lindex $tuple 1] [lindex $tuple 2] [set start [lindex $tuple 3]]
	}
	return $all
}

proc texiRevRegionFindAll {t l start end} {
	return [lreverse [texiRegionFindAll $t $l $start $end] 4]
}



# speed things up by inlining texiLineFind into texiLineFindAll
# no (other) client for texiLineFind
proc texiLineFindAll {t l start end} {
	set ws "\[ \t]"

	set alllines {}
	while {[set start [$t search -regexp -count startlen -- "^@$l$ws*( |\$)" $start $end]]!=""} {
		set tag [lindex [$t get $start+1c $start+${startlen}c-1c] 0]
		$t delete $start $start+${startlen}c
		lappend alllines $tag [string trim [$t get $start "$start lineend"]] $start
	}
	return $alllines
}

proc texiRevLineFindAll {t l start end} {
	return [lreverse [texiLineFindAll $t $l $start $end] 3]
}


# no separate texiTagFind because of tricky case of nested tags
# that's ok since no client for one anyway
proc texiTagFindAll {t l start end} {
#puts "texiTagFind $start $end"
	set all {}

	while {[set open [$t search -regexp -count openlen -- "@($l)\{" $start $end]]!=""} {
		set openend "$open+${openlen}c"
		set ocnt 1; set close $openend-1c
		# handle nested braces
		set closeadj ""
		while {$ocnt} {
			set newclose [$t search -regexp -count insidelen -- {@[a-z]+\{|\}} $close+1c $end]
			if {$newclose==""} break else {set close $newclose}
			if {[$t get $close]=="\}"} {incr ocnt -1} {incr ocnt; append closeadj "-${insidelen}c-1c"}
		}
		if {$ocnt} break
		set closeret [$t index $close]; if {[$t index [list $open linestart]]==[$t index [list $close linestart]]} {append closeret "-${openlen}c"}
		lappend all [$t get $open+1c $openend-1c] $open [$t index $closeret$closeadj]

		# delete labels
		$t delete $close
		$t delete $open $openend

		set start $open
	}

	return $all
}

proc texiRevTagFindAll {t l start end} {
	return [lreverse [texiTagFindAll $t $l $start $end] 3]
}



# c=concept, f=function, v=variable, k=keystroke, t=data type
# index category <whatever>+
array set texidef {
	deffn {findex category name arguments}
	defun {findex "Function" name arguments}
	defmac {findex "Macro" name arguments}
	defspec {findex "Special Form" name arguments}
	defvr {vindex category name}
	defvar {vindex "Variable" name}
	defopt {vindex "Option" name}
	deftypefn {tindex category datatype name arguments}
	deftypefun {tindex "Function" datatype name arguments}
	deftypevr {vindex category datatype name}
	deftypevar {vindex "Variable" datatype name}
	deftypecv {vindex category class name}
	deftypeivar {vindex "Instance Variable" class name}
	defop {findex category class name arguments}
	defmethod {findex "Method" class name arguments}
	defp {tindex category name attributes}
}


array set texitrans {
	"dots" "..."  "enddots" "...."  "bullet" "\xb7"  "copyright" "\xa9"  "TeX" "TEX"
	"result" "=>"  "equiv" "=="  "error" "error-->"  "expansion" "==>"  "point" "-!-"
	"print" "-|"   "result" "=>"   "minus" "-"  "sp" "\n"  "exclamdown" "\xa1"  "questiondown" "\xbf"
	"AA" "\xc5"  "aa" "\xe5"  "AE" "\xc6"  "ae" "\xe6"  "O" "\xd8"  "o" "\xf8"  "OE" "OE"  "oe" "oe"
	"pounds" "\xa3"
}
set texitrans(today) [clock format [clock seconds] -format "%d %B, %Y"]
set texitrans(regexp) "today"
foreach tag [array names texitrans] {append texitrans(regexp) "|" $tag}

array set texiaccent {
	`A "\xc0" 'A "\xc1" ^A "\xc2" ~A "\xc3" \"A "\xc4"
	,-C "\xc7"
	`E "\xc8" 'E "\xc9" ^E "\xca" \"E "\xcb"
	`I "\xcc" 'E "\xcd" ^I "\xce" \"I "\xcf"
	barD "\xd0"
	~N "\xd1"
	`O "\xd2" 'O "\xd3" ^O "\xd4" ~O "\xd5" \"O "\xd6"
	`U "\xd9" 'U "\xda" ^U "\xdb" \"U "\xdc"
	'Y "\xdd"
	`a "\xe0" 'a "\xe1" ^a "\xe2" ~a "\xe3" a-\" "\xe4"
	,c "\xe7"
	`e "\xe8" 'e "\xe9" ^e "\xea" e-\" "\xeb"
	`i "\xcc" 'i "\xcd" ^i "\xee" i-\" "\xef"
	bard "\xf0"
	~n "\xf1"
	`o "\xf2" 'o "\xf3" ^o "\xf4" ~o "\xf5" \"o "\xf6"
	`i "\xf9" 'i "\xfa" ^e "\xfb" \"e "\xfc"
	'y "\xfd" y-\" "\xfd"
}

proc texiMarkup {t start end {force 0}} {
#puts "texiMarkup $start .. $end"
	global texi texitrans texidef texiaccent

	cursorBusy

	foreach per $texi(persistent) {upvar #0 ${per}$t $per}

	### KILL IRRELEVANT TEXT
	# REGIONS: @ignore, @iftex/@tex, @menu (following printed manual)
	# delete that region
	foreach {tag arg s e} [texiRevRegionFindAll $t "(iftex|tex|ifhtml|ignore|menu)" $start $end] {$t delete $s $e}
	# delete markup only -- done as lines TOO in case not well nested
#	foreach {tag arg s e} [texiRevRegionFindAll $t "(ifinfo|group|titlepage|multitable)" $start $end] {$t insert $e "\n"}
	texiRevRegionFindAll $t "(ifinfo|group|titlepage|multitable|ifnothtml)" $start $end



	### SPECIAL HANDLING
	### LINES
	# add tag only
	foreach {tag ctnt s} [texiLineFindAll $t "(majorheading|heading|subheading|subsubheading)" $start $end] {$t tag add $tag $s $s+1l}
	# add tag + no line wrapping
	foreach {tag ctnt s} [texiLineFindAll $t "(subtitle|author|center)" $start $end] {$t insert $s " "; $t tag add $tag $s $s+1l}

	# special processing
	foreach {tag ctnt s} [texiRevLineFindAll $t "printindex" $start $end] {
		$t delete $s "$s lineend+1c"

		# name gives type to print: cp, fn, vr, ky, pg, tp
		$t mark set insert $s
		set last ""
		foreach tuple [lsort -dictionary -index 0 $texiindex($ctnt)] {
			foreach {entry title font} $tuple break
			if {$entry==$last} continue
			$t insert insert " @$font{$entry}\t"
			set mark [$t index insert]
			$t insert insert "$title\n"
			$t tag add texixref $mark insert-1c; # inherit invisibility tag
			set last $entry
		}
		$t tag add index $s insert
	}


	foreach {tag s e} [texiRevTagFindAll $t "value" $start $end] {
		set var [string trim [$t get $s $e]]
		$t delete $s $e
		if [catch {$t insert $s $texiinfovar($var)}] {
			$t insert $s "*No value for $var*"
		}
	}



	### KILL IRRELEVANT TEXT II
	# LINES: kill @comment, @node (use chapter/section/...), @[fc]index (just use full text search)
	foreach {tag ctnt s} [texiRevLineFindAll $t "(c|comment|\[a-z]+index|node|unnumbered|finalout|lowersections|need|vskip|sp|include|noindent|title|set|chapter|section|subsection|subsubsection|page|contents|shortcontents|summarycontents|top|footnotestyle|setchapternewpage|paragraphindent|nwnode|leftmargin|bye)" $start $end] {$t delete $s "$s lineend+1c"}
#	set x [$t search -exact {\input texinfo} 1.0 $end]
#	if {$x!=""} {$t delete $x "$x lineend+1c"}
	update idletasks


	### REGIONS: example, tables, lists
	# add tag only
	foreach {tag arg s e} [texiRegionFindAll $t "(quotation|display)" $start $end] {$t tag add $tag $s $e}
	# special markup with arg
	foreach {tag arg s e} [texiRevRegionFindAll $t "(example|smallexample|lisp|smalllisp|format|flushleft|flushright)" $start $end] {
		if {[string match "flush*" $tag]} {set ichar " "} elseif {$tag=="format"} {set ichar " "; set tag "r"} else {set ichar "\t"}
		$t tag add $tag $s $e
		for {scan $e "%d" i; incr i -1} {$i>=$s} {incr i -1} {
			$t insert $i.0 $ichar; # can't add $tag here too because want to pick up ALL existing tags
			$t tag add $tag $i.0
		}
		if {[$t get $e]!="\n"} {$t insert $e "\n"}
	}
	update idletasks

	# name, args left margin; category right margin; entered into index
	foreach {tag arg s e} [texiRevRegionFindAll $t "def\[^ \n\t]+" $start $end] {
		if {[string match "def*index" $tag] || $tag=="definfoenclose"} continue

		while 1 {
			set spec $texidef($tag)
			set inx 0
			set index [lfirst $spec]
			set category [lsecond $spec]; if {$category=="category"} {set category [lfirst $arg]; incr inx}
			$t insert $s " [lrange $arg $inx end]\n"
			$t tag add rmtab $s "$s lineend"
			set namelen [string length [lindex $arg $inx]]; $t tag add code $s+1c "$s+1c+${namelen}c"; $t tag add i "$s+1c+${namelen}c+1c" "$s+1c lineend"
			$t insert "$s lineend" "\t$category"

			# process def.*x
			append s "+1l"
			if {[$t get $s]=="@" && [string match "def*" [set tag [$t get $s+1c "$s+1c wordend-1c"]]]} {
				set arg [string trim [$t get "$s+1c wordend+1c" "$s lineend"]]; $t delete $s $s+1l
			} else {$t insert $s "\t"; break}
		}
	}
	update idletasks

	# special handling
	foreach {tag arg s e} [texiRevRegionFindAll $t "(ifset|ifclear)" $start $end] {
		if {$tag=="ifset" ^ [info exists texiinfovar([lfirst $arg])]} {$t delete $s $e}
	}
	# clean unpaired tags in this region
	foreach {tag ctnt s} [texiRevLineFindAll $t "(ifset|end ifset|ifclear|end ifclear|ifinfo|end ifinfo|iftex|end iftex|tex|end tex|ifhtml|end ifhtml|group|end group|titlepage|end titlepage)" $start $end] {$t delete $s "$s lineend+1c"}
	update idletasks


	# tables last before character ranges, as its tabs disable further region and line identification
	while {[set close [$t search -regexp -count closelen -- "^@end (.?table|itemize|enumerate)" $start $end]]!=""} {
		# handle nested tables by always marking up one with first *end*
		set type [$t get $close+5c $close+${closelen}c]
		set open [$t search -backwards -regexp "^@$type" $close $start]
#puts "$type |$open| .. |$close|"

		# tables well nested so this works fine
		foreach {tag type s e} [texiRegionFind $t "(.?table|itemize|enumerate)" $open $close+1l] break

		if {$tag=="itemize"} {
			switch -glob -- $type {
				@bullet* {set mark " \xb7\t"}
				@minus* {set mark " -\t"}
				default {set mark " $type"}
			}
		} elseif {$tag=="enumerate"} {
			set icnt 1
			set format " %d.\t"
			if [regexp {[0-9]+} $type num] {set icnt $num} \
			elseif [regexp {[A-Za-z]} $type letter] {scan $letter "%c" icnt; set format "%c.\t"}
		}

		set needindent 0
		for {scan $s "%d" i; scan $e "%d" ie} {$i<$ie} {incr i} {
			# replace @item, @itemx by @table's argument
			if {[$t get $i.0 $i.5]=="@item"} {
				set ex [expr {[$t get $i.5]=="x"}]
				$t delete $i.0 $i.6+${ex}c
				if {$tag=="itemize"} {
					$t insert $i.0 $mark
					$t delete "$i.0 lineend"; incr ie -1; #incr i -1; # preserves line end ("\n")
				} elseif {$tag=="enumerate"} {
					$t insert $i.0 [format $format $icnt]
					$t delete "$i.0 lineend"; incr ie -1; #incr i -1
					incr icnt
				} else {
					$t insert $i.0 "$type\{"
					$t insert "$i.0 lineend" "\}"
				}
				if {$ex} {$t insert $i.0-1c ", "; $t delete $i.0-1c; incr i -1; incr ie -1} else {$t insert $i.0 " "}
				set needindent 1
			} elseif {[set ch [$t get $i.0]]=="\n"} {
				set needindent 1
			} elseif {$needindent} {
				$t insert $i.0 "\t"
				set needindent 0
			}
		}

		if {[$t get $ie.0]!="\n"} {$t insert $ie.0 "\n"}
		$t tag add table $s $ie.0
	}
	update idletasks


	### TAGS
	### styles and replacement text
	# text styles
	foreach {tag s e} [texiTagFindAll $t "asis|sc|r|i|b|t|w|code|kbd|key|file|url|var|dfn|emph|strong|titlefont|subtitle|author|cite" $start $end] {$t tag add $tag $s $e}

	foreach {tag s e} [texiRevTagFindAll $t "footnote" $start $end] {
		set footnote [$t get $s $e]
		$t delete $s $e
		$t insert $s "*"
		$t tag add superscript $s

		$t mark set insert [set m [$t index $end-1l]]
		$t insert insert " *"; $t tag add superscript $m+1c
		$t insert insert "$footnote\n"
		$t tag add sc $m insert
	}
	update idletasks


	foreach {tag s e} [texiRevTagFindAll $t "(xref|ref|pxref|inforef)" $start $end] {
		foreach {nodename xrefname topic infofile printedmanual} [split [$t get $s $e] ","] break
		foreach var {nodename xrefname topic infofile printedmanual} {set $var [string trim [set $var]]}
#		set nodename [string trim [lfirst [split [$t get $s $e] ","]]]
		$t delete $s $e
		$t mark set insert $s
		if {$tag=="inforef"} {set tag "xref"}
		set txt ""; if {$tag=="xref"} {set txt "See "} elseif {$tag=="pxref"} {set txt "see "}
		$t insert insert $txt
		set mark [$t index insert]
		$t insert insert $nodename
		$t tag add texixref $mark insert; # want to inherit invisibility tag
#		if {$xrefname!=""} {$t insert insert $xrefname}
	}
	update idletasks


	# accents with immediate letter (no {})
	for {set s $start-1c} {[set s [$t search -regexp "@(\"|'|~|\\\^|`)" $s+1c $end]]!=""} {} {
		set txt [$t get $s+1c $s+3c]; $t delete $s $s+3c
		catch {set txt $texiaccent($txt)}
		$t insert $s $txt
	}

	# replacement text
	foreach {tag s e} [texiRevTagFindAll $t "$texitrans(regexp)|sc|uref|email|samp|dmn|math|,|v|H|u" $start $end] {
		set txt [$t get $s $e]; $t delete $s $e
		switch -exact $tag {
			# tags with no effect
			dmn -
			math {}

			# accented characters with alphabetic tags (these have {} around argument)
			v -
			, -
			H -
			ubaraccent -
			underdot -
			u {
				if [catch {set txt $texiaccent($tag$txt)}] {set txt "($tag)$txt"}
				$t insert $s $txt
			}

			sc {$t insert $s [string toupper $txt] sc}
			samp {$t insert $s "`$txt'"; $t tag add tt $s+1c $e+1c}
			email -
			uref {
				foreach {uref ureftxt} [split $txt ","] break
				$t insert $s "<$uref>" tt
				if {$ureftxt!=""} {$t insert $s "$ureftxt "}
			}
			TeX {$t insert $s "TEX"; $t tag add subscript $s+1c}

			default {$t insert $s $texitrans($tag)}
		}
	}

	foreach {tag s e} [texiRevTagFindAll $t $texienclose(enregexp) $start $end] {
		$t insert $e [lsecond $texienclose($tag)]
		$t insert $s [lfirst $texienclose($tag)]
	}



	### CLEAN UP
	# zap refill, which is not a region, line, or tag, so have to have separate
	set x $start
	while {[set x [$t search -regexp "@refill" $x $end]]!=""} {$t delete $x $x+7c}

	# finally unquote metachars and zap single newlines to let Tk word wrap
	for {set s $start-1c} {[set s [$t search -regexp "@(\{|@|\}|:|\\.|\\?|!|\\*| |\t|\n)" $s+1c $end]]!=""} {} {
		set ch [$t get $s+1c]
		if {$ch=="*"} {
			$t delete $s $s+2c
			if {[$t compare $s != [list $s lineend]]} {$t insert $s "\n"}
			$t insert $s+1c " "
		} else {$t delete $s}
	}
	for {set s $start-1c} {[set s [$t search -regexp "\\.:" $s+1c $end]]!=""} {} {$t delete $s+1c}

	if $force return

	scan [$t index $start+2l] "%d" stop
	# leave exactly one newline at end
	for {scan [$t index $end-2l] "%d" i} {$i>$stop && [$t get $i.0]=="\n"} {incr i -1} {$t delete $i.0}
	for {scan [$t index $end-1l] "%d" i} {$i>$stop} {incr i -1} {
		if {[string first [$t get $i.0] " \t\n"]==-1 && [string first [$t get $i.0-1l] "\n"]==-1} {$t insert $i.0-1c " "; $t delete $i.0-1c}
	}
	for {scan [$t index $start] "%d" i} {[$t get $i.0]=="\n"} {incr i} {}
	$t delete $start+1l $i.0

	cursorUnset
}


#--------------------------------------------------


# process file:
#	collect @include, @(chapter|{,sub,subsub}section), with byte offsets
#	process in order: on include, recurse; otherwise write out

set texilevelx(names) {chapter section subsection subsubsection}
array set texilevelx {
	0 chapter 1 section 2 subsection 3 subsubsection
	chapter 0 unnumbered 0 appendix 0 chapheading 0
	section 1 unnumberedsec 1 appendixsec 1
	subsection 2 unnumberedsubsec 2 appendixsubsec 2
	subsubsection 3 unnumberedsubsubsec 3 appendixsubsubsec 3
}

proc texiStruct {t type sectdelta name file byte} {
	global texi texilevel texilevelx

	set x [$t index end-1l]
	set lev [expr $texilevelx($type)+$sectdelta]
	if {$lev<0} {set lev 0}; if {$lev>3} {set lev 3}
	incr texilevel($lev)
	set n $texilevel(0)
	for {set i 1} {$i<=$lev} {incr i} {append n ".$texilevel($i)"}
	for {set i [expr $lev+1]} {$i<4} {incr i} {set texilevel($i) 0}
	$t insert end $name $texilevelx($lev) "\n"
#	$t insert end "$n  $name" $texilevelx($lev) "\n"

	set m "js$n"
	$t mark set $m $x

	return $m
}



proc texiProcess {t file {curnode "Front Matter"}} {
	global texi texised texiimap texiifont texidef man manx

	foreach per $texi(persistent) {upvar #0 ${per}$t $per}
# texiprof
	set sumbytes 0
	set sectdelta 0

	# at this point just ignore errors (usually for index.texi, which we generate ourselves)
#	if {[string match "*.tar.gz/*" $file]} {
#		# less error checking done with tar's
#		set realf $file
#	} else {
		set realf [lfirst [glob -nocomplain $file$manx(zoptglob)]]
		if {$realf=="" || ![file readable $realf]} {return $sumbytes}
#	}
	set fid [open "|[manManPipe $realf]"]; set filecontents [read $fid]; close $fid

	set on 1; set endsed ""
	set lastbyte -1; set printindex 0; set curmark ""
	set byte 0
	foreach line [split $filecontents "\n"] {
		set linelen [string length $line]; incr linelen
		set byte0 $byte
		incr byte $linelen

		if {[regexp "^@(set |clear |ifset |end ifset|ifclear |end ifclear|ignore|end ignore|iftex|end iftex|tex|end tex|ifhtml|end ifhtml|include |node |chapter |unnumbered |appendix |chapheading |section |unnumberedsec |appendixsec |subsection |unnumberedsubsec |appendixsubsec |subsubsection |unnumberedsubsubsec |appendixsubsubsec |titlepage|\[a-z]+index( |\$)|def|bye|raisesections|lowersections)" $line]} {

		# sed
		# one regexp that feeds the brood
		regexp {^@([^ 	]+)[ 	]*(([^ 	]+)[ 	]*([^ 	]+([ 	]+[^ 	]+)*)?)?} $line all type name arg1 arg2
		set typename "$type $arg1"
		if {$on} {
			if {[info exists texised($typename)]} {if {$texised($typename)!=""} {set on 0; set endsed $texised($typename); continue}
			} elseif {[info exists texised($type)]} {if {$texised($type)!=""} {set on 0; set endsed $texised($type); continue}
			}
		} else {
			if {$type==$endsed || $typename==$endsed} {set on 1}
			continue
		}


		# collect set/clear, add to regions to ignore according to @ifset/@ifclear
		# all matches here jump back to start of loop
		if {$type=="set"} {
			# set global so to communicate to possible include's
			set texiinfovar($arg1) $arg2; continue
		} elseif {$type=="clear"} {
			catch {unset texiinfovar($arg1)}; continue
		} elseif {$type=="ifset"} {
			if {![info exists texiinfovar($arg1)]} {set on 0; set endsed "end ifset"}
			continue
		} elseif {$type=="ifclear"} {
			if {[info exists texiinfovar($arg1)]} {set on 0; set endsed "end ifclear"}
			continue
		} elseif {$typename=="end ifset" || $typename=="end ifclear"} continue


		# meat
		# could use big switch, but that's slow right now, I think, or at least no faster
		if {$type=="node"} {
			# nodename => chapter title => Tk mark.  really slows things down, but unavoidable
			set nodename [string trim [lfirst [split $name ","]]]
#			regexp {[^,]+} $name nodename
			set texinode([set curnode $nodename]) ""; # mark set with following chapter/section/...
		} elseif {$type=="include"} {
			incr sumbytes [texiProcess $t $arg1 $curnode]
#		} elseif {[string match "*index" $type]} { -- slower
		} elseif {$type=="printindex"} {
#			# fault in sections with indexes right away so index is available to search
#			set printindex 1
		} elseif {$type=="synindex" || $type=="syncodeindex"} {
			# "Write an `@syncodeindex' command ... at the beginning of a Texinfo file"
			# merged-from always keeps its font
			set texiimap($arg1) $arg2
		} elseif {$type=="defindex"} {
			lappend texiimap(${arg1}index) $arg1
			lappend texiifont(${arg1}index) "asis"
		} elseif {$type=="defcodeindex"} {
			lappend texiimap(${arg1}index) $name
			lappend texiifont(${arg1}index) "code"
		} elseif {[string match "*index" $type]} {
			lappend texiindex($texiimap($type)) [list $name $curnode $texiifont($type)]
		} elseif {$type=="raisesections"} {
			incr sectdelta -1
		} elseif {$type=="lowersections"} {
			incr sectdelta
		} elseif {$type=="definfoenclose"} {
			# have to propagate these out to cache
			foreach {cmd before after} [split $name ","] break
			set texienclose([string trim $cmd]) [list [string trim $before] [string trim $after]]
		} elseif {[string match "def*" $type]} {
			if {[string match "*x" $type]} {set type [string range $type 0 [expr [string length $type]-1-1]]}
			set spec $texidef($type)
			set index [lfirst $spec]
			set inx [expr [lsearch $spec "name"]-2+1]; if {[lsecond $spec]=="category"} {incr inx}
			set name [lindex $all $inx]
			lappend texiindex($texiimap($index)) [list $name $curnode $texiifont($index)]
		} else {
			set skipbyte [expr $byte0+$linelen]
			if {$type=="titlepage"} {set type "chapter"; set name "Title Page"}

			if {$lastbyte!=-1} {lappend texix($curmark) [expr $byte0-$lastbyte]}
			if {$type=="bye"} break
			set curmark [texiStruct $t $type $sectdelta $name $file $skipbyte]
			set texix($curmark) [list $file $byte]; set texinode($curnode) $curmark
			set lastbyte $skipbyte
		}
		}
	}
	if {$lastbyte!=-1 && $type!="bye"} {lappend texix($curmark) [expr $byte0-$lastbyte]}

	$t see end
	update idletasks

	incr sumbytes $byte
	return $sumbytes
}


# process callback string: fault in text, mark it up
proc texiCallback {t sect next} {
	global texi manx
	upvar #0 texix$t texix

	if {$texix($sect)==""} return

	foreach {f s len} $texix($sect) break
	set point "$sect linestart+1l"
	if {$texi(lastfile$t)!=$f} {
		cd $texi(cd$t)
		set realf [lfirst [glob $f$manx(zoptglob)]]
		set fid [open "|[manManPipe $realf]"]; set texi(lastfilecontents$t) [read $fid]; close $fid
		set texi(lastfile$t) $f
	}

	$t insert $point "\n[string range $texi(lastfilecontents$t) $s [expr $s+$len-1]]\n" {}
	texiMarkup $t $point $next

	set taglist area$sect
	for {set sup $sect} {[regexp $manx(supregexp) $sup all supnum]} {set sup "js$supnum"} {
		lappend taglist areajs$supnum
	}
	foreach tag $taglist {$t tag add $tag $point-1c "$next linestart"}
	set texix($sect) ""; incr texi(sectcnt) -1
}


set texi(persistent) {texiindex texinode texiinfovar texix texienclose texiback}

# show one file, at least toplevel
proc texiShow {t f dir {force 0}} {
	global texi texised texiimap texiback texiifont texilevel texilevelx man manx
	foreach per $texi(persistent) {upvar #0 ${per}$t $per}

	manNewMode [winfo parent $t] texi; uplevel incr stat(texi)
	cd [set texi(cd$t) [file dirname $f]]


	# caller should have done this checking
	set realf [lfirst [glob -nocomplain $f$manx(zoptglob)]]
	#if {![regexp {(.*\.tar[^/]*)} $file all realf]} {set realf [lfirst [glob -nocomplain $f$manx(zoptglob)]]}
	if {$realf==""} {$t insert end "doesn't exist!" b; return
	} elseif {![file readable $realf]} {$t insert end "not readable!" b; return
	}

	set sumbytes 0

	foreach per $texi(persistent) {catch {unset $per}}

	# if in texinfodir, assume Tcl version already cached
	# use CPU over disk
	set cache [lfirst [glob -nocomplain "$dir/[file tail $f]$manx(zoptglob)"]]
	if {!$force && $cache!="" && [file readable $cache] && [file mtime $cache]>[file mtime $realf]} {
		set fid [open "|[manManPipe $cache]"]; set src [read $fid]; close $fid
		eval $src

	} else {

		array set texiimap {cindex cp findex fn vindex vr kindex ky pindex pg tindex tp}
		array set texiifont {cindex asis findex code vindex code kindex asis pindex code tindex code}
		for {set i 0} {$i<[llength $texilevelx(names)]} {incr i} {set texilevel($i) 0}
		catch {unset texised}
		array set texised {"ignore" "end ignore" "iftex" "end iftex" "tex" "end tex" "ifhtml" "end ifhtml"}

		cursorBusy
		set sumbytes [texiProcess $t [zapZ [file tail $f]]]
		cursorUnset

		if {[array names texienclose]==""} {
			set texienclose(enregexp) "^XXXNOMATCHESXXX\$"
		} else {
			set enregexp ""; foreach i [array names texienclose] {append enregexp $i "|"}
			set texienclose(enregexp) "@([string trimright $enregexp |])"
		}

		texiMarkup $t 1.0 end 1; # grr, embedded @code{...} bits


		# write out, if possible
		# catch needed on SunOS, not on Solaris, most others(?)
		catch {file delete -force $cache}
		if {![file writable $dir] || ($cache!="" && [file exists $cache])} {
			$t insert 1.0 "[file tail $f] not cached as $cache not writable\n" b "\n"
			$t see 1.0
			after 2000
		} else {
			# write out state: outline text+tags, node xref, numbered marks, index, 
			# creates long lines, but result not meant for human manipulation
			set cache $dir/[file tail $f]
			set wfid [open $cache "w"]
			puts -nonewline $wfid "\$t insert end"
# lsort -dictionary work?
			set ms [lsort -command "bytextposn $t " [lmatches "js*" [$t mark names]]]
#			set ms [lsortbytextposn $t [lmatches "js*" [$t mark names]]]
			foreach m $ms {
				set level [regsub -all {\.} $m {\.} junk]; set tag $texilevelx($level)
				puts -nonewline $wfid " [list [$t get [list $m linestart] [list $m lineend]]] $tag {\n} {}"
			}
			puts $wfid ""
			# save selected markup tags
			foreach tag {code subscript} {
				if {[$t tag ranges $tag]!=""} {puts $wfid "\$t tag add $tag [$t tag ranges $tag]"}
			}
			puts $wfid "update idletasks"
			puts $wfid "set line 1; foreach m [list $ms] {\$t mark set \$m \$line.0; incr line}"
			foreach per $texi(persistent) {
				puts $wfid "array set $per [list [array get $per]]"
			}
			close $wfid

			eval exec $man(compress) $cache &
		}
	}

	set texi(sectcnt) [llength [array names texix]]

	# back mapping file => line/section pairs + count of source files (so know when faulted them all)
	set texiback(files) {}
	foreach sect [lsort -dictionary [array names texix]] {
		foreach {file s l} $texix($sect) break
		if {![info exists hit($file)]} {lappend texiback(files) $file; set hit($file) 1}
		lappend texiback($file) [list $sect $s]
	}
	set texiback(filez) {}; foreach file $texiback(files) {lappend texiback(filez) [lfirst [glob $file$manx(zoptglob)]]}
	# sort + sentinal
	foreach file $texiback(files) {lappend texiback($file) {bogussect 100000000}}

	set texi(lastsearch$t) {}; set texi(lasthits$t) {}

#puts "out of show"
	return $sumbytes
}


# create database by iterating over dir, calling texiShow and writing out state
proc texiDatabase {dir t {force 0}} {
	global texi man manx curwin

	# show table of contents, take full paths from texi(texinfo,paths)
	set texi(texinfo,update) 0
	texiTop $dir $t; # just create texi(texinfo,paths)
	update idletasks; after 3000


	set sumbytes 0
	set time0 [clock seconds]
	if $force {texiClear $dir}

	set cnt 0
	foreach f $texi(texinfo,paths) {
		# check to see if up to date (vis-a-vis Tcl version and TkMan installation date--my version date? yes!) => database build forces reconstruction.  fast enough ahyhow(?)
		manTextOpen $curwin
#		$t insert end "$f\n" --> watch as pages load in.  that's a good show
		set bytes [texiShow $t $f $dir $force]
		if {$bytes!=0} {incr sumbytes; incr cnt}
		manTextClose $curwin
#		break; # just one while testing
	}

# not meaningful now that updates always incremental
#	set man(time-lasttexinfo) [clock format [expr [clock seconds]-$time0] -format "%M:%S"]
#	.occ.db entryconfigure "*Texinfo*" -label "Texinfo (last $man(time-lasttexinfo))"


	# cached ones take some time to load up, so doesn't immediately go to directory list, which would be confusing
#	texiTop $dir $t
	.vols invoke Texinfo
#	manTextOpen $curwin
#	$t insert end "Done\n\n" b "Time elapsed:  $man(time-lasttexinfo)\n$cnt files ([bytes2prefix $sumbytes]) indexed\n"
#	$t insert end "Done" b
#	manTextClose $curwin
}


# show list of files
set texi(texinfo,update) 0
proc texiTop {dir t} {
	global texi curwin manx
	# already manTextOpen
	set dirfile "$dir/dir.tkman"

	if ![file exists $dirfile] {$t insert end "$dirfile doesn't exist" b; return}
	if ![file readable $dirfile] {$t insert end "$dirfile unreadable" b; return}

	# cached one still up to date?
	if {[file mtime $dirfile]<=$texi(texinfo,update)} {
		set form $texi(texinfo,form)
	} else {
		set texi(texinfo,update) [file mtime $dirfile]
		set form {}
		set texi(texinfo,paths) {}

		set topregexp {^\*[ 	]+([^:]+): \((/[^\)]+\.(texi|texinfo|info.*))\)(\.z|\.gz\.Z)?\.[ 	]+(.*)}

		set cnt 0
		set fid [open $dirfile]
		while {![eof $fid]} {
			gets $fid line
			if [regexp $topregexp $line all name f suffix Zignore desc] {
				if {$suffix=="info"} {lappend form $f b "-- .info files are formatted text; you want .texi/.texinfo Texinfo " "" "source" i "\n" {}; continue}
				set realf [lfirst [glob -nocomplain [zapZ $f]$manx(zoptglob)]]
				#if {[regexp {(.*\.tar[^/]*)} $f all tar]} {set zapf $tar} else {set zapf $f}
				#set realf [lfirst [glob -nocomplain [zapZ $zapf]$manx(zoptglob)]]
				if ![file exists $realf] {lappend form $f "" " -- doesn't exist\n" b; continue
				} elseif ![file readable $realf] {lappend form $f "" " -- not readable\n" b; continue}

				# jsXXX defined after hyper, so higher priority, so called afterward, so overrides hotman var
				# point to root Texinfo
				lappend form $name [list js$cnt manref hyper] "\t$desc\n" {}
				lappend texi(texinfo,paths) $f
				incr cnt
			} else {lappend form "$line\n" {}}
		}
		close $fid
		set texi(texinfo,form) $form
	}

	eval $t insert end $texi(texinfo,form)
	set cnt 0
	foreach f $texi(texinfo,paths) {
		$t tag bind js$cnt <Button-1> "set manx(hotman$t) $f"
		incr cnt
	}
}


# search with GNU grep, map back to Texinfo source, fault in and format.  So slow first time,
# but get results in context, so much better than usual info one-at-a-time
# If all sections alread faulted in, skip grep.
# if this is slow, maybe just highlight section titles from hit chained to top
proc texiSearch {w} {
	global man manx texiback texi
	set MAXSECT 20

	if {$man(gzgrep)==""} return
	set t $w.show
	upvar #0 texiback$t texiback texix$t texix

	set pat $manx(search,string$w)
#	if {$texi(sectcnt)==0} {
#		# all sections loaded; don't have to search on disk
#	=>	# how to make visible all sections headers with hits?  regular search should do this
#		return 
#
#	} else
	if {$pat==$texi(lastsearch$t)} {
		set secthits $texi(lasthits$t)

	} else {

		set ignorecase ""; if {[string tolower $pat]==$pat} {set ignorecase "i"}
		cd [file dirname $manx(name$w)]

		cursorBusy
		set secthits {}
		if [catch {set hits [eval exec $man(gzgrep) -Eb$ignorecase $pat $texiback(filez) 2>/dev/null]} errinfo] {
			manWinstderr $w "ERROR: $errinfo"
			return -code break
		}

		set lasthit ""
		foreach hit [split $hits "\n"] {
			regexp {(([^:]+):)?([0-9]+)} $hit all junk file b; zapZ! file
			if {$file==""} {set file [lfirst $texiback(filez)]}; # only one file so grep doesn't report filename
			set lastsect ""
			foreach pair $texiback($file) {
				foreach {sect s} $pair break
				if {$s>$b} {
					if {$lastsect!="" && $lastsect!=$lasthit} {lappend secthits $lastsect}
					set lasthit $lastsect
					break
				}
				set lastsect $sect
			}
		}
		cursorUnset
	}
	set texi(lastsearch$t) $pat; set texi(lasthits$t) $secthits

	set newsecthits {}; foreach sect $secthits {if {$texix($sect)!=""} {lappend newsecthits $sect}}

	# always show hit's enclosing section
	after 100 "foreach sect [list $secthits] {$t tag add $man(search-show) \"\$sect linestart\" \"\$sect lineend\"}"

	foreach sect [lrange $newsecthits 0 $MAXSECT] {
		set next [lindex $manx(nextposns$w) [lsearch $manx(sectposns$w) $sect]]
		$t configure -state normal
		texiCallback $t $sect $next; update idletasks
		$t configure -state disabled
	}
	if {[llength $newsecthits]>$MAXSECT} {
		manWinstderr $w "Too many matches.  Showing many; search again to see more."
		set manx(search,oldstring$w) ""; # so can keep searching for same thing and getting KWIC soon enough
	}
}

proc texiClear {dir} {
	global manx
	catch {eval file delete -force [glob $dir/*.{texi,texinfo}$manx(zoptglob)]}
}