#! /usr/bin/env python3
#
# This file is part of secnet.
# See README for full list of copyright holders.
#
# secnet is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
# 
# secnet is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# version 3 along with secnet; if not, see
# https://www.gnu.org/licenses/gpl.html.

"""VPN sites file manipulation.

This program enables VPN site descriptions to be submitted for
inclusion in a central database, and allows the resulting database to
be turned into a secnet configuration file.

A database file can be turned into a secnet configuration file simply:
make-secnet-sites.py [infile [outfile]]

It would be wise to run secnet with the "--just-check-config" option
before installing the output on a live system.

The program expects to be invoked via userv to manage the database; it
relies on the USERV_USER and USERV_GROUP environment variables. The
command line arguments for this invocation are:

make-secnet-sites.py -u header-filename groupfiles-directory output-file \
  group

All but the last argument are expected to be set by userv; the 'group'
argument is provided by the user. A suitable userv configuration file
fragment is:

reset
no-disconnect-hup
no-suppress-args
cd ~/secnet/sites-test/
execute ~/secnet/make-secnet-sites.py -u vpnheader groupfiles sites

This program is part of secnet.

"""

from __future__ import print_function
from __future__ import unicode_literals
from builtins import int

import string
import time
import sys
import os
import getopt
import re
import argparse
import math

import ipaddress

# entry 0 is "near the executable", or maybe from PYTHONPATH=.,
# which we don't want to preempt
sys.path.insert(1,"/usr/local/share/secnet")
sys.path.insert(1,"/usr/share/secnet")
import ipaddrset
import base91

from argparseactionnoyes import ActionNoYes

VERSION="0.1.18"

max_version = 2

from sys import version_info
if version_info.major == 2:  # for python2
    import codecs
    sys.stdin = codecs.getreader('utf-8')(sys.stdin)
    sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
    import io
    open=lambda f,m='r': io.open(f,m,encoding='utf-8')

max={'rsa_bits':8200,'name':33,'dh_bits':8200,'algname':127}

def debugrepr(*args):
	if debug_level > 0:
		print(repr(args), file=sys.stderr)

def base91s_encode(bindata):
	return base91.encode(bindata).replace('"',"-")

def base91s_decode(string):
	return base91.decode(string.replace("-",'"'))

class Tainted:
	def __init__(self,s,tline=None,tfile=None):
		self._s=s
		self._ok=None
		self._line=line if tline is None else tline
		self._file=file if tfile is None else tfile
	def __eq__(self,e):
		return self._s==e
	def __ne__(self,e):
		# for Python2
		return not self.__eq__(e)
	def __str__(self):
		raise RuntimeError('direct use of Tainted value')
	def __repr__(self):
		return 'Tainted(%s)' % repr(self._s)

	def _bad(self,what,why):
		assert(self._ok is not True)
		self._ok=False
		complain('bad parameter: %s: %s' % (what, why))
		return False

	def _max_ok(self,what,maxlen):
		if len(self._s) > maxlen:
			return self._bad(what,'too long (max %d)' % maxlen)
		return True

	def _re_ok(self,bad,what,maxlen=None):
		if maxlen is None: maxlen=max[what]
		self._max_ok(what,maxlen)
		if self._ok is False: return False
		if bad.search(self._s):
			#print(repr(self), file=sys.stderr)
			return self._bad(what,'bad syntax')
		return True

	def _rtnval(self, is_ok, ifgood, ifbad=''):
		if is_ok:
			assert(self._ok is not False)
			self._ok=True
			return ifgood
		else:
			assert(self._ok is not True)
			self._ok=False
			return ifbad

	def _rtn(self, is_ok, ifbad=''):
		return self._rtnval(is_ok, self._s, ifbad)

	def raw(self):
		return self._s
	def raw_mark_ok(self):
		# caller promises to throw if syntax was dangeorus
		return self._rtn(True)

	def output(self):
		if self._ok is False: return ''
		if self._ok is True: return self._s
		print('%s:%d: unchecked/unknown additional data "%s"' %
		      (self._file,self._line,self._s),
		      file=sys.stderr)
		sys.exit(1)

	bad_name=re.compile(r'^[^a-zA-Z]|[^-_0-9a-zA-Z]')
	# secnet accepts _ at start of names, but we reserve that
	bad_name_counter=0
	def name(self,what='name'):
		ok=self._re_ok(Tainted.bad_name,what)
		return self._rtn(ok,
				 '_line%d_%s' % (self._line, id(self)))

	def keyword(self):
		ok=self._s in keywords or self._s in levels
		if not ok:
			complain('unknown keyword %s' % self._s)
		return self._rtn(ok)

	bad_hex=re.compile(r'[^0-9a-fA-F]')
	def bignum_16(self,kind,what):
		maxlen=(max[kind+'_bits']+3)/4
		ok=self._re_ok(Tainted.bad_hex,what,maxlen)
		return self._rtn(ok)

	bad_num=re.compile(r'[^0-9]')
	def bignum_10(self,kind,what):
		maxlen=math.ceil(max[kind+'_bits'] / math.log10(2))
		ok=self._re_ok(Tainted.bad_num,what,maxlen)
		return self._rtn(ok)

	def number(self,minn,maxx,what='number'):
		# not for bignums
		ok=self._re_ok(Tainted.bad_num,what,10)
		if ok:
			v=int(self._s)
			if v<minn or v>maxx:
				ok=self._bad(what,'out of range %d..%d'
					     % (minn,maxx))
		return self._rtnval(ok,v,minn)

	def hexid(self,byteslen,what):
		ok=self._re_ok(Tainted.bad_hex,what,byteslen*2)
		if ok:
			if len(self._s) < byteslen*2:
				ok=self._bad(what,'too short')
		return self._rtn(ok,ifbad='00'*byteslen)

	bad_host=re.compile(r'[^-\][_.:0-9a-zA-Z]')
	# We permit _ so we can refer to special non-host domains
	# which have A and AAAA RRs.  This is a crude check and we may
	# still produce config files with syntactically invalid
	# domains or addresses, but that is OK.
	def host(self):
		ok=self._re_ok(Tainted.bad_host,'host/address',255)
		return self._rtn(ok)

	bad_email=re.compile(r'[^-._0-9a-z@!$%^&*=+~/]')
	# ^ This does not accept all valid email addresses.  That's
	# not really possible with this input syntax.  It accepts
	# all ones that don't require quoting anywhere in email
	# protocols (and also accepts some invalid ones).
	def email(self):
		ok=self._re_ok(Tainted.bad_email,'email address',1023)
		return self._rtn(ok)

	bad_groupname=re.compile(r'^[^_A-Za-z]|[^-+_0-9A-Za-z]')
	def groupname(self):
		ok=self._re_ok(Tainted.bad_groupname,'group name',64)
		return self._rtn(ok)

	bad_base91=re.compile(r'[^!-~]|[\'\"\\]')
	def base91(self,what='base91'):
		ok=self._re_ok(Tainted.bad_base91,what,4096)
		return self._rtn(ok)

class ArgActionLambda(argparse.Action):
	def __init__(self, fn, **kwargs):
		self.fn=fn
		argparse.Action.__init__(self,**kwargs)
	def __call__(self,ap,ns,values,option_string):
		self.fn(values,ns,ap,option_string)

class PkmBase():
	def site_start(self,pubkeys_path):
		self._pa=pubkeys_path
		self._fs = FilterState()
	def site_serial(self,serial): pass
	def write_key(self,k): pass
	def site_finish(self,confw): pass

class PkmSingle(PkmBase):
	opt = 'single'
	help = 'write one public key per site to sites.conf'
	def site_start(self,pubkeys_path):
		PkmBase.site_start(self,pubkeys_path)
		self._outk = []
	def write_key(self,k):
		if k.okforonlykey(output_version,self._fs):
			self._outk.append(k)
	def site_finish(self,confw):
		if len(self._outk) == 0:
			complain("site with no public key");
		elif len(self._outk) != 1:
			debugrepr('outk ', self._outk)
			complain(
 "site with multiple public keys, without --pubkeys-install (maybe --output-version=1 would help"
			)
		else:
			confw.write("key %s;\n"%str(self._outk[0]))

class PkmInstall(PkmBase):
	opt = 'install'
	help = 'install public keys in public key directory'
	def site_start(self,pubkeys_path):
		PkmBase.site_start(self,pubkeys_path)
		self._pw=open(self._pa+'~tmp','w')
	def site_serial(self,serial):
		self._pw.write('serial %s\n' % serial)
	def write_key(self,k):
		wout=k.forpub(output_version,self._fs)
		self._pw.write(' '.join(wout))
		self._pw.write('\n')
	def site_finish(self,confw):
		self._pw.close()
		os.rename(self._pa+'~tmp',self._pa+'~update')
		PkmElide.site_finish(self,confw)

class PkmElide(PkmBase):
	opt = 'elide'
	help = 'no public keys in sites.conf output nor in directory'
	def site_finish(self,confw):
		confw.write("peer-keys \"%s\";\n"%self._pa);

class OpBase():
	# Base case is reading a sites file from self.inputfilee.
	# And writing a sites file to self.sitesfile.
	def check_group(self,group,w):
		if len(w) >= 3:
			w[2].groupname()
	def positional_args(self, av):
		if len(av.arg)>3:
			print("Too many arguments")
			sys.exit(1)
		(self.inputfile, self.outputfile) = (av.arg + [None]*2)[0:2]
	def read_in(self):
		if self.inputfile is None:
			self.inputlines = pfile("stdin",sys.stdin.readlines())
		else:
			self.inputlines = pfilepath(self.inputfile)
	def write_out(self):
		if self.outputfile is None:
			f=sys.stdout
		else:
			f=open(self.outputfile+"-tmp",'w')
		f.write("# sites file autogenerated by make-secnet-sites\n")
		self.write_out_heading(f)
		f.write("# use make-secnet-sites to turn this file into a\n")
		f.write("# valid /etc/secnet/sites.conf file\n\n")
		self.write_out_contents(f)
		f.write("# end of sites file\n")
		if self.outputfile is not None:
			f.close()
			os.rename(self.outputfile+"-tmp",self.outputfile)

class OpConf(OpBase):
	opts = ['--conf']
	help = 'sites.conf generation mode (default)'
	def write_out(self):
		if self.outputfile is None:
			of=sys.stdout
		else:
			tmp_outputfile=self.outputfile+'~tmp~'
			of=open(tmp_outputfile,'w')
		outputsites(of)
		if self.outputfile is not None:
			os.rename(tmp_outputfile,self.outputfile)

class OpFilter(OpBase):
	opts = ['--filter']
	help = 'sites file filtering mode'
	def positional_arXgs(self, av):
		if len(av.arg)!=1:
			print("Too many arguments")
		(self.inputfile,) = (av.arg + [None])[0:1]
		self.outputfile = None
	def write_out_heading(self,f):
		f.write("# --filter --output-version=%d\n"%output_version)
	def write_out_contents(self,f):
		for i in self.inputlines: f.write(i)

class OpUserv(OpBase):
	opts = ['--userv','-u']
	help = 'userv service fragment update mode'
	def positional_args(self, av):
		if len(av.arg)!=4:
			print("Wrong number of arguments")
			sys.exit(1)
		(self.header, self.groupfiledir,
		 self.outputfile, self.group) = av.arg
		self.group = Tainted(self.group,0,'command line')
		# untrusted argument from caller
		if "USERV_USER" not in os.environ:
			print("Environment variable USERV_USER not found")
			sys.exit(1)
		self.user=os.environ["USERV_USER"]
		# Check that group is in USERV_GROUP
		if "USERV_GROUP" not in os.environ:
			print("Environment variable USERV_GROUP not found")
			sys.exit(1)
		ugs=os.environ["USERV_GROUP"]
		ok=0
		for i in ugs.split():
			if self.group==i: ok=1
		if not ok:
			print("caller not in group %s"%self.group.groupname())
			sys.exit(1)
	def check_group(self,group,w):
		if group!=self.group: complain("Incorrect group!")
		OpBase.check_group(self,group,w)
	def read_in(self):
		self.headerinput=pfilepath(self.header,allow_include=True)
		self.userinput=sys.stdin.readlines()
		pfile("user input",self.userinput)
	def write_out(self):
		# Put the user's input into their group file, and
		# rebuild the main sites file
		f=open(self.groupfiledir+"/T"+self.group.groupname(),'w')
		f.write("# Section submitted by user %s, %s\n"%
			(self.user,time.asctime(time.localtime(time.time()))))
		f.write("# Checked by make-secnet-sites version %s\n\n"
			%VERSION)
		for i in self.userinput: f.write(i)
		f.write("\n")
		f.close()
		os.rename(self.groupfiledir+"/T"+self.group.groupname(),
			  self.groupfiledir+"/R"+self.group.groupname())
		OpBase.write_out(self)
	def write_out_heading(self,f):
		f.write("# generated %s, invoked by %s\n"%
			(time.asctime(time.localtime(time.time())),
			 self.user))
	def write_out_contents(self,f):
		for i in self.headerinput: f.write(i)
		files=os.listdir(self.groupfiledir)
		for i in files:
			if i[0]=='R':
				j=open(self.groupfiledir+"/"+i)
				f.write(j.read())
				j.close()

def parse_args():
	global opmode
	global prefix
	global key_prefix
	global debug_level
	global output_version
	global pubkeys_dir
	global pubkeys_mode

	ap = argparse.ArgumentParser(description='process secnet sites files')
	def add_opmode(how):
		ap.add_argument(*how().opts, action=ArgActionLambda,
			nargs=0,
			fn=(lambda v,ns,*x: setattr(ns,'opmode',how)),
			help=how().help)
	add_opmode(OpConf)
	add_opmode(OpFilter)
	add_opmode(OpUserv)
	ap.add_argument('--conf-key-prefix', action=ActionNoYes,
			default=True,
		 help='prefix conf file key names derived from sites data')
	def add_pkm(how):
		ap.add_argument('--pubkeys-'+how().opt, action=ArgActionLambda,
			nargs=0,
			fn=(lambda v,ns,*x: setattr(ns,'pkm',how)),
			help=how().help)
	add_pkm(PkmInstall)
	add_pkm(PkmSingle)
	add_pkm(PkmElide)
	ap.add_argument('--pubkeys-dir',  nargs=1,
			help='public key directory',
			default=['/var/lib/secnet/pubkeys'])
	ap.add_argument('--output-version', nargs=1, type=int,
			help='sites file output version',
			default=[max_version])
	ap.add_argument('--prefix', '-P', nargs=1,
			help='set prefix')
	ap.add_argument('--debug', '-D', action='count', default=0)
	ap.add_argument('arg',nargs=argparse.REMAINDER)
	av = ap.parse_args()
	debug_level = av.debug
	debugrepr('av',av)
	opmode = getattr(av,'opmode',OpConf)()
	prefix = '' if av.prefix is None else av.prefix[0]
	key_prefix = av.conf_key_prefix
	output_version = av.output_version[0]
	pubkeys_dir = av.pubkeys_dir[0]
	pubkeys_mode = getattr(av,'pkm',PkmSingle)
	opmode.positional_args(av)

parse_args()

# Classes describing possible datatypes in the configuration file

class basetype:
	"Common protocol for configuration types."
	def add(self,obj,w):
		complain("%s %s already has property %s defined"%
			(obj.type,obj.name,w[0].raw()))
	def forsites(self,version,copy,fs):
		return copy

class conflist:
	"A list of some kind of configuration type."
	def __init__(self,subtype,w):
		self.subtype=subtype
		self.list=[subtype(w)]
	def add(self,obj,w):
		self.list.append(self.subtype(w))
	def __str__(self):
		return ', '.join(map(str, self.list))
	def forsites(self,version,copy,fs):
		most_recent=self.list[len(self.list)-1]
		return most_recent.forsites(version,copy,fs)
def listof(subtype):
	return lambda w: conflist(subtype, w)

class single_ipaddr (basetype):
	"An IP address"
	def __init__(self,w):
		self.addr=ipaddress.ip_address(w[1].raw_mark_ok())
	def __str__(self):
		return '"%s"'%self.addr

class networks (basetype):
	"A set of IP addresses specified as a list of networks"
	def __init__(self,w):
		self.set=ipaddrset.IPAddressSet()
		for i in w[1:]:
			x=ipaddress.ip_network(i.raw_mark_ok(),strict=True)
			self.set.append([x])
	def __str__(self):
		return ",".join(map((lambda n: '"%s"'%n), self.set.networks()))

class dhgroup (basetype):
	"A Diffie-Hellman group"
	def __init__(self,w):
		self.mod=w[1].bignum_16('dh','dh mod')
		self.gen=w[2].bignum_16('dh','dh gen')
	def __str__(self):
		return 'diffie-hellman("%s","%s")'%(self.mod,self.gen)

class hash (basetype):
	"A choice of hash function"
	def __init__(self,w):
		hname=w[1]
		self.ht=hname.raw()
		if (self.ht!='md5' and self.ht!='sha1'):
			complain("unknown hash type %s"%(self.ht))
			self.ht=None
		else:
			hname.raw_mark_ok()
	def __str__(self):
		return '%s'%(self.ht)

class email (basetype):
	"An email address"
	def __init__(self,w):
		self.addr=w[1].email()
	def __str__(self):
		return '<%s>'%(self.addr)

class boolean (basetype):
	"A boolean"
	def __init__(self,w):
		v=w[1]
		if re.match('[TtYy1]',v.raw()):
			self.b=True
			v.raw_mark_ok()
		elif re.match('[FfNn0]',v.raw()):
			self.b=False
			v.raw_mark_ok()
		else:
			complain("invalid boolean value");
	def __str__(self):
		return ['False','True'][self.b]

class num (basetype):
	"A decimal number"
	def __init__(self,w):
		self.n=w[1].number(0,0x7fffffff)
	def __str__(self):
		return '%d'%(self.n)

class serial (basetype):
	def __init__(self,w):
		self.i=w[1].hexid(4,'serial')
	def __str__(self):
		return self.i
	def forsites(self,version,copy,fs):
		if version < 2: return []
		return copy

class address (basetype):
	"A DNS name and UDP port number"
	def __init__(self,w):
		self.adr=w[1].host()
		self.port=w[2].number(1,65536,'port')
	def __str__(self):
		return '"%s"; port %d'%(self.adr,self.port)

class inpub (basetype):
	def forsites(self,version,xcopy,fs):
		return self.forpub(version,fs)

class pubkey (inpub):
	"Some kind of publie key"
	def __init__(self,w):
		self.a=w[1].name('algname')
		self.d=w[2].base91();
	def __str__(self):
		return 'make-public("%s","%s")'%(self.a,self.d)
	def forpub(self,version,fs):
		if version < 2: return []
		return ['pub', self.a, self.d]
	def okforonlykey(self,version,fs):
		return len(self.forpub(version,fs)) != 0

class rsakey (pubkey):
	"An old-style RSA public key"
	def __init__(self,w):
		self.l=w[1].number(0,max['rsa_bits'],'rsa len')
		self.e=w[2].bignum_10('rsa','rsa e')
		self.n=w[3].bignum_10('rsa','rsa n')
		if len(w) >= 5: w[4].email()
		self.a='rsa1'
		self.d=base91s_encode(('%d %s %s' %
				       (self.l,
					self.e,
					self.n)).encode('ascii'))
		# ^ this allows us to use the pubkey.forsites()
		# method for output in versions>=2
	def __str__(self):
		return 'rsa-public("%s","%s")'%(self.e,self.n)
		# this specialisation means we can generate files
		# compatible with old secnet executables
	def forpub(self,version,fs):
		if version < 2:
			if fs.pkg != '00000000': return []
			return ['pubkey', str(self.l), self.e, self.n]
		return pubkey.forpub(self,version,fs)

class rsakey_newfmt(rsakey):
	"An old-style RSA public key in new-style sites format"
	# This is its own class simply to have its own constructor.
	def __init__(self,w):
		self.a=w[1].name()
		assert(self.a == 'rsa1')
		self.d=w[2].base91()
		try:
			w_inner=list(map(Tainted,
					['X-PUB-RSA1'] +
					base91s_decode(self.d)
					.decode('ascii')
					.split(' ')))
		except UnicodeDecodeError:
			complain('rsa1 key in new format has bad base91')
		#print(repr(w_inner), file=sys.stderr)
		rsakey.__init__(self,w_inner)

class pubkey_group(inpub):
	"Public key group introducer"
	# appears in the site's list of keys mixed in with the keys
	def __init__(self,w,fallback):
		self.i=w[1].hexid(4,'pkg-id')
		self.fallback=fallback
	def forpub(self,version,fs):
		fs.pkg=self.i
		if version < 2: return []
		return ['pkgf' if self.fallback else 'pkg', self.i]
	def okforonlykey(self,version,fs):
		self.forpub(version,fs)
		return False
	
def somepubkey(w):
	#print(repr(w), file=sys.stderr)
	if w[0]=='pubkey':
		return rsakey(w)
	elif w[0]=='pub' and w[1]=='rsa1':
		return rsakey_newfmt(w)
	elif w[0]=='pub':
		return pubkey(w)
	elif w[0]=='pkg':
		return pubkey_group(w,False)
	elif w[0]=='pkgf':
		return pubkey_group(w,True)
	else:
		assert(False)

# Possible properties of configuration nodes
keywords={
 'contact':(email,"Contact address"),
 'dh':(dhgroup,"Diffie-Hellman group"),
 'hash':(hash,"Hash function"),
 'key-lifetime':(num,"Maximum key lifetime (ms)"),
 'setup-timeout':(num,"Key setup timeout (ms)"),
 'setup-retries':(num,"Maximum key setup packet retries"),
 'wait-time':(num,"Time to wait after unsuccessful key setup (ms)"),
 'renegotiate-time':(num,"Time after key setup to begin renegotiation (ms)"),
 'restrict-nets':(networks,"Allowable networks"),
 'networks':(networks,"Claimed networks"),
 'serial':(serial,"public key set serial"),
 'pkg':(listof(somepubkey),"start of public key group",'pub'),
 'pkgf':(listof(somepubkey),"start of fallback public key group",'pub'),
 'pub':(listof(somepubkey),"new style public site key"),
 'pubkey':(listof(somepubkey),"Old-style RSA public site key",'pub'),
 'peer':(single_ipaddr,"Tunnel peer IP address"),
 'address':(address,"External contact address and port"),
 'mobile':(boolean,"Site is mobile"),
}

def sp(name,value):
	"Simply output a property - the default case"
	return "%s %s;\n"%(name,value)

# All levels support these properties
global_properties={
	'contact':(lambda name,value:"# Contact email address: %s\n"%(value)),
	'dh':sp,
	'hash':sp,
	'key-lifetime':sp,
	'setup-timeout':sp,
	'setup-retries':sp,
	'wait-time':sp,
	'renegotiate-time':sp,
	'restrict-nets':(lambda name,value:"# restrict-nets %s\n"%value),
}

class level:
	"A level in the configuration hierarchy"
	depth=0
	leaf=0
	allow_properties={}
	require_properties={}
	def __init__(self,w):
		self.type=w[0].keyword()
		self.name=w[1].name()
		self.properties={}
		self.children={}
	def indent(self,w,t):
		w.write("                 "[:t])
	def prop_out(self,n):
		return self.allow_properties[n](n,str(self.properties[n]))
	def output_props(self,w,ind):
		for i in sorted(self.properties.keys()):
			if self.allow_properties[i]:
				self.indent(w,ind)
				w.write("%s"%self.prop_out(i))
	def kname(self):
		return ((self.type[0].upper() if key_prefix else '')
			+ self.name)
	def output_data(self,w,path):
		ind = 2*len(path)
		self.indent(w,ind)
		w.write("%s {\n"%(self.kname()))
		self.output_props(w,ind+2)
		if self.depth==1: w.write("\n");
		for k in sorted(self.children.keys()):
			c=self.children[k]
			c.output_data(w,path+(c,))
		self.indent(w,ind)
		w.write("};\n")

class vpnlevel(level):
	"VPN level in the configuration hierarchy"
	depth=1
	leaf=0
	type="vpn"
	allow_properties=global_properties.copy()
	require_properties={
	 'contact':"VPN admin contact address"
	}
	def __init__(self,w):
		level.__init__(self,w)
	def output_vpnflat(self,w,path):
		"Output flattened list of site names for this VPN"
		ind=2*(len(path)+1)
		self.indent(w,ind)
		w.write("%s {\n"%(self.kname()))
		for i in self.children.keys():
			self.children[i].output_vpnflat(w,path+(self,))
		w.write("\n")
		self.indent(w,ind+2)
		w.write("all-sites %s;\n"%
			','.join(map(lambda i: i.kname(),
				     self.children.values())))
		self.indent(w,ind)
		w.write("};\n")

class locationlevel(level):
	"Location level in the configuration hierarchy"
	depth=2
	leaf=0
	type="location"
	allow_properties=global_properties.copy()
	require_properties={
	 'contact':"Location admin contact address",
	}
	def __init__(self,w):
		level.__init__(self,w)
		self.group=w[2].groupname()
	def output_vpnflat(self,w,path):
		ind=2*(len(path)+1)
		self.indent(w,ind)
		# The "path=path,self=self" abomination below exists because
		# Python didn't support nested_scopes until version 2.1
		#
		#"/"+self.name+"/"+i
		w.write("%s %s;\n"%(self.kname(),','.join(
			map(lambda x,path=path,self=self:
			    '/'.join([prefix+"vpn-data"] + list(map(
				    lambda i: i.kname(),
				    path+(self,x)))),
			    self.children.values()))))

class sitelevel(level):
	"Site level (i.e. a leafnode) in the configuration hierarchy"
	depth=3
	leaf=1
	type="site"
	allow_properties=global_properties.copy()
	allow_properties.update({
	 'address':sp,
	 'networks':None,
	 'peer':None,
         'serial':None,
	 'pkg':None,
	 'pkgf':None,
	 'pub':None,
	 'pubkey':None,
	 'mobile':sp,
	})
	require_properties={
	 'dh':"Diffie-Hellman group",
	 'contact':"Site admin contact address",
	 'networks':"Networks claimed by the site",
	 'hash':"hash function",
	 'peer':"Gateway address of the site",
	}
	def mangle_name(self):
		return self.name.replace('/',',')
	def pubkeys_path(self):
		return pubkeys_dir + '/peer.' + self.mangle_name()
	def __init__(self,w):
		level.__init__(self,w)
	def output_data(self,w,path):
		ind=2*len(path)
		np='/'.join(map(lambda i: i.name, path))
		self.indent(w,ind)
		w.write("%s {\n"%(self.kname()))
		self.indent(w,ind+2)
		w.write("name \"%s\";\n"%(np,))
		self.indent(w,ind+2)

		pkm = pubkeys_mode()
		debugrepr('pkm ',pkm)
		pkm.site_start(self.pubkeys_path())
		if 'serial' in self.properties:
			pkm.site_serial(self.properties['serial'])

		for k in self.properties["pub"].list:
			debugrepr('pubkeys ', k)
			pkm.write_key(k)

		pkm.site_finish(w)

		self.output_props(w,ind+2)
		self.indent(w,ind+2)
		w.write("link netlink {\n");
		self.indent(w,ind+4)
		w.write("routes %s;\n"%str(self.properties["networks"]))
		self.indent(w,ind+4)
		w.write("ptp-address %s;\n"%str(self.properties["peer"]))
		self.indent(w,ind+2)
		w.write("};\n")
		self.indent(w,ind)
		w.write("};\n")

# Levels in the configuration file
# (depth,properties)
levels={'vpn':vpnlevel, 'location':locationlevel, 'site':sitelevel}

def complain(msg):
	"Complain about a particular input line"
	moan(("%s line %d: "%(file,line))+msg)
def moan(msg):
	"Complain about something in general"
	global complaints
	print(msg);
	if complaints is None: sys.exit(1)
	complaints=complaints+1

class UntaintedRoot():
	def __init__(self,s): self._s=s
	def name(self): return self._s
	def keyword(self): return self._s

root=level([UntaintedRoot(x) for x in ['root','root']])
# All vpns are children of this node
obstack=[root]
allow_defs=0   # Level above which new definitions are permitted

def set_property(obj,w):
	"Set a property on a configuration node"
	prop=w[0]
	propname=prop.raw_mark_ok()
	kw=keywords[propname]
	if len(kw) >= 3: propname=kw[2] # for aliases
	if propname in obj.properties:
		obj.properties[propname].add(obj,w)
	else:
		obj.properties[propname]=kw[0](w)
	return obj.properties[propname]

class FilterState:
	def __init__(self):
		self.reset()
	def reset(self):
		# called when we enter a new node,
		# in particular, at the start of each site
		self.pkg = '00000000'

def pline(il,filterstate,allow_include=False):
	"Process a configuration file line"
	global allow_defs, obstack, root
	w=il.rstrip('\n').split()
	if len(w)==0: return ['']
	w=list([Tainted(x) for x in w])
	keyword=w[0]
	current=obstack[len(obstack)-1]
	copyout_core=lambda: ' '.join([ww.output() for ww in w])
	indent='    '*len(obstack)
	copyout=lambda: [indent + copyout_core() + '\n']
	if keyword=='end-definitions':
		keyword.raw_mark_ok()
		allow_defs=sitelevel.depth
		obstack=[root]
		return copyout()
	if keyword=='include':
		if not allow_include:
			complain("include not permitted here")
			return []
		if len(w) != 2:
			complain("include requires one argument")
			return []
		newfile=os.path.join(os.path.dirname(file),w[1].raw_mark_ok())
		# ^ user of "include" is trusted so raw_mark_ok is good
		return pfilepath(newfile,allow_include=allow_include)
	if keyword.raw() in levels:
		# We may go up any number of levels, but only down by one
		newdepth=levels[keyword.raw_mark_ok()].depth
		currentdepth=len(obstack) # actually +1...
		if newdepth<=currentdepth:
			obstack=obstack[:newdepth]
		if newdepth>currentdepth:
			complain("May not go from level %d to level %d"%
				(currentdepth-1,newdepth))
		# See if it's a new one (and whether that's permitted)
		# or an existing one
		current=obstack[len(obstack)-1]
		tname=w[1].name()
		if tname in current.children:
			# Not new
			current=current.children[tname]
			if current.depth==2:
				opmode.check_group(current.group, w)
		else:
			# New
			# Ignore depth check for now
			nl=levels[keyword.raw()](w)
			if nl.depth<allow_defs:
				complain("New definitions not allowed at "
					"level %d"%nl.depth)
				# we risk crashing if we continue
				sys.exit(1)
			current.children[tname]=nl
			current=nl
		filterstate.reset()
		obstack.append(current)
		return copyout()
	if keyword.raw() not in current.allow_properties:
		complain("Property %s not allowed at %s level"%
			(keyword.raw(),current.type))
		return []
	elif current.depth == vpnlevel.depth < allow_defs:
		complain("Not allowed to set VPN properties here")
		return []
	else:
		prop=set_property(current,w)
		out=[copyout_core()]
		out=prop.forsites(output_version,out,filterstate)
		if len(out)==0: return [indent + '#', copyout_core(), '\n']
		return [indent + ' '.join(out) + '\n']

	complain("unknown keyword '%s'"%(keyword.raw()))

def pfilepath(pathname,allow_include=False):
	f=open(pathname)
	outlines=pfile(pathname,f.readlines(),allow_include=allow_include)
	f.close()
	return outlines

def pfile(name,lines,allow_include=False):
	"Process a file"
	global file,line
	file=name
	line=0
	outlines=[]
	filterstate = FilterState()
	for i in lines:
		line=line+1
		if (i[0]=='#'): continue
		outlines += pline(i,filterstate,allow_include=allow_include)
	return outlines

def outputsites(w):
	"Output include file for secnet configuration"
	w.write("# secnet sites file autogenerated by make-secnet-sites "
		+"version %s\n"%VERSION)
	w.write("# %s\n"%time.asctime(time.localtime(time.time())))
	w.write("# Command line: %s\n\n"%' '.join(sys.argv))

	# Raw VPN data section of file
	w.write(prefix+"vpn-data {\n")
	for i in root.children.values():
		i.output_data(w,(i,))
	w.write("};\n")

	# Per-VPN flattened lists
	w.write(prefix+"vpn {\n")
	for i in root.children.values():
		i.output_vpnflat(w,())
	w.write("};\n")

	# Flattened list of sites
	w.write(prefix+"all-sites %s;\n"%",".join(
		map(lambda x:"%svpn/%s/all-sites"%(prefix,x.kname()),
			root.children.values())))

line=0
file=None
complaints=0

# Sanity check section
# Delete nodes where leaf=0 that have no children

def live(n):
	"Number of leafnodes below node n"
	if n.leaf: return 1
	for i in n.children.keys():
		if live(n.children[i]): return 1
	return 0
def delempty(n):
	"Delete nodes that have no leafnode children"
	for i in list(n.children.keys()):
		delempty(n.children[i])
		if not live(n.children[i]):
			del n.children[i]

# Check that all constraints are met (as far as I can tell
# restrict-nets/networks/peer are the only special cases)

def checkconstraints(n,p,ra):
	new_p=p.copy()
	new_p.update(n.properties)
	for i in n.require_properties.keys():
		if i not in new_p:
			moan("%s %s is missing property %s"%
				(n.type,n.name,i))
	for i in new_p.keys():
		if i not in n.allow_properties:
			moan("%s %s has forbidden property %s"%
				(n.type,n.name,i))
	# Check address range restrictions
	if "restrict-nets" in n.properties:
		new_ra=ra.intersection(n.properties["restrict-nets"].set)
	else:
		new_ra=ra
	if "networks" in n.properties:
		if not n.properties["networks"].set <= new_ra:
			moan("%s %s networks out of bounds"%(n.type,n.name))
		if "peer" in n.properties:
			if not n.properties["networks"].set.contains(
				n.properties["peer"].addr):
				moan("%s %s peer not in networks"%(n.type,n.name))
	for i in n.children.keys():
		checkconstraints(n.children[i],new_p,new_ra)

opmode.read_in()

delempty(root)
checkconstraints(root,{},ipaddrset.complete_set())

if complaints>0:
	if complaints==1: print("There was 1 problem.")
	else: print("There were %d problems."%(complaints))
	sys.exit(1)
complaints=None # arranges to crash if we complain later

opmode.write_out()