#!/usr/bin/env python3
#******************************************************************************
# imports.py, provides classes for a file import dialog and import functions
#
# TreeLine, an information storage program
# Copyright (C) 2019, Douglas W. Bell
#
# This is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License, either Version 2 or any later
# version. This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY. See the included LICENSE file for details.
#******************************************************************************
import pathlib
import re
import collections
import zipfile
import csv
import html.parser
import xml.sax.saxutils
from xml.etree import ElementTree
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QApplication, QDialog, QFileDialog, QMessageBox
import miscdialogs
import treenode
import treestructure
import treemodel
import nodeformat
import treeformats
import urltools
import globalref
methods = collections.OrderedDict()
methods.update([(_('Text'), None),
(_('&Tab indented text, one node per line'),
'importTabbedText'),
(_('Co&mma delimited (CSV) text table with level column && '
'header row'), 'importTableCsvLevels'),
(_('Comma delimited (CSV) text table &with header row'),
'importTableCsv'),
(_('Tab delimited text table with header &row'),
'importTableTabbed'),
(_('Plain text, one &node per line (CR delimited)'),
'importTextLines'),
(_('Plain text ¶graphs (blank line delimited)'),
'importTextPara'),
(_('Bookmarks'), None),
(_('&HTML bookmarks (Mozilla Format)'), 'importMozilla'),
(_('&XML bookmarks (XBEL format)'), 'importXbel'),
(_('Other'), None),
(_('Old Tree&Line File (1.x or 2.x)'), 'importOldTreeLine'),
(_('Treepad &file (text nodes only)'), 'importTreePad'),
(_('&Generic XML (non-TreeLine file)'), 'importXml'),
(_('Open &Document (ODF) outline'), 'importOdfText')])
fileFilters = {'importTabbedText': 'txt',
'importTableCsvLevels': 'csv',
'importTableCsv': 'csv',
'importTableTabbed': 'txt',
'importTextLines': 'txt',
'importTextPara': 'txt',
'importMozilla': 'html',
'importXbel': 'xml',
'importOldTreeLine': 'trl',
'importTreePad': 'hjt',
'importXml': 'xml',
'importOdfText': 'odt'}
oldDateTimeConv = {'d': '%-d', 'dd': '%d', 'ddd': '%a', 'dddd': '%A',
'M': '%-m', 'MM': '%m', 'MMM': '%b', 'MMMM': '%B',
'yy': '%y', 'yyyy': '%Y',
'H': '%-H', 'HH': '%H', 'h': '%-I', 'hh': '%I',
'm': '%-M', 'mm': '%M', 's': '%-S', 'ss': '%S',
'zzz': '%f', 'AP': '%p', 'ap': '%p'}
bookmarkFolderTypeName = _('FOLDER')
bookmarkLinkTypeName = _('BOOKMARK')
bookmarkSeparatorTypeName = _('SEPARATOR')
bookmarkLinkFieldName = _('Link')
textFieldName = _('Text')
genericXmlTextFieldName = 'Element_Data'
htmlUnescapeDict = {'amp': '&', 'lt': '<', 'gt': '>', 'quot': '"'}
class ImportControl:
"""Control file imports of alt file types.
"""
def __init__(self, pathObj=None):
"""Initialize the import control object.
Arguments:
pathObj -- the path object to import if given, o/w prompt user
"""
self.pathObj = pathObj
self.errorMessage = ''
# below members for old TreeLine file imports
self.treeLineImportVersion = []
self.treeLineRootAttrib = {}
self.treeLineOldFieldAttr = {}
def interactiveImport(self, addWarning=False):
"""Prompt the user for import type & proceed with import.
Return the structure if import is successful, otherwise None
Arguments:
addWarning - if True, add non-valid file warning to dialog
"""
dialog = miscdialogs.RadioChoiceDialog(_('Import File'),
_('Choose Import Method'),
methods.items(),
QApplication.
activeWindow())
if addWarning:
fileName = self.pathObj.name
dialog.addLabelBox(_('Invalid File'),
_('"{0}" is not a valid TreeLine file.\n\n'
'Use an import filter?').format(fileName))
if dialog.exec_() != QDialog.Accepted:
return None
method = dialog.selectedButton()
if not self.pathObj:
filters = ';;'.join((globalref.fileFilters[fileFilters[method]],
globalref.fileFilters['all']))
defaultFilePath = str(globalref.mainControl.defaultPathObj(True))
filePath, selFltr = QFileDialog.getOpenFileName(QApplication.
activeWindow(),
_('TreeLine - Import File'),
defaultFilePath, filters)
if not filePath:
return None
self.pathObj = pathlib.Path(filePath)
self.errorMessage = ''
try:
QApplication.setOverrideCursor(Qt.WaitCursor)
structure = getattr(self, method)()
QApplication.restoreOverrideCursor()
except IOError:
QApplication.restoreOverrideCursor()
QMessageBox.warning(QApplication.activeWindow(),
'TreeLine',
_('Error - could not read file {0}').
format(self.pathObj))
return None
except UnicodeDecodeError:
prevEncoding = globalref.localTextEncoding
globalref.localTextEncoding = 'utf-8'
structure = getattr(self, method)()
globalref.localTextEncoding = prevEncoding
QApplication.restoreOverrideCursor()
if not structure:
message = _('Error - improper format in {0}').format(self.pathObj)
if self.errorMessage:
message = '{0}\n{1}'.format(message, self.errorMessage)
self.errorMessage = ''
QMessageBox.warning(QApplication.activeWindow(), 'TreeLine',
message)
return structure
def importTabbedText(self):
"""Import a file with tabbed title structure.
Return the structure if import is successful, otherwise None
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
formatRef = structure.childList[0].formatRef
structure.removeNodeDictRef(structure.childList[0])
structure.childList = []
nodeList = []
with self.pathObj.open(encoding=globalref.localTextEncoding) as f:
for line in f:
text = line.strip()
if text:
level = line.count('\t', 0, len(line) - len(line.lstrip()))
node = treenode.TreeNode(formatRef)
node.setTitle(text)
structure.addNodeDictRef(node)
nodeList.append((node, level))
if nodeList and structure.loadChildNodeLevels(nodeList):
structure.generateSpots(None)
return structure
return None
def importTableCsvLevels(self):
"""Import a CSV-delimited table file with level column, header row.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addSpots=False)
tableFormat = nodeformat.NodeFormat(_('TABLE'), structure.treeFormats)
structure.treeFormats.addTypeIfMissing(tableFormat)
nodeList = []
with self.pathObj.open(newline='',
encoding=globalref.localTextEncoding) as f:
reader = csv.reader(f)
try:
headings = [self.correctFieldName(name) for name in
next(reader)][1:]
tableFormat.addFieldList(headings, True, True)
for entries in reader:
if entries:
node = treenode.TreeNode(tableFormat)
structure.addNodeDictRef(node)
try:
level = int(entries.pop(0))
except ValueError:
self.errorMessage = (_('Invalid level number on '
'line {0}').
format(reader.line_num))
return None # abort
nodeList.append((node, level))
try:
for heading in headings:
node.data[heading] = entries.pop(0)
except IndexError:
pass # fewer entries than headings is OK
if entries:
self.errorMessage = (_('Too many entries on '
'Line {0}').
format(reader.line_num))
return None # abort if too few headings
except csv.Error:
self.errorMessage = (_('Bad CSV format on Line {0}').
format(reader.line_num))
return None # abort
if nodeList:
if structure.loadChildNodeLevels(nodeList):
structure.generateSpots(None)
return structure
self.errorMessage = (_('Invalid level structure'))
return None
def importTableCsv(self):
"""Import a file with a CSV-delimited table with header row.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
tableFormat = nodeformat.NodeFormat(_('TABLE'), structure.treeFormats)
structure.treeFormats.addTypeIfMissing(tableFormat)
with self.pathObj.open(newline='',
encoding=globalref.localTextEncoding) as f:
reader = csv.reader(f)
try:
headings = [self.correctFieldName(name) for name in
next(reader)]
tableFormat.addFieldList(headings, True, True)
for entries in reader:
if entries:
node = treenode.TreeNode(tableFormat)
structure.childList[0].childList.append(node)
structure.addNodeDictRef(node)
try:
for heading in headings:
node.data[heading] = entries.pop(0)
except IndexError:
pass # fewer entries than headings is OK
if entries:
self.errorMessage = (_('Too many entries on '
'Line {0}').
format(reader.line_num))
return None # abort if too few headings
except csv.Error:
self.errorMessage = (_('Bad CSV format on Line {0}').
format(reader.line_num))
return None # abort
structure.generateSpots(None)
return structure
def importTableTabbed(self):
"""Import a file with a tab-delimited table with header row.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
tableFormat = nodeformat.NodeFormat(_('TABLE'), structure.treeFormats)
structure.treeFormats.addTypeIfMissing(tableFormat)
with self.pathObj.open(encoding=globalref.localTextEncoding) as f:
headings = [self.correctFieldName(name) for name in
f.readline().split('\t')]
tableFormat.addFieldList(headings, True, True)
lineNum = 1
for line in f:
lineNum += 1
if line.strip():
entries = line.split('\t')
node = treenode.TreeNode(tableFormat)
structure.childList[0].childList.append(node)
structure.addNodeDictRef(node)
try:
for heading in headings:
node.data[heading] = entries.pop(0)
except IndexError:
pass # fewer entries than headings is OK
if entries:
self.errorMessage = (_('Too many entries on Line {0}').
format(lineNum))
return None # abort if too few headings
structure.generateSpots(None)
return structure
@staticmethod
def correctFieldName(name):
"""Return the field name with any illegal characters removed.
Arguments:
name -- the name to modify
"""
name = re.sub(r'[^\w_\-.]', '_', name.strip())
if not name:
return 'X'
if not name[0].isalpha() or name[:3].lower() == 'xml':
name = 'X' + name
return name
def importTextLines(self):
"""Import a text file, creating one node per line.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
nodeFormat = structure.childList[0].formatRef
structure.removeNodeDictRef(structure.childList[0])
structure.childList = []
with self.pathObj.open(encoding=globalref.localTextEncoding) as f:
for line in f:
line = line.strip()
if line:
node = treenode.TreeNode(nodeFormat)
structure.childList.append(node)
structure.addNodeDictRef(node)
node.data[nodeformat.defaultFieldName] = line
structure.generateSpots(None)
return structure
def importTextPara(self):
"""Import a text file, creating one node per paragraph.
Blank line delimited.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
nodeFormat = structure.childList[0].formatRef
structure.removeNodeDictRef(structure.childList[0])
structure.childList = []
with self.pathObj.open(encoding=globalref.localTextEncoding) as f:
text = f.read()
paraList = text.split('\n\n')
for para in paraList:
para = para.strip()
if para:
node = treenode.TreeNode(nodeFormat)
structure.childList.append(node)
structure.addNodeDictRef(node)
node.data[nodeformat.defaultFieldName] = para
structure.generateSpots(None)
return structure
def importOldTreeLine(self):
"""Import an old TreeLine File (1.x or 2.x).
Return the structure if import is successful, otherwise None.
"""
tree = ElementTree.ElementTree()
try:
tree.parse(str(self.pathObj))
except ElementTree.ParseError:
tree = None
if not tree or not tree.getroot().get('item') == 'y':
fileObj = self.pathObj.open('rb')
# decompress before decrypt to support TreeLine 1.4 and earlier
fileObj, compressed = globalref.mainControl.decompressFile(fileObj)
fileObj, encrypted = globalref.mainControl.decryptFile(fileObj)
if not fileObj:
return None
if encrypted and not compressed:
fileObj, compressed = (globalref.mainControl.
decompressFile(fileObj))
if compressed or encrypted:
tree = ElementTree.ElementTree()
try:
tree.parse(fileObj)
except ElementTree.ParseError:
tree = None
fileObj.close()
if not tree or not tree.getroot().get('item') == 'y':
return None
version = tree.getroot().get('tlversion', '').split('.')
try:
self.treeLineImportVersion = [int(i) for i in version]
except ValueError:
pass
self.treeLineRootAttrib = self.convertPrintData(tree.getroot().attrib)
structure = treestructure.TreeStructure()
idRefDict = {}
linkList = []
self.loadOldTreeLineNode(tree.getroot(), structure, idRefDict,
linkList, None)
self.convertOldNodes(structure)
linkRe = re.compile(r']*href="#(.*?)"[^>]*>.*?', re.I | re.S)
for node, fieldName in linkList:
text = node.data[fieldName]
startPos = 0
while True:
match = linkRe.search(text, startPos)
if not match:
break
newId = idRefDict.get(match.group(1), '')
if newId:
text = text[:match.start(1)] + newId + text[match.end(1):]
startPos = match.start(1)
node.data[fieldName] = text
structure.generateSpots(None)
if nodeformat.FileInfoFormat.typeName in structure.treeFormats:
fileFormat = structure.treeFormats[nodeformat.FileInfoFormat.
typeName]
structure.treeFormats.fileInfoFormat.duplicateFileInfo(fileFormat)
del structure.treeFormats[nodeformat.FileInfoFormat.typeName]
structure.treeFormats.updateDerivedRefs()
for nodeFormat in structure.treeFormats.values():
nodeFormat.updateLineParsing()
return structure
def loadOldTreeLineNode(self, element, structure, idRefDict, linkList,
parent=None):
"""Recursively load an old TreeLine ElementTree node and its children.
Arguments:
element -- an ElementTree node
structure -- a ref to the new tree structure
idRefDict -- a dict to relate old to new unique node IDs
linkList -- internal link list ref with (node, fieldname) tuples
parent -- the parent TreeNode (None for the root node only)
"""
try:
typeFormat = structure.treeFormats[element.tag]
except KeyError:
formatData = self.convertOldNodeFormat(element.attrib)
typeFormat = nodeformat.NodeFormat(element.tag,
structure.treeFormats,
formatData)
structure.treeFormats[element.tag] = typeFormat
self.treeLineOldFieldAttr[typeFormat.name] = {}
if element.get('item') == 'y':
node = treenode.TreeNode(typeFormat)
oldId = element.attrib.get('uniqueid', '')
if oldId:
idRefDict[oldId] = node.uId
if parent:
parent.childList.append(node)
else:
structure.childList.append(node)
structure.nodeDict[node.uId] = node
cloneAttr = element.attrib.get('clones', '')
if cloneAttr:
for cloneId in cloneAttr.split(','):
if cloneId in idRefDict:
cloneNode = structure.nodeDict[idRefDict[cloneId]]
node.data = cloneNode.data.copy()
break
else: # bare format (no nodes)
node = None
for child in element:
if child.get('item') and node:
self.loadOldTreeLineNode(child, structure, idRefDict,
linkList, node)
else:
if node and child.text:
node.data[child.tag] = child.text
if child.get('linkcount'):
linkList.append((node, child.tag))
if child.tag not in typeFormat.fieldDict:
fieldData = self.convertOldFieldFormat(child.attrib)
oldFormatDict = self.treeLineOldFieldAttr[typeFormat.name]
oldFormatDict[child.tag] = fieldData
typeFormat.addField(child.tag, fieldData)
def convertPrintData(self, attrib):
"""Return JSON print data from old root attributes.
Arguments:
attrib -- old root print data attributes
"""
for key in ('printlines', 'printwidowcontrol', 'printportrait'):
if key in attrib:
attrib[key] = not attrib[key].startswith('n')
for key in ('printindentfactor', 'printpaperwidth', 'printpaperheight',
'printheadermargin', 'printfootermargin',
'printcolumnspace'):
if key in attrib:
attrib[key] = float(attrib[key])
if 'printmargins' in attrib:
attrib['printmargins'] = [float(margin) for margin in
attrib['printmargins'].split()]
if 'printnumcolumns' in attrib:
attrib['printnumcolumns'] = int(attrib['printnumcolumns'])
return attrib
def convertOldNodeFormat(self, attrib):
"""Return JSON format data from old node format attributes.
Arguments:
attrib -- old node format attrib dict
"""
for key in ('spacebetween', 'formathtml', 'bullets', 'tables'):
if key in attrib:
attrib[key] = attrib[key].startswith('y')
attrib['titleline'] = attrib.get('line0', '')
lineKeyRe = re.compile(r'line\d+$')
lineNums = sorted([int(key[4:]) for key in attrib.keys()
if lineKeyRe.match(key)])
if lineNums and lineNums[0] == 0:
del lineNums[0]
attrib['outputlines'] = [attrib['line{0}'.format(keyNum)] for
keyNum in lineNums]
if self.treeLineImportVersion < [1, 9]: # for very old TL versions
attrib['spacebetween'] = not (self.treeLineRootAttrib.
get('nospace', '').startswith('y'))
attrib['formathtml'] = not (self.treeLineRootAttrib.
get('nohtml', '').startswith('y'))
return attrib
def convertOldFieldFormat(self, attrib):
"""Return JSON format data from old field format attributes.
Arguments:
attrib -- old field node format attrib dict
"""
fieldType = attrib.get('type', '')
if fieldType:
attrib['fieldtype'] = fieldType
fieldFormat = attrib.get('format', '')
if self.treeLineImportVersion < [1, 9]: # for very old TL versions
if fieldType in ('URL', 'Path', 'ExecuteLink', 'Email'):
attrib['oldfieldtype'] = fieldType
fieldType = 'ExternalLink'
attrib['fieldtype'] = fieldType
if fieldType == 'Date':
fieldFormat = fieldFormat.replace('w', 'd')
fieldFormat = fieldFormat.replace('m', 'M')
if fieldType == 'Time':
fieldFormat = fieldFormat.replace('M', 'm')
fieldFormat = fieldFormat.replace('s', 'z')
fieldFormat = fieldFormat.replace('S', 's')
fieldFormat = fieldFormat.replace('AA', 'AP')
fieldFormat = fieldFormat.replace('aa', 'ap')
if 'lines' in attrib:
attrib['lines'] = int(attrib['lines'])
if 'sortkeynum' in attrib:
attrib['sortkeynum'] = int(attrib['sortkeynum'])
if 'sortkeydir' in attrib:
attrib['sortkeyfwd'] = not attrib['sortkeydir'].startswith('r')
if 'evalhtml' in attrib:
attrib['evalhtml'] = attrib['evalhtml'].startswith('y')
if fieldType in ('Date', 'Time', 'DateTime'):
origFormat = fieldFormat
fieldFormat = ''
while origFormat:
replLen = 4
while replLen > 0:
if origFormat[:replLen] in oldDateTimeConv:
fieldFormat += oldDateTimeConv[origFormat[:replLen]]
origFormat = origFormat[replLen:]
break
replLen -= 1
if replLen == 0:
fieldFormat += origFormat[0]
origFormat = origFormat[1:]
if fieldFormat:
attrib['format'] = fieldFormat
return attrib
def convertOldNodes(self, structure):
"""Convert node data to new date and time formats.
Arguments:
structure -- the ref structure containing the data
"""
for node in structure.nodeDict.values():
for field in node.formatRef.fields():
text = node.data.get(field.name, '')
if text:
if field.typeName in ('Date', 'DateTime'):
text = text.replace('/', '-')
if field.typeName in ('Time', 'DateTime'):
text = text + '.000000'
if self.treeLineImportVersion < [1, 9]: # very old TL ver
oldFormatDict = self.treeLineOldFieldAttr[node.
formatRef.name]
oldFieldAttr = oldFormatDict[field.name]
if (field.typeName == 'Text' and not
oldFieldAttr.get('html', '').startswith('y')):
text = text.strip()
text = xml.sax.saxutils.escape(text)
text = text.replace('\n', '
')
elif (field.typeName == 'ExternalLink' and
oldFieldAttr.get('oldfieldtype', '')):
oldType = oldFieldAttr['oldfieldtype']
linkAltField = oldFieldAttr.get('linkalt', '')
dispName = node.data.get(linkAltField, '')
if not dispName:
dispName = text
if oldType == 'URL':
if not urltools.extractScheme(text):
text = urltools.replaceScheme('http', text)
elif oldType == 'Path':
text = urltools.replaceScheme('file', text)
elif oldType == 'ExecuteLink':
if urltools.isRelative(text):
fullPath = urltools.which(text)
if fullPath:
text = fullPath
text = urltools.replaceScheme('file', text)
elif oldType == 'Email':
text = urltools.replaceScheme('mailto', text)
text = '{1}'.format(text,
dispName)
elif field.typeName == 'InternalLink':
linkAltField = oldFieldAttr.get('linkalt', '')
dispName = node.data.get(linkAltField, '')
if not dispName:
dispName = text
uniqueId = text.strip().split('\n', 1)[0]
uniqueId = uniqueId.replace(' ', '_').lower()
uniqueId = re.sub(r'[^a-zA-Z0-9_-]+', '', uniqueId)
text = '{1}'.format(uniqueId,
dispName)
elif field.typeName == 'Picture':
text = '
'.format(text)
node.data[field.name] = text
def importTreePad(self):
"""Import a Treepad file, text nodes only.
Return the model if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
structure.removeNodeDictRef(structure.childList[0])
structure.childList = []
tpFormat = structure.treeFormats[treeformats.defaultTypeName]
tpFormat.addFieldList([textFieldName], False, True)
tpFormat.fieldDict[textFieldName].changeType('SpacedText')
try:
with self.pathObj.open(encoding=globalref.localTextEncoding) as f:
textList = f.read().split(' 5P9i0s8y19Z')
except UnicodeDecodeError:
with self.pathObj.open(encoding='latin-1') as f:
textList = f.read().split(' 5P9i0s8y19Z')
except UnicodeDecodeError:
return None
nodeList = []
for text in textList:
text = text.strip()
if text:
try:
text = text.split('', 1)[1].lstrip()
lines = text.split('\n')
title = lines[0]
level = int(lines[1])
lines = lines[2:]
except (ValueError, IndexError):
return None
node = treenode.TreeNode(tpFormat)
node.data[nodeformat.defaultFieldName] = title
node.data[textFieldName] = '\n'.join(lines)
node.level = level
nodeList.append(node)
structure.addNodeDictRef(node)
parentList = []
for node in nodeList:
if node.level != 0:
parentList = parentList[:node.level]
node.parent = parentList[-1]
parentList[-1].childList.append(node)
parentList.append(node)
structure.childList = [nodeList[0]]
structure.generateSpots(None)
return structure
def importXml(self):
"""Import a non-treeline generic XML file.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure()
tree = ElementTree.ElementTree()
try:
tree.parse(str(self.pathObj))
self.loadXmlNode(tree.getroot(), structure, None)
except ElementTree.ParseError:
return None
for elemFormat in structure.treeFormats.values():
if not elemFormat.getTitleLine(): # fix formats if required
elemFormat.changeTitleLine(elemFormat.name)
for fieldName in elemFormat.fieldNames():
elemFormat.addOutputLine('{0}="{{*{1}*}}"'.
format(fieldName, fieldName))
if not elemFormat.fieldDict:
elemFormat.addField(genericXmlTextFieldName)
if structure.childList:
structure.generateSpots(None)
return structure
return None
def loadXmlNode(self, element, structure, parent=None):
"""Recursively load a generic XML ElementTree node and its children.
Arguments:
element -- an XML ElementTree node
structure -- a ref to the TreeLine structure
parent -- the parent TreeNode (None for the root node only)
"""
elemFormat = structure.treeFormats.get(element.tag, None)
if not elemFormat:
elemFormat = nodeformat.NodeFormat(element.tag,
structure.treeFormats)
structure.treeFormats[element.tag] = elemFormat
node = treenode.TreeNode(elemFormat)
structure.addNodeDictRef(node)
if not parent:
parent = structure
parent.childList.append(node)
if element.text and element.text.strip():
if genericXmlTextFieldName not in elemFormat.fieldDict:
elemFormat.addFieldList([genericXmlTextFieldName], True, True)
node.setTitle(element.text.strip())
for key, value in element.items():
elemFormat.addFieldIfNew(key)
node.data[key] = value
for child in element:
self.loadXmlNode(child, structure, node)
def importOdfText(self):
"""Import an ODF format text file outline.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure(addDefaults=True,
addSpots=False)
structure.removeNodeDictRef(structure.childList[0])
structure.childList = []
odfFormat = structure.treeFormats[treeformats.defaultTypeName]
odfFormat.addField(textFieldName)
odfFormat.changeOutputLines(['{{*{0}*}}'.
format(nodeformat.defaultFieldName),
'{{*{0}*}}'.format(textFieldName)])
odfFormat.formatHtml = True
try:
with zipfile.ZipFile(str(self.pathObj), 'r') as f:
text = f.read('content.xml')
except (zipfile.BadZipFile, KeyError):
return None
try:
rootElement = ElementTree.fromstring(text)
except ElementTree.ParseError:
return None
nameSpace = '{urn:oasis:names:tc:opendocument:xmlns:text:1.0}'
headerTag = '{0}h'.format(nameSpace)
paraTag = '{0}p'.format(nameSpace)
numRegExp = re.compile(r'.*?(\d+)$')
parents = [structure]
prevLevel = 0
for elem in rootElement.iter():
if elem.tag == headerTag:
style = elem.get('{0}style-name'.format(nameSpace), '')
try:
level = int(numRegExp.match(style).group(1))
except AttributeError:
return None
if level < 1 or level > prevLevel + 1:
return None
parents = parents[:level]
node = treenode.TreeNode(odfFormat)
structure.addNodeDictRef(node)
parents[-1].childList.append(node)
node.data[nodeformat.defaultFieldName] = ''.join(elem.
itertext())
parents.append(node)
prevLevel = level
elif elem.tag == paraTag:
text = ''.join(elem.itertext())
origText = node.data.get(textFieldName, '')
if origText:
text = '{0}
{1}'.format(origText, text)
node.data[textFieldName] = text
structure.generateSpots(None)
return structure
def createBookmarkFormat(self):
"""Return a set of node formats for bookmark imports.
"""
treeFormats = treeformats.TreeFormats()
folderFormat = nodeformat.NodeFormat(bookmarkFolderTypeName,
treeFormats, addDefaultField=True)
folderFormat.iconName = 'folder_3'
treeFormats[folderFormat.name] = folderFormat
linkFormat = nodeformat.NodeFormat(bookmarkLinkTypeName, treeFormats,
addDefaultField=True)
linkFormat.addField(bookmarkLinkFieldName,
{'fieldtype': 'ExternalLink'})
linkFormat.addOutputLine('{{*{0}*}}'.format(bookmarkLinkFieldName))
linkFormat.iconName = 'bookmark'
treeFormats[linkFormat.name] = linkFormat
sepFormat = nodeformat.NodeFormat(bookmarkSeparatorTypeName,
treeFormats, {'formathtml': True},
True)
sepFormat.changeTitleLine('------------------')
sepFormat.changeOutputLines(['
'])
treeFormats[sepFormat.name] = sepFormat
return treeFormats
def importMozilla(self):
"""Import an HTML mozilla-format bookmark file.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure()
structure.treeFormats = self.createBookmarkFormat()
with self.pathObj.open(encoding='utf-8') as f:
text = f.read()
try:
handler = HtmlBookmarkHandler(structure)
handler.feed(text)
handler.close()
except ValueError:
return None
structure.generateSpots(None)
return structure
def importXbel(self):
"""Import an XBEL format bookmark file.
Return the structure if import is successful, otherwise None.
"""
structure = treestructure.TreeStructure()
structure.treeFormats = self.createBookmarkFormat()
tree = ElementTree.ElementTree()
try:
tree.parse(str(self.pathObj))
except ElementTree.ParseError:
return None
self.loadXbelNode(tree.getroot(), structure, None)
if structure.childList:
structure.generateSpots(None)
return structure
return None
def loadXbelNode(self, element, structure, parent=None):
"""Recursively load an XBEL ElementTree node and its children.
Arguments:
element -- an XBEL ElementTree node
model -- a ref to the TreeLine model
parent -- the parent TreeNode (None for the root node only)
"""
if element.tag in ('xbel', 'folder'):
node = treenode.TreeNode(structure.
treeFormats[bookmarkFolderTypeName])
structure.addNodeDictRef(node)
if parent:
parent.childList.append(node)
else:
structure.childList.append(node)
for child in element:
self.loadXbelNode(child, structure, node)
elif element.tag == 'bookmark':
node = treenode.TreeNode(structure.
treeFormats[bookmarkLinkTypeName])
structure.addNodeDictRef(node)
parent.childList.append(node)
link = element.get('href').strip()
if link:
node.data[bookmarkLinkFieldName] = ('{1}'.
format(link, link))
for child in element:
self.loadXbelNode(child, structure, node)
elif element.tag == 'title':
parent.setTitle(element.text)
elif element.tag == 'separator':
node = treenode.TreeNode(structure.
treeFormats[bookmarkSeparatorTypeName])
structure.addNodeDictRef(node)
parent.childList.append(node)
else: # unsupported tags
pass
class HtmlBookmarkHandler(html.parser.HTMLParser):
"""Handler to parse HTML mozilla bookmark format.
"""
def __init__(self, structure):
"""Initialize the HTML parser object.
Arguments:
structure -- a reference to the tree structure
"""
super().__init__()
self.structure = structure
rootNode = treenode.TreeNode(self.structure.
treeFormats[bookmarkFolderTypeName])
rootNode.data[nodeformat.defaultFieldName] = _('Bookmarks')
self.structure.addNodeDictRef(rootNode)
self.structure.childList = [rootNode]
self.currentNode = rootNode
self.parents = []
self.text = ''
def handle_starttag(self, tag, attrs):
"""Called by the reader at each open tag.
Arguments:
tag -- the tag label
attrs -- any tag attributes
"""
if tag == 'dt' or tag == 'h1': # start any entry
self.text = ''
elif tag == 'dl': # start indent
self.parents.append(self.currentNode)
self.currentNode = None
elif tag == 'h3': # start folder
if not self.parents:
raise ValueError
self.currentNode = treenode.TreeNode(self.structure.
treeFormats[bookmarkFolderTypeName])
self.structure.addNodeDictRef(self.currentNode)
self.parents[-1].childList.append(self.currentNode)
elif tag == 'a': # start link
if not self.parents:
raise ValueError
self.currentNode = treenode.TreeNode(self.structure.
treeFormats[bookmarkLinkTypeName])
self.structure.addNodeDictRef(self.currentNode)
self.parents[-1].childList.append(self.currentNode)
for name, value in attrs:
if name == 'href':
link = '{0}'.format(value)
self.currentNode.data[bookmarkLinkFieldName] = link
elif tag == 'hr': # separator
if not self.parents:
raise ValueError
node = treenode.TreeNode(self.structure.
treeFormats[bookmarkSeparatorTypeName])
self.structure.addNodeDictRef(node)
self.parents[-1].childList.append(node)
self.currentNode = None
def handle_endtag(self, tag):
"""Called by the reader at each end tag.
Arguments:
tag -- the tag label
"""
if tag == 'dl': # end indented section
self.parents = self.parents[:-1]
self.currentNode = None
elif tag == 'h3' or tag == 'a': # end folder or link
if not self.currentNode:
raise ValueError
self.currentNode.data[nodeformat.defaultFieldName] = self.text
elif tag == 'h1': # end main title
self.structure.childList[0].data[nodeformat.
defaultFieldName] = self.text
def handle_data(self, data):
"""Called by the reader to process text.
Arguments:
data -- the new text
"""
self.text += data
def handle_entityref(self, name):
"""Convert escaped entity ref to char.
Arguments:
name -- the name of the escaped entity
"""
self.text += htmlUnescapeDict.get(name, '')