arsenetar/dupeguru
1
0
Fork 0
mirror of https://github.com/arsenetar/dupeguru.git synced 2026-01-22 06:37:17 +00:00
Code Releases Wiki Activity

Removed dependency on lxml (it made the final package much bigger, and building it on windows is not fun).

Browse Source
This commit is contained in:
Virgil Dupras
2010-08-15 14:42:55 +02:00
parent 12e6c400b9
commit c8827769b4
14 changed files with 71 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/directories.py
View File

@@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license
from lxml import etree
from xml.etree import ElementTree as ET
from hsutil import io
from hsutil.files import FileOrPath
@@ -126,10 +126,10 @@ class Directories(object):
def load_from_file(self, infile):
try:
root = etree.parse(infile).getroot()
except:
root = ET.parse(infile).getroot()
except Exception:
return
for rdn in root.iterchildren('root_directory'):
for rdn in root.getiterator('root_directory'):
attrib = rdn.attrib
if 'path' not in attrib:
continue
@@ -138,7 +138,7 @@ class Directories(object):
self.add_path(Path(path))
except (AlreadyThereError, InvalidPathError):
pass
for sn in root.iterchildren('state'):
for sn in root.getiterator('state'):
attrib = sn.attrib
if not ('path' in attrib and 'value' in attrib):
continue
@@ -148,15 +148,15 @@ class Directories(object):
def save_to_file(self, outfile):
with FileOrPath(outfile, 'wb') as fp:
root = etree.Element('directories')
root = ET.Element('directories')
for root_path in self:
root_path_node = etree.SubElement(root, 'root_directory')
root_path_node = ET.SubElement(root, 'root_directory')
root_path_node.set('path', str(root_path))
for path, state in self.states.items():
state_node = etree.SubElement(root, 'state')
state_node = ET.SubElement(root, 'state')
state_node.set('path', str(path))
state_node.set('value', str(state))
tree = etree.ElementTree(root)
tree = ET.ElementTree(root)
tree.write(fp, encoding='utf-8')
def set_state(self, path, state):

18
core/ignore.py
View File

@@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license
from lxml import etree
from xml.etree import ElementTree as ET
from hsutil.files import FileOrPath
@@ -77,14 +77,16 @@ class IgnoreList(object):
infile can be a file object or a filename.
"""
try:
root = etree.parse(infile).getroot()
root = ET.parse(infile).getroot()
except Exception:
return
for fn in root.iterchildren('file'):
file_elems = (e for e in root if e.tag == 'file')
for fn in file_elems:
file_path = fn.get('path')
if not file_path:
continue
for sfn in fn.iterchildren('file'):
subfile_elems = (e for e in fn if e.tag == 'file')
for sfn in subfile_elems:
subfile_path = sfn.get('path')
if subfile_path:
self.Ignore(file_path, subfile_path)
@@ -94,14 +96,14 @@ class IgnoreList(object):
outfile can be a file object or a filename.
"""
root = etree.Element('ignore_list')
root = ET.Element('ignore_list')
for filename, subfiles in self._ignored.items():
file_node = etree.SubElement(root, 'file')
file_node = ET.SubElement(root, 'file')
file_node.set('path', filename)
for subfilename in subfiles:
subfile_node = etree.SubElement(file_node, 'file')
subfile_node = ET.SubElement(file_node, 'file')
subfile_node.set('path', subfilename)
tree = etree.ElementTree(root)
tree = ET.ElementTree(root)
with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8')

20
core/results.py
View File

@@ -8,7 +8,7 @@
import logging
import re
from lxml import etree
from xml.etree import ElementTree as ET
from . import engine
from hscommon.job import nulljob
@@ -178,16 +178,16 @@ class Results(Markable):
self.apply_filter(None)
try:
root = etree.parse(infile).getroot()
root = ET.parse(infile).getroot()
except Exception:
return
group_elems = list(root.iterchildren('group'))
group_elems = list(root.getiterator('group'))
groups = []
marked = set()
for group_elem in j.iter_with_progress(group_elems, every=100):
group = engine.Group()
dupes = []
for file_elem in group_elem.iterchildren('file'):
for file_elem in group_elem.getiterator('file'):
path = file_elem.get('path')
words = file_elem.get('words', '')
if not path:
@@ -200,7 +200,7 @@ class Results(Markable):
dupes.append(file)
if file_elem.get('marked') == 'y':
marked.add(file)
for match_elem in group_elem.iterchildren('match'):
for match_elem in group_elem.getiterator('match'):
try:
attrs = match_elem.attrib
first_file = dupes[int(attrs['first'])]
@@ -277,10 +277,10 @@ class Results(Markable):
def save_to_xml(self, outfile):
self.apply_filter(None)
root = etree.Element('results')
root = ET.Element('results')
# writer = XMLGenerator(outfile, 'utf-8')
for g in self.groups:
group_elem = etree.SubElement(root, 'group')
group_elem = ET.SubElement(root, 'group')
dupe2index = {}
for index, d in enumerate(g):
dupe2index[d] = index
@@ -288,7 +288,7 @@ class Results(Markable):
words = engine.unpack_fields(d.words)
except AttributeError:
words = ()
file_elem = etree.SubElement(group_elem, 'file')
file_elem = ET.SubElement(group_elem, 'file')
try:
file_elem.set('path', str(d.path))
file_elem.set('words', ','.join(words))
@@ -297,11 +297,11 @@ class Results(Markable):
file_elem.set('is_ref', ('y' if d.is_ref else 'n'))
file_elem.set('marked', ('y' if self.is_marked(d) else 'n'))
for match in g.matches:
match_elem = etree.SubElement(group_elem, 'match')
match_elem = ET.SubElement(group_elem, 'match')
match_elem.set('first', str(dupe2index[match.first]))
match_elem.set('second', str(dupe2index[match.second]))
match_elem.set('percentage', str(int(match.percentage)))
tree = etree.ElementTree(root)
tree = ET.ElementTree(root)
with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8')
self.is_modified = False

5
core/tests/ignore_test.py
View File

@@ -7,7 +7,7 @@
# http://www.hardcoded.net/licenses/hs_license
import io
from lxml import etree
from xml.etree import ElementTree as ET
from hsutil.testutil import eq_
@@ -62,7 +62,7 @@ def test_save_to_xml():
f = io.BytesIO()
il.save_to_xml(f)
f.seek(0)
doc = etree.parse(f)
doc = ET.parse(f)
root = doc.getroot()
eq_(root.tag, 'ignore_list')
eq_(len(root), 2)
@@ -80,7 +80,6 @@ def test_SaveThenLoad():
f = io.BytesIO()
il.save_to_xml(f)
f.seek(0)
f.seek(0)
il = IgnoreList()
il.load_from_xml(f)
eq_(4,len(il))

40
core/tests/results_test.py
View File

@@ -10,7 +10,7 @@
import io
import os.path as op
from lxml import etree
from xml.etree import ElementTree as ET
from hsutil.path import Path
from hsutil.testutil import eq_
@@ -66,7 +66,7 @@ class TCResultsEmpty(TestCase):
f = io.BytesIO()
self.results.save_to_xml(f)
f.seek(0)
doc = etree.parse(f)
doc = ET.parse(f)
root = doc.getroot()
eq_('results', root.tag)
@@ -380,14 +380,14 @@ class TCResultsMarkings(TestCase):
f = io.BytesIO()
self.results.save_to_xml(f)
f.seek(0)
doc = etree.parse(f)
doc = ET.parse(f)
root = doc.getroot()
g1, g2 = root.iterchildren('group')
d1, d2, d3 = g1.iterchildren('file')
g1, g2 = root.getiterator('group')
d1, d2, d3 = g1.getiterator('file')
eq_('n', d1.get('marked'))
eq_('n', d2.get('marked'))
eq_('y', d3.get('marked'))
d1, d2 = g2.iterchildren('file')
d1, d2 = g2.getiterator('file')
eq_('n', d1.get('marked'))
eq_('y', d2.get('marked'))
@@ -425,7 +425,7 @@ class TCResultsXML(TestCase):
f = io.BytesIO()
self.results.save_to_xml(f)
f.seek(0)
doc = etree.parse(f)
doc = ET.parse(f)
root = doc.getroot()
eq_('results', root.tag)
eq_(2, len(root))
@@ -516,35 +516,35 @@ class TCResultsXML(TestCase):
def get_file(path):
return [f for f in self.objects if str(f.path) == path][0]
root = etree.Element('foobar') #The root element shouldn't matter, really.
group_node = etree.SubElement(root, 'group')
dupe_node = etree.SubElement(group_node, 'file') #Perfectly correct file
root = ET.Element('foobar') #The root element shouldn't matter, really.
group_node = ET.SubElement(root, 'group')
dupe_node = ET.SubElement(group_node, 'file') #Perfectly correct file
dupe_node.set('path', op.join('basepath','foo bar'))
dupe_node.set('is_ref', 'y')
dupe_node.set('words', 'foo,bar')
dupe_node = etree.SubElement(group_node, 'file') #is_ref missing, default to 'n'
dupe_node = ET.SubElement(group_node, 'file') #is_ref missing, default to 'n'
dupe_node.set('path',op.join('basepath','foo bleh'))
dupe_node.set('words','foo,bleh')
dupe_node = etree.SubElement(group_node, 'file') #words are missing, valid.
dupe_node = ET.SubElement(group_node, 'file') #words are missing, valid.
dupe_node.set('path',op.join('basepath','bar bleh'))
dupe_node = etree.SubElement(group_node, 'file') #path is missing, invalid.
dupe_node = ET.SubElement(group_node, 'file') #path is missing, invalid.
dupe_node.set('words','foo,bleh')
dupe_node = etree.SubElement(group_node, 'foobar') #Invalid element name
dupe_node = ET.SubElement(group_node, 'foobar') #Invalid element name
dupe_node.set('path',op.join('basepath','bar bleh'))
dupe_node.set('is_ref','y')
dupe_node.set('words','bar,bleh')
match_node = etree.SubElement(group_node, 'match') # match pointing to a bad index
match_node = ET.SubElement(group_node, 'match') # match pointing to a bad index
match_node.set('first', '42')
match_node.set('second', '45')
match_node = etree.SubElement(group_node, 'match') # match with missing attrs
match_node = etree.SubElement(group_node, 'match') # match with non-int values
match_node = ET.SubElement(group_node, 'match') # match with missing attrs
match_node = ET.SubElement(group_node, 'match') # match with non-int values
match_node.set('first', 'foo')
match_node.set('second', 'bar')
match_node.set('percentage', 'baz')
group_node = etree.SubElement(root, 'foobar') #invalid group
group_node = etree.SubElement(root, 'group') #empty group
group_node = ET.SubElement(root, 'foobar') #invalid group
group_node = ET.SubElement(root, 'group') #empty group
f = io.BytesIO()
tree = etree.ElementTree(root)
tree = ET.ElementTree(root)
tree.write(f, encoding='utf-8')
f.seek(0)
r = Results(data)