arsenetar/dupeguru
1
0
Fork 0
mirror of https://github.com/arsenetar/dupeguru.git synced 2026-01-22 14:41:39 +00:00
Code Releases Wiki Activity

Format files with black

Browse Source 
- Format all files with black
- Update tox.ini flake8 arguments to be compatible
- Add black to requirements-extra.txt
- Reduce ignored flake8 rules and fix a few violations
This commit is contained in:
Andrew Senetar
2019-12-31 20:16:27 -06:00
parent 359d6498f7
commit 7ba8aa3514
141 changed files with 5241 additions and 3648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

296
hscommon/build.py
View File

@@ -26,7 +26,8 @@ import modulefinder
from setuptools import setup, Extension
from .plat import ISWINDOWS
from .util import modified_after, find_in_path, ensure_folder, delete_files_with_pattern
from .util import ensure_folder, delete_files_with_pattern
def print_and_do(cmd):
"""Prints ``cmd`` and executes it in the shell.
@@ -35,6 +36,7 @@ def print_and_do(cmd):
p = Popen(cmd, shell=True)
return p.wait()
def _perform(src, dst, action, actionname):
if not op.lexists(src):
print("Copying %s failed: it doesn't exist." % src)
@@ -44,26 +46,32 @@ def _perform(src, dst, action, actionname):
shutil.rmtree(dst)
else:
os.remove(dst)
print('%s %s --> %s' % (actionname, src, dst))
print("%s %s --> %s" % (actionname, src, dst))
action(src, dst)
def copy_file_or_folder(src, dst):
if op.isdir(src):
shutil.copytree(src, dst, symlinks=True)
else:
shutil.copy(src, dst)
def move(src, dst):
_perform(src, dst, os.rename, 'Moving')
_perform(src, dst, os.rename, "Moving")
def copy(src, dst):
_perform(src, dst, copy_file_or_folder, 'Copying')
_perform(src, dst, copy_file_or_folder, "Copying")
def symlink(src, dst):
_perform(src, dst, os.symlink, 'Symlinking')
_perform(src, dst, os.symlink, "Symlinking")
def hardlink(src, dst):
_perform(src, dst, os.link, 'Hardlinking')
_perform(src, dst, os.link, "Hardlinking")
def _perform_on_all(pattern, dst, action):
# pattern is a glob pattern, example "folder/foo*". The file is moved directly in dst, no folder
@@ -73,12 +81,15 @@ def _perform_on_all(pattern, dst, action):
destpath = op.join(dst, op.basename(fn))
action(fn, destpath)
def move_all(pattern, dst):
_perform_on_all(pattern, dst, move)
def copy_all(pattern, dst):
_perform_on_all(pattern, dst, copy)
def ensure_empty_folder(path):
"""Make sure that the path exists and that it's an empty folder.
"""
@@ -86,43 +97,54 @@ def ensure_empty_folder(path):
shutil.rmtree(path)
os.mkdir(path)
def filereplace(filename, outfilename=None, **kwargs):
"""Reads `filename`, replaces all {variables} in kwargs, and writes the result to `outfilename`.
"""
if outfilename is None:
outfilename = filename
fp = open(filename, 'rt', encoding='utf-8')
fp = open(filename, "rt", encoding="utf-8")
contents = fp.read()
fp.close()
# We can't use str.format() because in some files, there might be {} characters that mess with it.
for key, item in kwargs.items():
contents = contents.replace('{{{}}}'.format(key), item)
fp = open(outfilename, 'wt', encoding='utf-8')
contents = contents.replace("{{{}}}".format(key), item)
fp = open(outfilename, "wt", encoding="utf-8")
fp.write(contents)
fp.close()
def get_module_version(modulename):
mod = importlib.import_module(modulename)
return mod.__version__
def setup_package_argparser(parser):
parser.add_argument(
'--sign', dest='sign_identity',
help="Sign app under specified identity before packaging (OS X only)"
"--sign",
dest="sign_identity",
help="Sign app under specified identity before packaging (OS X only)",
)
parser.add_argument(
'--nosign', action='store_true', dest='nosign',
help="Don't sign the packaged app (OS X only)"
"--nosign",
action="store_true",
dest="nosign",
help="Don't sign the packaged app (OS X only)",
)
parser.add_argument(
'--src-pkg', action='store_true', dest='src_pkg',
help="Build a tar.gz of the current source."
"--src-pkg",
action="store_true",
dest="src_pkg",
help="Build a tar.gz of the current source.",
)
parser.add_argument(
'--arch-pkg', action='store_true', dest='arch_pkg',
help="Force Arch Linux packaging type, regardless of distro name."
"--arch-pkg",
action="store_true",
dest="arch_pkg",
help="Force Arch Linux packaging type, regardless of distro name.",
)
# `args` come from an ArgumentParser updated with setup_package_argparser()
def package_cocoa_app_in_dmg(app_path, destfolder, args):
# Rather than signing our app in XCode during the build phase, we sign it during the package
@@ -130,7 +152,9 @@ def package_cocoa_app_in_dmg(app_path, destfolder, args):
# a valid signature.
if args.sign_identity:
sign_identity = "Developer ID Application: {}".format(args.sign_identity)
result = print_and_do('codesign --force --deep --sign "{}" "{}"'.format(sign_identity, app_path))
result = print_and_do(
'codesign --force --deep --sign "{}" "{}"'.format(sign_identity, app_path)
)
if result != 0:
print("ERROR: Signing failed. Aborting packaging.")
return
@@ -139,23 +163,31 @@ def package_cocoa_app_in_dmg(app_path, destfolder, args):
return
build_dmg(app_path, destfolder)
def build_dmg(app_path, destfolder):
"""Builds a DMG volume with application at ``app_path`` and puts it in ``dest_path``.
The name of the resulting DMG volume is determined by the app's name and version.
"""
print(repr(op.join(app_path, 'Contents', 'Info.plist')))
plist = plistlib.readPlist(op.join(app_path, 'Contents', 'Info.plist'))
print(repr(op.join(app_path, "Contents", "Info.plist")))
plist = plistlib.readPlist(op.join(app_path, "Contents", "Info.plist"))
workpath = tempfile.mkdtemp()
dmgpath = op.join(workpath, plist['CFBundleName'])
dmgpath = op.join(workpath, plist["CFBundleName"])
os.mkdir(dmgpath)
print_and_do('cp -R "%s" "%s"' % (app_path, dmgpath))
print_and_do('ln -s /Applications "%s"' % op.join(dmgpath, 'Applications'))
dmgname = '%s_osx_%s.dmg' % (plist['CFBundleName'].lower().replace(' ', '_'), plist['CFBundleVersion'].replace('.', '_'))
print('Building %s' % dmgname)
print_and_do('ln -s /Applications "%s"' % op.join(dmgpath, "Applications"))
dmgname = "%s_osx_%s.dmg" % (
plist["CFBundleName"].lower().replace(" ", "_"),
plist["CFBundleVersion"].replace(".", "_"),
)
print("Building %s" % dmgname)
# UDBZ = bzip compression. UDZO (zip compression) was used before, but it compresses much less.
print_and_do('hdiutil create "%s" -format UDBZ -nocrossdev -srcdir "%s"' % (op.join(destfolder, dmgname), dmgpath))
print('Build Complete')
print_and_do(
'hdiutil create "%s" -format UDBZ -nocrossdev -srcdir "%s"'
% (op.join(destfolder, dmgname), dmgpath)
)
print("Build Complete")
def copy_sysconfig_files_for_embed(destpath):
# This normally shouldn't be needed for Python 3.3+.
@@ -163,24 +195,28 @@ def copy_sysconfig_files_for_embed(destpath):
configh = sysconfig.get_config_h_filename()
shutil.copy(makefile, destpath)
shutil.copy(configh, destpath)
with open(op.join(destpath, 'site.py'), 'w') as fp:
fp.write("""
with open(op.join(destpath, "site.py"), "w") as fp:
fp.write(
"""
import os.path as op
from distutils import sysconfig
sysconfig.get_makefile_filename = lambda: op.join(op.dirname(__file__), 'Makefile')
sysconfig.get_config_h_filename = lambda: op.join(op.dirname(__file__), 'pyconfig.h')
""")
"""
)
def add_to_pythonpath(path):
"""Adds ``path`` to both ``PYTHONPATH`` env and ``sys.path``.
"""
abspath = op.abspath(path)
pythonpath = os.environ.get('PYTHONPATH', '')
pathsep = ';' if ISWINDOWS else ':'
pythonpath = os.environ.get("PYTHONPATH", "")
pathsep = ";" if ISWINDOWS else ":"
pythonpath = pathsep.join([abspath, pythonpath]) if pythonpath else abspath
os.environ['PYTHONPATH'] = pythonpath
os.environ["PYTHONPATH"] = pythonpath
sys.path.insert(1, abspath)
# This is a method to hack around those freakingly tricky data inclusion/exlusion rules
# in setuptools. We copy the packages *without data* in a build folder and then build the plugin
# from there.
@@ -195,14 +231,16 @@ def copy_packages(packages_names, dest, create_links=False, extra_ignores=None):
create_links = False
if not extra_ignores:
extra_ignores = []
ignore = shutil.ignore_patterns('.hg*', 'tests', 'testdata', 'modules', 'docs', 'locale', *extra_ignores)
ignore = shutil.ignore_patterns(
".hg*", "tests", "testdata", "modules", "docs", "locale", *extra_ignores
)
for package_name in packages_names:
if op.exists(package_name):
source_path = package_name
else:
mod = __import__(package_name)
source_path = mod.__file__
if mod.__file__.endswith('__init__.py'):
if mod.__file__.endswith("__init__.py"):
source_path = op.dirname(source_path)
dest_name = op.basename(source_path)
dest_path = op.join(dest, dest_name)
@@ -220,58 +258,81 @@ def copy_packages(packages_names, dest, create_links=False, extra_ignores=None):
else:
shutil.copy(source_path, dest_path)
def copy_qt_plugins(folder_names, dest): # This is only for Windows
def copy_qt_plugins(folder_names, dest): # This is only for Windows
from PyQt5.QtCore import QLibraryInfo
qt_plugin_dir = QLibraryInfo.location(QLibraryInfo.PluginsPath)
def ignore(path, names):
if path == qt_plugin_dir:
return [n for n in names if n not in folder_names]
else:
return [n for n in names if not n.endswith('.dll')]
return [n for n in names if not n.endswith(".dll")]
shutil.copytree(qt_plugin_dir, dest, ignore=ignore)
def build_debian_changelog(changelogpath, destfile, pkgname, from_version=None,
distribution='precise', fix_version=None):
def build_debian_changelog(
changelogpath,
destfile,
pkgname,
from_version=None,
distribution="precise",
fix_version=None,
):
"""Builds a debian changelog out of a YAML changelog.
Use fix_version to patch the top changelog to that version (if, for example, there was a
packaging error and you need to quickly fix it)
"""
def desc2list(desc):
# We take each item, enumerated with the '*' character, and transform it into a list.
desc = desc.replace('\n', ' ')
desc = desc.replace(' ', ' ')
result = desc.split('*')
desc = desc.replace("\n", " ")
desc = desc.replace(" ", " ")
result = desc.split("*")
return [s.strip() for s in result if s.strip()]
ENTRY_MODEL = "{pkg} ({version}-1) {distribution}; urgency=low\n\n{changes}\n -- Virgil Dupras <hsoft@hardcoded.net> {date}\n\n"
ENTRY_MODEL = (
"{pkg} ({version}-1) {distribution}; urgency=low\n\n{changes}\n "
"-- Virgil Dupras <hsoft@hardcoded.net> {date}\n\n"
)
CHANGE_MODEL = " * {description}\n"
changelogs = read_changelog_file(changelogpath)
if from_version:
# We only want logs from a particular version
for index, log in enumerate(changelogs):
if log['version'] == from_version:
changelogs = changelogs[:index+1]
if log["version"] == from_version:
changelogs = changelogs[: index + 1]
break
if fix_version:
changelogs[0]['version'] = fix_version
changelogs[0]["version"] = fix_version
rendered_logs = []
for log in changelogs:
version = log['version']
logdate = log['date']
desc = log['description']
rendered_date = logdate.strftime('%a, %d %b %Y 00:00:00 +0000')
version = log["version"]
logdate = log["date"]
desc = log["description"]
rendered_date = logdate.strftime("%a, %d %b %Y 00:00:00 +0000")
rendered_descs = [CHANGE_MODEL.format(description=d) for d in desc2list(desc)]
changes = ''.join(rendered_descs)
rendered_log = ENTRY_MODEL.format(pkg=pkgname, version=version, changes=changes,
date=rendered_date, distribution=distribution)
changes = "".join(rendered_descs)
rendered_log = ENTRY_MODEL.format(
pkg=pkgname,
version=version,
changes=changes,
date=rendered_date,
distribution=distribution,
)
rendered_logs.append(rendered_log)
result = ''.join(rendered_logs)
fp = open(destfile, 'w')
result = "".join(rendered_logs)
fp = open(destfile, "w")
fp.write(result)
fp.close()
re_changelog_header = re.compile(r'=== ([\d.b]*) \(([\d\-]*)\)')
re_changelog_header = re.compile(r"=== ([\d.b]*) \(([\d\-]*)\)")
def read_changelog_file(filename):
def iter_by_three(it):
while True:
@@ -283,25 +344,31 @@ def read_changelog_file(filename):
return
yield version, date, description
with open(filename, 'rt', encoding='utf-8') as fp:
with open(filename, "rt", encoding="utf-8") as fp:
contents = fp.read()
splitted = re_changelog_header.split(contents)[1:] # the first item is empty
splitted = re_changelog_header.split(contents)[1:] # the first item is empty
# splitted = [version1, date1, desc1, version2, date2, ...]
result = []
for version, date_str, description in iter_by_three(iter(splitted)):
date = datetime.strptime(date_str, '%Y-%m-%d').date()
d = {'date': date, 'date_str': date_str, 'version': version, 'description': description.strip()}
date = datetime.strptime(date_str, "%Y-%m-%d").date()
d = {
"date": date,
"date_str": date_str,
"version": version,
"description": description.strip(),
}
result.append(d)
return result
class OSXAppStructure:
def __init__(self, dest):
self.dest = dest
self.contents = op.join(dest, 'Contents')
self.macos = op.join(self.contents, 'MacOS')
self.resources = op.join(self.contents, 'Resources')
self.frameworks = op.join(self.contents, 'Frameworks')
self.infoplist = op.join(self.contents, 'Info.plist')
self.contents = op.join(dest, "Contents")
self.macos = op.join(self.contents, "MacOS")
self.resources = op.join(self.contents, "Resources")
self.frameworks = op.join(self.contents, "Frameworks")
self.infoplist = op.join(self.contents, "Info.plist")
def create(self, infoplist):
ensure_empty_folder(self.dest)
@@ -309,11 +376,11 @@ class OSXAppStructure:
os.mkdir(self.resources)
os.mkdir(self.frameworks)
copy(infoplist, self.infoplist)
open(op.join(self.contents, 'PkgInfo'), 'wt').write("APPLxxxx")
open(op.join(self.contents, "PkgInfo"), "wt").write("APPLxxxx")
def copy_executable(self, executable):
info = plistlib.readPlist(self.infoplist)
self.executablename = info['CFBundleExecutable']
self.executablename = info["CFBundleExecutable"]
self.executablepath = op.join(self.macos, self.executablename)
copy(executable, self.executablepath)
@@ -329,8 +396,14 @@ class OSXAppStructure:
copy(path, framework_dest)
def create_osx_app_structure(dest, executable, infoplist, resources=None, frameworks=None,
symlink_resources=False):
def create_osx_app_structure(
dest,
executable,
infoplist,
resources=None,
frameworks=None,
symlink_resources=False,
):
# `dest`: A path to the destination .app folder
# `executable`: the path of the executable file that goes in "MacOS"
# `infoplist`: The path to your Info.plist file.
@@ -343,13 +416,14 @@ def create_osx_app_structure(dest, executable, infoplist, resources=None, framew
app.copy_resources(*resources, use_symlinks=symlink_resources)
app.copy_frameworks(*frameworks)
class OSXFrameworkStructure:
def __init__(self, dest):
self.dest = dest
self.contents = op.join(dest, 'Versions', 'A')
self.resources = op.join(self.contents, 'Resources')
self.headers = op.join(self.contents, 'Headers')
self.infoplist = op.join(self.resources, 'Info.plist')
self.contents = op.join(dest, "Versions", "A")
self.resources = op.join(self.contents, "Resources")
self.headers = op.join(self.contents, "Headers")
self.infoplist = op.join(self.resources, "Info.plist")
self._update_executable_path()
def _update_executable_path(self):
@@ -357,7 +431,7 @@ class OSXFrameworkStructure:
self.executablename = self.executablepath = None
return
info = plistlib.readPlist(self.infoplist)
self.executablename = info['CFBundleExecutable']
self.executablename = info["CFBundleExecutable"]
self.executablepath = op.join(self.contents, self.executablename)
def create(self, infoplist):
@@ -371,10 +445,10 @@ class OSXFrameworkStructure:
def create_symlinks(self):
# Only call this after create() and copy_executable()
rel = lambda path: op.relpath(path, self.dest)
os.symlink('A', op.join(self.dest, 'Versions', 'Current'))
os.symlink("A", op.join(self.dest, "Versions", "Current"))
os.symlink(rel(self.executablepath), op.join(self.dest, self.executablename))
os.symlink(rel(self.headers), op.join(self.dest, 'Headers'))
os.symlink(rel(self.resources), op.join(self.dest, 'Resources'))
os.symlink(rel(self.headers), op.join(self.dest, "Headers"))
os.symlink(rel(self.resources), op.join(self.dest, "Resources"))
def copy_executable(self, executable):
copy(executable, self.executablepath)
@@ -393,23 +467,28 @@ class OSXFrameworkStructure:
def copy_embeddable_python_dylib(dst):
runtime = op.join(sysconfig.get_config_var('PYTHONFRAMEWORKPREFIX'), sysconfig.get_config_var('LDLIBRARY'))
filedest = op.join(dst, 'Python')
runtime = op.join(
sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX"),
sysconfig.get_config_var("LDLIBRARY"),
)
filedest = op.join(dst, "Python")
shutil.copy(runtime, filedest)
os.chmod(filedest, 0o774) # We need write permission to use install_name_tool
cmd = 'install_name_tool -id @rpath/Python %s' % filedest
os.chmod(filedest, 0o774) # We need write permission to use install_name_tool
cmd = "install_name_tool -id @rpath/Python %s" % filedest
print_and_do(cmd)
def collect_stdlib_dependencies(script, dest_folder, extra_deps=None):
sysprefix = sys.prefix # could be a virtualenv
real_lib_prefix = sysconfig.get_config_var('LIBDEST')
sysprefix = sys.prefix # could be a virtualenv
real_lib_prefix = sysconfig.get_config_var("LIBDEST")
def is_stdlib_path(path):
# A module path is only a stdlib path if it's in either sys.prefix or
# sysconfig.get_config_var('prefix') (the 2 are different if we are in a virtualenv) and if
# there's no "site-package in the path.
if not path:
return False
if 'site-package' in path:
if "site-package" in path:
return False
if not (path.startswith(sysprefix) or path.startswith(real_lib_prefix)):
return False
@@ -425,13 +504,17 @@ def collect_stdlib_dependencies(script, dest_folder, extra_deps=None):
relpath = op.relpath(p, real_lib_prefix)
elif p.startswith(sysprefix):
relpath = op.relpath(p, sysprefix)
assert relpath.startswith('lib/python3.') # we want to get rid of that lib/python3.x part
relpath = relpath[len('lib/python3.X/'):]
assert relpath.startswith(
"lib/python3."
) # we want to get rid of that lib/python3.x part
relpath = relpath[len("lib/python3.X/") :]
else:
raise AssertionError()
if relpath.startswith('lib-dynload'): # We copy .so files in lib-dynload directly in our dest
relpath = relpath[len('lib-dynload/'):]
if relpath.startswith('encodings') or relpath.startswith('distutils'):
if relpath.startswith(
"lib-dynload"
): # We copy .so files in lib-dynload directly in our dest
relpath = relpath[len("lib-dynload/") :]
if relpath.startswith("encodings") or relpath.startswith("distutils"):
# We force their inclusion later.
continue
dest_path = op.join(dest_folder, relpath)
@@ -440,34 +523,47 @@ def collect_stdlib_dependencies(script, dest_folder, extra_deps=None):
# stringprep is used by encodings.
# We use real_lib_prefix with distutils because virtualenv messes with it and we need to refer
# to the original distutils folder.
FORCED_INCLUSION = ['encodings', 'stringprep', op.join(real_lib_prefix, 'distutils')]
FORCED_INCLUSION = [
"encodings",
"stringprep",
op.join(real_lib_prefix, "distutils"),
]
if extra_deps:
FORCED_INCLUSION += extra_deps
copy_packages(FORCED_INCLUSION, dest_folder)
# There's a couple of rather big exe files in the distutils folder that we absolutely don't
# need. Remove them.
delete_files_with_pattern(op.join(dest_folder, 'distutils'), '*.exe')
delete_files_with_pattern(op.join(dest_folder, "distutils"), "*.exe")
# And, finally, create an empty "site.py" that Python needs around on startup.
open(op.join(dest_folder, 'site.py'), 'w').close()
open(op.join(dest_folder, "site.py"), "w").close()
def fix_qt_resource_file(path):
# pyrcc5 under Windows, if the locale is non-english, can produce a source file with a date
# containing accented characters. If it does, the encoding is wrong and it prevents the file
# from being correctly frozen by cx_freeze. To work around that, we open the file, strip all
# comments, and save.
with open(path, 'rb') as fp:
with open(path, "rb") as fp:
contents = fp.read()
lines = contents.split(b'\n')
lines = [l for l in lines if not l.startswith(b'#')]
with open(path, 'wb') as fp:
fp.write(b'\n'.join(lines))
lines = contents.split(b"\n")
lines = [l for l in lines if not l.startswith(b"#")]
with open(path, "wb") as fp:
fp.write(b"\n".join(lines))
def build_cocoa_ext(extname, dest, source_files, extra_frameworks=(), extra_includes=()):
def build_cocoa_ext(
extname, dest, source_files, extra_frameworks=(), extra_includes=()
):
extra_link_args = ["-framework", "CoreFoundation", "-framework", "Foundation"]
for extra in extra_frameworks:
extra_link_args += ['-framework', extra]
ext = Extension(extname, source_files, extra_link_args=extra_link_args, include_dirs=extra_includes)
setup(script_args=['build_ext', '--inplace'], ext_modules=[ext])
extra_link_args += ["-framework", extra]
ext = Extension(
extname,
source_files,
extra_link_args=extra_link_args,
include_dirs=extra_includes,
)
setup(script_args=["build_ext", "--inplace"], ext_modules=[ext])
# Our problem here is to get the fully qualified filename of the resulting .so but I couldn't
# find a documented way to do so. The only thing I could find is this below :(
fn = ext._file_name