Improved source packaging and bootstrapping

.gitignore

@@ -11,6 +11,7 @@ dist
 install
 installer_tmp-cache
 env
+/deps
 cocoa/autogen
 
 /run.py

README.md

@@ -76,11 +76,6 @@ Then, you can install pip requirements in your virtualenv:
     
 ([osx|win] depends, of course, on your platform. On other platforms, just use requirements.txt).
 
-**Temporary problem:** The author of polib, which is a dependency here,
-[hasn't yet uploaded packages to PyPI][polib], which will make the command fail if you use
-pip >= 1.5. You'll have to add `--allow-external polib --allow-unverified polib` to your command.
-This is, hopefully, a temporary situation.
-
 ## Actual building and running
 
 With your virtualenv activated, you can build and run dupeGuru with these commands:
@@ -101,4 +96,3 @@ You can also package dupeGuru into an installable package with:
 [pyqt]: http://www.riverbankcomputing.com
 [cxfreeze]: http://cx-freeze.sourceforge.net/
 [advinst]: http://www.advancedinstaller.com
-[polib]: https://bitbucket.org/izi/polib/issue/42

bootstrap.sh

@@ -2,30 +2,36 @@
 
 command -v python3 -m venv >/dev/null 2>&1 || { echo >&2 "Python 3.3 required. Install it and try again. Aborting"; exit 1; }
 
+if [ -d "deps" ]; then
+    # We have a collection of dependencies in our source package. We might as well use it instead
+    # of downloading it from PyPI.
+    PIPARGS="--no-index --find-links=deps"
+fi
+
 if [ ! -d "env" ]; then
     echo "No virtualenv. Creating one"
-    command -v curl >/dev/null 2>&1 || { echo >&2 "curl required. Install it and try again. Aborting"; exit 1; }
     python3 -m venv --system-site-packages env
     source env/bin/activate
-    curl https://bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py | python
-    easy_install pip
+    # With a new venv, we want to force (without checking if it exists first) installing a venv pip
+    # or else we'll end up with the system one.
+    python get-pip.py $PIPARGS --force-reinstall
 else
     echo "There's already an env. Activating it"
     source env/bin/activate
 fi
 
+command -v pip
+if [ $? -ne 0 ]; then
+    echo "pip not installed. Installing."
+    python get-pip.py $PIPARGS
+fi
+
 echo "Installing pip requirements"
 if [ "$(uname)" == "Darwin" ]; then
-    pip install -r requirements-osx.txt --allow-external polib --allow-unverified polib
+    pip install $PIPARGS -r requirements-osx.txt
 else
     python3 -c "import PyQt5" >/dev/null 2>&1 || { echo >&2 "PyQt 5.1+ required. Install it and try again. Aborting"; exit 1; }
-    if [ -d "deps" ]; then
-        # We have a collection of dependencies in our source package. We might as well use it instead
-        # of downloading it from PyPI.
-        pip install --no-index --find-links=deps -r requirements.txt
-    else
-        pip install -r requirements.txt --allow-external polib --allow-unverified polib
-    fi
+    pip install $PIPARGS -r requirements.txt
 fi
 
 echo "Bootstrapping complete! You can now configure, build and run dupeGuru with:"

download_deps.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# The goal here is to have a folder with all packages needed as dependencies by the project. The
+# obvious solution is "pip install --download deps -r requirements.txt", but this thing doesn't
+# follow sub-dependencies. The 2nd obvious solution would be to use the result of a `pip freeze`
+# instead of requirements.txt, but this command outputs everything on the system, which isn't cool.
+# So, what about "pip freeze -l"? That would work, unless one of the dependencies is installed
+# system-wide (Sphinx often is). We can't disable system site packages because we need PyQt, which
+# is always installed globally.
+
+# So, what we do here is that we create a brand new venv just for dependencies download, which
+# we'll pip freeze.
+
+rm -rf deps
+rm -rf depsenv
+mkdir deps
+
+python3 -m venv depsenv
+source depsenv/bin/activate
+python get-pip.py
+pip install -r requirements.txt
+pip freeze -l > deps/requirements.freeze
+pip install --download=deps -r deps/requirements.freeze setuptools pip
+deactivate
+rm -rf depsenv

package.py

@@ -155,7 +155,7 @@ def package_arch(edition):
 def package_source_tgz(edition):
     if not op.exists('deps'):
         print("Downloading PyPI dependencies")
-        print_and_do('pip install --download=deps -r requirements.txt')
+        print_and_do('./download_deps.sh')
     print("Creating git archive")
     app_version = get_module_version('core_{}'.format(edition))
     name = 'dupeguru-{}-src-{}.tar'.format(edition, app_version)