From 334f4dd2ae7f32753a4ea34cd3079235b5fc7e31 Mon Sep 17 00:00:00 2001 From: Virgil Dupras Date: Wed, 8 Jun 2016 12:23:10 -0400 Subject: [PATCH] Increase md5 reading buffer to 1mb This makes md5 computing faster without using too much memory. --- core/fs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/fs.py b/core/fs.py index 96ee93a7..4f2c95cf 100644 --- a/core/fs.py +++ b/core/fs.py @@ -121,7 +121,10 @@ class File: try: fp = self.path.open('rb') md5 = hashlib.md5() - CHUNK_SIZE = 8192 + # The goal here is to not run out of memory on really big files. However, the chunk + # size has to be large enough so that the python loop isn't too costly in terms of + # CPU. + CHUNK_SIZE = 1024 * 1024 # 1 mb filedata = fp.read(CHUNK_SIZE) while filedata: md5.update(filedata)