From 096e2bb78a609195fe9ede35fd3d7b53b650d5e3 Mon Sep 17 00:00:00 2001 From: Virgil Dupras Date: Sun, 20 Oct 2013 16:01:27 -0400 Subject: [PATCH 1/2] Updated hscommon --- hscommon/currency.py | 48 +++++++++++++++++++++++++++++++++++++++----- hscommon/util.py | 11 ++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/hscommon/currency.py b/hscommon/currency.py index 9ffcd313..e0b7a354 100644 --- a/hscommon/currency.py +++ b/hscommon/currency.py @@ -14,6 +14,7 @@ from queue import Queue, Empty from . import io from .path import Path +from .util import iterdaterange class Currency: all = [] @@ -270,6 +271,9 @@ EUR = Currency(code='EUR') class CurrencyNotSupportedException(Exception): """The current exchange rate provider doesn't support the requested currency.""" +def date2str(date): + return '%d%02d%02d' % (date.year, date.month, date.day) + class RatesDB: """Stores exchange rates for currencies. @@ -329,12 +333,35 @@ class RatesDB: return row[0] return seek('<=', 'desc') or seek('>=', '') or Currency(currency_code).latest_rate + def _ensure_filled(self, date_start, date_end, currency_code): + """Make sure that the cache contains *something* for each of the dates in the range. + + Sometimes, our provider doesn't return us the range we sought. When it does, it usually + means that it never will and to avoid repeatedly querying those ranges forever, we have to + fill them. We use the closest rate for this. + """ + # We don't want to fill today, because we want to repeatedly fetch that one until the + # provider gives it to us. + if date_end >= date.today(): + date_end = date.today() - timedelta(1) + sql = "select rate from rates where date = ? and currency = ?" + for curdate in iterdaterange(date_start, date_end): + cur = self._execute(sql, [date2str(curdate), currency_code]) + if cur.fetchone() is None: + nearby_rate = self._seek_value_in_CAD(date2str(curdate), currency_code) + self.set_CAD_value(curdate, currency_code, nearby_rate) + logging.debug("Filled currency void for %s at %s (value: %2.2f)", currency_code, curdate, nearby_rate) + def _save_fetched_rates(self): while True: try: - rates, currency = self._fetched_values.get_nowait() + rates, currency, fetch_start, fetch_end = self._fetched_values.get_nowait() + logging.debug("Saving %d rates for the currency %s", len(rates), currency) for rate_date, rate in rates: + logging.debug("Saving rate %2.2f for %s", rate, rate_date) self.set_CAD_value(rate_date, currency, rate) + self._ensure_filled(fetch_start, fetch_end, currency) + logging.debug("Finished saving rates for currency %s", currency) except Empty: break @@ -374,7 +401,7 @@ class RatesDB: else: value2 = self._cache.get((date, currency2_code)) if value1 is None or value2 is None: - str_date = '%d%02d%02d' % (date.year, date.month, date.day) + str_date = date2str(date) if value1 is None: value1 = self._seek_value_in_CAD(str_date, currency1_code) self._cache[(date, currency1_code)] = value1 @@ -388,7 +415,7 @@ class RatesDB: # we must clear the whole cache because there might be other dates affected by this change # (dates when the currency server has no rates). self.clear_cache() - str_date = '%d%02d%02d' % (date.year, date.month, date.day) + str_date = date2str(date) sql = "replace into rates(date, currency, rate) values(?, ?, ?)" self._execute(sql, [str_date, currency_code, value]) self.con.commit() @@ -419,6 +446,7 @@ class RatesDB: """ def do(): for currency, fetch_start, fetch_end in currencies_and_range: + logging.debug("Fetching rates for %s for date range %s to %s", currency, fetch_start, fetch_end) for rate_provider in self._rate_providers: try: values = rate_provider(currency, fetch_start, fetch_end) @@ -426,7 +454,11 @@ class RatesDB: continue else: if values: - self._fetched_values.put((values, currency)) + self._fetched_values.put((values, currency, fetch_start, fetch_end)) + logging.debug("Fetching successful!") + break + else: + logging.debug("Fetching failed!") currencies_and_range = [] for currency in currencies: @@ -437,7 +469,9 @@ class RatesDB: except KeyError: cached_range = self.date_range(currency) range_start = start_date - range_end = date.today() + # Don't try to fetch today's rate, it's never there and results in useless server + # hitting. + range_end = date.today() - timedelta(1) if cached_range is not None: cached_start, cached_end = cached_range if range_start >= cached_start: @@ -446,6 +480,10 @@ class RatesDB: else: # Make a backward fetch range_end = cached_start - timedelta(days=1) + # We don't want to fetch ranges that are too big. It can cause various problems, such + # as hangs. We prefer to take smaller bites. + if (range_end - range_start).days > 30: + range_start = range_end - timedelta(days=30) if range_start <= range_end: currencies_and_range.append((currency, range_start, range_end)) self._fetched_ranges[currency] = (start_date, date.today()) diff --git a/hscommon/util.py b/hscommon/util.py index 8d562d89..55858b1e 100644 --- a/hscommon/util.py +++ b/hscommon/util.py @@ -13,6 +13,7 @@ import re from math import ceil import glob import shutil +from datetime import timedelta from . import io from .path import Path @@ -254,6 +255,16 @@ def multi_replace(s, replace_from, replace_to=''): s = s.replace(r_from, r_to) return s +#--- Date related + +def iterdaterange(start, end): + """Yields every day between ``start`` and ``end``. + """ + date = start + while date <= end: + yield date + date += timedelta(1) + #--- Files related def modified_after(first_path, second_path): From a5633277234b71515eaef7c3c8b1a1287e07be93 Mon Sep 17 00:00:00 2001 From: Virgil Dupras Date: Sun, 20 Oct 2013 16:01:59 -0400 Subject: [PATCH 2/2] Updated cocoalib --- cocoalib/controllers/HSOutline.m | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cocoalib/controllers/HSOutline.m b/cocoalib/controllers/HSOutline.m index 41c52ed8..a39a566d 100644 --- a/cocoalib/controllers/HSOutline.m +++ b/cocoalib/controllers/HSOutline.m @@ -101,7 +101,13 @@ http://www.hardcoded.net/licenses/bsd_license [[self view] setDelegate:nil]; [[self view] reloadData]; [[self view] setDelegate:self]; - [oldRetainer release]; + /* Item retainer and releasing + + In theory, [oldRetainer release] should work, but in practice, doing so causes occasional + crashes during drag & drop, which I guess keep the reference of an item a bit longer than it + should. This is why we autorelease here. See #354. + */ + [oldRetainer autorelease]; [self updateSelection]; }