@ -9,12 +9,20 @@ import os.path as op
import logging
import sqlite3 as sqlite
from core . pe . cache import string _to_colors, colors_to_ string
from core . pe . cache import byte s_to_colors, colors_to_ byte s
class SqliteCache :
""" A class to cache picture blocks in a sqlite backend. """
schema_version = 1
schema_version_description = " Changed from string to bytes for blocks. "
create_table_query = " CREATE TABLE IF NOT EXISTS pictures(path TEXT, mtime_ns INTEGER, blocks BLOB) "
create_index_query = " CREATE INDEX IF NOT EXISTS idx_path on pictures (path) "
drop_table_query = " DROP TABLE IF EXISTS pictures "
drop_index_query = " DROP INDEX IF EXISTS idx_path "
def __init__ ( self , db = " :memory: " , readonly = False ) :
# readonly is not used in the sqlite version of the cache
self . dbname = db
@ -40,7 +48,7 @@ class SqliteCache:
sql = " select blocks from pictures where path = ? "
result = self . con . execute ( sql , [ key ] ) . fetchone ( )
if result :
result = string _to_colors( result [ 0 ] )
result = byte s_to_colors( result [ 0 ] )
return result
else :
raise KeyError ( key )
@ -56,15 +64,15 @@ class SqliteCache:
return result [ 0 ] [ 0 ]
def __setitem__ ( self , path_str , blocks ) :
blocks = colors_to_ string ( blocks )
blocks = colors_to_ byte s( blocks )
if op . exists ( path_str ) :
mtime = int ( os . stat ( path_str ) . st_mtime )
else :
mtime = 0
if path_str in self :
sql = " update pictures set blocks = ?, mtime = ? where path = ?"
sql = " update pictures set blocks = ?, mtime _ns = ? where path = ?"
else :
sql = " insert into pictures(blocks,mtime ,path) values(?,?,?)"
sql = " insert into pictures(blocks,mtime _ns ,path) values(?,?,?)"
try :
self . con . execute ( sql , [ blocks , mtime , path_str ] )
except sqlite . OperationalError :
@ -73,18 +81,9 @@ class SqliteCache:
logging . warning ( " DatabaseError while setting value for key %r : %s " , path_str , str ( e ) )
def _create_con ( self , second_try = False ) :
def create_tables ( ) :
logging . debug ( " Creating picture cache tables. " )
self . con . execute ( " drop table if exists pictures " )
self . con . execute ( " drop index if exists idx_path " )
self . con . execute ( " create table pictures(path TEXT, mtime INTEGER, blocks TEXT) " )
self . con . execute ( " create index idx_path on pictures (path) " )
self . con = sqlite . connect ( self . dbname , isolation_level = None )
try :
self . con . execute ( " select path, mtime, blocks from pictures where 1=2 " )
except sqlite . OperationalError : # new db
create_tables ( )
self . con = sqlite . connect ( self . dbname , isolation_level = None )
self . _check_upgrade ( )
except sqlite . DatabaseError as e : # corrupted db
if second_try :
raise # Something really strange is happening
@ -93,6 +92,25 @@ class SqliteCache:
os . remove ( self . dbname )
self . _create_con ( second_try = True )
def _check_upgrade ( self ) - > None :
with self . con as conn :
has_schema = conn . execute (
" SELECT NAME FROM sqlite_master WHERE type= ' table ' AND name= ' schema_version ' "
) . fetchall ( )
version = None
if has_schema :
version = conn . execute ( " SELECT version FROM schema_version ORDER BY version DESC " ) . fetchone ( ) [ 0 ]
else :
conn . execute ( " CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT) " )
if version != self . schema_version :
conn . execute ( self . drop_table_query )
conn . execute (
" INSERT OR REPLACE INTO schema_version VALUES (:version, :description) " ,
{ " version " : self . schema_version , " description " : self . schema_version_description } ,
)
conn . execute ( self . create_table_query )
conn . execute ( self . create_index_query )
def clear ( self ) :
self . close ( )
if self . dbname != " :memory: " :
@ -120,7 +138,7 @@ class SqliteCache:
def get_multiple ( self , rowids ) :
sql = " select rowid, blocks from pictures where rowid in ( %s ) " % " , " . join ( map ( str , rowids ) )
cur = self . con . execute ( sql )
return ( ( rowid , string _to_colors( blocks ) ) for rowid , blocks in cur )
return ( ( rowid , byte s_to_colors( blocks ) ) for rowid , blocks in cur )
def purge_outdated ( self ) :
""" Go through the cache and purge outdated records.
@ -129,12 +147,12 @@ class SqliteCache:
the db .
"""
todelete = [ ]
sql = " select rowid, path, mtime from pictures"
sql = " select rowid, path, mtime _ns from pictures"
cur = self . con . execute ( sql )
for rowid , path_str , mtime in cur :
if mtime and op . exists ( path_str ) :
for rowid , path_str , mtime _ns in cur :
if mtime _ns and op . exists ( path_str ) :
picture_mtime = os . stat ( path_str ) . st_mtime
if int ( picture_mtime ) < = mtime :
if int ( picture_mtime ) < = mtime _ns :
# not outdated
continue
todelete . append ( rowid )