# Copyright 2016 Virgil Dupras # # This software is licensed under the "GPLv3" License as described in the "LICENSE" file, # which should be included with this package. The terms are also available at # http://www.gnu.org/licenses/gpl-3.0.html import os import os.path as op import logging import sqlite3 as sqlite from core.pe.cache import bytes_to_colors, colors_to_bytes class SqliteCache: """A class to cache picture blocks in a sqlite backend.""" schema_version = 1 schema_version_description = "Changed from string to bytes for blocks." create_table_query = "CREATE TABLE IF NOT EXISTS pictures(path TEXT, mtime_ns INTEGER, blocks BLOB)" create_index_query = "CREATE INDEX IF NOT EXISTS idx_path on pictures (path)" drop_table_query = "DROP TABLE IF EXISTS pictures" drop_index_query = "DROP INDEX IF EXISTS idx_path" def __init__(self, db=":memory:", readonly=False): # readonly is not used in the sqlite version of the cache self.dbname = db self.con = None self._create_con() def __contains__(self, key): sql = "select count(*) from pictures where path = ?" result = self.con.execute(sql, [key]).fetchall() return result[0][0] > 0 def __delitem__(self, key): if key not in self: raise KeyError(key) sql = "delete from pictures where path = ?" self.con.execute(sql, [key]) # Optimized def __getitem__(self, key): if isinstance(key, int): sql = "select blocks from pictures where rowid = ?" else: sql = "select blocks from pictures where path = ?" result = self.con.execute(sql, [key]).fetchone() if result: result = bytes_to_colors(result[0]) return result else: raise KeyError(key) def __iter__(self): sql = "select path from pictures" result = self.con.execute(sql) return (row[0] for row in result) def __len__(self): sql = "select count(*) from pictures" result = self.con.execute(sql).fetchall() return result[0][0] def __setitem__(self, path_str, blocks): blocks = colors_to_bytes(blocks) if op.exists(path_str): mtime = int(os.stat(path_str).st_mtime) else: mtime = 0 if path_str in self: sql = "update pictures set blocks = ?, mtime_ns = ? where path = ?" else: sql = "insert into pictures(blocks,mtime_ns,path) values(?,?,?)" try: self.con.execute(sql, [blocks, mtime, path_str]) except sqlite.OperationalError: logging.warning("Picture cache could not set value for key %r", path_str) except sqlite.DatabaseError as e: logging.warning("DatabaseError while setting value for key %r: %s", path_str, str(e)) def _create_con(self, second_try=False): try: self.con = sqlite.connect(self.dbname, isolation_level=None) self._check_upgrade() except sqlite.DatabaseError as e: # corrupted db if second_try: raise # Something really strange is happening logging.warning("Could not create picture cache because of an error: %s", str(e)) self.con.close() os.remove(self.dbname) self._create_con(second_try=True) def _check_upgrade(self) -> None: with self.con as conn: has_schema = conn.execute( "SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'" ).fetchall() version = None if has_schema: version = conn.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0] else: conn.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)") if version != self.schema_version: conn.execute(self.drop_table_query) conn.execute( "INSERT OR REPLACE INTO schema_version VALUES (:version, :description)", {"version": self.schema_version, "description": self.schema_version_description}, ) conn.execute(self.create_table_query) conn.execute(self.create_index_query) def clear(self): self.close() if self.dbname != ":memory:": os.remove(self.dbname) self._create_con() def close(self): if self.con is not None: self.con.close() self.con = None def filter(self, func): to_delete = [key for key in self if not func(key)] for key in to_delete: del self[key] def get_id(self, path): sql = "select rowid from pictures where path = ?" result = self.con.execute(sql, [path]).fetchone() if result: return result[0] else: raise ValueError(path) def get_multiple(self, rowids): sql = "select rowid, blocks from pictures where rowid in (%s)" % ",".join(map(str, rowids)) cur = self.con.execute(sql) return ((rowid, bytes_to_colors(blocks)) for rowid, blocks in cur) def purge_outdated(self): """Go through the cache and purge outdated records. A record is outdated if the picture doesn't exist or if its mtime is greater than the one in the db. """ todelete = [] sql = "select rowid, path, mtime_ns from pictures" cur = self.con.execute(sql) for rowid, path_str, mtime_ns in cur: if mtime_ns and op.exists(path_str): picture_mtime = os.stat(path_str).st_mtime if int(picture_mtime) <= mtime_ns: # not outdated continue todelete.append(rowid) if todelete: sql = "delete from pictures where rowid in (%s)" % ",".join(map(str, todelete)) self.con.execute(sql)