From 46521c8af11cdf072a7fb0af8be70c528bd3261e Mon Sep 17 00:00:00 2001 From: Andrew Senetar Date: Fri, 13 Jan 2023 00:05:47 -0600 Subject: [PATCH] feat: Add migration for picture cache db - Add migration (just delete db and change to new schema) for picture cache following the same sort of strategy as the file digest cache - Rename mtime column to mtime_ns to match file cache for consistency --- core/pe/cache_sqlite.py | 52 +++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/core/pe/cache_sqlite.py b/core/pe/cache_sqlite.py index bf9f7c5b..4cb3c588 100644 --- a/core/pe/cache_sqlite.py +++ b/core/pe/cache_sqlite.py @@ -15,6 +15,14 @@ from core.pe.cache import bytes_to_colors, colors_to_bytes class SqliteCache: """A class to cache picture blocks in a sqlite backend.""" + schema_version = 1 + schema_version_description = "Changed from string to bytes for blocks." + + create_table_query = "CREATE TABLE IF NOT EXISTS pictures(path TEXT, mtime_ns INTEGER, blocks BLOB)" + create_index_query = "CREATE INDEX IF NOT EXISTS idx_path on pictures (path)" + drop_table_query = "DROP TABLE IF EXISTS pictures" + drop_index_query = "DROP INDEX IF EXISTS idx_path" + def __init__(self, db=":memory:", readonly=False): # readonly is not used in the sqlite version of the cache self.dbname = db @@ -62,9 +70,9 @@ class SqliteCache: else: mtime = 0 if path_str in self: - sql = "update pictures set blocks = ?, mtime = ? where path = ?" + sql = "update pictures set blocks = ?, mtime_ns = ? where path = ?" else: - sql = "insert into pictures(blocks,mtime,path) values(?,?,?)" + sql = "insert into pictures(blocks,mtime_ns,path) values(?,?,?)" try: self.con.execute(sql, [blocks, mtime, path_str]) except sqlite.OperationalError: @@ -73,18 +81,9 @@ class SqliteCache: logging.warning("DatabaseError while setting value for key %r: %s", path_str, str(e)) def _create_con(self, second_try=False): - def create_tables(): - logging.debug("Creating picture cache tables.") - self.con.execute("drop table if exists pictures") - self.con.execute("drop index if exists idx_path") - self.con.execute("create table pictures(path TEXT, mtime INTEGER, blocks BLOB)") - self.con.execute("create index idx_path on pictures (path)") - - self.con = sqlite.connect(self.dbname, isolation_level=None) try: - self.con.execute("select path, mtime, blocks from pictures where 1=2") - except sqlite.OperationalError: # new db - create_tables() + self.con = sqlite.connect(self.dbname, isolation_level=None) + self._check_upgrade() except sqlite.DatabaseError as e: # corrupted db if second_try: raise # Something really strange is happening @@ -93,6 +92,25 @@ class SqliteCache: os.remove(self.dbname) self._create_con(second_try=True) + def _check_upgrade(self) -> None: + with self.con as conn: + has_schema = conn.execute( + "SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'" + ).fetchall() + version = None + if has_schema: + version = conn.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0] + else: + conn.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)") + if version != self.schema_version: + conn.execute(self.drop_table_query) + conn.execute( + "INSERT OR REPLACE INTO schema_version VALUES (:version, :description)", + {"version": self.schema_version, "description": self.schema_version_description}, + ) + conn.execute(self.create_table_query) + conn.execute(self.create_index_query) + def clear(self): self.close() if self.dbname != ":memory:": @@ -129,12 +147,12 @@ class SqliteCache: the db. """ todelete = [] - sql = "select rowid, path, mtime from pictures" + sql = "select rowid, path, mtime_ns from pictures" cur = self.con.execute(sql) - for rowid, path_str, mtime in cur: - if mtime and op.exists(path_str): + for rowid, path_str, mtime_ns in cur: + if mtime_ns and op.exists(path_str): picture_mtime = os.stat(path_str).st_mtime - if int(picture_mtime) <= mtime: + if int(picture_mtime) <= mtime_ns: # not outdated continue todelete.append(rowid)