arsenetar
/
dupeguru
mirror of https://github.com/arsenetar/dupeguru.git
1
0
Fork 0
Code Releases Wiki Activity

feat: Add migration for picture cache db 

- Add migration (just delete db and change to new schema) for picture
  cache following the same sort of strategy as the file digest cache
- Rename mtime column to mtime_ns to match file cache for consistency
This commit is contained in:
Andrew Senetar 2023-01-13 00:05:47 -06:00
parent 8c5e18b980
commit 46521c8af1
Signed by: arsenetar
GPG Key ID: C63300DCE48AB2F1
1 changed files with 35 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
core/pe/cache_sqlite.py
View File

@ -15,6 +15,14 @@ from core.pe.cache import bytes_to_colors, colors_to_bytes
class SqliteCache:
"""A class to cache picture blocks in a sqlite backend."""
schema_version = 1
schema_version_description = "Changed from string to bytes for blocks."
create_table_query = "CREATE TABLE IF NOT EXISTS pictures(path TEXT, mtime_ns INTEGER, blocks BLOB)"
create_index_query = "CREATE INDEX IF NOT EXISTS idx_path on pictures (path)"
drop_table_query = "DROP TABLE IF EXISTS pictures"
drop_index_query = "DROP INDEX IF EXISTS idx_path"
def __init__(self, db=":memory:", readonly=False):
# readonly is not used in the sqlite version of the cache
self.dbname = db
@ -62,9 +70,9 @@ class SqliteCache:
else:
mtime = 0
if path_str in self:
sql = "update pictures set blocks = ?, mtime = ? where path = ?"
sql = "update pictures set blocks = ?, mtime_ns = ? where path = ?"
else:
sql = "insert into pictures(blocks,mtime,path) values(?,?,?)"
sql = "insert into pictures(blocks,mtime_ns,path) values(?,?,?)"
try:
self.con.execute(sql, [blocks, mtime, path_str])
except sqlite.OperationalError:
@ -73,18 +81,9 @@ class SqliteCache:
logging.warning("DatabaseError while setting value for key %r: %s", path_str, str(e))
def _create_con(self, second_try=False):
def create_tables():
logging.debug("Creating picture cache tables.")
self.con.execute("drop table if exists pictures")
self.con.execute("drop index if exists idx_path")
self.con.execute("create table pictures(path TEXT, mtime INTEGER, blocks BLOB)")
self.con.execute("create index idx_path on pictures (path)")
self.con = sqlite.connect(self.dbname, isolation_level=None)
try:
self.con.execute("select path, mtime, blocks from pictures where 1=2")
except sqlite.OperationalError: # new db
create_tables()
self.con = sqlite.connect(self.dbname, isolation_level=None)
self._check_upgrade()
except sqlite.DatabaseError as e: # corrupted db
if second_try:
raise # Something really strange is happening
@ -93,6 +92,25 @@ class SqliteCache:
os.remove(self.dbname)
self._create_con(second_try=True)
def _check_upgrade(self) -> None:
with self.con as conn:
has_schema = conn.execute(
"SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'"
).fetchall()
version = None
if has_schema:
version = conn.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0]
else:
conn.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)")
if version != self.schema_version:
conn.execute(self.drop_table_query)
conn.execute(
"INSERT OR REPLACE INTO schema_version VALUES (:version, :description)",
{"version": self.schema_version, "description": self.schema_version_description},
)
conn.execute(self.create_table_query)
conn.execute(self.create_index_query)
def clear(self):
self.close()
if self.dbname != ":memory:":
@ -129,12 +147,12 @@ class SqliteCache:
the db.
"""
todelete = []
sql = "select rowid, path, mtime from pictures"
sql = "select rowid, path, mtime_ns from pictures"
cur = self.con.execute(sql)
for rowid, path_str, mtime in cur:
if mtime and op.exists(path_str):
for rowid, path_str, mtime_ns in cur:
if mtime_ns and op.exists(path_str):
picture_mtime = os.stat(path_str).st_mtime
if int(picture_mtime) <= mtime:
if int(picture_mtime) <= mtime_ns:
# not outdated
continue
todelete.append(rowid)