|
|
|
@ -9,12 +9,20 @@ import os.path as op
|
|
|
|
|
import logging
|
|
|
|
|
import sqlite3 as sqlite
|
|
|
|
|
|
|
|
|
|
from core.pe.cache import string_to_colors, colors_to_string
|
|
|
|
|
from core.pe.cache import bytes_to_colors, colors_to_bytes
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SqliteCache:
|
|
|
|
|
"""A class to cache picture blocks in a sqlite backend."""
|
|
|
|
|
|
|
|
|
|
schema_version = 1
|
|
|
|
|
schema_version_description = "Changed from string to bytes for blocks."
|
|
|
|
|
|
|
|
|
|
create_table_query = "CREATE TABLE IF NOT EXISTS pictures(path TEXT, mtime_ns INTEGER, blocks BLOB)"
|
|
|
|
|
create_index_query = "CREATE INDEX IF NOT EXISTS idx_path on pictures (path)"
|
|
|
|
|
drop_table_query = "DROP TABLE IF EXISTS pictures"
|
|
|
|
|
drop_index_query = "DROP INDEX IF EXISTS idx_path"
|
|
|
|
|
|
|
|
|
|
def __init__(self, db=":memory:", readonly=False):
|
|
|
|
|
# readonly is not used in the sqlite version of the cache
|
|
|
|
|
self.dbname = db
|
|
|
|
@ -40,7 +48,7 @@ class SqliteCache:
|
|
|
|
|
sql = "select blocks from pictures where path = ?"
|
|
|
|
|
result = self.con.execute(sql, [key]).fetchone()
|
|
|
|
|
if result:
|
|
|
|
|
result = string_to_colors(result[0])
|
|
|
|
|
result = bytes_to_colors(result[0])
|
|
|
|
|
return result
|
|
|
|
|
else:
|
|
|
|
|
raise KeyError(key)
|
|
|
|
@ -56,15 +64,15 @@ class SqliteCache:
|
|
|
|
|
return result[0][0]
|
|
|
|
|
|
|
|
|
|
def __setitem__(self, path_str, blocks):
|
|
|
|
|
blocks = colors_to_string(blocks)
|
|
|
|
|
blocks = colors_to_bytes(blocks)
|
|
|
|
|
if op.exists(path_str):
|
|
|
|
|
mtime = int(os.stat(path_str).st_mtime)
|
|
|
|
|
else:
|
|
|
|
|
mtime = 0
|
|
|
|
|
if path_str in self:
|
|
|
|
|
sql = "update pictures set blocks = ?, mtime = ? where path = ?"
|
|
|
|
|
sql = "update pictures set blocks = ?, mtime_ns = ? where path = ?"
|
|
|
|
|
else:
|
|
|
|
|
sql = "insert into pictures(blocks,mtime,path) values(?,?,?)"
|
|
|
|
|
sql = "insert into pictures(blocks,mtime_ns,path) values(?,?,?)"
|
|
|
|
|
try:
|
|
|
|
|
self.con.execute(sql, [blocks, mtime, path_str])
|
|
|
|
|
except sqlite.OperationalError:
|
|
|
|
@ -73,18 +81,9 @@ class SqliteCache:
|
|
|
|
|
logging.warning("DatabaseError while setting value for key %r: %s", path_str, str(e))
|
|
|
|
|
|
|
|
|
|
def _create_con(self, second_try=False):
|
|
|
|
|
def create_tables():
|
|
|
|
|
logging.debug("Creating picture cache tables.")
|
|
|
|
|
self.con.execute("drop table if exists pictures")
|
|
|
|
|
self.con.execute("drop index if exists idx_path")
|
|
|
|
|
self.con.execute("create table pictures(path TEXT, mtime INTEGER, blocks TEXT)")
|
|
|
|
|
self.con.execute("create index idx_path on pictures (path)")
|
|
|
|
|
|
|
|
|
|
self.con = sqlite.connect(self.dbname, isolation_level=None)
|
|
|
|
|
try:
|
|
|
|
|
self.con.execute("select path, mtime, blocks from pictures where 1=2")
|
|
|
|
|
except sqlite.OperationalError: # new db
|
|
|
|
|
create_tables()
|
|
|
|
|
self.con = sqlite.connect(self.dbname, isolation_level=None)
|
|
|
|
|
self._check_upgrade()
|
|
|
|
|
except sqlite.DatabaseError as e: # corrupted db
|
|
|
|
|
if second_try:
|
|
|
|
|
raise # Something really strange is happening
|
|
|
|
@ -93,6 +92,25 @@ class SqliteCache:
|
|
|
|
|
os.remove(self.dbname)
|
|
|
|
|
self._create_con(second_try=True)
|
|
|
|
|
|
|
|
|
|
def _check_upgrade(self) -> None:
|
|
|
|
|
with self.con as conn:
|
|
|
|
|
has_schema = conn.execute(
|
|
|
|
|
"SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'"
|
|
|
|
|
).fetchall()
|
|
|
|
|
version = None
|
|
|
|
|
if has_schema:
|
|
|
|
|
version = conn.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0]
|
|
|
|
|
else:
|
|
|
|
|
conn.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)")
|
|
|
|
|
if version != self.schema_version:
|
|
|
|
|
conn.execute(self.drop_table_query)
|
|
|
|
|
conn.execute(
|
|
|
|
|
"INSERT OR REPLACE INTO schema_version VALUES (:version, :description)",
|
|
|
|
|
{"version": self.schema_version, "description": self.schema_version_description},
|
|
|
|
|
)
|
|
|
|
|
conn.execute(self.create_table_query)
|
|
|
|
|
conn.execute(self.create_index_query)
|
|
|
|
|
|
|
|
|
|
def clear(self):
|
|
|
|
|
self.close()
|
|
|
|
|
if self.dbname != ":memory:":
|
|
|
|
@ -120,7 +138,7 @@ class SqliteCache:
|
|
|
|
|
def get_multiple(self, rowids):
|
|
|
|
|
sql = "select rowid, blocks from pictures where rowid in (%s)" % ",".join(map(str, rowids))
|
|
|
|
|
cur = self.con.execute(sql)
|
|
|
|
|
return ((rowid, string_to_colors(blocks)) for rowid, blocks in cur)
|
|
|
|
|
return ((rowid, bytes_to_colors(blocks)) for rowid, blocks in cur)
|
|
|
|
|
|
|
|
|
|
def purge_outdated(self):
|
|
|
|
|
"""Go through the cache and purge outdated records.
|
|
|
|
@ -129,12 +147,12 @@ class SqliteCache:
|
|
|
|
|
the db.
|
|
|
|
|
"""
|
|
|
|
|
todelete = []
|
|
|
|
|
sql = "select rowid, path, mtime from pictures"
|
|
|
|
|
sql = "select rowid, path, mtime_ns from pictures"
|
|
|
|
|
cur = self.con.execute(sql)
|
|
|
|
|
for rowid, path_str, mtime in cur:
|
|
|
|
|
if mtime and op.exists(path_str):
|
|
|
|
|
for rowid, path_str, mtime_ns in cur:
|
|
|
|
|
if mtime_ns and op.exists(path_str):
|
|
|
|
|
picture_mtime = os.stat(path_str).st_mtime
|
|
|
|
|
if int(picture_mtime) <= mtime:
|
|
|
|
|
if int(picture_mtime) <= mtime_ns:
|
|
|
|
|
# not outdated
|
|
|
|
|
continue
|
|
|
|
|
todelete.append(rowid)
|
|
|
|
|