From 20abd1b733b36a72ce0d219b0c3a3a2cde0651e5 Mon Sep 17 00:00:00 2001 From: Emily Frost Date: Tue, 19 May 2020 13:21:58 -0500 Subject: [PATCH] Initial conversion to new db schema. --- hashdb.py | 139 +++++++++++++++++++++------------------------------- lark | 55 +++++++++++++-------- metadata.md | 1 + 3 files changed, 91 insertions(+), 104 deletions(-) diff --git a/hashdb.py b/hashdb.py index 02b1b48..d85efbb 100644 --- a/hashdb.py +++ b/hashdb.py @@ -4,19 +4,20 @@ hashdb.py import collections import hashlib import sqlite3 +import uuid -# TODO: Decide on a way to auto-download DATs. # TODO: l o g g i n g HASH_CHUNK_SIZE = 10485760 # 10mb SQL_AND = ' AND ' SQL_OR = ' OR ' # TODO: Figure out how to do some kind of type checking for these named tuples. -RomInfo = collections.namedtuple('RomInfo', 'sha1sum, filename, platform, datorigin') -DatInfo = collections.namedtuple('DatInfo', 'name, description, platform, version') -PlatformInfo = collections.namedtuple('PlatformInfo', 'shortcode, fullname, aliases') - -ORPHAN_DAT = DatInfo('', 'Orphaned hashes', 'nonexistent', '1') +# TODO: Switch these over to using traditional classes. +# TODO: UUID generation/editing is probably best done in here. +ImageData = collections.namedtuple('ImageData', 'UUID, sha1sum, filename, release_group') +ReleaseGroupData = collections.namedtuple('ReleaseGroupData', 'UUID, name, platform') +PlatformData = collections.namedtuple('PlatformData', 'UUID, name, shortcode') +DatData = collections.namedtuple('DatData', 'UUID, name, website, version, image_list') # TODO: This should go in the eventual romdb class. def get_file_sha1sum(filename): @@ -57,114 +58,84 @@ def _build_sql_constraints(inclusive, constraints): return (sql_constraint_string, sql_parameter_list) -class HashDB: - '''Store and retrieve hash metadata from an SQLite database.''' - # TODO: Low-priority: Probably design this around using multiple hash algorithms eventually. - def __init__(self, filename): + +class MetadataDB: + def __init__(self, db_path): ''' If db file does not exist, create it and create necessary tables. Either way, create a connection object. ''' # TODO: This process needs real error handling. - self._connection = sqlite3.connect(filename) + self._connection = sqlite3.connect(db_path) with self._connection: - # TODO: sha1sums.datorigin should be treated as a list. - self._connection.execute('CREATE TABLE IF NOT EXISTS sha1sums (sha1sum PRIMARY KEY, ' - 'filename NOT NULL, platform NOT NULL, datorigin);') + self._connection.execute('CREATE TABLE IF NOT EXISTS images (uuid PRIMARY KEY, ' + 'sha1sum UNIQUE NOT NULL, filename NOT NULL, release_group);') + + self._connection.execute('CREATE TABLE IF NOT EXISTS release_groups (UUID PRIMARY KEY, ' + 'name NOT NULL, platform NOT NULL);') - # TODO: Consider moving image-dat association to dats table. - self._connection.execute('CREATE TABLE IF NOT EXISTS dats (name PRIMARY KEY, ' - 'description, platform NOT NULL, version NOT NULL);') + self._connection.execute('CREATE TABLE IF NOT EXISTS platforms (UUID PRIMARY KEY, ' + 'name NOT NULL, shortcode UNIQUE NOT NULL);') - # TODO: Add support for custom roms not tracked in DAT releases. - # INSERT INTO dats (name="custom", description="Personally added hashes.", version=1); + # TODO: Add DAT import support. - self._connection.execute('CREATE TABLE IF NOT EXISTS platforms (shortcode PRIMARY KEY, ' - 'fullname NOT NULL, aliases );') print('Database initialized.') + def add_image(self, image_data): + pass - def add_hash(self, rom_info): - ''' Add a hash to the database. ''' - # INSERT INTO sha1sums (sha1sum, filename, platform, datorigin); - with self._connection: - self._connection.execute('INSERT INTO sha1sums VALUES (?, ?, ?, ?)', rom_info) + def update_image(self, image_data): + pass - def add_hash_list(self, rom_info_list): - '''Add many hashes to the database. ''' - with self._connection: - for rom_info in rom_info_list: - self._connection.execute('INSERT INTO sha1sums VALUES (?, ?, ?, ?)', rom_info) + def remove_image(self, image_data): + pass - def remove_hash(self, rom_info): - ''' Remove a hash from the database. ''' - # DELETE FROM sha1sums WHERE sha1sum=sha1sum; - with self._connection: - self._connection.execute('DELETE FROM sha1sums WHERE sha1sum=?;', [rom_info.sha1sum]) + def add_release_group(self, release_group_data): + pass - def remove_hash_list(self, rom_info_list): - '''Remove many hashes from the database. ''' - with self._connection: - for rom_info in rom_info_list: - self._connection.execute('DELETE FROM sha1sums WHERE sha1sum=?;', [rom_info.sha1sum]) + def update_release_group(self, release_group_data): + pass + + def remove_release_group(self, release_group_data): + pass - def add_platform(self, platform_info): + def add_platform(self, platform_data): ''' Add a platform shortcode to the database. ''' - # TODO: Collisions need user input to resolve, so remove this try block later. - try: - with self._connection: - self._connection.execute('INSERT INTO platforms VALUES (?, ?, ?);', platform_info) - except sqlite3.IntegrityError: - print('Warning: %s is already in database.' % platform_info.shortcode) - - def update_platform_aliases(self, shortcode, aliases): - ''' Change the list of aliases for a platform shortcode ''' - # UPDATE platforms SET aliases=aliases WHERE shortcode=shortcode; - - def remove_platform(self, platform_info): - ''' Remove a platform and all associated DATs and hashes from the database. ''' - # DELETE FROM sha1sums WHERE platform=shortcode; - # DELETE FROM dats WHERE platform=shortcode; - # DELETE FROM platform WHERE platform=shortcode; - with self._connection: - self._connection.execute('DELETE FROM sha1sums WHERE platform=?;', - [platform_info.shortcode]) - self._connection.execute('DELETE FROM dats WHERE platform=?;', - [platform_info.shortcode]) - self._connection.execute('DELETE FROM platforms WHERE shortcode=?;', - [platform_info.shortcode]) - - def add_dat(self, dat_info): - '''Add a DAT's metadata to the database. ''' with self._connection: - self._connection.execute('INSERT INTO platforms VALUES (?, ?, ?, ?);', dat_info) + self._connection.execute('INSERT INTO platforms VALUES (?, ?, ?);', platform_data) - def remove_dat(self, dat_info): - ''' Delete a DAT and all of its' hashes from the database. ''' - # DELETE FROM sha1sums WHERE datorigin=name; - # DELETE FROM dats WHERE name=name; + def update_platform(self, platform_data): + pass + # TODO: Switch this to use more generic constraints language + def remove_platform(self, platform_shortcode): with self._connection: - # TODO: Support multiple dat sources for the same hash. - self._connection.execute('DELETE FROM sha1sums WHERE datorigin=?;', [dat_info.name]) - self._connection.execute('DELETE FROM dats WHERE name=?;', [dat_info.name]) + self._connection.execute('DELETE FROM platforms WHERE shortcode=?;', platform_shortcode) - def hash_search(self, inclusive=True, **constraints): - '''Search for hashes, given the parameters. ''' + def search_platforms(self, inclusive=True, **constraints): + '''Search for platforms, given the parameters. ''' sql_where_clause, sql_parameters = _build_sql_constraints(inclusive, constraints) - rom_info_list = [] + platform_data_list = [] with self._connection: cursor = self._connection.cursor() - sql_query = 'SELECT * FROM sha1sums %s;' % sql_where_clause + sql_query = 'SELECT * FROM platforms %s;' % sql_where_clause cursor.execute(sql_query, sql_parameters) - print(sql_query) rows = cursor.fetchall() for row in rows: - rom_info = RomInfo(*row) - rom_info_list.append(rom_info) + platform_data = PlatformData(*row) + platform_data_list.append(platform_data) + + return platform_data_list + + def add_dat(self, dat_data): + pass + + def update_dat(self, dat_data): + pass - return rom_info_list + def remove_dat(self, dat_data): + pass diff --git a/lark b/lark index bb13842..42c068e 100755 --- a/lark +++ b/lark @@ -35,13 +35,14 @@ UI notes import hashlib import sys import os +import uuid import xdg.BaseDirectory import dat import hashdb HASH_CHUNK_SIZE = 10485760 # 10mb -SQLITE_FILENAME = 'lark.db' +SQLITE_FILENAME = 'metadata.db' data_path = os.path.join(xdg.BaseDirectory.xdg_data_home, 'lark') @@ -78,22 +79,36 @@ for filename in os.listdir(search_dir): # TODO: Write test code that doesn't depend on external resources. SMD_DAT_PATH = '/home/lumia/Downloads/Sega - Mega Drive - Genesis (20200303-035539).dat' TEST_HASH = 'cfbf98c36c776677290a872547ac47c53d2761d6' -smd_platform= hashdb.PlatformInfo(shortcode='smd', fullname='Sega - Genesis - Megadrive', - aliases='') -db = hashdb.HashDB(os.path.join(data_path, SQLITE_FILENAME)) -db.add_platform(smd_platform) - -smd_dat = dat.Dat(SMD_DAT_PATH) -smd_dat.set_platform(smd_platform) -#hashes = smd_dat.read_all_hashes() - -#db.add_hash_list(hashes) - -smd_hashes = db.hash_search(datorigin=smd_dat.info.name) -print(len(smd_hashes)) -#print(hashdb._build_sql_constraints(hashdb.SQL_OR, {'butt':'yes', 'platform':'smd'})) -#print(db.hash_search(platform='smd')) -#db.remove_platform(smd_platform) -#db.remove_dat(smd_dat.info) -print(hashdb._build_sql_constraints(True, {'sha1sum':TEST_HASH.upper()})) -print(db.hash_search(sha1sum=TEST_HASH.upper())) + +action_object = sys.argv[1] +action = sys.argv[2] + +db = hashdb.MetadataDB(os.path.join(data_path, SQLITE_FILENAME)) + +# TODO: Use a real UI library. This one is just intended for development. +if action_object == 'platform': + if action == 'add': + print('add a platform') + platform_uuid = uuid.uuid4() + platform_shortcode = sys.argv[3] + platform_name = sys.argv[4] + + platform_data = hashdb.PlatformData(UUID=str(platform_uuid), shortcode=platform_shortcode, + name=platform_name) + + print(platform_data) + + db.add_platform(platform_data) + + if action == 'list': + # TODO: convert this into a dict before passing. + # TODO: Abstract out constraint parsing etc. + constraints = sys.argv[3:] + platform_results = db.search_platforms() + print(platform_results) + + if action == 'remove': + constraints = sys.argv[3:] + db.remove_platform(constraints) +else: + print('Unknown object.') diff --git a/metadata.md b/metadata.md index d8d0620..f76e297 100644 --- a/metadata.md +++ b/metadata.md @@ -29,6 +29,7 @@ * UUID * name + * shortcode ## DAT credits - A large list of hashes imported from other sources.