diff --git a/.gitignore b/.gitignore index 13d1490..f2c19bc 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ dmypy.json # Pyre type checker .pyre/ +# Kate swap files +*.kate-swp + diff --git a/dat.py b/dat.py deleted file mode 100644 index bf382d1..0000000 --- a/dat.py +++ /dev/null @@ -1,62 +0,0 @@ -import xml.etree.ElementTree -import hashdb - -class DatImportError(Exception): - '''This error is raised when a DAT import fails.''' - -# TODO: l o g g i n g - -# TODO: Consider using a context object to avoid keeping large XML trees in memory. -class Dat: - '''A Dat object processes DAT files into the data structures defined in hashdb. ''' - def __init__(self, filename): - '''Open the given DAT file and gather metadata from it.''' - - xml_tree = xml.etree.ElementTree.parse(filename) - self._xml_root = xml_tree.getroot() - - dat_header = self._xml_root.find('header') - self.info = hashdb.DatInfo(name=dat_header.find('name').text, - description=dat_header.find('description').text, - platform=None, - version=dat_header.find('version').text) - - def set_platform(self, platform_info): - ''' - Set a platform for this DAT file. - DAT files don't include platform metadata, but are all platform-specific. - ''' - new_info = hashdb.DatInfo(name=self.info.name, - description=self.info.description, - platform=platform_info.shortcode, - version=self.info.version) - self.info = new_info - - def set_name(self, new_name): - ''' - Override the DAT file's name. - DAT files often have less-than-helpful names. - ''' - new_info = hashdb.DatInfo(name=new_name, - description=self.info.description, - platform=self.info.platform, - version=self.info.version) - self.info = new_info - - def read_all_hashes(self): - '''Read every hash in the DAT file and return it as a large list of RomInfo tuples.''' - if self.info.platform is None: - raise DatImportError('DAT platform not set.') - - rom_info_list = [] - all_rom_entries = self._xml_root.findall('.//rom') - - for rom in all_rom_entries: - rom_info = hashdb.RomInfo(sha1sum=rom.get('sha1'), - filename=rom.get('name'), - platform=self.info.platform, - datorigin=self.info.name) - - rom_info_list.append(rom_info) - - return rom_info_list diff --git a/lark b/lark index 4b05ce7..8fbb302 100755 --- a/lark +++ b/lark @@ -20,15 +20,15 @@ SQLITE_FILENAME = 'metadata.db' data_path = os.path.join(xdg.BaseDirectory.xdg_data_home, 'lark') def get_sha1sum(filename): - sha1sum = hashlib.sha1() - with open(filename, 'rb') as file_contents: - while True: - chunk = file_contents.read(HASH_CHUNK_SIZE) - if not chunk: - break - sha1sum.update(chunk) + sha1sum = hashlib.sha1() + with open(filename, 'rb') as file_contents: + while True: + chunk = file_contents.read(HASH_CHUNK_SIZE) + if not chunk: + break + sha1sum.update(chunk) - return sha1sum.hexdigest() + return sha1sum.hexdigest() ''' @@ -37,16 +37,16 @@ smd_dat = dat(SMD_DAT_PATH) # TODO: Use a proper arg parser. search_dir = sys.argv[1] for filename in os.listdir(search_dir): - # TODO: Ignore or descend into directories. - # TODO: Compare hashes - file_path = os.path.abspath(os.path.join(search_dir, filename)) - file_sha1 = get_sha1sum(file_path) - search_result = smd_dat.search_by_sha1(file_sha1) - if search_result: - rom_data = search_result[0] - print('File %s matches database entry for %s.' % (filename, rom_data.filename)) - else: - print('File %s is not in database.' % filename) + # TODO: Ignore or descend into directories. + # TODO: Compare hashes + file_path = os.path.abspath(os.path.join(search_dir, filename)) + file_sha1 = get_sha1sum(file_path) + search_result = smd_dat.search_by_sha1(file_sha1) + if search_result: + rom_data = search_result[0] + print('File %s matches database entry for %s.' % (filename, rom_data.filename)) + else: + print('File %s is not in database.' % filename) ''' # Test code! :D # TODO: Write test code that doesn't depend on external resources. @@ -54,12 +54,12 @@ SMD_DAT_PATH = '/home/lumia/Downloads/Sega - Mega Drive - Genesis (20200303-0355 TEST_HASH = 'cfbf98c36c776677290a872547ac47c53d2761d6' def _kwargs_parse(kwargs_list): - kwargs = {} - for kwarg_string in kwargs_list: - key, value = kwarg_string.split('=') - kwargs[key] = value + kwargs = {} + for kwarg_string in kwargs_list: + key, value = kwarg_string.split('=') + kwargs[key] = value - return kwargs + return kwargs action_object = sys.argv[1] action = sys.argv[2] @@ -68,102 +68,102 @@ metadata.configure(os.path.join(data_path, SQLITE_FILENAME)) # TODO: Use a real UI library. This mess is just intended for development. if action_object == 'platform': - if action == 'add': - print('add a platform') - platform_shortcode = sys.argv[3] - platform_name = sys.argv[4] - - platform_data = metadata.Platform(shortcode=platform_shortcode, - fullname=platform_name) - with metadata.get_db_session() as session: - session.add(platform_data) - - elif action == 'list': - # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure - # out a good way to include other filters. - filters = _kwargs_parse(sys.argv[3:]) - with metadata.get_db_session() as session: - print(metadata.search(session, metadata.Platform, **filters)) - - elif action == 'remove': - constraints = sys.argv[3:] - filters = _kwargs_parse(sys.argv[3:]) - - with metadata.get_db_session() as session: - platforms = metadata.search(session, metadata.Platform, **filters) - for platform in platforms: - print('Removing %s.' % platform.fullname) - session.delete(platform) - - elif action == 'test': - # TODO: Delete this action before merging into dev. It's just for ugly testing. - platform_shortcode = sys.argv[3] - platform_name = sys.argv[4] - - platform_data = metadata.Platform(shortcode=platform_shortcode, - fullname=platform_name) - - with metadata.get_db_session() as session: - print(metadata.search(session, metadata.Platform)) + if action == 'add': + print('add a platform') + platform_shortcode = sys.argv[3] + platform_name = sys.argv[4] + + platform_data = metadata.Platform(shortcode=platform_shortcode, + fullname=platform_name) + with metadata.get_db_session() as session: + session.add(platform_data) + + elif action == 'list': + # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure + # out a good way to include other filters. + filters = _kwargs_parse(sys.argv[3:]) + with metadata.get_db_session() as session: + print(metadata.search(session, metadata.Platform, **filters)) + + elif action == 'remove': + constraints = sys.argv[3:] + filters = _kwargs_parse(sys.argv[3:]) + + with metadata.get_db_session() as session: + platforms = metadata.search(session, metadata.Platform, **filters) + for platform in platforms: + print('Removing %s.' % platform.fullname) + session.delete(platform) + + elif action == 'test': + # TODO: Delete this action before merging into dev. It's just for ugly testing. + platform_shortcode = sys.argv[3] + platform_name = sys.argv[4] + + platform_data = metadata.Platform(shortcode=platform_shortcode, + fullname=platform_name) + + with metadata.get_db_session() as session: + print(metadata.search(session, metadata.Platform)) elif action_object == 'release-group': - if action == 'add': - properties = _kwargs_parse(sys.argv[3:]) - release_group = metadata.ReleaseGroup(name=properties['name']) - - with metadata.get_db_session() as session: - if properties['platform']: - platform = metadata.search(session, metadata.Platform, - shortcode=properties['platform'])[0] - release_group.platform = platform - session.add(release_group) - - if action == 'list': - # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure - # out a good way to include other filters. - print('Listing release groups.') - filters = _kwargs_parse(sys.argv[3:]) - with metadata.get_db_session() as session: - print(metadata.search(session, metadata.ReleaseGroup, **filters)) - - elif action == 'remove': - constraints = sys.argv[3:] - filters = _kwargs_parse(sys.argv[3:]) - - with metadata.get_db_session() as session: - release_groups = metadata.search(session, metadata.ReleaseGroup, **filters) - for release_group in release_groups: - print('Removing %s.' % release_group.name) - session.delete(release_group) + if action == 'add': + properties = _kwargs_parse(sys.argv[3:]) + release_group = metadata.ReleaseGroup(name=properties['name']) + + with metadata.get_db_session() as session: + if properties['platform']: + platform = metadata.search(session, metadata.Platform, + shortcode=properties['platform'])[0] + release_group.platform = platform + session.add(release_group) + + if action == 'list': + # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure + # out a good way to include other filters. + print('Listing release groups.') + filters = _kwargs_parse(sys.argv[3:]) + with metadata.get_db_session() as session: + print(metadata.search(session, metadata.ReleaseGroup, **filters)) + + elif action == 'remove': + constraints = sys.argv[3:] + filters = _kwargs_parse(sys.argv[3:]) + + with metadata.get_db_session() as session: + release_groups = metadata.search(session, metadata.ReleaseGroup, **filters) + for release_group in release_groups: + print('Removing %s.' % release_group.name) + session.delete(release_group) elif action_object == 'release': - if action == 'add': - properties = _kwargs_parse(sys.argv[3:]) - release = metadata.Release(**properties) - - with metadata.get_db_session() as session: - if properties['release-group']: - release_group = metadata.search(session, metadata.ReleaseGroup, - name=properties['release-group'])[0] - release.release_group = release_group - session.add(release) - - if action == 'list': - # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure - # out a good way to include other filters. - print('Listing releases.') - filters = _kwargs_parse(sys.argv[3:]) - with metadata.get_db_session() as session: - print(metadata.search(session, metadata.Release, **filters)) - - elif action == 'remove': - constraints = sys.argv[3:] - filters = _kwargs_parse(sys.argv[3:]) - - with metadata.get_db_session() as session: - release_groups = metadata.search(session, metadata.Release, **filters) - for release in release_groups: - print('Removing %s.' % release.name) - session.delete(release) + if action == 'add': + properties = _kwargs_parse(sys.argv[3:]) + release = metadata.Release(**properties) + + with metadata.get_db_session() as session: + if properties['release-group']: + release_group = metadata.search(session, metadata.ReleaseGroup, + name=properties['release-group'])[0] + release.release_group = release_group + session.add(release) + + if action == 'list': + # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure + # out a good way to include other filters. + print('Listing releases.') + filters = _kwargs_parse(sys.argv[3:]) + with metadata.get_db_session() as session: + print(metadata.search(session, metadata.Release, **filters)) + + elif action == 'remove': + constraints = sys.argv[3:] + filters = _kwargs_parse(sys.argv[3:]) + + with metadata.get_db_session() as session: + release_groups = metadata.search(session, metadata.Release, **filters) + for release in release_groups: + print('Removing %s.' % release.name) + session.delete(release) else: - print('Unknown object.') + print('Unknown object.') diff --git a/metadata.py b/metadata.py index 2b81151..1305535 100644 --- a/metadata.py +++ b/metadata.py @@ -13,6 +13,9 @@ import sqlalchemy.orm # TODO: l o g g i n g HASH_CHUNK_SIZE = 10485760 # 10mb + +db_session = None + _db_session_maker = sqlalchemy.orm.sessionmaker() _engine = None _configured = False @@ -25,6 +28,11 @@ class MetadataDBSessionException(Exception): def _uuidgen(): return str(uuid.uuid4()) +''' +Metadata ORM classes for SQLAlchemy. For a detailed description of each piece of data, refer to +metadata/README.md +''' + class Platform(_SQLBase): '''SQLAlchemy ORM class for platform metadata.''' __tablename__ = 'platforms' @@ -33,6 +41,7 @@ class Platform(_SQLBase): ) fullname = sqlalchemy.Column(sqlalchemy.String, nullable=False) shortcode = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False) + regional_names = sqlalchemy.Column(sqlalchemy.String) release_groups = sqlalchemy.orm.relationship( 'ReleaseGroup', order_by=ReleaseGroup.id, back_populates='platform' @@ -53,7 +62,6 @@ class ReleaseGroup(_SQLBase): name = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False) platform_id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('platforms.id')) - platform = sqlalchemy.orm.relationship('Platform', back_populates='release_groups') releases = sqlalchemy.orm.relationship('Release', back_populates='release_group') def __repr__(self): @@ -76,7 +84,7 @@ class Release(_SQLBase): release_group = sqlalchemy.orm.relationship('ReleaseGroup', back_populates='releases') def __repr__(self): - return ('ROM Image: id: %s, en_name: %s' % (self.id, self.en_name ) + return ('Release: id: %s, en_name: %s' % (self.id, self.en_name ) class Image(_SQLBase): '''SQLAlchemy ORM class for ROM image metadata.''' @@ -105,6 +113,11 @@ class Image(_SQLBase): ) +''' +# TODO: I was attempting to avoid writing a class to see if I could, but it is not worth the clunk. +Rewrite these functions into a class. +''' + def configure(db_path): ''' Configure and initialize the database for the entire module. @@ -118,7 +131,6 @@ def configure(db_path): _db_session_maker.configure(bind=_engine) _configured = True -# TODO: Passing the session object is a little clunky. Maybe there's a way to infer it somehow? def search(session, table_object, **constraints): ''' Search the database for entries matching the given constraints. diff --git a/readme.md b/readme.md index e8329bf..f9d8940 100644 --- a/readme.md +++ b/readme.md @@ -8,17 +8,11 @@ directory structure. ## Planned features * Validate ROM images. -* Import DAT files * Rename/move ROM files * Maintain a database of present ROMs * A nice, Beets-like interface * Grouping ROMS in archive files ## Known issues -* This probably isn't terribly efficient. It's Python parsing XML into an SQLite database and I only - know pretty basic database design. - -* Python's `xml.etree` module has a couple of known security issues[1]. Stick to importing DATs from -known places and it shouldn't be an issue. - -[1] - https://docs.python.org/3/library/xml.html#xml-vulnerabilities +* This probably isn't terribly efficient. It's Python parsing JSON into an SQLite database and I +onlyknow pretty basic database design.