Emily
/
Lark
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed indentation in lark and updated comments.

Browse Source
metadata-restructure
Emily Frost 4 years ago
parent c53b9d81f5
commit e55e31d5bb
Signed by: Emily
GPG Key ID: AA5D42849F1CBDC9
5 changed files with 137 additions and 190 deletions
Show Stats Download Patch File Download Diff File

3
.gitignore vendored
Unescape Escape View File

@ -129,3 +129,6 @@ dmypy.json
# Pyre type checker # Pyre type checker
.pyre/ .pyre/
# Kate swap files
*.kate-swp

62
dat.py
Unescape Escape View File

@ -1,62 +0,0 @@
import xml.etree.ElementTree
import hashdb
class DatImportError(Exception):
'''This error is raised when a DAT import fails.'''
# TODO: l o g g i n g
# TODO: Consider using a context object to avoid keeping large XML trees in memory.
class Dat:
'''A Dat object processes DAT files into the data structures defined in hashdb. '''
def __init__(self, filename):
'''Open the given DAT file and gather metadata from it.'''
xml_tree = xml.etree.ElementTree.parse(filename)
self._xml_root = xml_tree.getroot()
dat_header = self._xml_root.find('header')
self.info = hashdb.DatInfo(name=dat_header.find('name').text,
description=dat_header.find('description').text,
platform=None,
version=dat_header.find('version').text)
def set_platform(self, platform_info):
'''
Set a platform for this DAT file.
DAT files don't include platform metadata, but are all platform-specific.
'''
new_info = hashdb.DatInfo(name=self.info.name,
description=self.info.description,
platform=platform_info.shortcode,
version=self.info.version)
self.info = new_info
def set_name(self, new_name):
'''
Override the DAT file's name.
DAT files often have less-than-helpful names.
'''
new_info = hashdb.DatInfo(name=new_name,
description=self.info.description,
platform=self.info.platform,
version=self.info.version)
self.info = new_info
def read_all_hashes(self):
'''Read every hash in the DAT file and return it as a large list of RomInfo tuples.'''
if self.info.platform is None:
raise DatImportError('DAT platform not set.')
rom_info_list = []
all_rom_entries = self._xml_root.findall('.//rom')
for rom in all_rom_entries:
rom_info = hashdb.RomInfo(sha1sum=rom.get('sha1'),
filename=rom.get('name'),
platform=self.info.platform,
datorigin=self.info.name)
rom_info_list.append(rom_info)
return rom_info_list

234
lark
Unescape Escape View File

@ -20,15 +20,15 @@ SQLITE_FILENAME = 'metadata.db'
data_path = os.path.join(xdg.BaseDirectory.xdg_data_home, 'lark') data_path = os.path.join(xdg.BaseDirectory.xdg_data_home, 'lark')
def get_sha1sum(filename): def get_sha1sum(filename):
sha1sum = hashlib.sha1() sha1sum = hashlib.sha1()
with open(filename, 'rb') as file_contents: with open(filename, 'rb') as file_contents:
while True: while True:
chunk = file_contents.read(HASH_CHUNK_SIZE) chunk = file_contents.read(HASH_CHUNK_SIZE)
if not chunk: if not chunk:
break break
sha1sum.update(chunk) sha1sum.update(chunk)
return sha1sum.hexdigest() return sha1sum.hexdigest()
''' '''
@ -37,16 +37,16 @@ smd_dat = dat(SMD_DAT_PATH)
# TODO: Use a proper arg parser. # TODO: Use a proper arg parser.
search_dir = sys.argv[1] search_dir = sys.argv[1]
for filename in os.listdir(search_dir): for filename in os.listdir(search_dir):
# TODO: Ignore or descend into directories. # TODO: Ignore or descend into directories.
# TODO: Compare hashes # TODO: Compare hashes
file_path = os.path.abspath(os.path.join(search_dir, filename)) file_path = os.path.abspath(os.path.join(search_dir, filename))
file_sha1 = get_sha1sum(file_path) file_sha1 = get_sha1sum(file_path)
search_result = smd_dat.search_by_sha1(file_sha1) search_result = smd_dat.search_by_sha1(file_sha1)
if search_result: if search_result:
rom_data = search_result[0] rom_data = search_result[0]
print('File %s matches database entry for %s.' % (filename, rom_data.filename)) print('File %s matches database entry for %s.' % (filename, rom_data.filename))
else: else:
print('File %s is not in database.' % filename) print('File %s is not in database.' % filename)
''' '''
# Test code! :D # Test code! :D
# TODO: Write test code that doesn't depend on external resources. # TODO: Write test code that doesn't depend on external resources.
@ -54,12 +54,12 @@ SMD_DAT_PATH = '/home/lumia/Downloads/Sega - Mega Drive - Genesis (20200303-0355
TEST_HASH = 'cfbf98c36c776677290a872547ac47c53d2761d6' TEST_HASH = 'cfbf98c36c776677290a872547ac47c53d2761d6'
def _kwargs_parse(kwargs_list): def _kwargs_parse(kwargs_list):
kwargs = {} kwargs = {}
for kwarg_string in kwargs_list: for kwarg_string in kwargs_list:
key, value = kwarg_string.split('=') key, value = kwarg_string.split('=')
kwargs[key] = value kwargs[key] = value
return kwargs return kwargs
action_object = sys.argv[1] action_object = sys.argv[1]
action = sys.argv[2] action = sys.argv[2]
@ -68,102 +68,102 @@ metadata.configure(os.path.join(data_path, SQLITE_FILENAME))
# TODO: Use a real UI library. This mess is just intended for development. # TODO: Use a real UI library. This mess is just intended for development.
if action_object == 'platform': if action_object == 'platform':
if action == 'add': if action == 'add':
print('add a platform') print('add a platform')
platform_shortcode = sys.argv[3] platform_shortcode = sys.argv[3]
platform_name = sys.argv[4] platform_name = sys.argv[4]
platform_data = metadata.Platform(shortcode=platform_shortcode, platform_data = metadata.Platform(shortcode=platform_shortcode,
fullname=platform_name) fullname=platform_name)
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
session.add(platform_data) session.add(platform_data)
elif action == 'list': elif action == 'list':
# TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure
# out a good way to include other filters. # out a good way to include other filters.
filters = _kwargs_parse(sys.argv[3:]) filters = _kwargs_parse(sys.argv[3:])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
print(metadata.search(session, metadata.Platform, **filters)) print(metadata.search(session, metadata.Platform, **filters))
elif action == 'remove': elif action == 'remove':
constraints = sys.argv[3:] constraints = sys.argv[3:]
filters = _kwargs_parse(sys.argv[3:]) filters = _kwargs_parse(sys.argv[3:])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
platforms = metadata.search(session, metadata.Platform, **filters) platforms = metadata.search(session, metadata.Platform, **filters)
for platform in platforms: for platform in platforms:
print('Removing %s.' % platform.fullname) print('Removing %s.' % platform.fullname)
session.delete(platform) session.delete(platform)
elif action == 'test': elif action == 'test':
# TODO: Delete this action before merging into dev. It's just for ugly testing. # TODO: Delete this action before merging into dev. It's just for ugly testing.
platform_shortcode = sys.argv[3] platform_shortcode = sys.argv[3]
platform_name = sys.argv[4] platform_name = sys.argv[4]
platform_data = metadata.Platform(shortcode=platform_shortcode, platform_data = metadata.Platform(shortcode=platform_shortcode,
fullname=platform_name) fullname=platform_name)
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
print(metadata.search(session, metadata.Platform)) print(metadata.search(session, metadata.Platform))
elif action_object == 'release-group': elif action_object == 'release-group':
if action == 'add': if action == 'add':
properties = _kwargs_parse(sys.argv[3:]) properties = _kwargs_parse(sys.argv[3:])
release_group = metadata.ReleaseGroup(name=properties['name']) release_group = metadata.ReleaseGroup(name=properties['name'])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
if properties['platform']: if properties['platform']:
platform = metadata.search(session, metadata.Platform, platform = metadata.search(session, metadata.Platform,
shortcode=properties['platform'])[0] shortcode=properties['platform'])[0]
release_group.platform = platform release_group.platform = platform
session.add(release_group) session.add(release_group)
if action == 'list': if action == 'list':
# TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure
# out a good way to include other filters. # out a good way to include other filters.
print('Listing release groups.') print('Listing release groups.')
filters = _kwargs_parse(sys.argv[3:]) filters = _kwargs_parse(sys.argv[3:])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
print(metadata.search(session, metadata.ReleaseGroup, **filters)) print(metadata.search(session, metadata.ReleaseGroup, **filters))
elif action == 'remove': elif action == 'remove':
constraints = sys.argv[3:] constraints = sys.argv[3:]
filters = _kwargs_parse(sys.argv[3:]) filters = _kwargs_parse(sys.argv[3:])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
release_groups = metadata.search(session, metadata.ReleaseGroup, **filters) release_groups = metadata.search(session, metadata.ReleaseGroup, **filters)
for release_group in release_groups: for release_group in release_groups:
print('Removing %s.' % release_group.name) print('Removing %s.' % release_group.name)
session.delete(release_group) session.delete(release_group)
elif action_object == 'release': elif action_object == 'release':
if action == 'add': if action == 'add':
properties = _kwargs_parse(sys.argv[3:]) properties = _kwargs_parse(sys.argv[3:])
release = metadata.Release(**properties) release = metadata.Release(**properties)
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
if properties['release-group']: if properties['release-group']:
release_group = metadata.search(session, metadata.ReleaseGroup, release_group = metadata.search(session, metadata.ReleaseGroup,
name=properties['release-group'])[0] name=properties['release-group'])[0]
release.release_group = release_group release.release_group = release_group
session.add(release) session.add(release)
if action == 'list': if action == 'list':
# TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure # TODO: Filter support is exclusively limited to SQLAlchemy's filter.ilike function. Figure
# out a good way to include other filters. # out a good way to include other filters.
print('Listing releases.') print('Listing releases.')
filters = _kwargs_parse(sys.argv[3:]) filters = _kwargs_parse(sys.argv[3:])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
print(metadata.search(session, metadata.Release, **filters)) print(metadata.search(session, metadata.Release, **filters))
elif action == 'remove': elif action == 'remove':
constraints = sys.argv[3:] constraints = sys.argv[3:]
filters = _kwargs_parse(sys.argv[3:]) filters = _kwargs_parse(sys.argv[3:])
with metadata.get_db_session() as session: with metadata.get_db_session() as session:
release_groups = metadata.search(session, metadata.Release, **filters) release_groups = metadata.search(session, metadata.Release, **filters)
for release in release_groups: for release in release_groups:
print('Removing %s.' % release.name) print('Removing %s.' % release.name)
session.delete(release) session.delete(release)
else: else:
print('Unknown object.') print('Unknown object.')

18
metadata.py
Unescape Escape View File

@ -13,6 +13,9 @@ import sqlalchemy.orm
# TODO: l o g g i n g # TODO: l o g g i n g
HASH_CHUNK_SIZE = 10485760 # 10mb HASH_CHUNK_SIZE = 10485760 # 10mb
db_session = None
_db_session_maker = sqlalchemy.orm.sessionmaker() _db_session_maker = sqlalchemy.orm.sessionmaker()
_engine = None _engine = None
_configured = False _configured = False
@ -25,6 +28,11 @@ class MetadataDBSessionException(Exception):
def _uuidgen(): def _uuidgen():
return str(uuid.uuid4()) return str(uuid.uuid4())
'''
Metadata ORM classes for SQLAlchemy. For a detailed description of each piece of data, refer to
metadata/README.md
'''
class Platform(_SQLBase): class Platform(_SQLBase):
'''SQLAlchemy ORM class for platform metadata.''' '''SQLAlchemy ORM class for platform metadata.'''
__tablename__ = 'platforms' __tablename__ = 'platforms'
@ -33,6 +41,7 @@ class Platform(_SQLBase):
) )
fullname = sqlalchemy.Column(sqlalchemy.String, nullable=False) fullname = sqlalchemy.Column(sqlalchemy.String, nullable=False)
shortcode = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False) shortcode = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
regional_names = sqlalchemy.Column(sqlalchemy.String)
release_groups = sqlalchemy.orm.relationship( release_groups = sqlalchemy.orm.relationship(
'ReleaseGroup', order_by=ReleaseGroup.id, back_populates='platform' 'ReleaseGroup', order_by=ReleaseGroup.id, back_populates='platform'
@ -53,7 +62,6 @@ class ReleaseGroup(_SQLBase):
name = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False) name = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
platform_id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('platforms.id')) platform_id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('platforms.id'))
platform = sqlalchemy.orm.relationship('Platform', back_populates='release_groups')
releases = sqlalchemy.orm.relationship('Release', back_populates='release_group') releases = sqlalchemy.orm.relationship('Release', back_populates='release_group')
def __repr__(self): def __repr__(self):
@ -76,7 +84,7 @@ class Release(_SQLBase):
release_group = sqlalchemy.orm.relationship('ReleaseGroup', back_populates='releases') release_group = sqlalchemy.orm.relationship('ReleaseGroup', back_populates='releases')
def __repr__(self): def __repr__(self):
return ('ROM Image: id: %s, en_name: %s' % (self.id, self.en_name ) return ('Release: id: %s, en_name: %s' % (self.id, self.en_name )
class Image(_SQLBase): class Image(_SQLBase):
'''SQLAlchemy ORM class for ROM image metadata.''' '''SQLAlchemy ORM class for ROM image metadata.'''
@ -105,6 +113,11 @@ class Image(_SQLBase):
) )
'''
# TODO: I was attempting to avoid writing a class to see if I could, but it is not worth the clunk.
Rewrite these functions into a class.
'''
def configure(db_path): def configure(db_path):
''' '''
Configure and initialize the database for the entire module. Configure and initialize the database for the entire module.
@ -118,7 +131,6 @@ def configure(db_path):
_db_session_maker.configure(bind=_engine) _db_session_maker.configure(bind=_engine)
_configured = True _configured = True
# TODO: Passing the session object is a little clunky. Maybe there's a way to infer it somehow?
def search(session, table_object, **constraints): def search(session, table_object, **constraints):
''' '''
Search the database for entries matching the given constraints. Search the database for entries matching the given constraints.

10
readme.md
Unescape Escape View File

@ -8,17 +8,11 @@ directory structure.
## Planned features ## Planned features
* Validate ROM images. * Validate ROM images.
* Import DAT files
* Rename/move ROM files * Rename/move ROM files
* Maintain a database of present ROMs * Maintain a database of present ROMs
* A nice, Beets-like interface * A nice, Beets-like interface
* Grouping ROMS in archive files * Grouping ROMS in archive files
## Known issues ## Known issues
* This probably isn't terribly efficient. It's Python parsing XML into an SQLite database and I only * This probably isn't terribly efficient. It's Python parsing JSON into an SQLite database and I
know pretty basic database design. onlyknow pretty basic database design.
* Python's `xml.etree` module has a couple of known security issues[1]. Stick to importing DATs from
known places and it shouldn't be an issue.
[1] - https://docs.python.org/3/library/xml.html#xml-vulnerabilities

Write Preview
Loading…
Cancel
Save