.gitignore

@@ -129,6 +129,3 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# Kate swap files
-*.kate-swp
-

dat.py

@@ -0,0 +1,62 @@
+import xml.etree.ElementTree
+import hashdb
+
+class DatImportError(Exception):
+    '''This error is raised when a DAT import fails.'''
+
+# TODO: l o g g i n g
+
+# TODO: Consider using a context object to avoid keeping large XML trees in memory.
+class Dat:
+    '''A Dat object processes DAT files into the data structures defined in hashdb. '''
+    def __init__(self, filename):
+        '''Open the given DAT file and gather metadata from it.'''
+
+        xml_tree = xml.etree.ElementTree.parse(filename)
+        self._xml_root = xml_tree.getroot()
+
+        dat_header = self._xml_root.find('header')
+        self.info = hashdb.DatInfo(name=dat_header.find('name').text,
+                                   description=dat_header.find('description').text,
+                                   platform=None,
+                                   version=dat_header.find('version').text)
+
+    def set_platform(self, platform_info):
+        '''
+        Set a platform for this DAT file.
+        DAT files don't include platform metadata, but are all platform-specific.
+        '''
+        new_info = hashdb.DatInfo(name=self.info.name,
+                                  description=self.info.description,
+                                  platform=platform_info.shortcode,
+                                  version=self.info.version)
+        self.info = new_info
+
+    def set_name(self, new_name):
+        '''
+        Override the DAT file's name.
+        DAT files often have less-than-helpful names.
+        '''
+        new_info = hashdb.DatInfo(name=new_name,
+                                  description=self.info.description,
+                                  platform=self.info.platform,
+                                  version=self.info.version)
+        self.info = new_info
+
+    def read_all_hashes(self):
+        '''Read every hash in the DAT file and return it as a large list of RomInfo tuples.'''
+        if self.info.platform is None:
+            raise DatImportError('DAT platform not set.')
+
+        rom_info_list = []
+        all_rom_entries = self._xml_root.findall('.//rom')
+
+        for rom in all_rom_entries:
+            rom_info = hashdb.RomInfo(sha1sum=rom.get('sha1'),
+                                      filename=rom.get('name'),
+                                      platform=self.info.platform,
+                                      datorigin=self.info.name)
+
+            rom_info_list.append(rom_info)
+
+        return rom_info_list

lark

@@ -11,6 +11,7 @@ import os
 import uuid
 import xdg.BaseDirectory
 
+#import dat
 import metadata
 
 HASH_CHUNK_SIZE = 10485760 # 10mb
@@ -29,7 +30,28 @@ def get_sha1sum(filename):
 
     return sha1sum.hexdigest()
 
+
+'''
+smd_dat = dat(SMD_DAT_PATH)
+# TODO: Default to '.'
+# TODO: Use a proper arg parser.
+search_dir = sys.argv[1]
+for filename in os.listdir(search_dir):
+    # TODO: Ignore or descend into directories.
+    # TODO: Compare hashes
+    file_path = os.path.abspath(os.path.join(search_dir, filename))
+    file_sha1 = get_sha1sum(file_path)
+    search_result = smd_dat.search_by_sha1(file_sha1)
+    if search_result:
+        rom_data = search_result[0]
+        print('File %s matches database entry for %s.' % (filename, rom_data.filename))
+    else:
+        print('File %s is not in database.' % filename)
+'''
+# Test code! :D
 # TODO: Write test code that doesn't depend on external resources.
+SMD_DAT_PATH = '/home/lumia/Downloads/Sega - Mega Drive - Genesis (20200303-035539).dat'
+TEST_HASH = 'cfbf98c36c776677290a872547ac47c53d2761d6'
 
 def _kwargs_parse(kwargs_list):
     kwargs = {}

metadata.md

@@ -1,79 +1,41 @@
 # Metadata definitions
-
-## Platform
-A single set of hardware and/or software that shares compatibility. For example:
-	- Nintendo Entertainment System
-	- Microsoft Windows 7
-	- Microsoft Windows 98
-
-	* name
-		- The English name of the platform. As with release group names, this is mostly for
-			hand-editing data.
-	* regional_names
-		- A hash map with the keys being language identifers (en-US, jp, fr, etc) and the values
-			being the platform's name in the language and script it was released in.
-	* shortcode
-		- A small, three to five letter code for the platform. This must be unique among other
-			platforms.
-	* release groups
-		- A list containing release groups for this platform.
-
-## Release group
-A container for all languages and versions of a release. For example, the Star Fox 64 release
-	group would	contain all of the following releases:
-	- Star Fox 64 (U) v1.2
-	- Star Fox 64 (U) v1.0
-	- Star Fox 64 (J) v1.0
-	- Lylat Wars (E) v1.0
-
-	* name
-		- The English name for the release group. This is largely for contributors to quickly
-			understand the raw data, as frontends should ideally pull the name from a configured
-			region or something.
-	* releases
-		- A list containing all releases in this release group.
-
 ## Release
-A single release of a game. For example: Star Fox 64 (U) v1.2
+- A single release of a game. For example: Star Fox 64 (U) v1.2
 
-	* name
-		- The release's name in the language and script it was released in.
+    * UUID
+    * sha1sum
+    * format (For now this is just the file extension)
     * region
-		- The official release code for the game.
     * version
-		- The release version of the game. Some are verison numbers, some are just sequential
-			release numbers.
-	* disambiguation (can be empty)
-		- If this release is different in a way that isn't region or version, that information goes
-			here.
-	* images
-		- A list of all images of this release.
-## Image
-A stored copy of a game's disk image. For example:
-	- Star Fox 64 (U) v1.2.z64
-	- Star Fox 64 (U) v1.2.v64
-
-	* format
-		- The format this image is stored in. For example: bin, iso, chd, z64, v64
-	* sha1sum
-		- The sha1sum hash of this specific image.
-	* dump credit
-		- Where the hash came from, who dumped it, etc
-	* patches (can be empty)
-		- A list of patches that are intended to apply to this image.
-
-## Patches
-Unofficial patches or mods for a game, usually called romhacks.
+    * disambiguation
+    * release group
 
+## Release group
+- A container for all versions of a release.
+  For example, the Star Fox 64 release group would contain:
+    Star Fox 64 (U) v1.2
+    Star Fox 64 (U) v1.0
+    Star Fox 64 (J) v1.0
+    etc.
+
+    * UUID
     * name
-		- The patch's name in it's original language.
-	* file sha1sum
-		- The patch file's sha1sum.
+    * platform
 
-# Design notes
+## Platform
+- A single set of hardware and/or software that shares compatibility
+  Examples:
+    Nintendo Entertainment System
+    Microsoft Windows 7
+    Microsoft Windows 98
 
-* This spec is designed with the intent to be imported into databases while also being reasonably
-hand-editable for contributors.
+    * UUID
+    * name
+    * shortcode
 
-* Releases have multiple images to account for different formats. The z64 and v64 images of the
-same N64 game are different files with different hashes.
+## DAT credits
+- A large list of hashes imported from other sources.
+    * DAT name
+    * DAT website
+    * DAT version
+    * image UUID list

metadata.py

@@ -13,122 +13,101 @@ import sqlalchemy.orm
 
 # TODO: l o g g i n g
 HASH_CHUNK_SIZE = 10485760 # 10mb
+_db_session_maker = sqlalchemy.orm.sessionmaker()
+_engine = None
+_configured = False
 
-_SQLBase = sqlalchemy.ext.declarative.declarative_base()
+# TODO: Support DAT credit, DAT filenames, and checking DAT completeness.
+#DatData = collections.namedtuple('DatData', 'UUID, name, website, version, image_list')
 
-'''
-Metadata ORM classes for SQLAlchemy. For a detailed description of each piece of data, refer to
-metadata/README.md
-'''
+class MetadataDBSessionException(Exception):
+    '''This exception is raised when something goes wrong with a database session.'''
 
-class Platform(_SQLBase):
-	'''SQLAlchemy ORM class for platform metadata.'''
-	__tablename__ = 'platforms'
-	id = sqlalchemy.Column(
-		sqlalchemy.Integer, sqlalchemy.Sequence('platform_id_sequence'), primary_key=True
-		)
-	fullname = sqlalchemy.Column(sqlalchemy.String, nullable=False)
-	shortcode = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
-	regional_names = sqlalchemy.Column(sqlalchemy.String)
 
-	release_groups = sqlalchemy.orm.relationship(
-		'ReleaseGroup', order_by=ReleaseGroup.id, back_populates='platform'
-		)
+def _uuidgen():
+    return str(uuid.uuid4())
 
-	def __repr__(self):
-		return 'Platform: id: %s, fullname: %s, shortcode: %s' % (
-			self.id, self.fullname, self.shortcode
-			)
+_SQLBase = sqlalchemy.ext.declarative.declarative_base()
 
+class Release(_SQLBase):
+    '''SQLAlchemy ORM class for ROM image metadata.'''
+    __tablename__ = 'images'
+    id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.Sequence('image_id_sequence'),
+                           primary_key=True)
+    uuid = sqlalchemy.Column(sqlalchemy.String, nullable=False, default=_uuidgen)
+    sha1sum = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
+    format = sqlalchemy.Column(sqlalchemy.String, nullable=False)
+    region = sqlalchemy.Column(sqlalchemy.String)
+    version = sqlalchemy.Column(sqlalchemy.String, nullable=False)
+    disambiguation = sqlalchemy.Column(sqlalchemy.String)
+    release_group_id = sqlalchemy.Column(sqlalchemy.Integer,
+                                         sqlalchemy.ForeignKey('release_groups.id'))
+
+    release_group = sqlalchemy.orm.relationship('ReleaseGroup', back_populates='releases')
+
+    def __repr__(self):
+        return ('ROM Image: id: %s, uuid: %s, sha1sum: %s, release-group: %s, region: %s, '
+                'version: %s, disambiguation: %s' % (
+                    self.id, self.uuid, self.sha1sum, self.release_group.name, self.region,
+                    self.version, self.disambiguation))
 
 class ReleaseGroup(_SQLBase):
     '''SQLAlchemy ORM class for release group metadata.'''
     __tablename__ = 'release_groups'
-	id = sqlalchemy.Column(
-		sqlalchemy.Integer, sqlalchemy.Sequence('release_group_id_sequence'), primary_key=True
-		)
+    id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.Sequence('image_id_sequence'),
+                           primary_key=True)
+    uuid = sqlalchemy.Column(sqlalchemy.String, nullable=False, default=_uuidgen)
     name = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
     platform_id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('platforms.id'))
 
-	releases = sqlalchemy.orm.relationship('Release', back_populates='release_group')
+    platform = sqlalchemy.orm.relationship('Platform', back_populates='release_groups')
+    images = sqlalchemy.orm.relationship('Release', back_populates='release_group')
 
     def __repr__(self):
-		return 'Release Group: id: %s, name: %s, platform:%s' % (
-			self.id, self.name, self.platform.fullname
-			)
-
-class Release(_SQLBase):
-	'''SQLAlchemy ORM class for release metadata.'''
-	__tablename__ = 'releases'
-	id = sqlalchemy.Column(
-		sqlalchemy.Integer, sqlalchemy.Sequence('release_id_sequence'), primary_key=True
-		)
-	sha1sum = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
-	en_name = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
-	release_group_id = sqlalchemy.Column(
-		sqlalchemy.Integer, sqlalchemy.ForeignKey('release_groups.id')
-		)
-
-	release_group = sqlalchemy.orm.relationship('ReleaseGroup', back_populates='releases')
+        return 'Release Group: id: %s, uuid: %s, name: %s, platform:%s' % (self.id, self.uuid,
+                self.name, self.platform.fullname)
 
-	def __repr__(self):
-		return ('Release: id: %s, en_name: %s' % (self.id, self.en_name )
 
-class Image(_SQLBase):
-	'''SQLAlchemy ORM class for ROM image metadata.'''
-	__tablename__ = 'images'
-	id = sqlalchemy.Column(
-		sqlalchemy.Integer, sqlalchemy.Sequence('image_id_sequence'), primary_key=True
-		)
-	sha1sum = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
-	format = sqlalchemy.Column(sqlalchemy.String, nullable=False)
-	region = sqlalchemy.Column(sqlalchemy.String, nullable=False)
-	version = sqlalchemy.Column(sqlalchemy.String, nullable=False)
-	disambiguation = sqlalchemy.Column(sqlalchemy.String)
-	release_group_id = sqlalchemy.Column(
-		sqlalchemy.Integer,sqlalchemy.ForeignKey('release_groups.id')
-		)
+class Platform(_SQLBase):
+    '''SQLAlchemy ORM class for platform metadata.'''
+    __tablename__ = 'platforms'
+    id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.Sequence('platform_id_sequence'),
+                           primary_key=True)
+    uuid = sqlalchemy.Column(sqlalchemy.String, nullable=False, default=_uuidgen)
+    fullname = sqlalchemy.Column(sqlalchemy.String, nullable=False)
+    shortcode = sqlalchemy.Column(sqlalchemy.String, unique=True, nullable=False)
 
-	release_group = sqlalchemy.orm.relationship('ReleaseGroup', back_populates='releases')
+    release_groups = sqlalchemy.orm.relationship('ReleaseGroup', order_by=ReleaseGroup.id,
+                                                 back_populates='platform')
 
     def __repr__(self):
-		return (
-			'ROM Image: id: %s, sha1sum: %s, release-group: %s, region: %s, version: %s, '
-			'disambiguation: %s' % (
-				self.id, self.sha1sum, self.release_group.name, self.region, self.version,
-				self.disambiguation
-				)
-			)
-
-class hashdb:
-	_engine = None
+        return 'Platform: id: %s, uuid: %s, fullname: %s, shortcode: %s' % (self.id, self.uuid,
+                self.fullname, self.shortcode)
 
-	# TODO: db_path's default should be set here, not in frontend.
-	def __init__(self, db_path):
+def configure(db_path):
     '''
     Configure and initialize the database for the entire module.
     Currently, only SQLite is supported.
 
     db_path: Path for the SQLite database
     '''
-		self._engine = sqlalchemy.create_engine('sqlite:///%s' % db_path)
+    _engine = sqlalchemy.create_engine('sqlite:///%s' % db_path)
 
-		_SQLBase.metadata.create_all(self._engine)
-		self._session_maker.configure(bind=self._engine)
+    _SQLBase.metadata.create_all(_engine)
+    _db_session_maker.configure(bind=_engine)
+    _configured = True
 
-	def search(self, table_object, **constraints):
+# TODO: Passing the session object is a little clunky. Maybe there's a way to infer it somehow?
+def search(session, table_object, **constraints):
     '''
     Search the database for entries matching the given constraints.
 
+    session: SQLAlchemy session, presumably from get_db_session
     table_object: SQLAlchemy ORM table object, defined in the file above
     constraints: key-value pairs to match against specific fields in the database
                 Note: Currently, only the query.ilike method is supported. This is intended to
                 eventually support the entire range of available filters.
     '''
-
-		with self._get_db_session() as session:
-			# TODO: Consider making this return data recursively on items that reference other
-			#	tables.
     query = session.query(table_object)
 
     for key, value in constraints.items():
@@ -140,20 +119,14 @@ class hashdb:
 
     return item_list
 
-	def import_json(self, json_file):
-		'''
-		Import metadata from a json file.
-		'''
-
-		# json files can be large, but not too large for ram
-		# do not exit on invalid metadata, log the error and skip the object
-		pass
-
 @contextlib.contextmanager
-	def _get_db_session():
+def get_db_session():
     '''Get a SQLAlchemy database session with a proper context object. '''
+    # TODO: There's probably a more reliable way of knowing whether the database was configured.
+    if not _configured:
+        raise MetadataDBSessionException('Tried to get session without configuring a database.')
 
-		session = sqlalchemy.orm.sessionmaker()
+    session = _db_session_maker()
     try:
         yield session
 

readme.md

@@ -8,11 +8,17 @@ directory structure.
 
 ## Planned features
 * Validate ROM images.
+* Import DAT files
 * Rename/move ROM files
 * Maintain a database of present ROMs
 * A nice, Beets-like interface
 * Grouping ROMS in archive files
 
 ## Known issues
-* This probably isn't terribly efficient. It's Python parsing JSON into an SQLite database and I
-onlyknow pretty basic database design.
+* This probably isn't terribly efficient. It's Python parsing XML into an SQLite database and I only
+  know pretty basic database design.
+
+* Python's `xml.etree` module has a couple of known security issues[1]. Stick to importing DATs from
+known places and it shouldn't be an issue.
+
+[1] - https://docs.python.org/3/library/xml.html#xml-vulnerabilities