From 7784520c2e7e148f85f150017100e7ad35a36227 Mon Sep 17 00:00:00 2001 From: luxick Date: Fri, 7 Jul 2017 23:30:22 +0200 Subject: [PATCH] Store all data in sqlite3 database. --- cardvault/application.py | 87 ++++++++++------- cardvault/database.py | 169 +++++++++++++++++++++++++++++++++ cardvault/gui/mainwindow.glade | 47 ++++++--- cardvault/handlers.py | 28 +++++- 4 files changed, 280 insertions(+), 51 deletions(-) diff --git a/cardvault/application.py b/cardvault/application.py index ec82520..d55e642 100644 --- a/cardvault/application.py +++ b/cardvault/application.py @@ -14,6 +14,7 @@ import os import copy import re import mtgsdk +import time from typing import Type, Dict, List from cardvault import handlers @@ -69,7 +70,8 @@ class Application: self.library = Dict[str, Type[mtgsdk.Card]] self.tags = Dict[str, str] self.wants = Dict[str, List[Type[mtgsdk.Card]]] - self.load_library() + + self.load_data() self.handlers = handlers.Handlers(self) self.ui.connect_signals(self.handlers) @@ -191,41 +193,26 @@ class Application: else: return value - def save_library(self): - # Save library file - util.save_file(util.get_root_filename("library"), self.library) - # Save tags file - util.save_file(util.get_root_filename("tags"), self.tags) - # Save wants file - util.save_file(util.get_root_filename("wants"), self.wants) + def save_data(self): + util.log("Saving Data to database", util.LogLevel.Info) + start = time.time() + self.db.save_library(self.library) + self.db.save_tags(self.tags) + self.db.save_wants(self.wants) + end = time.time() + util.log("Finished in {}s".format(str(round(end - start, 3))), util.LogLevel.Info) self.unsaved_changes = False - self.push_status("Library saved") + self.push_status("All data saved.") - def load_library(self): - all_existing = True - - # Load library file - self.library = util.load_file(util.get_root_filename("library")) - if not self.library: - all_existing = False - self.library = {} - - # Load tags file - self.tags = util.load_file(util.get_root_filename("tags")) - if not self.tags: - all_existing = False - self.tags = {} - - # Load wants lists - self.wants = util.load_file(util.get_root_filename("wants")) - if not self.wants: - all_existing = False - self.wants = {} - - # If parts were missing save to create the files - if not all_existing: - self.save_library() - self.push_status("Library loaded") + def load_data(self): + util.log("Loading Data from database", util.LogLevel.Info) + start = time.time() + self.library = self.db.get_library() + self.tags = self.db.get_tags() + self.wants = self.db.get_wants() + end = time.time() + util.log("Finished in {}s".format(str(round(end-start, 3))), util.LogLevel.Info) + self.push_status("All data loaded.") def get_untagged_cards(self): lib = copy.copy(self.library) @@ -364,6 +351,38 @@ class Application: else: return filter_text.lower() in model[iter][1].lower() + def load_data_legacy(self): + all_existing = True + # Load library file + self.library = util.load_file(util.get_root_filename("library")) + if not self.library: + all_existing = False + self.library = {} + # Load tags file + self.tags = util.load_file(util.get_root_filename("tags")) + if not self.tags: + all_existing = False + self.tags = {} + # Load wants lists + self.wants = util.load_file(util.get_root_filename("wants")) + if not self.wants: + all_existing = False + self.wants = {} + # If parts were missing save to create the files + if not all_existing: + self.save_library_legacy() + self.push_status("Library loaded") + + def save_library_legacy(self): + # Save library file + util.save_file(util.get_root_filename("library"), self.library) + # Save tags file + util.save_file(util.get_root_filename("tags"), self.tags) + # Save wants file + util.save_file(util.get_root_filename("wants"), self.wants) + + self.unsaved_changes = False + self.push_status("Library saved") def main(): Application() diff --git a/cardvault/database.py b/cardvault/database.py index b9faa87..ea49415 100644 --- a/cardvault/database.py +++ b/cardvault/database.py @@ -1,5 +1,8 @@ import sqlite3 import ast + +from pygments.lexers.robotframework import _Table + from mtgsdk import Card from cardvault import util @@ -29,6 +32,8 @@ class CardVaultDB: "`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT, " "PRIMARY KEY(`multiverseid`) )") con.execute("CREATE TABLE IF NOT EXISTS library ( multiverse_id INT PRIMARY KEY, copies INT )") + con.execute("CREATE TABLE IF NOT EXISTS tags ( tag TEXT, multiverseid INT )") + con.execute("CREATE TABLE IF NOT EXISTS wants ( listName TEXT, multiverseid INT )") def insert_card(self, card: Card): # Connect to database @@ -59,6 +64,17 @@ class CardVaultDB: util.log("Database Error", util.LogLevel.Error) util.log(str(err), util.LogLevel.Error) + def clear_database(self): + con = sqlite3.connect(self.db_file) + try: + with con: + con.execute('DELETE FROM library') + con.execute('DELETE FROM wants') + con.execute('DELETE FROM tags') + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + def search_cards_by_name_filtered(self, term: str, filters: dict, list_size: int) -> dict: """Search for cards based on the cards name with filter constrains""" filter_rarity = filters["rarity"] @@ -106,12 +122,165 @@ class CardVaultDB: rows = cur.fetchall() con.close() + return self.rows_to_card_dict(rows) + + def add_card_to_lib(self, card: Card): + """Insert card into library""" + con = sqlite3.connect(self.db_file) + try: + with con: + con.execute("INSERT INTO `library` (`copies`, `multiverseid`) VALUES (?, ?)", (1, card.multiverse_id)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def get_library(self) -> dict: + """Load library from database""" + con = sqlite3.connect(self.db_file) + cur = con.cursor() + cur.row_factory = sqlite3.Row + cur.execute('SELECT * FROM `library` INNER JOIN `cards` ON library.multiverseid = cards.multiverseid') + rows = cur.fetchall() + con.close() + + return self.rows_to_card_dict(rows) + + def get_tags(self): + """Loads a dict from database with all tags and the card ids tagged""" + con = sqlite3.connect(self.db_file) + cur = con.cursor() + cur.row_factory = sqlite3.Row + + # First load all tags + cur.execute("SELECT `tag` FROM tags GROUP BY `tag`") + rows = cur.fetchall() + tags = {} + for row in rows: + tags[row["tag"]] = [] + + # Go trough all tags an load the card ids + for tag in tags.keys(): + cur.execute('SELECT `multiverseid` FROM `tags` WHERE tags.tag = ? AND multiverseid NOT NULL', (tag, )) + rows = cur.fetchall() + for row in rows: + tags[tag].append(row["multiverseid"]) + + return tags + + def add_tag(self, tag: str): + """Add a new tag to the database""" + con = sqlite3.connect(self.db_file) + try: + with con: + con.execute("INSERT INTO `tags` VALUES (?, NULL)", (tag, )) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def tag_card(self, tag: str, card_id: int): + """Add an entry for a tagged card""" + con = sqlite3.connect(self.db_file) + try: + with con: + con.execute("INSERT INTO `tags` VALUES (?, ?)", (tag, card_id)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def rows_to_card_dict(self, rows): + """Convert database rows to a card dict""" output = {} for row in rows: card = self.table_to_card_mapping(row) output[card.multiverse_id] = card return output + def add_wants_list(self, name: str): + """Add a new wants list to the database""" + con = sqlite3.connect(self.db_file) + try: + with con: + con.execute("INSERT INTO `wants` VALUES (?, NULL)", (name,)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def add_card_to_wants(self, list_name: str, card_id: int): + """Add a card entry to a wants list""" + con = sqlite3.connect(self.db_file) + try: + with con: + con.execute("INSERT INTO `wants` VALUES (?, ?)", (list_name, card_id)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def get_wants(self): + """Load all wants lists from database""" + con = sqlite3.connect(self.db_file) + cur = con.cursor() + cur.row_factory = sqlite3.Row + + # First load all lists + cur.execute("SELECT `listName` FROM wants GROUP BY `listName`") + rows = cur.fetchall() + wants = {} + for row in rows: + wants[row["listName"]] = [] + + # Go trough all tags an load the card ids + for list_name in wants.keys(): + cur.execute('SELECT * FROM ' + '(SELECT `multiverseid` FROM `wants` WHERE `listName` = ? AND multiverseid NOT NULL) tagged ' + 'INNER JOIN `cards` ON tagged.multiverseid = cards.multiverseid', (list_name, )) + rows = cur.fetchall() + for row in rows: + wants[list_name].append(self.table_to_card_mapping(row)) + + return wants + + def save_library(self, cards: dict): + """Updates the library, adds new cards""" + con = sqlite3.connect(self.db_file) + try: + with con: + for multiverse_id in cards.keys(): + con.execute('UPDATE `library` SET `copies`=?, `multiverseid`=? WHERE multiverseid = ?;', (1, multiverse_id, multiverse_id)) + con.execute('INSERT INTO `library` (`copies`, `multiverseid`) ' + 'SELECT ?,? WHERE (Select Changes() = 0);', (1, multiverse_id)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def save_tags(self, tags: dict): + """Updates the tags table""" + con = sqlite3.connect(self.db_file) + try: + with con: + for tag, id_list in tags.items(): + for id in id_list: + con.execute('UPDATE `tags` SET `tag`=?, `multiverseid`=? WHERE tags.multiverseid = ?;', (tag, id, id)) + con.execute('INSERT INTO `tags` (`tag`, `multiverseid`) ' + 'SELECT ?,? WHERE (Select Changes() = 0);', (tag, id)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + + def save_wants(self, wants: dict): + """Updates the wants table""" + con = sqlite3.connect(self.db_file) + try: + with con: + for name, cards in wants.items(): + for card in cards: + con.execute('UPDATE `wants` SET `listName`=?, `multiverseid`=? WHERE wants.multiverseid = ?;', + (name, card.multiverse_id, card.multiverse_id)) + con.execute('INSERT INTO `wants` (`listName`, `multiverseid`) ' + 'SELECT ?,? WHERE (Select Changes() = 0);', (name, card.multiverse_id)) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + @staticmethod def card_to_table_mapping(card: Card): """Return the database representation of a card object""" diff --git a/cardvault/gui/mainwindow.glade b/cardvault/gui/mainwindow.glade index 2031fe3..708086c 100644 --- a/cardvault/gui/mainwindow.glade +++ b/cardvault/gui/mainwindow.glade @@ -166,12 +166,44 @@ True False - + True False - Clear Card Data + Clear DB Data True - + + + True + False + + + True + False + Clear Library Data + True + + + + + + True + False + Clear Card Data + True + + + + + + + + + + True + False + Load Library to DB + True + @@ -183,15 +215,6 @@ - - - True - False - Load Library to DB - True - - - diff --git a/cardvault/handlers.py b/cardvault/handlers.py index d8febf4..f82f287 100644 --- a/cardvault/handlers.py +++ b/cardvault/handlers.py @@ -26,7 +26,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): # --------------------------------- Main Window Handlers ---------------------------------------------- def do_save_library(self, item): - self.app.save_library() + self.app.save_data() def do_export_library(self, item): dialog = Gtk.FileChooserDialog("Export Library", self.app.ui.get_object("mainWindow"), @@ -88,19 +88,32 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): response = self.app.show_question_dialog("Unsaved Changes", "You have unsaved changes in your library. " "Save before exiting?") if response == Gtk.ResponseType.YES: - self.app.save_library() + self.app.save_data() return False elif response == Gtk.ResponseType.CANCEL: return True # ---------------------- Debug actions ------------------------------- - def do_load_lib_to_db(self, menu_item): + def do_load_data_to_db(self, item): util.log("Attempt loading library to database", util.LogLevel.Info) start = datetime.datetime.now() - self.app.db.bulk_insert_card(self.app.library.values()) + + for card in self.app.library.values(): + self.app.db.add_card_to_lib(card) + + for tag, card_ids in self.app.tags.items(): + self.app.db.add_tag(tag) + for card_id in card_ids: + self.app.db.tag_card(tag, card_id) + + for list_name, cards in self.app.wants.items(): + self.app.db.add_wants_list(list_name) + for card in cards: + self.app.db.add_card_to_wants(list_name, card.multiverse_id) + end = datetime.datetime.now() - util.log("Finished in {}s".format(str(end-start)), util.LogLevel.Info) + util.log("Finished in {}s".format(str(end - start)), util.LogLevel.Info) def do_load_all_cards(self, menu_item): util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info) @@ -122,4 +135,9 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): def do_clear_card_data(self, menu_item): util.log("Deleting all local card data", util.LogLevel.Info) self.app.db.clear_card_data() + util.log("Done", util.LogLevel.Info) + + def do_clear_data(self, item): + util.log("Deleting all library data", util.LogLevel.Info) + self.app.db.clear_database() util.log("Done", util.LogLevel.Info) \ No newline at end of file