Store all data in sqlite3 database.

cardvault/application.py

@@ -14,6 +14,7 @@ import os
 import copy
 import re
 import mtgsdk
+import time
 from typing import Type, Dict, List
 
 from cardvault import handlers
@@ -69,7 +70,8 @@ class Application:
         self.library = Dict[str, Type[mtgsdk.Card]]
         self.tags = Dict[str, str]
         self.wants = Dict[str, List[Type[mtgsdk.Card]]]
-        self.load_library()
+
+        self.load_data()
 
         self.handlers = handlers.Handlers(self)
         self.ui.connect_signals(self.handlers)
@@ -191,41 +193,26 @@ class Application:
         else:
             return value
 
-    def save_library(self):
+    def save_data(self):
-        # Save library file
+        util.log("Saving Data to database", util.LogLevel.Info)
-        util.save_file(util.get_root_filename("library"), self.library)
+        start = time.time()
-        # Save tags file
+        self.db.save_library(self.library)
-        util.save_file(util.get_root_filename("tags"), self.tags)
+        self.db.save_tags(self.tags)
-        # Save wants file
+        self.db.save_wants(self.wants)
-        util.save_file(util.get_root_filename("wants"), self.wants)
+        end = time.time()
+        util.log("Finished in {}s".format(str(round(end - start, 3))), util.LogLevel.Info)
         self.unsaved_changes = False
-        self.push_status("Library saved")
+        self.push_status("All data saved.")
 
-    def load_library(self):
+    def load_data(self):
-        all_existing = True
+        util.log("Loading Data from database", util.LogLevel.Info)
-
+        start = time.time()
-        # Load library file
+        self.library = self.db.get_library()
-        self.library = util.load_file(util.get_root_filename("library"))
+        self.tags = self.db.get_tags()
-        if not self.library:
+        self.wants = self.db.get_wants()
-            all_existing = False
+        end = time.time()
-            self.library = {}
+        util.log("Finished in {}s".format(str(round(end-start, 3))), util.LogLevel.Info)
-
+        self.push_status("All data loaded.")
-        # Load tags file
-        self.tags = util.load_file(util.get_root_filename("tags"))
-        if not self.tags:
-            all_existing = False
-            self.tags = {}
-
-        # Load wants lists
-        self.wants = util.load_file(util.get_root_filename("wants"))
-        if not self.wants:
-            all_existing = False
-            self.wants = {}
-
-        # If parts were missing save to create the files
-        if not all_existing:
-            self.save_library()
-        self.push_status("Library loaded")
 
     def get_untagged_cards(self):
         lib = copy.copy(self.library)
@@ -364,6 +351,38 @@ class Application:
         else:
             return filter_text.lower() in model[iter][1].lower()
 
+    def load_data_legacy(self):
+        all_existing = True
+        # Load library file
+        self.library = util.load_file(util.get_root_filename("library"))
+        if not self.library:
+            all_existing = False
+            self.library = {}
+        # Load tags file
+        self.tags = util.load_file(util.get_root_filename("tags"))
+        if not self.tags:
+            all_existing = False
+            self.tags = {}
+        # Load wants lists
+        self.wants = util.load_file(util.get_root_filename("wants"))
+        if not self.wants:
+            all_existing = False
+            self.wants = {}
+        # If parts were missing save to create the files
+        if not all_existing:
+            self.save_library_legacy()
+        self.push_status("Library loaded")
+
+    def save_library_legacy(self):
+        # Save library file
+        util.save_file(util.get_root_filename("library"), self.library)
+        # Save tags file
+        util.save_file(util.get_root_filename("tags"), self.tags)
+        # Save wants file
+        util.save_file(util.get_root_filename("wants"), self.wants)
+
+        self.unsaved_changes = False
+        self.push_status("Library saved")
 
 def main():
     Application()

cardvault/database.py

@@ -1,5 +1,8 @@
 import sqlite3
 import ast
+
+from pygments.lexers.robotframework import _Table
+
 from mtgsdk import Card
 
 from cardvault import util
@@ -29,6 +32,8 @@ class CardVaultDB:
                         "`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT, "
                         "PRIMARY KEY(`multiverseid`) )")
             con.execute("CREATE TABLE IF NOT EXISTS library ( multiverse_id INT PRIMARY KEY, copies INT )")
+            con.execute("CREATE TABLE IF NOT EXISTS tags ( tag TEXT, multiverseid INT )")
+            con.execute("CREATE TABLE IF NOT EXISTS wants ( listName TEXT, multiverseid INT )")
 
     def insert_card(self, card: Card):
         # Connect to database
@@ -59,6 +64,17 @@ class CardVaultDB:
             util.log("Database Error", util.LogLevel.Error)
             util.log(str(err), util.LogLevel.Error)
 
+    def clear_database(self):
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                con.execute('DELETE FROM library')
+                con.execute('DELETE FROM wants')
+                con.execute('DELETE FROM tags')
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
     def search_cards_by_name_filtered(self, term: str, filters: dict, list_size: int) -> dict:
         """Search for cards based on the cards name with filter constrains"""
         filter_rarity = filters["rarity"]
@@ -106,12 +122,165 @@ class CardVaultDB:
         rows = cur.fetchall()
         con.close()
 
+        return self.rows_to_card_dict(rows)
+
+    def add_card_to_lib(self, card: Card):
+        """Insert card into library"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                con.execute("INSERT INTO `library` (`copies`, `multiverseid`) VALUES (?, ?)", (1, card.multiverse_id))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def get_library(self) -> dict:
+        """Load library from database"""
+        con = sqlite3.connect(self.db_file)
+        cur = con.cursor()
+        cur.row_factory = sqlite3.Row
+        cur.execute('SELECT * FROM `library` INNER JOIN `cards` ON library.multiverseid = cards.multiverseid')
+        rows = cur.fetchall()
+        con.close()
+
+        return self.rows_to_card_dict(rows)
+
+    def get_tags(self):
+        """Loads a dict from database with all tags and the card ids tagged"""
+        con = sqlite3.connect(self.db_file)
+        cur = con.cursor()
+        cur.row_factory = sqlite3.Row
+
+        # First load all tags
+        cur.execute("SELECT `tag` FROM tags GROUP BY `tag`")
+        rows = cur.fetchall()
+        tags = {}
+        for row in rows:
+            tags[row["tag"]] = []
+
+        # Go trough all tags an load the card ids
+        for tag in tags.keys():
+            cur.execute('SELECT `multiverseid` FROM `tags` WHERE tags.tag = ? AND multiverseid NOT NULL', (tag, ))
+            rows = cur.fetchall()
+            for row in rows:
+                tags[tag].append(row["multiverseid"])
+
+        return tags
+
+    def add_tag(self, tag: str):
+        """Add a new tag to the database"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                con.execute("INSERT INTO `tags` VALUES (?, NULL)", (tag, ))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def tag_card(self, tag: str, card_id: int):
+        """Add an entry for a tagged card"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                con.execute("INSERT INTO `tags` VALUES (?, ?)", (tag, card_id))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def rows_to_card_dict(self, rows):
+        """Convert database rows to a card dict"""
         output = {}
         for row in rows:
             card = self.table_to_card_mapping(row)
             output[card.multiverse_id] = card
         return output
 
+    def add_wants_list(self, name: str):
+        """Add a new wants list to the database"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                con.execute("INSERT INTO `wants` VALUES (?, NULL)", (name,))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def add_card_to_wants(self, list_name: str, card_id: int):
+        """Add a card entry to a wants list"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                con.execute("INSERT INTO `wants` VALUES (?, ?)", (list_name, card_id))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def get_wants(self):
+        """Load all wants lists from database"""
+        con = sqlite3.connect(self.db_file)
+        cur = con.cursor()
+        cur.row_factory = sqlite3.Row
+
+        # First load all lists
+        cur.execute("SELECT `listName` FROM wants GROUP BY `listName`")
+        rows = cur.fetchall()
+        wants = {}
+        for row in rows:
+            wants[row["listName"]] = []
+
+        # Go trough all tags an load the card ids
+        for list_name in wants.keys():
+            cur.execute('SELECT * FROM '
+                        '(SELECT `multiverseid` FROM `wants` WHERE `listName` = ? AND multiverseid NOT NULL) tagged '
+                        'INNER JOIN `cards` ON tagged.multiverseid = cards.multiverseid', (list_name, ))
+            rows = cur.fetchall()
+            for row in rows:
+                wants[list_name].append(self.table_to_card_mapping(row))
+
+        return wants
+
+    def save_library(self, cards: dict):
+        """Updates the library, adds new cards"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                for multiverse_id in cards.keys():
+                    con.execute('UPDATE `library` SET `copies`=?, `multiverseid`=? WHERE multiverseid = ?;', (1, multiverse_id, multiverse_id))
+                    con.execute('INSERT INTO `library` (`copies`, `multiverseid`) '
+                                'SELECT ?,? WHERE (Select Changes() = 0);', (1, multiverse_id))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def save_tags(self, tags: dict):
+        """Updates the tags table"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                for tag, id_list in tags.items():
+                    for id in id_list:
+                        con.execute('UPDATE `tags` SET `tag`=?, `multiverseid`=? WHERE tags.multiverseid = ?;', (tag, id, id))
+                        con.execute('INSERT INTO `tags` (`tag`, `multiverseid`) '
+                                    'SELECT ?,? WHERE (Select Changes() = 0);', (tag, id))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
+    def save_wants(self, wants: dict):
+        """Updates the wants table"""
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                for name, cards in wants.items():
+                    for card in cards:
+                        con.execute('UPDATE `wants` SET `listName`=?, `multiverseid`=? WHERE wants.multiverseid = ?;',
+                                    (name, card.multiverse_id, card.multiverse_id))
+                        con.execute('INSERT INTO `wants` (`listName`, `multiverseid`) '
+                                    'SELECT ?,? WHERE (Select Changes() = 0);', (name, card.multiverse_id))
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
     @staticmethod
     def card_to_table_mapping(card: Card):
         """Return the database representation of a card object"""

cardvault/gui/mainwindow.glade

@@ -166,12 +166,44 @@
                     <property name="visible">True</property>
                     <property name="can_focus">False</property>
                     <child>
-                      <object class="GtkMenuItem" id="del_card_db">
+                      <object class="GtkMenuItem">
                         <property name="visible">True</property>
                         <property name="can_focus">False</property>
-                        <property name="label" translatable="yes">Clear Card Data</property>
+                        <property name="label" translatable="yes">Clear DB Data</property>
                         <property name="use_underline">True</property>
-                        <signal name="activate" handler="do_clear_card_data" swapped="no"/>
+                        <child type="submenu">
+                          <object class="GtkMenu">
+                            <property name="visible">True</property>
+                            <property name="can_focus">False</property>
+                            <child>
+                              <object class="GtkMenuItem" id="del_data">
+                                <property name="visible">True</property>
+                                <property name="can_focus">False</property>
+                                <property name="label" translatable="yes">Clear Library Data</property>
+                                <property name="use_underline">True</property>
+                                <signal name="activate" handler="do_clear_data" swapped="no"/>
+                              </object>
+                            </child>
+                            <child>
+                              <object class="GtkMenuItem" id="del_cards">
+                                <property name="visible">True</property>
+                                <property name="can_focus">False</property>
+                                <property name="label" translatable="yes">Clear Card Data</property>
+                                <property name="use_underline">True</property>
+                                <signal name="activate" handler="do_clear_card_data" swapped="no"/>
+                              </object>
+                            </child>
+                          </object>
+                        </child>
+                      </object>
+                    </child>
+                    <child>
+                      <object class="GtkMenuItem" id="load_data_to_db">
+                        <property name="visible">True</property>
+                        <property name="can_focus">False</property>
+                        <property name="label" translatable="yes">Load Library to DB</property>
+                        <property name="use_underline">True</property>
+                        <signal name="activate" handler="do_load_data_to_db" swapped="no"/>
                       </object>
                     </child>
                     <child>
@@ -183,15 +215,6 @@
                         <signal name="activate" handler="do_load_all_cards" swapped="no"/>
                       </object>
                     </child>
-                    <child>
-                      <object class="GtkMenuItem" id="lib_to_db">
-                        <property name="visible">True</property>
-                        <property name="can_focus">False</property>
-                        <property name="label" translatable="yes">Load Library to DB</property>
-                        <property name="use_underline">True</property>
-                        <signal name="activate" handler="do_load_lib_to_db" swapped="no"/>
-                      </object>
-                    </child>
                   </object>
                 </child>
               </object>

cardvault/handlers.py

@@ -26,7 +26,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
     # --------------------------------- Main Window Handlers ----------------------------------------------
 
     def do_save_library(self, item):
-        self.app.save_library()
+        self.app.save_data()
 
     def do_export_library(self, item):
         dialog = Gtk.FileChooserDialog("Export Library", self.app.ui.get_object("mainWindow"),
@@ -88,19 +88,32 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
             response = self.app.show_question_dialog("Unsaved Changes", "You have unsaved changes in your library. "
                                                                         "Save before exiting?")
             if response == Gtk.ResponseType.YES:
-                self.app.save_library()
+                self.app.save_data()
                 return False
             elif response == Gtk.ResponseType.CANCEL:
                 return True
 
     # ---------------------- Debug actions -------------------------------
 
-    def do_load_lib_to_db(self, menu_item):
+    def do_load_data_to_db(self, item):
         util.log("Attempt loading library to database", util.LogLevel.Info)
         start = datetime.datetime.now()
-        self.app.db.bulk_insert_card(self.app.library.values())
+
+        for card in self.app.library.values():
+            self.app.db.add_card_to_lib(card)
+
+        for tag, card_ids in self.app.tags.items():
+            self.app.db.add_tag(tag)
+            for card_id in card_ids:
+                self.app.db.tag_card(tag, card_id)
+
+        for list_name, cards in self.app.wants.items():
+            self.app.db.add_wants_list(list_name)
+            for card in cards:
+                self.app.db.add_card_to_wants(list_name, card.multiverse_id)
+
         end = datetime.datetime.now()
-        util.log("Finished in {}s".format(str(end-start)), util.LogLevel.Info)
+        util.log("Finished in {}s".format(str(end - start)), util.LogLevel.Info)
 
     def do_load_all_cards(self, menu_item):
         util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info)
@@ -123,3 +136,8 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
         util.log("Deleting all local card data", util.LogLevel.Info)
         self.app.db.clear_card_data()
         util.log("Done", util.LogLevel.Info)
+
+    def do_clear_data(self, item):
+        util.log("Deleting all library data", util.LogLevel.Info)
+        self.app.db.clear_database()
+        util.log("Done", util.LogLevel.Info)