Replace card download from Gatherer with mtgjson.com.

cardvault/application.py

@@ -357,7 +357,7 @@ class Application:
         util.log("Removed {} from library".format(card.name), util.LogLevel.Info)
         self.push_status(card.name + " removed from library")
 
-    def override_user_data(self):
+    def db_override_user_data(self):
         """Called after import of user data. Overrides existing user data in database"""
         util.log("Clearing old user data", util.LogLevel.Info)
         self.db.db_clear_data_user()
@@ -380,6 +380,12 @@ class Application:
         util.log("Finished in {}s".format(str(round(end - start, 3))), util.LogLevel.Info)
         self.push_status("User data imported")
 
+    def db_delete_card_data(self):
+        """Called before before rebuilding local data storage"""
+        util.log("Clearing local card data", util.LogLevel.Info)
+        self.db.db_clear_data_card()
+        util.log("Done", util.LogLevel.Info)
+
     def get_mana_icons(self, mana_string):
         if not mana_string:
             util.log("No mana string provided", util.LogLevel.Info)

cardvault/database.py

@@ -23,14 +23,13 @@ class CardVaultDB:
                         "`colors` TEXT, `names` TEXT, `type` TEXT, `supertypes` TEXT, "
                         "`subtypes` TEXT, `types` TEXT, `rarity` TEXT, `text` TEXT, "
                         "`flavor` TEXT, `artist` TEXT, `number` INTEGER, `power` TEXT, "
-                        "`toughness` TEXT, `loyalty` INTEGER, `multiverseid` INTEGER UNIQUE , "
+                        "`toughness` TEXT, `loyalty` INTEGER, `multiverseid` INTEGER , "
                         "`variations` TEXT, `watermark` TEXT, `border` TEXT, `timeshifted` "
                         "TEXT, `hand` TEXT, `life` TEXT, `releaseDate` TEXT, `starter` TEXT, "
                         "`printings` TEXT, `originalText` TEXT, `originalType` TEXT, "
                         "`source` TEXT, `imageUrl` TEXT, `set` TEXT, `setName` TEXT, `id` TEXT, "
-                        "`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT, "
+                        "`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT) ")
-                        "PRIMARY KEY(`multiverseid`) )")
+            con.execute("CREATE TABLE IF NOT EXISTS library ( multiverseid INT PRIMARY KEY, copies INT )")
-            con.execute("CREATE TABLE IF NOT EXISTS library ( multiverse_id INT PRIMARY KEY, copies INT )")
             con.execute("CREATE TABLE IF NOT EXISTS tags ( tag TEXT, multiverseid INT )")
             con.execute("CREATE TABLE IF NOT EXISTS wants ( listName TEXT, multiverseid INT )")
 
@@ -41,7 +40,8 @@ class CardVaultDB:
             with con:
                 # Map card object to database tables
                 db_values = self.card_to_table_mapping(card)
-                sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+                sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \
+                             "?,?,?,?,?,?,?,?,?,?)"
                 # Insert into database
                 con.execute(sql_string, db_values)
         except sqlite3.OperationalError as err:
@@ -50,10 +50,6 @@ class CardVaultDB:
         except sqlite3.IntegrityError:
             pass
 
-    def db_card_insert_bulk(self, card_list: list):
-        for card in card_list:
-            self.db_card_insert(card)
-
     def db_get_all(self):
         sql = 'SELECT * FROM cards'
         con = sqlite3.connect(self.db_file)
@@ -68,6 +64,34 @@ class CardVaultDB:
             output.append(card)
         return output
 
+    def db_insert_data_card(self, cards_json):
+        """Insert download from mtgjson"""
+        rows = []
+        for data in cards_json.values():
+            cards = []
+            for raw in data["cards"]:
+                c = Card(raw)
+                c.image_url = util.CARD_IMAGE_URL.format(c.multiverse_id)
+                c.set = data["code"]
+                c.set_name = data["name"]
+                cards.append(c)
+
+            for c in cards:
+                rows.append(self.card_to_table_mapping(c))
+        # Connect to database
+        con = sqlite3.connect(self.db_file)
+        try:
+            with con:
+                sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \
+                             "?,?,?,?,?,?,?,?,?,?)"
+                con.executemany(sql_string, rows)
+        except sqlite3.OperationalError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+        except sqlite3.IntegrityError as err:
+            util.log("Database Error", util.LogLevel.Error)
+            util.log(str(err), util.LogLevel.Error)
+
     def db_clear_data_card(self):
         """Delete all resource data from database"""
         self.db_operation("DELETE FROM cards")

cardvault/handlers.py

@@ -70,12 +70,11 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
                 self.app.tags = imports[1]
                 self.app.wants = imports[2]
                 # Save imported data to database
-                self.app.override_user_data()
+                self.app.db_override_user_data()
                 # Cause current page to reload with imported data
                 self.app.current_page.emit('show')
         dialog.destroy()
 
-
     def on_view_changed(self, item):
         if item.get_active():
             container = self.app.ui.get_object("contentPage")
@@ -105,7 +104,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
         """The cancel button was pressed, set cancel_token to stop download thread"""
         self.cancel_token = True
         # Delete Dialog
-        self.app.ui.get_object("loadDataDialog").destroy()
+        self.app.ui.get_object("loadDataDialog").hide()
         self.app.push_status("Download canceled")
         util.log("Download canceled by user", util.LogLevel.Info)
 
@@ -116,23 +115,22 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
 
     def download_failed(self, err: MtgException):
         # Delete Dialog
-        self.app.ui.get_object("loadDataDialog").destroy()
+        self.app.ui.get_object("loadDataDialog").hide()
         self.app.push_status("Download canceled")
-        self.app.show_message("Download Faild", str(err))
+        self.app.show_message("Download Failed", str(err))
 
     def download_finished(self):
         """Download thread finished without errors"""
         self.cancel_token = False
         self.app.config["local_db"] = True
         self.app.save_config()
-        self.app.ui.get_object("loadDataDialog").destroy()
+        self.app.ui.get_object("loadDataDialog").hide()
         self.app.push_status("Card data downloaded")
         util.log("Card data download finished", util.LogLevel.Info)
 
     def do_download_card_data(self, item: Gtk.MenuItem):
         """Download button was pressed in the menu bar. Starts a thread to load data from the internet"""
-        info_string = "Start downloading card information from the internet?\n" \
+        info_string = "Start downloading card information from the internet?\n"  \
-                      "This process can take up to 10 minutes.\n" \
                       "You can cancel the download at any point."
         response = self.app.show_dialog_yn("Download Card Data", info_string)
         if response == Gtk.ResponseType.NO:
@@ -150,16 +148,40 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
         thread = threading.Thread(target=self.load_thread)
         thread.daemon = True
         thread.start()
-        util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info)
+        util.log("Attempt downloading all cards. This may take a while...", util.LogLevel.Info)
 
     def load_thread(self):
         """Worker thread to download info using the mtgsdk"""
-        all_cards = []
+
-        # Request total number of cards we are going to download
+        # Gatherer uses rate limit on Card.all()
+        # Takes ~10 minutes to download all cards
+        # all = self.load_thread_gatherer()
+
+        # Download from mtgjson.com
+        GObject.idle_add(self.load_show_insert_ui, "Downloading...")
+        util.log("Starting download", util.LogLevel.Info)
+        s = time.time()
+        all_json = util.net_all_cards_mtgjson()
+        e = time.time()
+        util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info)
+
+        self.app.db_delete_card_data()
+
+        GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...")
+        util.log("Saving to sqlite", util.LogLevel.Info)
+        s = time.time()
+        self.app.db.db_insert_data_card(all_json)
+        e = time.time()
+        util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info)
+
+        self.download_finished()
+
+    def load_thread_gatherer(self):
+        all = []
         all_num = util.get_all_cards_num()
         all_pages = int(math.ceil(all_num / 100))
 
-        # Download cards in pages until no new cards are added
+        # Paging for ui control between downloads
         for i in range(all_pages):
             req_start = time.time()
             try:
@@ -167,7 +189,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
             except MtgException as err:
                 util.log(str(err), util.LogLevel.Error)
                 return
-            all_cards = all_cards + new_cards
+            all = all + new_cards
             req_end = time.time()
 
             # Check if the action was canceled during download
@@ -180,13 +202,9 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
             self.app.ui.get_object("dl_progress_bar").set_visible(True)
             self.app.ui.get_object("dl_progress_label").set_visible(True)
             passed = str(round(req_end - req_start, 3))
-            GObject.idle_add(self.load_update_ui, all_cards, all_num, passed)
+            GObject.idle_add(self.load_update_ui, all, all_num, passed)
 
-        # All cards have been downloaded
+        return all
-        GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...")
-        self.app.db.db_card_insert_bulk(all_cards)
-
-        self.download_finished()
 
     def load_update_ui(self, current_list: list, max_cards: int, time_passed: str):
         """Called from withing the worker thread. Updates the download dialog with infos."""

cardvault/util.py

@@ -4,19 +4,17 @@ import json
 import os
 import re
 import sys
+import six.moves.cPickle as pickle
+from time import localtime, strftime, time
+from PIL import Image as PImage
+import urllib.request
 from urllib import request
+from mtgsdk import Set, Card, MtgException
 
 import gi
-from time import localtime, strftime, time
 gi.require_version('Gtk', '3.0')
 from gi.repository import GdkPixbuf, GLib
-import six.moves.cPickle as pickle
-from PIL import Image as PImage
 
-from urllib.request import Request, urlopen
-
-from mtgsdk import Set
-from mtgsdk import MtgException
 
 # Title of the Program Window
 APPLICATION_TITLE = "Card Vault"
@@ -46,21 +44,25 @@ DB_NAME = "cardvault.db"
 
 ALL_NUM_URL = 'https://api.magicthegathering.io/v1/cards?page=0&pageSize=100'
 
+ALL_SETS_JSON_URL = 'https://mtgjson.com/json/AllSets-x.json'
+
+# URL for card images. Insert card.multiverse_id.
+CARD_IMAGE_URL = 'http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid={}&type=card'
+
 # Colors for card rows in search view
-SEARCH_TREE_COLORS ={
+SEARCH_TREE_COLORS = {
     "unowned": "black",
     "wanted": "#D39F30",
     "owned": "#62B62F"
 }
 
 # Colors for card rows in every default view
-GENERIC_TREE_COLORS ={
+GENERIC_TREE_COLORS = {
     "unowned": "black",
     "wanted": "black",
     "owned": "black"
 }
 
-
 default_config = {
     "hide_duplicates_in_search": False,
     "start_page": "search",
@@ -73,7 +75,7 @@ default_config = {
     }
 }
 
-legality_colors ={
+legality_colors = {
     "Banned": "#C65642",
     "Restricted": "#D39F30",
     "Legal": "#62B62F"
@@ -187,19 +189,19 @@ def reload_preconstructed_icons(path: str) -> dict:
     if not os.path.exists(path):
         os.makedirs(path)
 
-    iconfiles = os.listdir(path)
+    files = os.listdir(path)
-    for file in iconfiles:
+    for file in files:
         # Split filename into single icon names and remove extension
         without_ext = file.split(".")[0]
-        list = without_ext.split("_")
+        names = without_ext.split("_")
         # Compute size of the finished icon
-        pic_width = len(list) * 105
+        pic_width = len(names) * 105
         pic_height = 105
         try:
             pixbuf = GdkPixbuf.Pixbuf.new_from_file(ICON_CACHE_PATH + file)
             pixbuf = pixbuf.scale_simple(pic_width / 5, pic_height / 5, GdkPixbuf.InterpType.HYPER)
             # Set name for icon
-            iconname = "_".join(list)
+            iconname = "_".join(names)
             cache[iconname] = pixbuf
         except OSError as err:
             log("Error loading image: " + str(err), LogLevel.Error)
@@ -211,8 +213,8 @@ def load_mana_icons(path: str) -> dict:
         log("Directory for mana icons not found " + path, LogLevel.Error)
         return {}
     icons = {}
-    filenames = os.listdir(path)
+    files = os.listdir(path)
-    for file in filenames:
+    for file in files:
         img = PImage.open(path + file)
         # Strip file extension
         name = os.path.splitext(file)[0]
@@ -220,13 +222,19 @@ def load_mana_icons(path: str) -> dict:
     return icons
 
 
+def net_all_cards_mtgjson() -> dict:
+    with urllib.request.urlopen(ALL_SETS_JSON_URL) as url:
+        data = json.loads(url.read().decode())
+        return data
+
+
 def net_load_set_list() -> dict:
     """ Load the list of all MTG sets from the Gather"""
     try:
         start = time()
         sets = Set.all()
         stop = time()
-        log("Fetched set list in {}s".format(round(stop-start, 3)), LogLevel.Info)
+        log("Fetched set list in {}s".format(round(stop - start, 3)), LogLevel.Info)
     except MtgException as err:
         log(str(err), LogLevel.Error)
         return {}
@@ -305,11 +313,11 @@ def load_dummy_image(size_x: int, size_y: int) -> GdkPixbuf:
                                                   + '/resources/images/dummy.jpg', size_x, size_y)
 
 
-def load_card_image(card: 'mtgsdk.Card', size_x: int, size_y: int, cache: dict) -> GdkPixbuf:
+def load_card_image(card: Card, size_x: int, size_y: int, cache: dict) -> GdkPixbuf:
     """ Retrieve an card image from cache or alternatively load from gatherer"""
     try:
         image = cache[card.multiverse_id]
-    except KeyError as err:
+    except KeyError:
         log("No local image for " + card.name + ". Loading from " + card.image_url, LogLevel.Info)
         filename, image = net_load_card_image(card, size_x, size_y)
         cache[card.multiverse_id] = image
@@ -329,27 +337,27 @@ def net_load_card_image(card, size_x: int, size_y: int) -> (str, GdkPixbuf):
 def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf:
     # Convert the string to a List
     safe_string = mana_string.replace("/", "-")
-    list = re.findall("{(.*?)}", safe_string)
+    glyphs = re.findall("{(.*?)}", safe_string)
-    if len(list) == 0:
+    if len(glyphs) == 0:
         return
     # Compute horizontal size for the final image
-    imagesize = len(list) * 105
+    size = len(glyphs) * 105
-    image = PImage.new("RGBA", (imagesize, 105))
+    image = PImage.new("RGBA", (size, 105))
     # Increment for each position of an icon
     # (Workaround: 2 or more of the same icon will be rendered in the same position)
-    poscounter = 0
+    c = 0
     # Go through all entries an add the correspondent icon to the final image
-    for icon in list:
+    for icon in glyphs:
-        xpos = poscounter * 105
+        x_pos = c * 105
         try:
             loaded = icons[icon]
-        except KeyError as err:
+        except KeyError:
             log("No icon file named '" + icon + "' found.", LogLevel.Warning)
             return
-        image.paste(loaded, (xpos, 0))
+        image.paste(loaded, (x_pos, 0))
-        poscounter += 1
+        c += 1
     # Save Icon file
-    path = ICON_CACHE_PATH + "_".join(list) + ".png"
+    path = ICON_CACHE_PATH + "_".join(glyphs) + ".png"
     image.save(path)
     try:
         pixbuf = GdkPixbuf.Pixbuf.new_from_file(path)
@@ -360,7 +368,7 @@ def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf:
 
 
 def sizeof_fmt(num, suffix='B'):
-    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
+    for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
         if abs(num) < 1024.0:
             return "%3.1f%s%s" % (num, unit, suffix)
         num /= 1024.0
@@ -368,11 +376,9 @@ def sizeof_fmt(num, suffix='B'):
 
 
 def get_all_cards_num() -> int:
-    req = Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'})
+    req = urllib.request.Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'})
-    response = urlopen(req)
+    response = urllib.request.urlopen(req)
     headers = response.info()._headers
     for header, value in headers:
         if header == 'Total-Count':
             return int(value)
-# endregion
-

setup.py

@@ -37,5 +37,5 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'License :: OSI Approved :: MIT License',
-    ], install_requires=['gi', 'pillow']
+    ], install_requires=['gi', 'pillow', 'six']
 )