From 2b1f3b940f464f724fdcb6d9d1d7ea7432ea8c0a Mon Sep 17 00:00:00 2001 From: luxick Date: Wed, 26 Jul 2017 00:01:21 +0200 Subject: [PATCH] Replace card download from Gatherer with mtgjson.com. --- cardvault/application.py | 8 +++- cardvault/database.py | 42 ++++++++++++++++----- cardvault/handlers.py | 56 ++++++++++++++++++---------- cardvault/util.py | 80 +++++++++++++++++++++------------------- setup.py | 2 +- 5 files changed, 121 insertions(+), 67 deletions(-) diff --git a/cardvault/application.py b/cardvault/application.py index 2400c74..d442842 100644 --- a/cardvault/application.py +++ b/cardvault/application.py @@ -357,7 +357,7 @@ class Application: util.log("Removed {} from library".format(card.name), util.LogLevel.Info) self.push_status(card.name + " removed from library") - def override_user_data(self): + def db_override_user_data(self): """Called after import of user data. Overrides existing user data in database""" util.log("Clearing old user data", util.LogLevel.Info) self.db.db_clear_data_user() @@ -380,6 +380,12 @@ class Application: util.log("Finished in {}s".format(str(round(end - start, 3))), util.LogLevel.Info) self.push_status("User data imported") + def db_delete_card_data(self): + """Called before before rebuilding local data storage""" + util.log("Clearing local card data", util.LogLevel.Info) + self.db.db_clear_data_card() + util.log("Done", util.LogLevel.Info) + def get_mana_icons(self, mana_string): if not mana_string: util.log("No mana string provided", util.LogLevel.Info) diff --git a/cardvault/database.py b/cardvault/database.py index 711e5d1..52f0d41 100644 --- a/cardvault/database.py +++ b/cardvault/database.py @@ -23,14 +23,13 @@ class CardVaultDB: "`colors` TEXT, `names` TEXT, `type` TEXT, `supertypes` TEXT, " "`subtypes` TEXT, `types` TEXT, `rarity` TEXT, `text` TEXT, " "`flavor` TEXT, `artist` TEXT, `number` INTEGER, `power` TEXT, " - "`toughness` TEXT, `loyalty` INTEGER, `multiverseid` INTEGER UNIQUE , " + "`toughness` TEXT, `loyalty` INTEGER, `multiverseid` INTEGER , " "`variations` TEXT, `watermark` TEXT, `border` TEXT, `timeshifted` " "TEXT, `hand` TEXT, `life` TEXT, `releaseDate` TEXT, `starter` TEXT, " "`printings` TEXT, `originalText` TEXT, `originalType` TEXT, " "`source` TEXT, `imageUrl` TEXT, `set` TEXT, `setName` TEXT, `id` TEXT, " - "`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT, " - "PRIMARY KEY(`multiverseid`) )") - con.execute("CREATE TABLE IF NOT EXISTS library ( multiverse_id INT PRIMARY KEY, copies INT )") + "`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT) ") + con.execute("CREATE TABLE IF NOT EXISTS library ( multiverseid INT PRIMARY KEY, copies INT )") con.execute("CREATE TABLE IF NOT EXISTS tags ( tag TEXT, multiverseid INT )") con.execute("CREATE TABLE IF NOT EXISTS wants ( listName TEXT, multiverseid INT )") @@ -41,7 +40,8 @@ class CardVaultDB: with con: # Map card object to database tables db_values = self.card_to_table_mapping(card) - sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \ + "?,?,?,?,?,?,?,?,?,?)" # Insert into database con.execute(sql_string, db_values) except sqlite3.OperationalError as err: @@ -50,10 +50,6 @@ class CardVaultDB: except sqlite3.IntegrityError: pass - def db_card_insert_bulk(self, card_list: list): - for card in card_list: - self.db_card_insert(card) - def db_get_all(self): sql = 'SELECT * FROM cards' con = sqlite3.connect(self.db_file) @@ -68,6 +64,34 @@ class CardVaultDB: output.append(card) return output + def db_insert_data_card(self, cards_json): + """Insert download from mtgjson""" + rows = [] + for data in cards_json.values(): + cards = [] + for raw in data["cards"]: + c = Card(raw) + c.image_url = util.CARD_IMAGE_URL.format(c.multiverse_id) + c.set = data["code"] + c.set_name = data["name"] + cards.append(c) + + for c in cards: + rows.append(self.card_to_table_mapping(c)) + # Connect to database + con = sqlite3.connect(self.db_file) + try: + with con: + sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \ + "?,?,?,?,?,?,?,?,?,?)" + con.executemany(sql_string, rows) + except sqlite3.OperationalError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + except sqlite3.IntegrityError as err: + util.log("Database Error", util.LogLevel.Error) + util.log(str(err), util.LogLevel.Error) + def db_clear_data_card(self): """Delete all resource data from database""" self.db_operation("DELETE FROM cards") diff --git a/cardvault/handlers.py b/cardvault/handlers.py index 0887df3..6002dc6 100644 --- a/cardvault/handlers.py +++ b/cardvault/handlers.py @@ -70,12 +70,11 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): self.app.tags = imports[1] self.app.wants = imports[2] # Save imported data to database - self.app.override_user_data() + self.app.db_override_user_data() # Cause current page to reload with imported data self.app.current_page.emit('show') dialog.destroy() - def on_view_changed(self, item): if item.get_active(): container = self.app.ui.get_object("contentPage") @@ -105,7 +104,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): """The cancel button was pressed, set cancel_token to stop download thread""" self.cancel_token = True # Delete Dialog - self.app.ui.get_object("loadDataDialog").destroy() + self.app.ui.get_object("loadDataDialog").hide() self.app.push_status("Download canceled") util.log("Download canceled by user", util.LogLevel.Info) @@ -116,23 +115,22 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): def download_failed(self, err: MtgException): # Delete Dialog - self.app.ui.get_object("loadDataDialog").destroy() + self.app.ui.get_object("loadDataDialog").hide() self.app.push_status("Download canceled") - self.app.show_message("Download Faild", str(err)) + self.app.show_message("Download Failed", str(err)) def download_finished(self): """Download thread finished without errors""" self.cancel_token = False self.app.config["local_db"] = True self.app.save_config() - self.app.ui.get_object("loadDataDialog").destroy() + self.app.ui.get_object("loadDataDialog").hide() self.app.push_status("Card data downloaded") util.log("Card data download finished", util.LogLevel.Info) def do_download_card_data(self, item: Gtk.MenuItem): """Download button was pressed in the menu bar. Starts a thread to load data from the internet""" - info_string = "Start downloading card information from the internet?\n" \ - "This process can take up to 10 minutes.\n" \ + info_string = "Start downloading card information from the internet?\n" \ "You can cancel the download at any point." response = self.app.show_dialog_yn("Download Card Data", info_string) if response == Gtk.ResponseType.NO: @@ -150,16 +148,40 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): thread = threading.Thread(target=self.load_thread) thread.daemon = True thread.start() - util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info) + util.log("Attempt downloading all cards. This may take a while...", util.LogLevel.Info) def load_thread(self): """Worker thread to download info using the mtgsdk""" - all_cards = [] - # Request total number of cards we are going to download + + # Gatherer uses rate limit on Card.all() + # Takes ~10 minutes to download all cards + # all = self.load_thread_gatherer() + + # Download from mtgjson.com + GObject.idle_add(self.load_show_insert_ui, "Downloading...") + util.log("Starting download", util.LogLevel.Info) + s = time.time() + all_json = util.net_all_cards_mtgjson() + e = time.time() + util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info) + + self.app.db_delete_card_data() + + GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...") + util.log("Saving to sqlite", util.LogLevel.Info) + s = time.time() + self.app.db.db_insert_data_card(all_json) + e = time.time() + util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info) + + self.download_finished() + + def load_thread_gatherer(self): + all = [] all_num = util.get_all_cards_num() all_pages = int(math.ceil(all_num / 100)) - # Download cards in pages until no new cards are added + # Paging for ui control between downloads for i in range(all_pages): req_start = time.time() try: @@ -167,7 +189,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): except MtgException as err: util.log(str(err), util.LogLevel.Error) return - all_cards = all_cards + new_cards + all = all + new_cards req_end = time.time() # Check if the action was canceled during download @@ -180,13 +202,9 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers): self.app.ui.get_object("dl_progress_bar").set_visible(True) self.app.ui.get_object("dl_progress_label").set_visible(True) passed = str(round(req_end - req_start, 3)) - GObject.idle_add(self.load_update_ui, all_cards, all_num, passed) + GObject.idle_add(self.load_update_ui, all, all_num, passed) - # All cards have been downloaded - GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...") - self.app.db.db_card_insert_bulk(all_cards) - - self.download_finished() + return all def load_update_ui(self, current_list: list, max_cards: int, time_passed: str): """Called from withing the worker thread. Updates the download dialog with infos.""" diff --git a/cardvault/util.py b/cardvault/util.py index 5235114..356ff0c 100644 --- a/cardvault/util.py +++ b/cardvault/util.py @@ -4,19 +4,17 @@ import json import os import re import sys +import six.moves.cPickle as pickle +from time import localtime, strftime, time +from PIL import Image as PImage +import urllib.request from urllib import request +from mtgsdk import Set, Card, MtgException import gi -from time import localtime, strftime, time gi.require_version('Gtk', '3.0') from gi.repository import GdkPixbuf, GLib -import six.moves.cPickle as pickle -from PIL import Image as PImage -from urllib.request import Request, urlopen - -from mtgsdk import Set -from mtgsdk import MtgException # Title of the Program Window APPLICATION_TITLE = "Card Vault" @@ -46,21 +44,25 @@ DB_NAME = "cardvault.db" ALL_NUM_URL = 'https://api.magicthegathering.io/v1/cards?page=0&pageSize=100' +ALL_SETS_JSON_URL = 'https://mtgjson.com/json/AllSets-x.json' + +# URL for card images. Insert card.multiverse_id. +CARD_IMAGE_URL = 'http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid={}&type=card' + # Colors for card rows in search view -SEARCH_TREE_COLORS ={ +SEARCH_TREE_COLORS = { "unowned": "black", "wanted": "#D39F30", "owned": "#62B62F" } # Colors for card rows in every default view -GENERIC_TREE_COLORS ={ +GENERIC_TREE_COLORS = { "unowned": "black", "wanted": "black", "owned": "black" } - default_config = { "hide_duplicates_in_search": False, "start_page": "search", @@ -73,7 +75,7 @@ default_config = { } } -legality_colors ={ +legality_colors = { "Banned": "#C65642", "Restricted": "#D39F30", "Legal": "#62B62F" @@ -187,19 +189,19 @@ def reload_preconstructed_icons(path: str) -> dict: if not os.path.exists(path): os.makedirs(path) - iconfiles = os.listdir(path) - for file in iconfiles: + files = os.listdir(path) + for file in files: # Split filename into single icon names and remove extension without_ext = file.split(".")[0] - list = without_ext.split("_") + names = without_ext.split("_") # Compute size of the finished icon - pic_width = len(list) * 105 + pic_width = len(names) * 105 pic_height = 105 try: pixbuf = GdkPixbuf.Pixbuf.new_from_file(ICON_CACHE_PATH + file) pixbuf = pixbuf.scale_simple(pic_width / 5, pic_height / 5, GdkPixbuf.InterpType.HYPER) # Set name for icon - iconname = "_".join(list) + iconname = "_".join(names) cache[iconname] = pixbuf except OSError as err: log("Error loading image: " + str(err), LogLevel.Error) @@ -211,8 +213,8 @@ def load_mana_icons(path: str) -> dict: log("Directory for mana icons not found " + path, LogLevel.Error) return {} icons = {} - filenames = os.listdir(path) - for file in filenames: + files = os.listdir(path) + for file in files: img = PImage.open(path + file) # Strip file extension name = os.path.splitext(file)[0] @@ -220,13 +222,19 @@ def load_mana_icons(path: str) -> dict: return icons +def net_all_cards_mtgjson() -> dict: + with urllib.request.urlopen(ALL_SETS_JSON_URL) as url: + data = json.loads(url.read().decode()) + return data + + def net_load_set_list() -> dict: """ Load the list of all MTG sets from the Gather""" try: start = time() sets = Set.all() stop = time() - log("Fetched set list in {}s".format(round(stop-start, 3)), LogLevel.Info) + log("Fetched set list in {}s".format(round(stop - start, 3)), LogLevel.Info) except MtgException as err: log(str(err), LogLevel.Error) return {} @@ -305,11 +313,11 @@ def load_dummy_image(size_x: int, size_y: int) -> GdkPixbuf: + '/resources/images/dummy.jpg', size_x, size_y) -def load_card_image(card: 'mtgsdk.Card', size_x: int, size_y: int, cache: dict) -> GdkPixbuf: +def load_card_image(card: Card, size_x: int, size_y: int, cache: dict) -> GdkPixbuf: """ Retrieve an card image from cache or alternatively load from gatherer""" try: image = cache[card.multiverse_id] - except KeyError as err: + except KeyError: log("No local image for " + card.name + ". Loading from " + card.image_url, LogLevel.Info) filename, image = net_load_card_image(card, size_x, size_y) cache[card.multiverse_id] = image @@ -329,27 +337,27 @@ def net_load_card_image(card, size_x: int, size_y: int) -> (str, GdkPixbuf): def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf: # Convert the string to a List safe_string = mana_string.replace("/", "-") - list = re.findall("{(.*?)}", safe_string) - if len(list) == 0: + glyphs = re.findall("{(.*?)}", safe_string) + if len(glyphs) == 0: return # Compute horizontal size for the final image - imagesize = len(list) * 105 - image = PImage.new("RGBA", (imagesize, 105)) + size = len(glyphs) * 105 + image = PImage.new("RGBA", (size, 105)) # Increment for each position of an icon # (Workaround: 2 or more of the same icon will be rendered in the same position) - poscounter = 0 + c = 0 # Go through all entries an add the correspondent icon to the final image - for icon in list: - xpos = poscounter * 105 + for icon in glyphs: + x_pos = c * 105 try: loaded = icons[icon] - except KeyError as err: + except KeyError: log("No icon file named '" + icon + "' found.", LogLevel.Warning) return - image.paste(loaded, (xpos, 0)) - poscounter += 1 + image.paste(loaded, (x_pos, 0)) + c += 1 # Save Icon file - path = ICON_CACHE_PATH + "_".join(list) + ".png" + path = ICON_CACHE_PATH + "_".join(glyphs) + ".png" image.save(path) try: pixbuf = GdkPixbuf.Pixbuf.new_from_file(path) @@ -360,7 +368,7 @@ def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf: def sizeof_fmt(num, suffix='B'): - for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 @@ -368,11 +376,9 @@ def sizeof_fmt(num, suffix='B'): def get_all_cards_num() -> int: - req = Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'}) - response = urlopen(req) + req = urllib.request.Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'}) + response = urllib.request.urlopen(req) headers = response.info()._headers for header, value in headers: if header == 'Total-Count': return int(value) -# endregion - diff --git a/setup.py b/setup.py index fa1565a..8b8cb3a 100755 --- a/setup.py +++ b/setup.py @@ -37,5 +37,5 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'License :: OSI Approved :: MIT License', - ], install_requires=['gi', 'pillow'] + ], install_requires=['gi', 'pillow', 'six'] )