luxick/cardvault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace card download from Gatherer with mtgjson.com.

Browse Source
This commit is contained in:
luxick
2017-07-26 00:01:21 +02:00
parent 1db6ad0f2d
commit 2b1f3b940f
5 changed files with 121 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cardvault/application.py
View File

@@ -357,7 +357,7 @@ class Application:
util.log("Removed {} from library".format(card.name), util.LogLevel.Info)
self.push_status(card.name + " removed from library")
def override_user_data(self):
def db_override_user_data(self):
"""Called after import of user data. Overrides existing user data in database"""
util.log("Clearing old user data", util.LogLevel.Info)
self.db.db_clear_data_user()
@@ -380,6 +380,12 @@ class Application:
util.log("Finished in {}s".format(str(round(end - start, 3))), util.LogLevel.Info)
self.push_status("User data imported")
def db_delete_card_data(self):
"""Called before before rebuilding local data storage"""
util.log("Clearing local card data", util.LogLevel.Info)
self.db.db_clear_data_card()
util.log("Done", util.LogLevel.Info)
def get_mana_icons(self, mana_string):
if not mana_string:
util.log("No mana string provided", util.LogLevel.Info)

42
cardvault/database.py
View File

@@ -23,14 +23,13 @@ class CardVaultDB:
"`colors` TEXT, `names` TEXT, `type` TEXT, `supertypes` TEXT, "
"`subtypes` TEXT, `types` TEXT, `rarity` TEXT, `text` TEXT, "
"`flavor` TEXT, `artist` TEXT, `number` INTEGER, `power` TEXT, "
"`toughness` TEXT, `loyalty` INTEGER, `multiverseid` INTEGER UNIQUE , "
"`toughness` TEXT, `loyalty` INTEGER, `multiverseid` INTEGER , "
"`variations` TEXT, `watermark` TEXT, `border` TEXT, `timeshifted` "
"TEXT, `hand` TEXT, `life` TEXT, `releaseDate` TEXT, `starter` TEXT, "
"`printings` TEXT, `originalText` TEXT, `originalType` TEXT, "
"`source` TEXT, `imageUrl` TEXT, `set` TEXT, `setName` TEXT, `id` TEXT, "
"`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT, "
"PRIMARY KEY(`multiverseid`) )")
con.execute("CREATE TABLE IF NOT EXISTS library ( multiverse_id INT PRIMARY KEY, copies INT )")
"`legalities` TEXT, `rulings` TEXT, `foreignNames` TEXT) ")
con.execute("CREATE TABLE IF NOT EXISTS library ( multiverseid INT PRIMARY KEY, copies INT )")
con.execute("CREATE TABLE IF NOT EXISTS tags ( tag TEXT, multiverseid INT )")
con.execute("CREATE TABLE IF NOT EXISTS wants ( listName TEXT, multiverseid INT )")
@@ -41,7 +40,8 @@ class CardVaultDB:
with con:
# Map card object to database tables
db_values = self.card_to_table_mapping(card)
sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \
"?,?,?,?,?,?,?,?,?,?)"
# Insert into database
con.execute(sql_string, db_values)
except sqlite3.OperationalError as err:
@@ -50,10 +50,6 @@ class CardVaultDB:
except sqlite3.IntegrityError:
pass
def db_card_insert_bulk(self, card_list: list):
for card in card_list:
self.db_card_insert(card)
def db_get_all(self):
sql = 'SELECT * FROM cards'
con = sqlite3.connect(self.db_file)
@@ -68,6 +64,34 @@ class CardVaultDB:
output.append(card)
return output
def db_insert_data_card(self, cards_json):
"""Insert download from mtgjson"""
rows = []
for data in cards_json.values():
cards = []
for raw in data["cards"]:
c = Card(raw)
c.image_url = util.CARD_IMAGE_URL.format(c.multiverse_id)
c.set = data["code"]
c.set_name = data["name"]
cards.append(c)
for c in cards:
rows.append(self.card_to_table_mapping(c))
# Connect to database
con = sqlite3.connect(self.db_file)
try:
with con:
sql_string = "INSERT INTO `cards` VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?," \
"?,?,?,?,?,?,?,?,?,?)"
con.executemany(sql_string, rows)
except sqlite3.OperationalError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
except sqlite3.IntegrityError as err:
util.log("Database Error", util.LogLevel.Error)
util.log(str(err), util.LogLevel.Error)
def db_clear_data_card(self):
"""Delete all resource data from database"""
self.db_operation("DELETE FROM cards")

56
cardvault/handlers.py
View File

@@ -70,12 +70,11 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
self.app.tags = imports[1]
self.app.wants = imports[2]
# Save imported data to database
self.app.override_user_data()
self.app.db_override_user_data()
# Cause current page to reload with imported data
self.app.current_page.emit('show')
dialog.destroy()
def on_view_changed(self, item):
if item.get_active():
container = self.app.ui.get_object("contentPage")
@@ -105,7 +104,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
"""The cancel button was pressed, set cancel_token to stop download thread"""
self.cancel_token = True
# Delete Dialog
self.app.ui.get_object("loadDataDialog").destroy()
self.app.ui.get_object("loadDataDialog").hide()
self.app.push_status("Download canceled")
util.log("Download canceled by user", util.LogLevel.Info)
@@ -116,23 +115,22 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
def download_failed(self, err: MtgException):
# Delete Dialog
self.app.ui.get_object("loadDataDialog").destroy()
self.app.ui.get_object("loadDataDialog").hide()
self.app.push_status("Download canceled")
self.app.show_message("Download Faild", str(err))
self.app.show_message("Download Failed", str(err))
def download_finished(self):
"""Download thread finished without errors"""
self.cancel_token = False
self.app.config["local_db"] = True
self.app.save_config()
self.app.ui.get_object("loadDataDialog").destroy()
self.app.ui.get_object("loadDataDialog").hide()
self.app.push_status("Card data downloaded")
util.log("Card data download finished", util.LogLevel.Info)
def do_download_card_data(self, item: Gtk.MenuItem):
"""Download button was pressed in the menu bar. Starts a thread to load data from the internet"""
info_string = "Start downloading card information from the internet?\n" \
"This process can take up to 10 minutes.\n" \
info_string = "Start downloading card information from the internet?\n" \
"You can cancel the download at any point."
response = self.app.show_dialog_yn("Download Card Data", info_string)
if response == Gtk.ResponseType.NO:
@@ -150,16 +148,40 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
thread = threading.Thread(target=self.load_thread)
thread.daemon = True
thread.start()
util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info)
util.log("Attempt downloading all cards. This may take a while...", util.LogLevel.Info)
def load_thread(self):
"""Worker thread to download info using the mtgsdk"""
all_cards = []
# Request total number of cards we are going to download
# Gatherer uses rate limit on Card.all()
# Takes ~10 minutes to download all cards
# all = self.load_thread_gatherer()
# Download from mtgjson.com
GObject.idle_add(self.load_show_insert_ui, "Downloading...")
util.log("Starting download", util.LogLevel.Info)
s = time.time()
all_json = util.net_all_cards_mtgjson()
e = time.time()
util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info)
self.app.db_delete_card_data()
GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...")
util.log("Saving to sqlite", util.LogLevel.Info)
s = time.time()
self.app.db.db_insert_data_card(all_json)
e = time.time()
util.log("Finished in {}s".format(round(e - s, 3)), util.LogLevel.Info)
self.download_finished()
def load_thread_gatherer(self):
all = []
all_num = util.get_all_cards_num()
all_pages = int(math.ceil(all_num / 100))
# Download cards in pages until no new cards are added
# Paging for ui control between downloads
for i in range(all_pages):
req_start = time.time()
try:
@@ -167,7 +189,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
except MtgException as err:
util.log(str(err), util.LogLevel.Error)
return
all_cards = all_cards + new_cards
all = all + new_cards
req_end = time.time()
# Check if the action was canceled during download
@@ -180,13 +202,9 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
self.app.ui.get_object("dl_progress_bar").set_visible(True)
self.app.ui.get_object("dl_progress_label").set_visible(True)
passed = str(round(req_end - req_start, 3))
GObject.idle_add(self.load_update_ui, all_cards, all_num, passed)
GObject.idle_add(self.load_update_ui, all, all_num, passed)
# All cards have been downloaded
GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...")
self.app.db.db_card_insert_bulk(all_cards)
self.download_finished()
return all
def load_update_ui(self, current_list: list, max_cards: int, time_passed: str):
"""Called from withing the worker thread. Updates the download dialog with infos."""

80
cardvault/util.py
View File

@@ -4,19 +4,17 @@ import json
import os
import re
import sys
import six.moves.cPickle as pickle
from time import localtime, strftime, time
from PIL import Image as PImage
import urllib.request
from urllib import request
from mtgsdk import Set, Card, MtgException
import gi
from time import localtime, strftime, time
gi.require_version('Gtk', '3.0')
from gi.repository import GdkPixbuf, GLib
import six.moves.cPickle as pickle
from PIL import Image as PImage
from urllib.request import Request, urlopen
from mtgsdk import Set
from mtgsdk import MtgException
# Title of the Program Window
APPLICATION_TITLE = "Card Vault"
@@ -46,21 +44,25 @@ DB_NAME = "cardvault.db"
ALL_NUM_URL = 'https://api.magicthegathering.io/v1/cards?page=0&pageSize=100'
ALL_SETS_JSON_URL = 'https://mtgjson.com/json/AllSets-x.json'
# URL for card images. Insert card.multiverse_id.
CARD_IMAGE_URL = 'http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid={}&type=card'
# Colors for card rows in search view
SEARCH_TREE_COLORS ={
SEARCH_TREE_COLORS = {
"unowned": "black",
"wanted": "#D39F30",
"owned": "#62B62F"
}
# Colors for card rows in every default view
GENERIC_TREE_COLORS ={
GENERIC_TREE_COLORS = {
"unowned": "black",
"wanted": "black",
"owned": "black"
}
default_config = {
"hide_duplicates_in_search": False,
"start_page": "search",
@@ -73,7 +75,7 @@ default_config = {
}
}
legality_colors ={
legality_colors = {
"Banned": "#C65642",
"Restricted": "#D39F30",
"Legal": "#62B62F"
@@ -187,19 +189,19 @@ def reload_preconstructed_icons(path: str) -> dict:
if not os.path.exists(path):
os.makedirs(path)
iconfiles = os.listdir(path)
for file in iconfiles:
files = os.listdir(path)
for file in files:
# Split filename into single icon names and remove extension
without_ext = file.split(".")[0]
list = without_ext.split("_")
names = without_ext.split("_")
# Compute size of the finished icon
pic_width = len(list) * 105
pic_width = len(names) * 105
pic_height = 105
try:
pixbuf = GdkPixbuf.Pixbuf.new_from_file(ICON_CACHE_PATH + file)
pixbuf = pixbuf.scale_simple(pic_width / 5, pic_height / 5, GdkPixbuf.InterpType.HYPER)
# Set name for icon
iconname = "_".join(list)
iconname = "_".join(names)
cache[iconname] = pixbuf
except OSError as err:
log("Error loading image: " + str(err), LogLevel.Error)
@@ -211,8 +213,8 @@ def load_mana_icons(path: str) -> dict:
log("Directory for mana icons not found " + path, LogLevel.Error)
return {}
icons = {}
filenames = os.listdir(path)
for file in filenames:
files = os.listdir(path)
for file in files:
img = PImage.open(path + file)
# Strip file extension
name = os.path.splitext(file)[0]
@@ -220,13 +222,19 @@ def load_mana_icons(path: str) -> dict:
return icons
def net_all_cards_mtgjson() -> dict:
with urllib.request.urlopen(ALL_SETS_JSON_URL) as url:
data = json.loads(url.read().decode())
return data
def net_load_set_list() -> dict:
""" Load the list of all MTG sets from the Gather"""
try:
start = time()
sets = Set.all()
stop = time()
log("Fetched set list in {}s".format(round(stop-start, 3)), LogLevel.Info)
log("Fetched set list in {}s".format(round(stop - start, 3)), LogLevel.Info)
except MtgException as err:
log(str(err), LogLevel.Error)
return {}
@@ -305,11 +313,11 @@ def load_dummy_image(size_x: int, size_y: int) -> GdkPixbuf:
+ '/resources/images/dummy.jpg', size_x, size_y)
def load_card_image(card: 'mtgsdk.Card', size_x: int, size_y: int, cache: dict) -> GdkPixbuf:
def load_card_image(card: Card, size_x: int, size_y: int, cache: dict) -> GdkPixbuf:
""" Retrieve an card image from cache or alternatively load from gatherer"""
try:
image = cache[card.multiverse_id]
except KeyError as err:
except KeyError:
log("No local image for " + card.name + ". Loading from " + card.image_url, LogLevel.Info)
filename, image = net_load_card_image(card, size_x, size_y)
cache[card.multiverse_id] = image
@@ -329,27 +337,27 @@ def net_load_card_image(card, size_x: int, size_y: int) -> (str, GdkPixbuf):
def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf:
# Convert the string to a List
safe_string = mana_string.replace("/", "-")
list = re.findall("{(.*?)}", safe_string)
if len(list) == 0:
glyphs = re.findall("{(.*?)}", safe_string)
if len(glyphs) == 0:
return
# Compute horizontal size for the final image
imagesize = len(list) * 105
image = PImage.new("RGBA", (imagesize, 105))
size = len(glyphs) * 105
image = PImage.new("RGBA", (size, 105))
# Increment for each position of an icon
# (Workaround: 2 or more of the same icon will be rendered in the same position)
poscounter = 0
c = 0
# Go through all entries an add the correspondent icon to the final image
for icon in list:
xpos = poscounter * 105
for icon in glyphs:
x_pos = c * 105
try:
loaded = icons[icon]
except KeyError as err:
except KeyError:
log("No icon file named '" + icon + "' found.", LogLevel.Warning)
return
image.paste(loaded, (xpos, 0))
poscounter += 1
image.paste(loaded, (x_pos, 0))
c += 1
# Save Icon file
path = ICON_CACHE_PATH + "_".join(list) + ".png"
path = ICON_CACHE_PATH + "_".join(glyphs) + ".png"
image.save(path)
try:
pixbuf = GdkPixbuf.Pixbuf.new_from_file(path)
@@ -360,7 +368,7 @@ def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf:
def sizeof_fmt(num, suffix='B'):
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
@@ -368,11 +376,9 @@ def sizeof_fmt(num, suffix='B'):
def get_all_cards_num() -> int:
req = Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'})
response = urlopen(req)
req = urllib.request.Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'})
response = urllib.request.urlopen(req)
headers = response.info()._headers
for header, value in headers:
if header == 'Total-Count':
return int(value)
# endregion

2
setup.py
View File

@@ -37,5 +37,5 @@ setup(
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'License :: OSI Approved :: MIT License',
], install_requires=['gi', 'pillow']
], install_requires=['gi', 'pillow', 'six']
)