Download function for card data.

This commit is contained in:
luxick
2017-07-12 12:04:46 +02:00
parent 7784520c2e
commit e7c2d9199d
10 changed files with 467 additions and 95 deletions

View File

@@ -158,15 +158,24 @@ class Application:
window.connect("key-press-event", eval_key_pressed)
def show_question_dialog(self, title, message):
dialog = Gtk.MessageDialog(self.ui.get_object("mainWindow"), 0, Gtk.MessageType.WARNING,
Gtk.ButtonsType.NONE, title)
dialog.add_buttons(Gtk.STOCK_YES, Gtk.ResponseType.YES,
Gtk.STOCK_NO, Gtk.ResponseType.NO,
Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL)
dialog.format_secondary_text(message)
def show_dialog_yes_no_cancel(self, title: str, message: str) -> Gtk.ResponseType:
"""Display a simple Yes/No Question dialog and return the result"""
dialog = self.ui.get_object("ync_dialog")
dialog.set_transient_for(self.ui.get_object("mainWindow"))
dialog.set_title(title)
self.ui.get_object("ync_label").set_text(message)
response = dialog.run()
dialog.destroy()
dialog.hide()
return response
def show_dialog_yes_no(self, title: str, message: str) -> Gtk.ResponseType:
"""Display a simple Yes/No Question dialog and return the result"""
dialog = self.ui.get_object("yn_dialog")
dialog.set_transient_for(self.ui.get_object("mainWindow"))
dialog.set_title(title)
self.ui.get_object("yn_label").set_text(message)
response = dialog.run()
dialog.hide()
return response
def show_message(self, title, message):

View File

@@ -16,7 +16,7 @@ gi.require_version('Gdk', '3.0')
class CardList(Gtk.ScrolledWindow):
def __init__(self, filtered, app: 'application.Application', row_colors: Dict[str, str]):
Gtk.ScrolledWindow.__init__(self)
self.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
#self.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
self.set_hexpand(True)
self.set_vexpand(True)

View File

@@ -89,7 +89,7 @@ class CardVaultDB:
parameters.append(filter_rarity)
if filer_type != "":
sql += ' AND `types` LIKE ?'
parameters.append(filer_type)
parameters.append('%'+filer_type+'%')
if filter_set != "":
sql += ' AND `set` = ?'
parameters.append(filter_set)

View File

@@ -2,6 +2,133 @@
<!-- Generated with glade 3.20.0 -->
<interface>
<requires lib="gtk+" version="3.20"/>
<object class="GtkDialog" id="loadDataDialog">
<property name="can_focus">False</property>
<property name="title" translatable="yes">Download</property>
<property name="resizable">False</property>
<property name="modal">True</property>
<property name="window_position">center</property>
<property name="type_hint">dialog</property>
<property name="deletable">False</property>
<child internal-child="vbox">
<object class="GtkBox">
<property name="can_focus">False</property>
<property name="margin_left">5</property>
<property name="margin_right">5</property>
<property name="margin_top">5</property>
<property name="margin_bottom">5</property>
<property name="hexpand">True</property>
<property name="vexpand">True</property>
<property name="orientation">vertical</property>
<property name="spacing">15</property>
<child internal-child="action_area">
<object class="GtkButtonBox">
<property name="can_focus">False</property>
<property name="layout_style">end</property>
<child>
<placeholder/>
</child>
<child>
<object class="GtkButton">
<property name="label">gtk-cancel</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
<signal name="clicked" handler="do_cancel_download" swapped="no"/>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">False</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkBox">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="orientation">vertical</property>
<property name="spacing">5</property>
<child>
<object class="GtkBox">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="spacing">5</property>
<child>
<object class="GtkLabel" id="dl_info_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="label" translatable="yes">Connecting to the Internet</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkSpinner" id="dl_spinner">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="active">True</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="pack_type">end</property>
<property name="position">1</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkProgressBar" id="dl_progress_bar">
<property name="visible">True</property>
<property name="can_focus">False</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
<child>
<object class="GtkLabel" id="dl_progress_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">2</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
</object>
</child>
<child>
<placeholder/>
</child>
</object>
<object class="GtkDialog" id="nameEnterDialog">
<property name="can_focus">False</property>
<property name="resizable">False</property>
@@ -84,36 +211,59 @@
<placeholder/>
</child>
</object>
<object class="GtkWindow" id="renameWindow">
<object class="GtkDialog" id="yn_dialog">
<property name="can_focus">False</property>
<property name="title" translatable="yes">Rename</property>
<property name="resizable">False</property>
<property name="modal">True</property>
<property name="skip_taskbar_hint">True</property>
<child>
<property name="type_hint">dialog</property>
<property name="deletable">False</property>
<child internal-child="vbox">
<object class="GtkBox">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">center</property>
<property name="valign">center</property>
<property name="spacing">5</property>
<property name="orientation">vertical</property>
<property name="spacing">15</property>
<child internal-child="action_area">
<object class="GtkButtonBox">
<property name="can_focus">False</property>
<property name="layout_style">end</property>
<child>
<object class="GtkEntry" id="renameEntry">
<object class="GtkButton" id="yn_yes">
<property name="label">gtk-yes</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
</object>
<packing>
<property name="expand">False</property>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkButton" id="renameButton">
<property name="label" translatable="yes">Rename</property>
<object class="GtkButton" id="yn_no">
<property name="label">gtk-no</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">False</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkLabel" id="yn_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
</object>
<packing>
<property name="expand">False</property>
@@ -123,7 +273,96 @@
</child>
</object>
</child>
<child type="titlebar">
<action-widgets>
<action-widget response="-8">yn_yes</action-widget>
<action-widget response="-9">yn_no</action-widget>
</action-widgets>
<child>
<placeholder/>
</child>
</object>
<object class="GtkDialog" id="ync_dialog">
<property name="can_focus">False</property>
<property name="resizable">False</property>
<property name="type_hint">dialog</property>
<property name="deletable">False</property>
<child internal-child="vbox">
<object class="GtkBox">
<property name="can_focus">False</property>
<property name="orientation">vertical</property>
<property name="spacing">15</property>
<child internal-child="action_area">
<object class="GtkButtonBox">
<property name="can_focus">False</property>
<property name="layout_style">end</property>
<child>
<object class="GtkButton" id="ync_yes">
<property name="label">gtk-yes</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkButton" id="ync_no">
<property name="label">gtk-no</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
<child>
<object class="GtkButton" id="ync_cancel">
<property name="label">gtk-cancel</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">True</property>
<property name="use_stock">True</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">2</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">False</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkLabel" id="ync_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
</object>
</child>
<action-widgets>
<action-widget response="-8">ync_yes</action-widget>
<action-widget response="-9">ync_no</action-widget>
<action-widget response="-6">ync_cancel</action-widget>
</action-widgets>
<child>
<placeholder/>
</child>
</object>

View File

@@ -74,6 +74,21 @@
<property name="can_focus">False</property>
</object>
</child>
<child>
<object class="GtkMenuItem" id="main_load_data">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="label" translatable="yes">Download Card Data</property>
<property name="use_underline">True</property>
<signal name="activate" handler="do_download_card_data" swapped="no"/>
</object>
</child>
<child>
<object class="GtkSeparatorMenuItem">
<property name="visible">True</property>
<property name="can_focus">False</property>
</object>
</child>
<child>
<object class="GtkImageMenuItem" id="quitProgram">
<property name="label">gtk-quit</property>
@@ -129,6 +144,7 @@
<object class="GtkRadioMenuItem" id="decksViewItem">
<property name="name">decks</property>
<property name="visible">True</property>
<property name="sensitive">False</property>
<property name="can_focus">False</property>
<property name="label" translatable="yes">Decks</property>
<property name="use_underline">True</property>
@@ -206,15 +222,6 @@
<signal name="activate" handler="do_load_data_to_db" swapped="no"/>
</object>
</child>
<child>
<object class="GtkMenuItem" id="load_all_cards">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="label" translatable="yes">Load All Cards From Gatherer</property>
<property name="use_underline">True</property>
<signal name="activate" handler="do_load_all_cards" swapped="no"/>
</object>
</child>
</object>
</child>
</object>

View File

@@ -1,12 +1,16 @@
import gi
import sys
import math
gi.require_version('Gtk', '3.0')
import datetime
import itertools
import time, datetime
import os
from gi.repository import Gtk
import threading
from gi.repository import Gtk, GObject
from cardvault import util, application
from mtgsdk import Card
from mtgsdk import Card, MtgException
from search import SearchHandlers
from library import LibraryHandlers
@@ -17,6 +21,8 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
def __init__(self, app: 'application.Application'):
"""Initialize handlers for UI signals"""
self.app = app
# Token to cancel a running download
self.cancel_token = False
# Call constructor of view handlers classes
SearchHandlers.__init__(self, app)
@@ -55,7 +61,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
response = dialog.run()
if response == Gtk.ResponseType.OK:
# Show confirmation message
override_question = self.app.show_question_dialog("Import Library",
override_question = self.app.show_dialog_yes_no_cancel("Import Library",
"Importing a library will override your current library. "
"Proceed?")
if override_question == Gtk.ResponseType.YES:
@@ -85,7 +91,7 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
def do_delete_event(self, arg1, arg2):
if self.app.unsaved_changes:
response = self.app.show_question_dialog("Unsaved Changes", "You have unsaved changes in your library. "
response = self.app.show_dialog_yes_no_cancel("Unsaved Changes", "You have unsaved changes in your library. "
"Save before exiting?")
if response == Gtk.ResponseType.YES:
self.app.save_data()
@@ -93,6 +99,114 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
elif response == Gtk.ResponseType.CANCEL:
return True
def do_cancel_download(self, item: Gtk.MenuItem):
"""The cancel button was pressed, set cancel_token to stop download thread"""
self.cancel_token = True
# Delete Dialog
self.app.ui.get_object("loadDataDialog").destroy()
self.app.push_status("Download canceled")
util.log("Download canceled by user", util.LogLevel.Info)
def download_canceled(self):
"""The download thread was canceled and finished executing"""
self.cancel_token = False
util.log("Download thread ended", util.LogLevel.Info)
def download_failed(self, err: MtgException):
# Delete Dialog
self.app.ui.get_object("loadDataDialog").destroy()
self.app.push_status("Download canceled")
self.app.show_message("Download Faild", str(err))
def download_finished(self):
"""Download thread finished without errors"""
self.cancel_token = False
self.app.ui.get_object("loadDataDialog").destroy()
self.app.push_status("Card data downloaded")
util.log("Card data download finished", util.LogLevel.Info)
def do_download_card_data(self, item: Gtk.MenuItem):
"""Download button was pressed in the menu bar. Starts a thread to load data from the internet"""
info_string = "Start downloading card information from the internet?\n" \
"This process can take up to 10 minutes.\n" \
"You can cancel the download at any point."
response = self.app.show_dialog_yes_no("Download Card Data", info_string)
if response == Gtk.ResponseType.NO:
return
# Launch download info dialog
dl_dialog = self.app.ui.get_object("loadDataDialog")
dl_dialog.set_transient_for(self.app.ui.get_object("mainWindow"))
dl_dialog.show()
# Hide Progress UI until download started
self.app.ui.get_object("dl_progress_bar").set_visible(False)
self.app.ui.get_object("dl_progress_label").set_visible(False)
# Create and start the download in a separate thread so it will not block the UI
thread = threading.Thread(target=self.load_thread)
thread.daemon = True
thread.start()
util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info)
def load_thread(self):
"""Worker thread to download info using the mtgsdk"""
all_cards = []
# Request total number of cards we are going to download
all_num = util.get_all_cards_num()
all_pages = int(math.ceil(all_num/100))
# Download cards in pages until no new cards are added
for i in range(all_pages):
req_start = time.time()
try:
new_cards = Card.where(page=i).where(pageSize=100).all()
except MtgException as err:
util.log(str(err), util.LogLevel.Error)
return
all_cards = all_cards + new_cards
req_end = time.time()
# Check if the action was canceled during download
if self.cancel_token:
GObject.idle_add(self.download_canceled)
return
# Activate download UI
self.app.ui.get_object("dl_spinner").set_visible(False)
self.app.ui.get_object("dl_progress_bar").set_visible(True)
self.app.ui.get_object("dl_progress_label").set_visible(True)
passed = str(round(req_end - req_start, 3))
GObject.idle_add(self.load_update_ui, all_cards, all_num, passed)
# All cards have been downloaded
GObject.idle_add(self.load_show_insert_ui, "Saving data to disk...")
self.app.db.bulk_insert_card(all_cards)
self.download_finished()
def load_update_ui(self, current_list: list, max_cards: int, time_passed: str):
"""Called from withing the worker thread. Updates the download dialog with infos."""
# Get info widgets
info_label = self.app.ui.get_object("dl_info_label")
progress_label = self.app.ui.get_object("dl_progress_label")
bar = self.app.ui.get_object("dl_progress_bar")
# Compute numbers for display
size_human = util.sizeof_fmt(sys.getsizeof(current_list))
size_bytes = sys.getsizeof(current_list)
percent = len(current_list) / max_cards
# Update UI
info_label.set_text("Downloading Cards...")
progress_label.set_text("{:.1%} ({})".format(percent, size_human))
bar.set_fraction(percent)
util.log("Downloading: {:.1%} | {} Bytes | {}s".format(percent, size_bytes, time_passed), util.LogLevel.Info)
def load_show_insert_ui(self, info: str):
"""Called from worker thread after download finished. Sets UI to display the passed string"""
self.app.ui.get_object("dl_info_label").set_text(info)
self.app.ui.get_object("dl_spinner").set_visible(True)
self.app.ui.get_object("dl_progress_bar").set_visible(False)
self.app.ui.get_object("dl_progress_label").set_visible(False)
# ---------------------- Debug actions -------------------------------
def do_load_data_to_db(self, item):
@@ -115,23 +229,6 @@ class Handlers(SearchHandlers, LibraryHandlers, WantsHandlers):
end = datetime.datetime.now()
util.log("Finished in {}s".format(str(end - start)), util.LogLevel.Info)
def do_load_all_cards(self, menu_item):
util.log("Attempt fetching all cards from Gatherer. This may take a while...", util.LogLevel.Info)
start = datetime.datetime.now()
all_cards = []
for i in itertools.count():
new_cards = Card.where(page=i).where(pageSize=100).all()
if len(new_cards) == 0:
break
all_cards = all_cards + new_cards
util.log("Fetched page {}, {} cards so far".format(str(i), str(len(all_cards))), util.LogLevel.Info)
end = datetime.datetime.now()
util.log("Finished fetching {} cards in {}".format(str(len(all_cards)), (end - start)), util.LogLevel.Info)
util.log("Inserting cards into library...", util.LogLevel.Info)
self.app.db.bulk_insert_card(all_cards)
util.log("Done", util.LogLevel.Info)
def do_clear_card_data(self, menu_item):
util.log("Deleting all local card data", util.LogLevel.Info)
self.app.db.clear_card_data()

View File

@@ -9,10 +9,25 @@ from cardvault import cardlist
class LibraryHandlers:
def __init__(self, app: 'application.Application'):
"""Initialize the library view"""
self.app = app
self.init_library_view()
# Create Tree View for library
container = self.app.ui.get_object("libraryContainer")
card_list = cardlist.CardList(True, self.app, util.GENERIC_TREE_COLORS)
card_list.set_name("libScroller")
# Show details
card_list.tree.connect("row-activated", self.on_library_card_selected)
# Show Context menu
card_list.tree.connect("button-press-event", self.on_library_tree_press_event)
card_list.filter.set_visible_func(self.app.filter_lib_func)
container.add(card_list)
container.add_overlay(self.app.ui.get_object("noResults"))
container.show_all()
self.app.ui.get_object("noResults").set_visible(False)
def do_reload_library(self, view):
"""Handler for the 'show' signal"""
self.reload_library()
def do_tag_entry_changed(self, entry):
@@ -156,23 +171,6 @@ class LibraryHandlers:
# -------------------------- Class Functions -------------------------------
def init_library_view(self):
"""Initialize the library view"""
# Create Tree View for library
container = self.app.ui.get_object("libraryContainer")
card_list = cardlist.CardList(True, self.app, util.GENERIC_TREE_COLORS)
card_list.set_name("libScroller")
# Show details
card_list.tree.connect("row-activated", self.on_library_card_selected)
# Show Context menu
card_list.tree.connect("button-press-event", self.on_library_tree_press_event)
card_list.filter.set_visible_func(self.app.filter_lib_func)
container.add(card_list)
container.add_overlay(self.app.ui.get_object("noResults"))
container.show_all()
self.app.ui.get_object("noResults").set_visible(False)
def reload_library(self, tag="All"):
if tag == "Untagged":
lib = self.app.get_untagged_cards()

View File

@@ -15,7 +15,19 @@ from cardvault import util
class SearchHandlers:
def __init__(self, app: 'application.Application'):
self.app = app
self.init_search_view()
# set mana icons on filter buttons
buttons = [x for x in self.app.ui.get_object("manaFilterGrid").get_children()
if isinstance(x, Gtk.ToggleButton)]
self._init_mana_buttons(buttons)
# set auto completion for filter entry
self._init_set_entry(self.app.ui.get_object("setEntry"))
# Fill rarity box
self._init_combo_box(self.app.ui.get_object("rarityCombo"), util.rarity_dict.keys())
# Fill type box
self._init_combo_box(self.app.ui.get_object("typeCombo"), util.card_types)
# Create Model for search results
self._init_results_tree()
def do_search_cards(self, sender):
search_term = self.app.ui.get_object("searchEntry").get_text()
@@ -159,20 +171,6 @@ class SearchHandlers:
self.app.add_card_to_lib(card)
self.reload_search_view()
def init_search_view(self):
# set mana icons on filter buttons
buttons = [x for x in self.app.ui.get_object("manaFilterGrid").get_children()
if isinstance(x, Gtk.ToggleButton)]
self._init_mana_buttons(buttons)
# set auto completion for filter entry
self._init_set_entry(self.app.ui.get_object("setEntry"))
# Fill rarity box
self._init_combo_box(self.app.ui.get_object("rarityCombo"), util.rarity_dict.keys())
# Fill type box
self._init_combo_box(self.app.ui.get_object("typeCombo"), util.card_types)
# Create Model for search results
self._init_results_tree()
def reload_search_view(self):
""" Reload the card tree """
results_tree = self.app.ui.get_object("searchResults").get_child()

View File

@@ -14,6 +14,10 @@ from gi.repository import GdkPixbuf, GLib
import six.moves.cPickle as pickle
from PIL import Image as PImage
from urllib.request import Request, urlopen
from urllib.error import HTTPError
from urllib.parse import urlencode
from mtgsdk import Set
from mtgsdk import MtgException
@@ -43,6 +47,8 @@ LOG_LEVEL = 1
# Name of the database
DB_NAME = "cardvault.db"
ALL_NUM_URL = 'https://api.magicthegathering.io/v1/cards?page=0&pageSize=100'
# Colors for card rows in search view
SEARCH_TREE_COLORS ={
"unowned": "black",
@@ -354,5 +360,21 @@ def create_mana_icons(icons: dict, mana_string: str) -> GdkPixbuf:
return
return pixbuf
def sizeof_fmt(num, suffix='B'):
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
def get_all_cards_num() -> int:
req = Request(ALL_NUM_URL, headers={'User-Agent': 'Mozilla/5.0'})
response = urlopen(req)
headers = response.info()._headers
for header, value in headers:
if header == 'Total-Count':
return int(value)
# endregion

View File

@@ -10,7 +10,7 @@
import json
from urllib.request import Request, urlopen
from urllib.error import HTTPError
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
@@ -37,6 +37,8 @@ class RestClient(object):
return response
except HTTPError as err:
raise MtgException(err.read())
except URLError as err:
raise MtgException(str(err.reason))
class MtgException(Exception):