Implement caching framework

This PR includes: - Caching and fallback framework for listings (e.g. A-Z, or categories) - Replace Mock with xbmc library replacement For testing the caching-functionality the use of Mock() became an even more complex issue. So this replaces a quite complex Mock() setup to mock kodiwrapper with an alternative implementation of the xbmc libraries. The benefit is that we don't have to fake every possible kodiwrapper functionality, but only the xbmc functionality that kodiwrapper is using. This means we are now also testing kodiwrapper to a greater extent.
add-ons · May 20, 2019 · c5f7e1f · c5f7e1f
1 parent 3e5f2df
commit c5f7e1f
Show file tree

Hide file tree

Showing 25 changed files with 614 additions and 225 deletions.
diff --git a/addon.py b/addon.py
@@ -29,6 +29,9 @@ def router(params_string):
     _kodi = kodiwrapper.KodiWrapper(_ADDON_HANDLE, _ADDON_URL, addon)
     _kodi.log_access(_ADDON_URL, params_string)
 
+    if action == actions.INVALIDATE_CACHES:
+        _kodi.invalidate_caches()
+        return
     if action == actions.CLEAR_COOKIES:
         from resources.lib.vrtplayer import tokenresolver
         _tokenresolver = tokenresolver.TokenResolver(_kodi)
@@ -52,7 +55,7 @@ def router(params_string):
         _favorites.unfollow(program=params.get('program'), path=params.get('path'))
         return
     if action == actions.REFRESH_FAVORITES:
-        _favorites.update_favorites()
+        _favorites.get_favorites(ttl=0)
         return
 
     from resources.lib.vrtplayer import vrtapihelper, vrtplayer

diff --git a/pylintrc b/pylintrc
@@ -12,12 +12,13 @@ disable=
     no-self-use,
     old-style-class,
     too-few-public-methods,
-    too-many-public-methods,
     too-many-arguments,
     too-many-branches,
     too-many-function-args,
     too-many-instance-attributes,
     too-many-locals,
+    too-many-public-methods,
+    too-many-return-statements,
     too-many-statements,
     unnecessary-lambda,
     unused-argument,

diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po
@@ -286,6 +286,10 @@ msgctxt "#30830"
 msgid "When enabled, menus are being cached so they work more quickly. Only in very specific use-cases this is problematic as new episodes may not appear when they are expected."
 msgstr ""
 
+msgctxt "#30831"
+msgid "Enable HTTP caching [COLOR gray][I](experimental)[/I][/COLOR]"
+msgstr ""
+
 msgctxt "#30840"
 msgid "Playback"
 msgstr ""
@@ -339,26 +343,30 @@ msgid "Refresh favorites"
 msgstr ""
 
 msgctxt "#30867"
-msgid "Streaming"
+msgid "Invalidate HTTP caches"
 msgstr ""
 
 msgctxt "#30869"
-msgid "Use InputStream Adaptive"
+msgid "Streaming"
 msgstr ""
 
 msgctxt "#30871"
-msgid "InputStream Adaptive settings..."
+msgid "Use InputStream Adaptive"
 msgstr ""
 
 msgctxt "#30873"
-msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
+msgid "InputStream Adaptive settings..."
 msgstr ""
 
 msgctxt "#30875"
-msgid "Logging"
+msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
 msgstr ""
 
 msgctxt "#30877"
+msgid "Logging"
+msgstr ""
+
+msgctxt "#30879"
 msgid "Log level"
 msgstr ""
 

diff --git a/resources/language/resource.language.nl_nl/strings.po b/resources/language/resource.language.nl_nl/strings.po
@@ -275,6 +275,10 @@ msgctxt "#30829"
 msgid "Enable menu caching"
 msgstr "Gebruik menu caching"
 
+msgctxt "#30831"
+msgid "Enable HTTP caching [COLOR gray][I](experimental)[/I][/COLOR]"
+msgstr "Gebruik HTTP caching [COLOR gray][I](experimenteel)[/I][/COLOR]"
+
 msgctxt "#30840"
 msgid "Playback"
 msgstr "Afspelen"
@@ -315,27 +319,31 @@ msgctxt "#30065"
 msgid "Refresh favorites"
 msgstr "Ververs gevolgde programma's"
 
-msgctxt "#30067"
+msgctxt "#30867"
+msgid "Invalidate HTTP caches"
+msgstr "Invalideer HTTP caches"
+
+msgctxt "#30069"
 msgid "Streaming"
 msgstr "Streaming"
 
-msgctxt "#30869"
+msgctxt "#30870"
 msgid "Use InputStream Adaptive"
 msgstr "Gebruik InputStream Adaptive"
 
-msgctxt "#30071"
+msgctxt "#30073"
 msgid "InputStream Adaptive settings..."
 msgstr "InputStream Adaptive instellingen..."
 
-msgctxt "#30073"
+msgctxt "#30075"
 msgid "Install Widevine... [COLOR gray][I](needed for DRM content)[/I][/COLOR]"
 msgstr "Installeer Widevine... [COLOR gray][I](nodig voor DRM content)[/I][/COLOR]"
 
-msgctxt "#30075"
+msgctxt "#30077"
 msgid "Logging"
 msgstr "Logboek"
 
-msgctxt "#30077"
+msgctxt "#30079"
 msgid "Log level"
 msgstr "Log level"
 

diff --git a/resources/lib/kodiwrappers/kodiwrapper.py b/resources/lib/kodiwrappers/kodiwrapper.py
@@ -106,11 +106,11 @@ def __init__(self, handle, url, addon):
         self._addon_id = addon.getAddonInfo('id')
         self._max_log_level = log_levels.get(self.get_setting('max_log_level'), 3)
         self._usemenucaching = self.get_setting('usemenucaching') == 'true'
+        self._cache_path = self.get_userdata_path() + 'cache/'
         self._system_locale_works = self.set_locale()
 
     def install_widevine(self):
-        import xbmcgui
-        ok = xbmcgui.Dialog().yesno(self.localize(30971), self.localize(30972))
+        ok = self.show_yesno_dialog(heading=self.localize(30971), message=self.localize(30972))
         if not ok:
             return
         try:
@@ -227,14 +227,20 @@ def show_ok_dialog(self, heading='', message=''):
         import xbmcgui
         if not heading:
             heading = self._addon.getAddonInfo('name')
-        xbmcgui.Dialog().ok(heading=heading, message=message)
+        xbmcgui.Dialog().ok(heading=heading, line1=message)
 
     def show_notification(self, heading='', message='', icon='info', time=4000):
         import xbmcgui
         if not heading:
             heading = self._addon.getAddonInfo('name')
         xbmcgui.Dialog().notification(heading=heading, message=message, icon=icon, time=time)
 
+    def show_yesno_dialog(self, heading='', message=''):
+        import xbmcgui
+        if not heading:
+            heading = self._addon.getAddonInfo('name')
+        return xbmcgui.Dialog().yesno(heading=self.localize(30971), line1=self.localize(30972))
+
     def set_locale(self):
         import locale
         locale_lang = self.get_global_setting('locale.language').split('.')[-1]
@@ -243,7 +249,7 @@ def set_locale(self):
             locale.setlocale(locale.LC_ALL, locale_lang)
             return True
         except Exception as e:
-            self.log_notice(e, 'Debug')
+            self.log_notice('Failed to set locale: %s' % e, 'Debug')
             return False
 
     def localize(self, string_id):
@@ -353,9 +359,19 @@ def get_addon_path(self):
     def get_path(self, path):
         return xbmc.translatePath(path)
 
-    def make_dir(self, path):
+    def listdir(self, path):
+        import xbmcvfs
+        return xbmcvfs.listdir(path)
+
+    def mkdir(self, path):
+        import xbmcvfs
+        self.log_notice("Create directory '%s'." % path, 'Debug')
+        return xbmcvfs.mkdir(path)
+
+    def mkdirs(self, path):
         import xbmcvfs
-        xbmcvfs.mkdir(path)
+        self.log_notice("Recursively create directory '%s'." % path, 'Debug')
+        return xbmcvfs.mkdirs(path)
 
     def check_if_path_exists(self, path):
         import xbmcvfs
@@ -374,8 +390,72 @@ def stat_file(self, path):
 
     def delete_file(self, path):
         import xbmcvfs
+        self.log_notice("Delete file '%s'." % path, 'Debug')
         return xbmcvfs.delete(path)
 
+    def md5(self, path):
+        import hashlib
+        with self.open_file(path) as f:
+            return hashlib.md5(f.read().encode('utf-8'))
+
+    def get_cache(self, path, ttl=None):
+        if self.get_setting('usehttpcaching') == 'false':
+            return None
+
+        path = self._cache_path + path
+        if not self.check_if_path_exists(path):
+            return None
+
+        import time
+        if ttl is None or self.stat_file(path).st_mtime() > time.mktime(time.localtime()) - ttl:
+            import json
+            if ttl is None:
+                self.log_notice("Cache '%s' is forced from cache." % path, 'Debug')
+            else:
+                self.log_notice("Cache '%s' is fresh, within ttl of %s seconds." % (path, ttl), 'Debug')
+            with self.open_file(path, 'r') as f:
+                try:
+                    return json.load(f, encoding='utf-8')
+                except ValueError:
+                    return None
+
+        return None
+
+    def update_cache(self, path, data):
+        if self.get_setting('usehttpcaching') == 'false':
+            return
+
+        import hashlib
+        import json
+        path = self._cache_path + path
+        if self.check_if_path_exists(path):
+            md5 = self.md5(path)
+        else:
+            md5 = 0
+            # Create cache directory if missing
+            if not self.check_if_path_exists(self._cache_path):
+                self.log_notice("Create path '%s'." % self._cache_path, 'Debug')
+                self.mkdirs(self._cache_path)
+
+        # Avoid writes if possible (i.e. SD cards)
+        if md5 != hashlib.md5(json.dumps(data, encoding='utf-8')):
+            self.log_notice("Write cache '%s'." % path, 'Debug')
+            with self.open_file(path, 'w') as f:
+                json.dump(data, f, encoding='utf-8')
+        else:
+            # Update timestamp
+            import os
+            self.log_notice("Cache '%s' has not changed, updating mtime only." % path, 'Debug')
+            os.utime(path)
+
+    def invalidate_cache(self, path):
+        self.delete_file(self._cache_path + path)
+
+    def invalidate_caches(self):
+        _, files = self.listdir(self._cache_path)
+        for f in files:
+            self.delete_file(self._cache_path + f)
+
     def container_refresh(self):
         self.log_notice('Execute: Container.Refresh', 'Debug')
         xbmc.executebuiltin('Container.Refresh')

diff --git a/resources/lib/vrtplayer/__init__.py b/resources/lib/vrtplayer/__init__.py
@@ -119,6 +119,7 @@ class actions:
     CLEAR_COOKIES = 'clearcookies'
     FOLLOW = 'follow'
     INSTALL_WIDEVINE = 'installwidevine'
+    INVALIDATE_CACHES = 'invalidatecaches'
     LISTING_ALL_EPISODES = 'listingallepisodes'
     LISTING_AZ_TVSHOWS = 'listingaztvshows'
     LISTING_CATEGORIES = 'listingcategories'

diff --git a/resources/lib/vrtplayer/favorites.py b/resources/lib/vrtplayer/favorites.py
@@ -3,8 +3,6 @@
 # GNU General Public License v3.0 (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import absolute_import, division, unicode_literals
-import json
-import time
 
 from resources.lib.vrtplayer import tokenresolver
 
@@ -21,36 +19,38 @@ def __init__(self, _kodi):
         self._tokenresolver = tokenresolver.TokenResolver(_kodi)
         self._proxies = _kodi.get_proxies()
         install_opener(build_opener(ProxyHandler(self._proxies)))
-        self._cache_file = _kodi.get_userdata_path() + 'favorites.json'
         self._favorites = None
         if _kodi.get_setting('usefavorites') == 'true' and _kodi.has_credentials():
-            self.get_favorites()
+            # Get favorites from cache if fresh
+            self.get_favorites(ttl=60 * 60)
 
     def is_activated(self):
         return self._favorites is not None
 
-    def get_favorites(self):
-        if self._kodi.check_if_path_exists(self._cache_file):
-            if self._kodi.stat_file(self._cache_file).st_mtime() > time.mktime(time.localtime()) - (2 * 60):
-                self._kodi.log_notice('CACHE: %s vs %s' % (self._kodi.stat_file(self._cache_file).st_mtime(), time.mktime(time.localtime()) - (5 * 60)), 'Debug')
-                with self._kodi.open_file(self._cache_file) as f:
-                    self._favorites = json.loads(f.read())
-                return
-        self.update_favorites()
-
-    def update_favorites(self):
-        xvrttoken = self._tokenresolver.get_fav_xvrttoken()
-        headers = {
-            'authorization': 'Bearer ' + xvrttoken,
-            'content-type': 'application/json',
-            # 'Cookie': 'X-VRT-Token=' + xvrttoken,
-            'Referer': 'https://www.vrt.be/vrtnu',
-        }
-        req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
-        self._favorites = json.loads(urlopen(req).read())
-        self.write_favorites()
+    def get_favorites(self, ttl=None):
+        import json
+        api_json = self._kodi.get_cache('favorites.json', ttl)
+        if not api_json:
+            xvrttoken = self._tokenresolver.get_fav_xvrttoken()
+            headers = {
+                'authorization': 'Bearer ' + xvrttoken,
+                'content-type': 'application/json',
+                # 'Cookie': 'X-VRT-Token=' + xvrttoken,
+                'Referer': 'https://www.vrt.be/vrtnu',
+            }
+            req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
+            self._kodi.log_notice('URL post: https://video-user-data.vrt.be/favorites', 'Verbose')
+            try:
+                api_json = json.load(urlopen(req))
+            except Exception:
+                # Force favorites from cache
+                api_json = self._kodi.get_cache('favorites.json', ttl=None)
+            else:
+                self._kodi.update_cache('favorites.json', api_json)
+        self._favorites = api_json
 
     def set_favorite(self, program, path, value=True):
+        import json
         if value is not self.is_favorite(path):
             xvrttoken = self._tokenresolver.get_fav_xvrttoken()
             headers = {
@@ -69,11 +69,7 @@ def set_favorite(self, program, path, value=True):
                 self._kodi.log_error("Failed to follow program '%s' at VRT NU" % path)
             # NOTE: Updates to favorites take a longer time to take effect, so we keep our own cache and use it
             self._favorites[self.uuid(path)] = dict(value=payload)
-            self.write_favorites()
-
-    def write_favorites(self):
-        with self._kodi.open_file(self._cache_file, 'w') as f:
-            f.write(json.dumps(self._favorites))
+            self._kodi.update_cache('favorites.json', self._favorites)
 
     def is_favorite(self, path):
         value = False
@@ -103,3 +99,7 @@ def names(self):
 
     def titles(self):
         return [p.get('value').get('title') for p in self._favorites.values() if p.get('value').get('isFavorite')]
+
+    def invalidate_cache(self):
+        self._kodi.invalidate_cache('offline-filtered.json')
+        self._kodi.invalidate_cache('recent-filtered.json')
diff --git a/resources/lib/vrtplayer/streamservice.py b/resources/lib/vrtplayer/streamservice.py
@@ -34,13 +34,13 @@ def __init__(self, _kodi, _tokenresolver):
         self._vualto_license_url = None
 
     def _get_vualto_license_url(self):
-        self._vualto_license_url = json.loads(urlopen(self._VUPLAY_API_URL).read()).get('drm_providers', dict()).get('widevine', dict()).get('la_url')
+        self._vualto_license_url = json.load(urlopen(self._VUPLAY_API_URL)).get('drm_providers', dict()).get('widevine', dict()).get('la_url')
         self._kodi.log_notice('URL get: ' + unquote(self._VUPLAY_API_URL), 'Verbose')
 
     def _create_settings_dir(self):
         settingsdir = self._kodi.get_userdata_path()
         if not self._kodi.check_if_path_exists(settingsdir):
-            self._kodi.make_dir(settingsdir)
+            self._kodi.mkdir(settingsdir)
 
     def _get_license_key(self, key_url, key_type='R', key_headers=None, key_value=None):
         ''' Generates a propery license key value
@@ -152,7 +152,7 @@ def _get_stream_json(self, api_data):
                 api_data.video_id + '?vrtPlayerToken=' + playertoken + '&client=' + api_data.client
             self._kodi.log_notice('URL get: ' + unquote(api_url), 'Verbose')
             try:
-                stream_json = json.loads(urlopen(api_url).read())
+                stream_json = json.load(urlopen(api_url))
             except HTTPError as e:
                 stream_json = json.loads(e.read())