more error handling

[svtplaydump.git] / svtplaydump.py

diff --git a/svtplaydump.py b/svtplaydump.py
index e9a4a3b9c0d6cf29f57539c01670de268570a64c..4a90e28e73e74ab9a740bda1a2a12a3a978d4984 100755 (executable)
--- a/svtplaydump.py
+++ b/svtplaydump.py
@@ -31,7 +31,8 @@ import argparse
 import requests
 import sys, os
 import socket
-
+import feedparser
+from datetime import datetime, timezone
 class Video(dict):
     def __init__(self, *args, **kwargs):
         self.update(dict(*args, **kwargs))  # use the free update to set keys
@@ -76,6 +77,11 @@ def scrape_player_page(video):
         rtmp = flashvars['pathflv'][0]
         filename = video['title']+".flv"
         print(Popen(["mplayer","-dumpstream","-dumpfile",filename, rtmp], stdout=PIPE).communicate()[0])
+    if not 'timestamp' in video:
+        if soup.find_all(datetime=True):
+            xmldate_str = soup.find_all(datetime=True)[0].attrs['datetime']
+            video['timestamp'] = datetime(*feedparser._parse_date_w3dtf(xmldate_str)[:6]) #naive in utc
+            video['timestamp'] = video['timestamp'].replace(tzinfo=timezone.utc).astimezone(tz=None) #convert to local time
     if 'video' in flashvars:
         for reference in flashvars['video']['videoReferences']:
             if 'm3u8' in reference['url']:
@@ -83,7 +89,8 @@ def scrape_player_page(video):
                 video['filename'] = video['title']+'.ts'
                 if 'statistics' in flashvars:
                     video['category'] = flashvars['statistics']['category']
-        download_from_playlist(video)
+        if not download_from_playlist(video):
+            return False
     if not 'url' in video:
         print("Could not find any streams")
         return False
@@ -106,7 +113,12 @@ def download_from_playlist(video):
         segment=0
         size = 0
         for url in segments:
-            ufile = requests.get(url, stream=True).raw
+            try:
+                ufile = requests.get(url, stream=True).raw
+            except:
+                print("Error reading, skipping file") #FIXME mark file as failed
+                print(sys.exc_info()[1])
+                return False
             print("\r{0:.2f} MB".format(size/1024/1024),end="")
             sys.stdout.flush()
             if decrypt:
@@ -115,10 +127,10 @@ def download_from_playlist(video):
             while(True):
                 try:
                     buf = ufile.read(4096)
-                except socket.error as e:
-                    print("Error reading, skipping file")
-                    print(e)
-                    return
+                except:
+                    print("Error reading, skipping file") #FIXME mark file as failed
+                    print(sys.exc_info()[1])
+                    return False
                 if not buf:
                     break
                 if decrypt:
@@ -129,6 +141,7 @@ def download_from_playlist(video):
 
     if 'thumb-url' in video:
         video['thumb'] = requests.get(video['thumb-url'],stream=True).raw
+    return True
 
 def parse_playlist(playlist):
     if not playlist.startswith("#EXTM3U"):
@@ -200,7 +213,7 @@ def remux(video, xml=None):
             os.mkdir(video['genre'])
         video['path'] = os.path.join(video['genre'],basename+'.mkv')
     else:
-        video['path'] = basename+'.mkv'
+        video['path'] = basename+'.mkv' 
     command = ["mkvmerge","-o",video['path'], '--title',video['title']]
 
     if xml:
@@ -220,6 +233,12 @@ def remux(video, xml=None):
             os.unlink(fname)
         except:
             pass
+    if 'timestamp' in video:
+        try:
+            os.utime(video['path'], times=(video['timestamp'].timestamp(),video['timestamp'].timestamp()))
+        except FileNotFoundError as e:
+            print(e)
+
     
 def mkv_metadata(video):
     root = BeautifulSoup(features='xml')
@@ -258,7 +277,6 @@ if __name__ == "__main__":
     
     args = parser.parse_args()
     if args.rss: 
-        import feedparser
         d = feedparser.parse(args.rss)
         for e in d.entries:
             print(("Downloading: %s"%e.title))
@@ -284,7 +302,13 @@ if __name__ == "__main__":
             if args.no_act:
                 continue
             open(os.path.join('.seen',video['title']),'w').close() #touch
-            video = scrape_player_page(video)
+            ret = scrape_player_page(video)
+            if not ret:
+                if not os.path.exists('.failed'):
+                    os.mkdir('.failed')
+                open(os.path.join('.failed',video['title']),'w').close() #touch
+                continue
+            video = ret
             if args.no_remux:
                 continue
             xml = mkv_metadata(video)