add youtube upload task

back/tasks.py

@@ -1,6 +1,8 @@
 import time
 from subprocess import Popen,PIPE
 from rq import get_current_job
+import os.path
+
 def long_job(arg):
     for i in range(20):
         time.sleep(1)
@@ -11,8 +13,42 @@ def long_job(arg):
     return "37"
 
 
-def run_process(process=None):
-    process=['ffmpeg', "-loglevel","error","-stats",'-i', '/home/david/movie-1467810207.flv', '-ss', '0','-t','145', '-c:v','libx264','-movflags','+faststart','-y','output.mp4']
+def upload_video(title,thumb_path,video_path):
+    import youtube_upload.main
+    j=get_current_job()
+    return youtube_upload.main.main(title,thumb_path,video_path,job=j)
+
+def cut_video(in_fname,start_time,end_time,out_fname):
+    if not os.path.isfile(in_fname):
+        return { "status":"error",
+                 "error": "El archivo %s no existe" % in_fname
+               }
+    if os.path.isfile(out_fname):
+        return { "status":"error",
+                 "error": "El archivo %s existe" % out_fname
+               }
+
+    process = [ 'ffmpeg',
+                '-loglevel', 'error',
+                '-stats',
+                '-i', in_fname,
+                '-ss', start_time,
+                '-t', end_time,
+                '-c', 'copy',
+                '-movflags', '+faststart',
+                out_fname
+              ]
+
+    out = run_process(process)
+    if out == "":
+        out = { "status": "Proceso finalizado" }
+    return out
+
+def run_process(process = None):
+    if process is None or type(process) is not list:
+        return { "status": "error",
+                 "error": "Argumento process invalido"
+               }
     proc=Popen(process, stderr=PIPE, universal_newlines=True)
     while True:
         line = proc.stderr.readline()

back/web.py

@@ -30,9 +30,15 @@ def raw_list():
 
 @app.route(BASE_PATH + '/job')
 def job():
+    # func=tasks.cut_video , args=("/home/david/movie-1467033043.flv","0:45","1:48:00","/home/david/salida-cortada.mp4",), result_ttl=5000
     job = q.enqueue_call(
-            func=tasks.long_job, args=("testarg",), result_ttl=5000
-    )
+            func=tasks.upload_video,
+            args=("titulo","/home/david/photo_2016-07-10_13-49-56.jpg","/home/david/movie-1467033043.flv",),
+            timeout=3600*2,
+            result_ttl=3600*2
+        )
+    #con decorator seria:
+    #tasks.upload_video.delay(..args..)
     ret = {"id":job.get_id()}
 
     return jsonify(ret)
@@ -47,7 +53,9 @@ def get_results(job_key):
     if job.is_finished:
         return str(job.result), 200
     else:
-        return "Nay! %s" % job.meta.get('progress') , 202
+        return "%s" % json.dumps(job.meta) , 202
+        #return "Nay! %s - %s" % (job.meta.get('video_id'),job.meta.get('status')) , 202
+        #return "Nay! %s - %s" % (job.meta.get('video_id'),job.meta.get('status')) , 202
 
 
 @app.route(BASE_PATH + '/cut')

back/youtube_upload/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.8.0"

back/youtube_upload/auth/__init__.py

@@ -0,0 +1,42 @@
+"""Wrapper for Google OAuth2 API."""
+import sys
+import json
+
+import googleapiclient.discovery
+import oauth2client
+import httplib2 
+
+from youtube_upload import lib
+from youtube_upload.auth import console
+from youtube_upload.auth import browser
+
+YOUTUBE_UPLOAD_SCOPE = ["https://www.googleapis.com/auth/youtube.upload", "https://www.googleapis.com/auth/youtube"]
+
+def _get_credentials_interactively(flow, storage, get_code_callback):
+    """Return the credentials asking the user."""
+    flow.redirect_uri = oauth2client.client.OOB_CALLBACK_URN
+    authorize_url = flow.step1_get_authorize_url()
+    code = get_code_callback(authorize_url)
+    if code:
+        credential = flow.step2_exchange(code, http=None)
+        storage.put(credential)
+        credential.set_store(storage)
+        return credential
+
+def _get_credentials(flow, storage, get_code_callback):
+    """Return the user credentials. If not found, run the interactive flow."""
+    existing_credentials = storage.get()
+    if existing_credentials and not existing_credentials.invalid:
+        return existing_credentials
+    else:
+        return _get_credentials_interactively(flow, storage, get_code_callback)
+
+def get_resource(client_secrets_file, credentials_file, get_code_callback):
+    """Authenticate and return a googleapiclient.discovery.Resource object."""
+    get_flow = oauth2client.client.flow_from_clientsecrets
+    flow = get_flow(client_secrets_file, scope=YOUTUBE_UPLOAD_SCOPE)
+    storage = oauth2client.file.Storage(credentials_file)
+    credentials = _get_credentials(flow, storage, get_code_callback)
+    if credentials:
+        http = credentials.authorize(httplib2.Http())
+        return googleapiclient.discovery.build("youtube", "v3", http=http)

back/youtube_upload/auth/browser.py

@@ -0,0 +1,19 @@
+from .. import lib
+
+try:
+    from youtube_upload.auth import webkit_qt as backend
+    WEBKIT_BACKEND = "qt"
+except ImportError:
+    try:
+        from youtube_upload.auth import webkit_gtk as backend
+        WEBKIT_BACKEND = "gtk"
+    except ImportError:
+        WEBKIT_BACKEND = None
+
+def get_code(url, size=(640, 480), title="Google authentication"):
+    if WEBKIT_BACKEND:
+        lib.debug("Using webkit backend: " + WEBKIT_BACKEND)
+        with lib.default_sigint():
+            return backend.get_code(url, size=size, title=title)
+    else:
+        raise NotImplementedError("GUI auth requires pywebkitgtk or qtwebkit")

back/youtube_upload/auth/console.py

@@ -0,0 +1,13 @@
+import sys
+
+def get_code(authorize_url):
+    
+    """Show authorization URL and return the code the user wrote."""
+    message = "Check this link in your browser: {0}".format(authorize_url)
+    sys.stderr.write(message + "\n")
+    try: input = raw_input #For Python2 compatability
+    except NameError: 
+        #For Python3 on Windows compatability
+        try: from builtins import input as input 
+        except ImportError: pass
+    return input("Enter verification code: ")

back/youtube_upload/auth/webkit_gtk.py

@@ -0,0 +1,48 @@
+import json
+
+CHECK_AUTH_JS = """
+    var code = document.getElementById("code");
+    var access_denied = document.getElementById("access_denied");
+    var result;
+    
+    if (code) {
+        result = {authorized: true, code: code.value};
+    } else if (access_denied) {
+        result = {authorized: false, message: access_denied.innerText};
+    } else {
+        result = {};
+    }
+    window.status = JSON.stringify(result);
+"""
+
+def _on_webview_status_bar_changed(webview, status, dialog):
+    if status:
+        authorization = json.loads(status)
+        if authorization.has_key("authorized"):
+            dialog.set_data("authorization_code", authorization["code"])
+            dialog.response(0)
+
+def get_code(url, size=(640, 480), title="Google authentication"):
+    """Open a GTK webkit window and return the access code."""
+    import gtk
+    import webkit
+    dialog = gtk.Dialog(title=title)
+    webview = webkit.WebView()
+    scrolled = gtk.ScrolledWindow()
+    scrolled.add(webview)
+    dialog.get_children()[0].add(scrolled)
+    webview.load_uri(url)    
+    dialog.resize(*size)
+    dialog.show_all()
+    dialog.connect("delete-event", 
+        lambda event, data: dialog.response(1))
+    webview.connect("load-finished", 
+        lambda view, frame: view.execute_script(CHECK_AUTH_JS))       
+    webview.connect("status-bar-text-changed", 
+        _on_webview_status_bar_changed, dialog)
+    dialog.set_data("authorization_code", None)
+    status = dialog.run()
+    dialog.destroy()
+    while gtk.events_pending():
+        gtk.main_iteration(False)
+    return dialog.get_data("authorization_code")

back/youtube_upload/auth/webkit_qt.py

@@ -0,0 +1,54 @@
+CHECK_AUTH_JS = """
+    var code = document.getElementById("code");
+    var access_denied = document.getElementById("access_denied");
+    var result;
+    
+    if (code) {
+        result = {authorized: true, code: code.value};
+    } else if (access_denied) {
+        result = {authorized: false, message: access_denied.innerText};
+    } else {
+        result = {};
+    }
+    result;
+"""
+
+def _on_qt_page_load_finished(dialog, webview):
+    to_s = lambda x: (str(x.toUtf8()) if hasattr(x,'toUtf8') else x)
+    frame = webview.page().currentFrame()
+    try: #PySide does not QStrings
+        from QtCore import QString
+        jscode = QString(CHECK_AUTH_JS)
+    except ImportError:
+        jscode = CHECK_AUTH_JS
+    res = frame.evaluateJavaScript(jscode)
+    try:
+        authorization = dict((to_s(k), to_s(v)) for (k, v) in res.toPyObject().items())
+    except AttributeError: #PySide returns the result in pure Python
+        authorization = dict((to_s(k), to_s(v)) for (k, v) in res.items())
+    if "authorized" in authorization:
+        dialog.authorization_code = authorization.get("code")
+        dialog.close()
+   
+def get_code(url, size=(640, 480), title="Google authentication"):
+    """Open a QT webkit window and return the access code."""
+    try:
+        from PyQt4 import QtCore, QtGui, QtWebKit
+    except ImportError:
+        from PySide import QtCore, QtGui, QtWebKit
+    app = QtGui.QApplication([])
+    dialog = QtGui.QDialog()
+    dialog.setWindowTitle(title)
+    dialog.resize(*size)
+    webview = QtWebKit.QWebView()
+    webpage = QtWebKit.QWebPage()
+    webview.setPage(webpage)           
+    webpage.loadFinished.connect(lambda: _on_qt_page_load_finished(dialog, webview))
+    webview.setUrl(QtCore.QUrl.fromEncoded(url))
+    layout = QtGui.QGridLayout()
+    layout.addWidget(webview)
+    dialog.setLayout(layout)
+    dialog.authorization_code = None
+    dialog.show()
+    app.exec_()
+    return dialog.authorization_code

back/youtube_upload/categories.py

@@ -0,0 +1,51 @@
+try:
+    #import urllib2 
+    from urllib2 import urlopen    
+    import urllib
+except ImportError:
+    from urllib.request import urlopen
+import json
+
+URL = "https://www.googleapis.com/youtube/v3/videoCategories"
+
+IDS = {
+    "Film & Animation": 1,
+    "Autos & Vehicles": 2,
+    "Music": 10,
+    "Pets & Animals": 15,
+    "Sports": 17,
+    "Short Movies": 18,
+    "Travel & Events": 19,
+    "Gaming": 20,
+    "Videoblogging": 21,
+    "People & Blogs": 22,
+    "Comedy": 34,
+    "Entertainment": 24,
+    "News & Politics": 25,
+    "Howto & Style": 26,
+    "Education": 27,
+    "Science & Technology": 28,
+    "Nonprofits & Activism": 29,
+    "Movies": 30,
+    "Anime/Animation": 31,
+    "Action/Adventure": 32,
+    "Classics": 33,
+    "Documentary": 35,
+    "Drama": 36,
+    "Family": 37,
+    "Foreign": 38,
+    "Horror": 39,
+    "Sci-Fi/Fantasy": 40,
+    "Thriller": 41,
+    "Shorts": 42,
+    "Shows": 43,
+    "Trailers": 44,
+}
+
+def get(region_code="us", api_key=None):
+    params = dict(part="snippet", regionCode=region_code, key=api_key)  
+    full_url = URL + "?" + urllib.urlencode(params)
+    response = urlopen(full_url)
+    categories_info = json.loads(response.read())
+    items = categories_info["items"]
+    return dict((item["snippet"]["title"], item["id"]) for item in items)

back/youtube_upload/lib.py

@@ -0,0 +1,90 @@
+from __future__ import print_function
+import os
+import sys
+import locale
+import random
+import time
+import signal
+from contextlib import contextmanager
+
+@contextmanager
+def default_sigint():
+    original_sigint_handler = signal.getsignal(signal.SIGINT)
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    try:
+        yield
+    finally:
+        signal.signal(signal.SIGINT, original_sigint_handler)
+        
+def to_utf8(s):
+    """Re-encode string from the default system encoding to UTF-8."""
+    current = locale.getpreferredencoding()
+    if hasattr(s, 'decode'): #Python 3 workaround
+        return (s.decode(current).encode("UTF-8") if s and current != "UTF-8" else s)
+    elif isinstance(s, bytes):
+        return bytes.decode(s)
+    else:
+        return s
+       
+def debug(obj, fd=sys.stderr):
+    """Write obj to standard error."""
+    print(obj, file=fd)
+
+def catch_exceptions(exit_codes, fun, *args, **kwargs):
+    """
+    Catch exceptions on fun(*args, **kwargs) and return the exit code specified
+    in the exit_codes dictionary. Return 0 if no exception is raised.
+    """
+    try:
+        fun(*args, **kwargs)
+        return 0
+    except tuple(exit_codes.keys()) as exc:
+        debug("[{0}] {1}".format(exc.__class__.__name__, exc))
+        return exit_codes[exc.__class__]
+
+def get_encoding(fd):
+    """Guess terminal encoding."""
+    return fd.encoding or locale.getpreferredencoding()
+
+def first(it):
+    """Return first element in iterable."""
+    return it.next()
+
+def string_to_dict(string):
+    """Return dictionary from string "key1=value1, key2=value2"."""
+    if string:
+        pairs = [s.strip() for s in string.split(",")]
+        return dict(pair.split("=") for pair in pairs)
+
+def get_first_existing_filename(prefixes, relative_path):
+    """Get the first existing filename of relative_path seeking on prefixes directories."""
+    for prefix in prefixes:
+        path = os.path.join(prefix, relative_path)
+        if os.path.exists(path):
+            return path
+
+def retriable_exceptions(fun, retriable_exceptions, max_retries=None):
+    """Run function and retry on some exceptions (with exponential backoff)."""
+    retry = 0
+    while 1:
+        try:
+            return fun()
+        except tuple(retriable_exceptions) as exc:
+            retry += 1
+            if type(exc) not in retriable_exceptions:
+                raise exc
+            elif max_retries is not None and retry > max_retries:
+                debug("[Retryable errors] Retry limit reached")
+                raise exc
+            else:
+                seconds = random.uniform(0, 2**retry)
+                message = ("[Retryable error {current_retry}/{total_retries}] " +
+                    "{error_type} ({error_msg}). Wait {wait_time} seconds").format(
+                    current_retry=retry, 
+                    total_retries=max_retries or "-", 
+                    error_type=type(exc).__name__, 
+                    error_msg=str(exc) or "-", 
+                    wait_time="%.1f" % seconds,
+                )
+                debug(message)
+                time.sleep(seconds)

back/youtube_upload/main.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+import os
+import sys
+import collections
+
+import googleapiclient.errors
+import oauth2client
+
+from . import auth
+from . import upload_video
+from . import categories
+from . import lib
+from . import playlists
+
+class InvalidCategory(Exception): pass
+class OptionsError(Exception): pass
+class AuthenticationError(Exception): pass
+class RequestError(Exception): pass
+
+EXIT_CODES = {
+    OptionsError: 2,
+    InvalidCategory: 3,
+    RequestError: 3,
+    AuthenticationError: 4,
+    oauth2client.client.FlowExchangeError: 4,
+    NotImplementedError: 5,
+}
+
+USE_DEBUG = False
+JOB = None
+#debug = lib.debug
+def debug(args):
+    if USE_DEBUG:
+        print(args)
+struct = collections.namedtuple
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+def get_progress_info():
+    """Return a function callback to update the progressbar."""
+    progressinfo = struct("ProgressInfo", ["callback", "finish"])
+    def _callback(total_size, completed):
+        if JOB is not None:
+            JOB.meta['status'] = 'uploading'
+            JOB.meta['progress'] = (completed/total_size)*100
+            JOB.save()
+    def _finish():
+        if JOB is not None:
+            JOB.meta['status'] = 'finished'
+            JOB.meta['progress'] = 100
+            JOB.save()
+    return progressinfo(callback=_callback, finish=_finish)
+
+def get_category_id(category):
+    """Return category ID from its name."""
+    if category:
+        if category in categories.IDS:
+            ncategory = categories.IDS[category]
+            debug("Using category: {0} (id={1})".format(category, ncategory))
+            return str(categories.IDS[category])
+        else:
+            msg = "{0} is not a valid category".format(category)
+            raise InvalidCategory(msg)
+
+def upload_youtube_video(youtube, options, video_path, total_videos, index):
+    """Upload video with index (for split videos)."""
+    u = lib.to_utf8
+    title = u(options.title)
+    if hasattr(u('string'), 'decode'):   
+        description = u(options.description or "").decode("string-escape")
+    else:
+        description = options.description
+    if options.publish_at:    
+      debug("Your video will remain private until specified date.")
+      
+    tags = [u(s.strip()) for s in (options.tags or "").split(",")]
+    ns = dict(title=title, n=index+1, total=total_videos)
+    title_template = u(options.title_template)
+    complete_title = (title_template.format(**ns) if total_videos > 1 else title)
+    progress = get_progress_info()
+    category_id = get_category_id(options.category)
+    request_body = {
+        "snippet": {
+            "title": complete_title,
+            "description": description,
+            "categoryId": category_id,
+            "tags": tags,
+            "defaultLanguage": options.default_language,
+            "defaultAudioLanguage": options.default_audio_language,
+
+        },
+        "status": {
+            "privacyStatus": ("private" if options.publish_at else options.privacy),
+            "publishAt": options.publish_at,
+
+        },
+        "recordingDetails": {
+            "location": lib.string_to_dict(options.location),
+            "recordingDate": options.recording_date,
+        },
+    }
+
+    debug("Start upload: {0}".format(video_path))
+    if JOB is not None:
+        JOB.meta['status'] = 'starting upload'
+        JOB.save()
+    try:
+        video_id = upload_video.upload(youtube, video_path, 
+            request_body, progress_callback=progress.callback)
+    finally:
+        progress.finish()
+    return video_id
+
+def get_youtube_handler(options):
+    """Return the API Youtube object."""
+    home = os.path.expanduser("~")
+    default_client_secrets = lib.get_first_existing_filename(
+        [sys.prefix, os.path.join(sys.prefix, "local")],
+        "share/youtube_upload/client_secrets.json")  
+    default_credentials = os.path.join(home, ".youtube-upload-credentials.json")
+    client_secrets = options.client_secrets or default_client_secrets or \
+        os.path.join(home, ".client_secrets.json")
+    credentials = options.credentials_file or default_credentials
+    debug("Using client secrets: {0}".format(client_secrets))
+    debug("Using credentials file: {0}".format(credentials))
+    get_code_callback = (auth.browser.get_code 
+        if options.auth_browser else auth.console.get_code)
+    return auth.get_resource(client_secrets, credentials,
+        get_code_callback=get_code_callback)
+
+def run_main(title,thumb,path,output=sys.stdout):
+    """Run the main scripts from the parsed options/args."""
+    options=AttrDict({ 'title': title, 'thumb': thumb ,
+                    'privacy': 'unlisted', 'category': None, 
+                    'client_secrets': None, 'playlist': None, 'credentials_file': None, 'description': None, 'recording_date': None,
+                    'tags': None,  'open_link': None, 'default_language': None, 'title_template': '{title} [{n}/{total}]', 'location': None,
+                    'default_audio_language': None, 'publish_at': None, 'auth_browser': None
+                    })
+
+    youtube = get_youtube_handler(options)
+    video_path = path
+    video_id=None
+    if youtube:
+        video_id = upload_youtube_video(youtube, options, video_path, 1, 1)
+        if options.thumb:
+            youtube.thumbnails().set(videoId=video_id, media_body=options.thumb).execute()
+
+        output.write(video_id + "\n") #FIXME
+        if JOB is not None:
+            JOB.meta["video_id"]=video_id
+            JOB.save()
+    else:
+        raise AuthenticationError("Cannot get youtube resource")
+
+    return video_id
+
+def main(title,thumb,path,job=None):
+    global JOB
+    try:
+        JOB = job
+        return run_main(title,thumb,path)
+    except googleapiclient.errors.HttpError as error:
+        raise RequestError("Server response: {0}".format(bytes.decode(error.content).strip()))

back/youtube_upload/playlists.py

@@ -0,0 +1,48 @@
+from .lib import debug
+
+def get_playlist(youtube, title):
+    """Return users's playlist ID by title (None if not found)"""
+    playlists = youtube.playlists()
+    request = playlists.list(mine=True, part="id,snippet")
+    while request:
+        results = request.execute()
+        for item in results["items"]:
+            existing_playlist_title = item.get("snippet", {}).get("title")
+            if existing_playlist_title.encode("utf8") == title.encode("utf-8"):
+                return item.get("id")
+        request = playlists.list_next(request, results)
+
+def create_playlist(youtube, title, privacy):
+    """Create a playlist by title and return its ID"""
+    debug("Creating playlist: {0}".format(title))
+    response = youtube.playlists().insert(part="snippet,status", body={
+        "snippet": {
+            "title": title,
+        },
+        "status": {
+            "privacyStatus": privacy,
+        }
+    }).execute()
+    return response.get("id")
+
+def add_video_to_existing_playlist(youtube, playlist_id, video_id):
+    """Add video to playlist (by identifier) and return the playlist ID."""
+    debug("Adding video to playlist: {0}".format(playlist_id))
+    return youtube.playlistItems().insert(part="snippet", body={
+        "snippet": {
+            "playlistId": playlist_id,
+            "resourceId": {
+                "kind": "youtube#video",
+                "videoId": video_id,
+            }
+        }
+    }).execute()
+    
+def add_video_to_playlist(youtube, video_id, title, privacy="public"):
+    """Add video to playlist (by title) and return the full response."""
+    playlist_id = get_playlist(youtube, title) or \
+        create_playlist(youtube, title, privacy)
+    if playlist_id:
+        return add_video_to_existing_playlist(youtube, playlist_id, video_id)
+    else:
+        debug("Error adding video to playlist")

back/youtube_upload/upload_video.py

@@ -0,0 +1,40 @@
+try:
+    import httplib
+except ImportError:
+    import http.client as httplib
+
+import googleapiclient.errors
+import apiclient.http
+import httplib2
+
+from . import lib
+
+RETRIABLE_EXCEPTIONS = [
+    IOError, httplib2.HttpLib2Error, httplib.NotConnected,
+    httplib.IncompleteRead, httplib.ImproperConnectionState,
+    httplib.CannotSendRequest, httplib.CannotSendHeader,
+    httplib.ResponseNotReady, httplib.BadStatusLine,
+]
+
+def _upload_to_request(request, progress_callback):
+    """Upload a video to a Youtube request. Return video ID."""
+    while 1:
+        status, response = request.next_chunk()
+        if status and progress_callback:
+            progress_callback(status.total_size, status.resumable_progress)
+        if response:
+            if "id" in response:
+                return response['id']
+            else:
+                raise KeyError("Expected field 'id' not found in response")
+
+def upload(resource, path, body, chunksize=3*1024*1024, 
+        progress_callback=None, max_retries=10):
+    """Upload video to Youtube. Return video ID."""
+    body_keys = ",".join(body.keys())
+    media = apiclient.http.MediaFileUpload(path, chunksize=chunksize, 
+        resumable=True, mimetype="application/octet-stream")
+    request = resource.videos().insert(part=body_keys, body=body, media_body=media)
+    upload_fun = lambda: _upload_to_request(request, progress_callback)
+    return lib.retriable_exceptions(upload_fun, 
+        RETRIABLE_EXCEPTIONS, max_retries=max_retries)

back/youtube_upload/yt.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+if __name__ == '__main__':
+    
+    #Allows you to a relative import from the parent folder
+    import os.path, sys
+    sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))    
+    
+    from youtube_upload import main    
+    main.run("titulo", '/tmp/tmp.qpW37PKFQe.png', "/home/david/movie-1467810207.flv")
+