refactor to make use of websockets instead of polling, automatically convert EPUB to MOBI

bot.py

@@ -3,6 +3,7 @@
 import random
 import threading
 import socket
+import subprocess
 import sys
 from unzipper import unar
 
@@ -42,7 +43,7 @@ class IRCClient():
     joined = False
     connected = False
 
-    def __init__(self, book, extension, t, logging=False, path="/tmp/"):
+    def __init__(self, book, extension, t, cb, logging=False, path="/tmp/"):
 
         self.TYPE = t
         self.EXTENSION = extension
@@ -54,6 +55,7 @@ class IRCClient():
         self.NICK = name
         self.IDENT = name
         self.REALNAME = name
+        self.callback = cb
 
         self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.socket.connect((self.HOST, self.PORT))
@@ -69,13 +71,13 @@ class IRCClient():
 
     def handle_connect(self):
         self.log("connected")
-        self.STATUS = "CONNECTED"
+        self.set_status("CONNECTED")
         self.connected = True
 
     def handle_close(self):
         self.NOT_EXITED = False
         self.log("closed")
-        self.STATUS = "DISCONNECTED"
+        self.set_status("DISCONNECTED")
         self.connected = False
         self.socket.close()
 
@@ -84,7 +86,7 @@ class IRCClient():
             return
         self.NOT_EXITED = False
         self.log("Abort mission")
-        self.STATUS = "TIMED OUT"
+        self.set_status("TIMED OUT")
         self.connected = False
         self.socket.close()
 
@@ -122,7 +124,7 @@ class IRCClient():
                 if comm == "JOIN":
                     if self.NICK not in msg_from:  # msg "NICK joined the channel" not about me
                         continue
-                    self.STATUS = "JOINED"
+                    self.set_status("JOINED")
                     self.log("Joined channel %s" % self.CHANNEL)
                     self.run_query()
                     self.joined = True
@@ -146,7 +148,7 @@ class IRCClient():
                     continue
 
     def run_query(self):
-        self.STATUS = "WAITING"
+        self.set_status("WAITING")
         if self.TYPE == "SEARCH":
             self.search(self.LOOKING_FOR)
             return
@@ -166,32 +168,49 @@ class IRCClient():
         port = int(args.pop())
         ip = ip_from_decimal(int(args.pop()))
         filename = "_".join(args).replace('"', '')
-        self.STATUS = "RECEIVING"
+        self.set_status("RECEIVING")
 
         n = self.netcat(ip, port, size, filename)
         files = unar(n, self.PATH)
+        print("Unarchived files", files)
 
         out = []
         for f in files:
             if "searchbot" in f.lower() or "searchook" in f.lower():
                 out.append(self.list_books(f))
 
+        print("Current output", out)
         self.OUTPUT = []
         if len(out) > 0:
             if isinstance(out[0], list):
+                print("1")
                 self.OUTPUT = [item for sublist in out for item in sublist]
             else:
+                print("2")
                 self.OUTPUT = out
+            print("3")
             self.OUTPUT = list(set(self.OUTPUT))
         else:
+            print("4")
             self.OUTPUT = files
 
+        print(self.OUTPUT)
+
         if self.TYPE == "BOOK":
             self.log("Files: ")
             self.log(self.OUTPUT)
+            for f in self.OUTPUT:
+                if f.lower().endswith(".epub"):
+                    print("EPUB %s" % f)
+                    new_fname = f.replace("epub", "mobi")
+                    p = subprocess.Popen(["ebook-convert", f, new_fname])
+                    p.wait()
+                    self.OUTPUT.append(new_fname)
+                    break
 
         self.OUTPUT = [i.replace(self.PATH, "") for i in self.OUTPUT]
         self.handle_close()
+        self.set_status("FINISHED")
 
     def list_books(self, f):
         f = open(f, "r")
@@ -217,8 +236,8 @@ class IRCClient():
             count += len(data)
             f.write(data)
             perc = int(100 * count / size)
-            self.STATUS = "DOWNLOADING"  # % perc #progress
             self.PROGRESS = perc
+            self.set_status("DOWNLOADING")  # % perc #progress
 
             self.log("\r%d%%" % perc, instant=True)
             if count >= size:
@@ -258,6 +277,10 @@ class IRCClient():
             else:
                 sys.stdout.write(val)
 
+    def set_status(self, value):
+        self.STATUS = value
+        self.callback(value=="FINISHED")
+
 
 def ip_from_decimal(dec):
     return ".".join([str(int(b, 2)) for b in b_octets(bin(dec)[2:])])

front/index.html

@@ -4,6 +4,7 @@
 		<script src="js/angular.min.js"></script>
 		<script src="js/app.js"></script>
 		<link href="styles.css" rel="stylesheet" type="text/css"></link>
+        <meta name="viewport" content="width=device-width, initial-scale=1">
 	</head>
 <body>
 	<div ng-controller="main">
@@ -35,7 +36,7 @@
 									<th></th>
 									<th>Query</th>
 									<th>Status</th>
-									<th>Elapsed</th>
+									<th class="nomobile">Elapsed</th>
 									<th class="output">Output</th>
 								</tr>
 							</thead>
@@ -44,7 +45,7 @@
 									<td><a ng-click="showMore(l)">{{limit[l.ID] ? '&#9650;' : '&#9660;'}}</a></td>
 									<td> <span>{{l.QUERY}}</span> </td>
 									<td> <label class="{{l.STATUS}}">{{l.STATUS}}</label> </td>
-									<td> <p ng-show="l.STATUS=='WAITING'|| l.STATUS=='DOWNLOADING'">{{l.ELAPSED}}</p> </td>
+									<td class="nomobile"> <p ng-show="l.STATUS=='WAITING'|| l.STATUS=='DOWNLOADING'">{{l.ELAPSED}}</p> </td>
 									<td>
 										<i ng-show="l.STATUS=='DISCONNECTED'">Found {{l.OUT.length}} matches:</i>
 										<ul>
@@ -64,7 +65,7 @@
 									<th>Title</th>
 									<th>Status</th>
 									<th>Progress</th>
-									<th>Elapsed</th>
+									<th class="nomobile">Elapsed</th>
 									<th>Output</th>
 								</tr>
 							</thead>
@@ -75,12 +76,11 @@
 									<td>
 									<progress-bar progress="l.PROGRESS"></progress-bar>
 									</td>
-									<td> <p ng-hide="l.STATUS=='DISCONNECTED'|| l.STATUS=='TIMED OUT'">{{l.ELAPSED}}</p> </td>
+									<td class="nomobile"> <p ng-hide="l.STATUS=='DISCONNECTED'|| l.STATUS=='TIMED OUT'">{{l.ELAPSED}}</p> </td>
 									<td>
-										<i>Found {{l.OUT.length}} file{{l.OUT.length == 1 ? '' : 's'}}:</i>
 										<ul>
 											<li ng-repeat="o in l.OUT">
-												<a ng-href="/backend/{{o}}" target="_blank">{{o}}  &#8675;</a>
+												<a ng-href="/backend/{{o}}" target="_blank">{{clean(o)}}  &#8675;</a>
 											</li>
 										</ul>
 									</td>

front/js/app.js

@@ -8,52 +8,26 @@ app.controller('main', function($scope,$http,$interval) {
 	$scope.extension="";
 
 	$scope.activeTab = 'SEARCH';
-	$scope.getList = function() {
-		$http.get(ROOT_PATH+$scope.activeTab).then(
-		function(data) {
-			/*
-			data.data=data.data.map(function(el) {
-				if (el.STATUS!="DISCONNECTED")
-					el.STATUS=el.STATUS+" "+el.ELAPSED;
-				return el;
-			});
-			*/
-			if($scope.activeTab === "SEARCH")
-				$scope.searchlist=data.data;
-			else
-				$scope.results=data.data;
-		},
-		function(data){
-			console.log("error");
-
-		});
-	}
-	$scope.getList();
+    ws = new WebSocket("ws://192.168.1.12:8099/");
+    ws.onmessage = function(event) {
+        var j = JSON.parse(event.data);
+        console.log(j);
+        $scope.$apply(function() {
+            $scope.searchlist=j['SEARCH'];
+            $scope.results=j['BOOKS'];
+        });
+    };
 
 	$scope.download = function(book) {
-		$http.post(ROOT_PATH+"?adasasd",{"type":"BOOK","book":book}).then(
-		function(data) {
-			$scope.newElement(true);
-			$scope.getList();
-		},
-		function(data){
-			console.log("error");
-			console.log(data);
-		});
-	};
+		ws.send(JSON.stringify({"type":"BOOK","book":book}));
+    };
 
 	$scope.newElement = function(bool){
 		$scope.new = bool;
 	}
+
 	$scope.searchBook = function(book,extension){
-		$http.post(ROOT_PATH+"?ewqewqewq", {"type":"search","book":book,"extension":extension}).then(
-		function(data) {
-			console.log("success");
-			$scope.getList();
-		},
-		function(data){
-			console.log("err");
-		});
+		ws.send(JSON.stringify({"type":"search","book":book,"extension":extension}));
 	};
 
 	$scope.isActiveTab = function(tab){
@@ -68,11 +42,12 @@ app.controller('main', function($scope,$http,$interval) {
 	$scope.showMore = function(l){
 		$scope.limit[l.ID] = $scope.limit[l.ID] ? undefined : l.OUT.length;
 	}
-
-	var promise =  $interval($scope.getList, 3500);
-	$scope.$on('destroy', function() {
-		$interval.cancel(promise);
-	});
+    $scope.clean = function(s) {
+        s = s.replace(/\(.*?\)/g, '');
+        s.replace(/\s+/g, " ");
+        s.replace(/\s+\./g, ".");
+        return s;
+    }
 });
 
 app.directive('progressBar', function(){

front/styles.css

@@ -175,8 +175,8 @@ td > i {
 }
 
 td > ul > li {
-  margin-bottom: 5px;
-  margin-left: 10px;
+  padding-bottom: 5px;
+  padding-left: 10px;
 }
 
 td > ul > li:hover {
@@ -247,3 +247,19 @@ th.output{
   background-color: rgb(239, 199, 207);
   border-color: rgb(230, 190, 198);
 }
+.FINISHED{
+  background-color: rgb(200, 199, 207);
+  border-color: rgb(190, 190, 198);
+}
+@media only screen and (max-width: 600px) {
+    td, tr {
+      padding: 5px;
+    }
+    td > ul > li {
+        padding: 5px !important;
+        padding-left: 0px !important;
+    }
+    a { font-size: 0.9em; }
+    .tab-content{padding:10px}
+    .nomobile { display: none}
+}

qmanager.py

@@ -15,9 +15,10 @@ class qManager:
     last_id = 0
     PATH = ""
 
-    def __init__(self, path="/tmp/"):
+    def __init__(self, update_cb, path="/tmp/"):
         """Init the manager. Set output path"""
         self.PATH = path
+        self.update_cb = update_cb
 
     def new_dl(self, string):
         """Download a book"""
@@ -25,6 +26,7 @@ class qManager:
             string,
             "",
             "BOOK",
+            cb=self.update_cb,
             logging=False,
             path=self.PATH)
         self.new_task(string, irc_client)
@@ -35,6 +37,7 @@ class qManager:
             keywords,
             fmt,
             "SEARCH",
+            cb=self.update_cb,
             logging=False,
             path=self.PATH)
         self.new_task(keywords, irc_client)

requirements.txt

@@ -0,0 +1,2 @@
+pkg-resources==0.0.0
+websocket-server==0.4

unzipper.py

@@ -3,7 +3,7 @@ import zipfile
 import os.path
 import subprocess
 
-USE_UNRAR = True
+USE_UNRAR = False
 
 
 def unar(source, dest_dir):
@@ -38,8 +38,9 @@ def unzip(source_filename, dest_dir):
                 path = os.path.join(path, word)
             target = os.path.join(path, member.filename.split('/')[-1])
             out.append(target)
-            print("Extracted %s" % target)
+            print("Extracting %s" % target)
             zfile.extract(member, path)
+            print("Extracted %s" % target)
     return out
 
 
@@ -63,7 +64,6 @@ def unrar(source, dest_dir):
             del out[0]  # header
         if len(out[-1]) == 0:
             del out[-1]
-        # print(out)
     subprocess.run(extract_files, stdout=subprocess.DEVNULL)
 
     return [os.path.join(dest_dir, file) for file in out]

web.py

@@ -1,129 +1,50 @@
 #!/usr/bin/env python3
 import os
 import json
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from datetime import datetime
+from websocket_server import WebsocketServer
 from qmanager import qManager
 import urllib
-#import urllib.parse
 
 # Port on which server will run.
 PORT = 8080
-BASE_PATH="/backend/" #changes based on webserver
-FILE_PATH="/tmp/"
-
-
-class HTTPRequestHandler(BaseHTTPRequestHandler):
-    q = qManager()
-
-    def do_GET(self):
-        if self.path==BASE_PATH+"SEARCH":
-            self.send_200()
-            out=json.dumps(self.q.search_status())
-            self.wfile.write(bytes(out,"utf-8")) #fileHandle.read().encode()
-            return
-        if self.path==BASE_PATH+"list":
-            self.send_response(200)
-            mime="text/html"
-            self.send_header('Content-type', mime)
-            self.end_headers()
-            out = []
-            for book in self.q.books_status():
-                if 'OUT' not in book:
-                    continue
-                if len(book['OUT']) == 0:
-                    continue
-                out.append("<li><a href='%s'>%s</a>" % (urllib.parse.quote(book['OUT'][0]), book['QUERY']))
-            self.wfile.write(bytes("".join(out),"utf-8")) #fileHandle.read().encode()
+BASE_PATH='/backend/' #changes based on webserver
+FILE_PATH='/tmp/'
+last_msg = datetime.now()
+
+def send_status(server, client=None, force=False):
+    global last_msg
+    if client is None and not force:
+        if (datetime.now() - last_msg).total_seconds() < 2:
             return
-        if self.path==BASE_PATH+"RESULTS":
-            self.send_200()
-            out=json.dumps(self.q.books_status())
-            self.wfile.write(bytes(out,"utf-8")) #fileHandle.read().encode()
-            return
-        if self.path==BASE_PATH:
-            self.send_200()
-            out=json.dumps(self.q.task_status())
-            self.wfile.write(bytes(out,"utf-8")) #fileHandle.read().encode()
-            return
-
-        p=urllib.parse.unquote(self.path.replace(BASE_PATH,""))
-        path=FILE_PATH+p
-        #mime=mimetypes.guess_type(path) #fails on mobi, epub
-        mime=""
-        if p.endswith("mobi"):
-            mime="application/x-mobipocket-ebook"
-        if p.endswith("epub"):
-            mime="application/epub+zip"
-        if p.endswith("pdf"):
-            mime="application/pdf"
-        if p.endswith("html"):
-            mime="text/html"
-        
-        self.send_response(200)
-        self.send_header('Content-type', mime)
-        self.send_header('Content-Disposition', 'attachment;filename="%s"'%p)
-        self.end_headers()
+        last_msg = datetime.now()
+    status = { 'SEARCH': q.search_status(), 'BOOKS': q.books_status()}
+    if client is None:
+        ws.send_message_to_all(json.dumps(status))
+    else:
+        server.send_message(client, json.dumps(status))
 
-        fp = open(path,'rb')
-        while True:
-            b = fp.read(8192)
-            if b:
-                self.wfile.write(b) #fileHandle.read().encode()
-            else:
-                break
+def new_client(client, server):
+    send_status(server, client)
 
-    def send_400(self):
-        self.send_response(400, 'NOT OK')
-        self.send_header('Content-type', 'text/json')
-        self.end_headers()
+def message_received(client, server, message):
+    j = json.loads(message)
+    print(j)
+    if j['type'].upper() == 'SEARCH':
+        q.new_search(j['book'], j['extension'])
 
-    def send_200(self):
-        self.send_response(200)
-        self.send_header('Content-type', 'text/json')
-        self.end_headers()
+    if j['type'].upper() == 'BOOK':
+        q.new_dl(j['book'])
 
-    def do_POST(self):
-        # Check if path is there.
-        if self.path:
-            length = self.headers['content-length']
-            if length is None:
-               self.send_400()
-               return
+    send_status(server)
 
-
-            data = self.rfile.read(int(length))
-            data = data.decode("utf-8")
-            try:
-                d=json.loads(data)
-            except:
-                print("no json")
-                self.send_400()
-                return
-
-            if not "type" in d or not "book" in d:
-                self.send_400()
-                return
-            if d["type"].upper()=="SEARCH":
-                self.q.new_search(d["book"],d["extension"])
-                self.send_200()
-                return
-
-            if d["type"]=="BOOK":
-                self.q.new_dl(d["book"])
-                self.send_200()
-                return
+def updated(force=False):
+    send_status(ws, force=force)
 
 
 if __name__ == '__main__':
-
-    HTTPDeamon = HTTPServer(('', PORT), HTTPRequestHandler)
-
-    print("Listening at port", PORT)
-
-    try:
-        HTTPDeamon.serve_forever()
-    except KeyboardInterrupt:
-        pass
-
-    HTTPDeamon.server_close()
-    print("Server stopped")
+    q = qManager(updated)
+    ws = WebsocketServer(8081, host='127.0.0.1')
+    ws.set_fn_new_client(new_client)
+    ws.set_fn_message_received(message_received)
+    ws.run_forever()