initial refactoring

.gitignore

@@ -1,3 +1,4 @@
-*.pyc
+.venv
 *.swp
-*/.*.swp
+__pycache__
+bookworm.egg-info

README.md

@@ -1,46 +1,6 @@
 # CLI
 
-```bash
-$ ./bot.py
-USAGE:
-	./bot.py SEARCH <BOOK> <FORMAT>
-	./bot.py BOOK <BOT COMMAND>
-
-$ ./bot.py SEARCH "revival stephen king" "epub"
-Looking (SEARCH) for revival stephen king in format epub
-connected
-Joined channel #ebooks
-searching for revival stephen king
-!Trainfiles Stephen King - Revival (epub).rar  ::INFO:: 395.0KB
-!Trainfiles Stephen King - Revival (retail) (epub).rar  ::INFO:: 1.4MB
-!Xon Stephen King - Revival (retail) (epub).rar
-!Xon Stephen King - Revival (epub).rar
-!Ook Stephen King - Revival (epub).rar  ::INFO:: 395.0KB
-!Ook Stephen King - Revival (retail) (epub).rar  ::INFO:: 1.4MB
-!Mysfyt Stephen King - Revival (epub).rar  ::INFO:: 395.0KB
-!Mysfyt Stephen King - Revival (retail) (epub).rar  ::INFO:: 1.4MB
-!Pondering Stephen King - Revival (epub).rar  ::INFO:: 395.0KB
-!Pondering Stephen King - Revival (retail) (epub).rar  ::INFO:: 1.4MB
-closed
-
-$ ./bot.py BOOK "\!Ook Stephen King - Revival (epub).rar  ::INFO:: 395.0KB"
-Downloading !Ook Stephen King - Revival (epub).rar  ::INFO:: 395.0KB
-connected
-Joined channel #ebooks
-Asking for '!Ook Stephen King - Revival (epub).rar  ::INFO:: 395.0KB'
-Receiving file
-100%
-Files: 
-['/tmp/Stephen King - Revival (epub).epub']
-closed
-```
-
-Note: Most likely you'll have to escape '!' on your shell.
-
-requirements.txt
-```
-websocket_server
-```
+To be re-implemented
 
 # Basic Web interface
 Work in progress

bookclient.py

@@ -1,106 +0,0 @@
-from threading import Thread
-from ircclient import IRCClient, MODE_SEARCH, MODE_BOOK
-import logging
-import queue
-import subprocess
-
-logging.basicConfig(level=logging.DEBUG)
-log = logging.getLogger(__name__)
-log.setLevel(logging.DEBUG)
-
-def get_books_from_list(filename):
-    f = open(filename, "r")
-    ret = []
-    for line in f.readlines():
-        line = line.lower().strip()
-        if not line.startswith('!') or not any(_type in line for _type in ['epub', 'mobi']):
-            continue
-        ret.append(line)
-        log.info("Book matches: %s", line)
-    ret = list(set(ret)) # dedup
-    return ret
-
-def mode_from_files(files):
-    for f in files:
-        f = f.lower()
-        if 'searchbot' in f or 'searchook' in f.lower():
-            return MODE_SEARCH
-    return MODE_BOOK
-
-def handle_files(files):
-    log.info("Unarchived files %s", files)
-    out = []
-    mode = mode_from_files(files)
-    if mode == MODE_SEARCH:
-        for f in files:
-            if "searchbot" not in f.lower() and "searchook" not in f.lower():
-                continue
-            out.extend(get_books_from_list(f))
-        return out
-
-    for f in files:
-        if f.lower().endswith(".epub"):
-            log.info("EPUB %s", f)
-            new_fname = f.replace("epub", "mobi")
-            p = subprocess.Popen(["ebook-convert", f, new_fname], stdout=subprocess.DEVNULL)
-            # TODO log to file?
-            p.wait()
-            out.append(new_fname)
-        out.append(f)
-    # TODO make paths absolute?
-    return out
-
-def handle_results(q, cb):
-    while True:
-        try:
-            item = q.get(timeout=1)
-            if item is None:
-                break
-        except queue.Empty:
-            continue
-        log.info("Got a result! %s", item)
-        if item['type'] == 'status':
-            cb((item['key'], item['status']))
-        elif item['type'] == 'files':
-            cb(handle_files(item['files']))
-
-class BookClient:
-    workers = []
-    def __init__(self, cb):
-        self.q = queue.Queue()
-        self.rq = queue.Queue()
-
-        self.results_t = Thread(target=handle_results, args=(self.rq, cb))
-        self.results_t.daemon = True
-        self.results_t.start()
-
-        self.create_worker()
-
-    def create_worker(self):
-        print("Spawning a new worker")
-        worker = IRCClient(command_queue=self.q, results_queue=self.rq)
-        worker.start()
-        self.workers.append(worker)
-
-    def free_workers(self):
-        return any([not worker.busy for worker in self.workers])
-
-    def request(self, mode, query):
-        if not self.free_workers():
-            self.create_worker()
-
-        if mode not in [MODE_SEARCH, MODE_BOOK]:
-            print('Invalid mode')
-            return
-
-        self.q.put({'query': query, 'mode': mode})
-        print("Command acknowledged")
-
-    def __del__(self):
-        print("Cleaning up workers")
-        for worker in self.workers:
-            worker.stop()
-            worker.join(timeout=5)
-        print("Cleaning up results")
-        self.rq.put(None)
-        self.results_t.join(timeout=5)

bookworm/__init__.py

@@ -0,0 +1 @@
+

bookworm/cli.py

@@ -0,0 +1,12 @@
+import json
+import sys
+from bookworm.constants import REDIS_BOOK_COMMANDS
+
+import pg_simple
+import redis
+
+bot = sys.argv[1]
+book = sys.argv[2]
+
+r = redis.StrictRedis(host='localhost', port=6379)
+r.rpush(REDIS_BOOK_COMMANDS, json.dumps({'bot': bot, 'book': book}))

bookworm/constants.py

@@ -0,0 +1,8 @@
+RAW_FILE_BUCKET = 'rawfiles'
+IRC_TIME_TO_FIRST_COMMAND = 30
+IRC_CHANNEL = "#ebooks"
+PROCESSED_FILE_BUCKET = 'files'
+UNPACKABLE_EXTENSIONS = ['epub', 'mobi', 'azw3']
+REDIS_BOOK_COMMANDS = 'BOOK_COMMANDS'
+REDIS_FETCH_FILE = 'FETCH_FILE'
+REDIS_UNPACK_FILE = 'UNPACK_FILE'

bookworm/file_fetcher.py

@@ -0,0 +1,62 @@
+import json
+import socket
+import sys
+import time
+from threading import Thread
+from bookworm.constants import RAW_FILE_BUCKET, REDIS_UNPACK_FILE, REDIS_FETCH_FILE
+from bookworm.logger import log, setup_logger
+from bookworm import s3
+
+import redis
+
+def netcat(filename, ip, port, size, job_key, s3client, redis):
+    log.info('netcat: %s %d %d %s', ip, port, size, filename)
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    log.info("Fetching %s", filename)
+    s.connect((ip, port))
+    log.info("Receiving file %s", filename)
+
+    buff = b''
+    count = 0
+    last_perc = 0
+    while True:
+        data = s.recv(16384)
+        if len(data) == 0:
+            log.info("No data received - finished")
+            break
+        count += len(data)
+        buff += data
+        perc = int(100 * count / size)
+        if perc % 5 == 0 and perc != last_perc:
+            log.info("Download percentage: %d", perc)
+            # TODO set_job_state(job_key, 'DOWNLOADING', "%d%%" % perc)
+            last_perc = perc
+        if count >= size:
+            break
+    log.info("Download complete")
+    s.close()
+    log.info("Putting file in s3")
+    # TODO set_job_state(job_key, 'DOWNLOAD_DONE', job_key)
+    s3client.put_object(Body=buff, Bucket=RAW_FILE_BUCKET, Key=job_key)
+    log.info("File %s in s3 with key %s", filename, job_key)
+    redis.rpush(REDIS_UNPACK_FILE, json.dumps({'job_key': job_key}))
+
+def main():
+    r = redis.StrictRedis(host='localhost', port=6379)
+    setup_logger()
+
+    while True:
+        log.info('Waiting for message...')
+        topic, message = r.blpop(REDIS_FETCH_FILE)
+
+        log.info('got message: %s', message)
+        params = json.loads(message.decode('utf-8'))
+        log.info('params for netcat: %s', params)
+        params['s3client'] = s3.client()
+        params['redis'] = r
+
+        t = Thread(target=netcat, kwargs=params)
+        t.daemon = True
+        t.start()
+
+main()

bookworm/ircclient.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import json
+import sys
+import time
+import shlex
+
+import utils
+from threading import Thread
+from bookworm.constants import IRC_TIME_TO_FIRST_COMMAND, IRC_CHANNEL, REDIS_BOOK_COMMANDS, REDIS_FETCH_FILE
+from bookworm.logger import log, setup_logger
+
+import redis
+import irc.client
+import jaraco.stream.buffer
+
+
+class IRCClient(irc.client.SimpleIRCClient):
+    def __init__(self, target, name):
+        log.info('startup to %s, as %s', target, name)
+        irc.client.SimpleIRCClient.__init__(self)
+        irc.client.ServerConnection.buffer_class = jaraco.stream.buffer.LenientDecodingLineBuffer
+        self.target = target
+        self.name = name
+        self.startup = time.time()
+        self.r = redis.StrictRedis(host='localhost', port=6379)
+
+        self.connect("irc.irchighway.net", 6667, name)
+
+        t = Thread(target=self.start)
+        t.daemon = True
+        t.start()
+
+    def on_welcome(self, connection, event):
+        if irc.client.is_channel(self.target):
+            connection.join(self.target)
+
+    def wait_for_commands(self):
+        while True:
+            log.info('Waiting for message...')
+            topic, message = self.r.blpop(REDIS_BOOK_COMMANDS)
+            log.info('got message: %s', message)
+
+            delta = IRC_TIME_TO_FIRST_COMMAND - (time.time() - self.startup)
+            while delta > 0:
+                delta = IRC_TIME_TO_FIRST_COMMAND - (time.time() - self.startup)
+                log.info("I am not ready yet, still %d to go", delta)
+                time.sleep(max(min(delta, 2), 0))
+
+            data = message.decode('utf-8')
+            log.info('data: %s', data)
+            command = json.loads(data)
+            log.info('Command: %s', command)
+
+            bot = command['bot'].strip()
+            book = command['book'].strip()
+            # set_job_state(job_key, 'waiting', time.time())
+            self.connection.privmsg(self.target, f'!{bot} {book}')
+
+    def on_pubmsg(self, connection, event):
+        log.debug('pubmsg %s', event)
+
+    def on_privmsg(self, connection, event):
+        log.info('privmsg %s', event)
+    
+    def on_ctcp(self, connection, event):
+        if event.target != self.name:
+            log.debug('ctcp event: %s', event)
+            log.debug('ctcp event for someone else')
+            return
+
+        log.info('ctcp event: %s', event)
+        payload = event.arguments[1]
+        parts = shlex.split(payload) # quotes
+        log.info(parts)
+
+        command = parts.pop(0)
+        if command != "SEND":
+            return
+        log.info('fname %s', parts[-4])
+        log.info('peer_address %s', irc.client.ip_numstr_to_quad(parts[-3]))
+        log.info('Port %s', parts[-2])
+        log.info('size %s', parts[-1])
+
+        filename, peer_address, peer_port, size = parts
+        peer_address = irc.client.ip_numstr_to_quad(peer_address)
+        peer_port = int(peer_port)
+        job_key = filename
+        data = json.dumps({'ip': peer_address,
+                           'port': peer_port,
+                           'size': int(size),
+                           'filename': filename,
+                           "job_key": job_key})
+        log.info('Publishing to FETCH_FILE: %s', data)
+        self.r.rpush(REDIS_FETCH_FILE, data)
+
+    def on_disconnect(self, connection, event):
+        sys.exit(0)
+
+
+def main():
+
+    setup_logger()
+    name = "bookbot" + utils.random_hash()
+
+    c = IRCClient(IRC_CHANNEL, name)
+    c.wait_for_commands()
+main()

bookworm/logger.py

@@ -0,0 +1,16 @@
+import sys
+import logging
+
+FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+log = logging.getLogger(__name__)
+def setup_logger():
+    global log
+    formatter = logging.Formatter(FORMAT)
+    handler = logging.StreamHandler(sys.stdout)
+    
+    handler.setLevel(logging.INFO)
+    handler.setFormatter(formatter)
+    log.setLevel(logging.INFO)
+    log.addHandler(handler)
+    

bookworm/parse.py

@@ -0,0 +1,44 @@
+import time
+import re
+from collections import namedtuple
+import pg_simple
+
+ONE_GB = 2**30
+ONE_MB = 2**20
+ONE_KB = 2**10
+
+connection_pool = pg_simple.config_pool(dsn='dbname=david user=david')
+
+r = re.compile(r'^!(?P<bot>[a-z0-9-]+?) (?P<book>.+)\s(::INFO::|-+)\s+(?P<size>[0-9.]+\s*[BKMG]+)\s*$', re.I)
+tags_re = re.compile(r'[\[\(](?P<tag>.*?)[\]\)]')
+books = []
+Book = namedtuple('Book', 'raw bot book size')
+
+for line in open('all_books_2.txt', 'r'):
+    line = line.strip()
+    match = r.match(line)
+    bot = match.group('bot').strip()
+    book = match.group('book').strip()
+    size = match.group('size').strip().lower()
+
+    if 'gb' in size:
+        size = float(size.replace('gb', '')) * ONE_GB
+    elif 'mb' in size:
+        size = float(size.replace('mb', '')) * ONE_MB
+    elif 'kb' in size:
+        size = float(size.replace('kb', '')) * ONE_KB
+    elif 'b' in size:
+        size = float(size.replace('b', ''))
+
+    books.append(Book(line, bot, book, size)._asdict())
+
+print('db..')
+with pg_simple.PgSimple(connection_pool) as db:
+    t = time.time()
+    vals = [tuple(book.values()) for book in books]
+    print('making tuples', time.time() - t)
+
+    db.insert_many('books', keys=books[0].keys(), values=vals, page_size=1000)
+    print('commiting..')
+    db.commit()
+    print('done..')

bookworm/s3.py

@@ -0,0 +1,13 @@
+import boto3
+import botocore
+
+def client():
+    config = botocore.client.Config(connect_timeout=3, read_timeout=3, retries={'max_attempts': 0})
+    session = boto3.session.Session()
+    return session.client(
+        service_name='s3',
+        aws_access_key_id='3207900NM6AZ01AN02O1',
+        aws_secret_access_key='pmWuVRye20yiPbRk5tau1L6ggSueeaU5KXh2n0aZ',
+        endpoint_url='http://localhost:9000',
+        config=config
+    )

bookworm/unpacker.py

@@ -0,0 +1,88 @@
+import json
+import socket
+import sys
+import time
+import tempfile
+from threading import Thread
+from subprocess import check_output
+from bookworm import s3
+from bookworm.logger import log, setup_logger
+from bookworm.constants import RAW_FILE_BUCKET, PROCESSED_FILE_BUCKET, UNPACKABLE_EXTENSIONS, REDIS_UNPACK_FILE
+import redis
+
+def should_unpack(fname):
+    fname = fname.lower()
+    return fname.endswith('rar') or fname.endswith('zip')
+
+def archive_contents(fd):
+    to_extract = {}
+    contents = check_output(['lsar', '-j', fd.name]).decode('utf-8')
+    contents = json.loads(contents)
+    
+    log.debug(contents['lsarContents'])
+    for f in contents['lsarContents']:
+        fname = f['XADFileName']
+        if any([extension in fname.lower() for extension in UNPACKABLE_EXTENSIONS]):
+            log.info('Extracting %s from the archive', fname)
+            to_extract[f['XADIndex']] = fname
+    return to_extract
+
+def store_file(s3client, fname, file_contents):
+    log.info('Puttin in s3 under bucket %s with key %s', PROCESSED_FILE_BUCKET, fname)
+    s3client.put_object(Body=file_contents, Bucket=PROCESSED_FILE_BUCKET, Key=fname)
+    log.info('Put in s3 under bucket %s with key %s', PROCESSED_FILE_BUCKET, fname)
+
+def delete_raw_file(s3client, job_key):
+    log.info('Deleting %s from %s', job_key, RAW_FILE_BUCKET)
+    s3client.delete_object(Bucket=RAW_FILE_BUCKET, Key=job_key)
+
+def unpack(job_key, s3client, redis):
+    log.info('Got a request to unpack %s', job_key)
+    data = s3client.get_object(Key=job_key, Bucket=RAW_FILE_BUCKET)
+    with tempfile.NamedTemporaryFile() as fd:
+        raw_file_contents = data['Body'].read()
+        fd.write(raw_file_contents)
+        fd.flush()
+
+        if should_unpack(job_key):
+            to_extract = archive_contents(fd)
+        else:
+            log.info("Not unpacking %s", job_key)
+            store_file(s3client, job_key, raw_file_contents)
+            delete_raw_file(s3client, job_key)
+            return
+
+        if not to_extract:
+            log.info(contents['lsarContents'])
+            log.error("Could not find any valid file")
+            return
+
+        for index, fname in to_extract.items():
+            log.info('Processing %s %s', index, fname)
+            file_contents = check_output(['unar', '-o', '-', '-i', fd.name, str(index)])
+            log.info('Got %d bytes', len(file_contents))
+            store_file(s3client, fname, file_contents)
+
+    # TODO set_job_state(job_key, 'UNPACK_DONE', job_key)
+    delete_raw_file(s3client, job_key)
+    log.info('Done with job %s', job_key)
+
+def main():
+    r = redis.StrictRedis(host='localhost', port=6379)
+    setup_logger()
+    s3client = s3.client()
+    while True:
+        log.info('Waiting for message...')
+        topic, message = r.blpop(REDIS_UNPACK_FILE)
+
+        log.info('got message: %s', message)
+        params = json.loads(message.decode('utf-8'))
+        log.info('params for unpacker: %s', params)
+        params['s3client'] = s3client
+        params['redis'] = r
+
+        t = Thread(target=unpack, kwargs=params)
+        t.daemon = True
+        t.start()
+
+main()

web.py

@@ -17,7 +17,8 @@ def send_status(server, client=None, force=False):
         if (datetime.now() - last_msg).total_seconds() < 2:
             return
         last_msg = datetime.now()
-    status = { 'SEARCH': q.search_status(), 'BOOKS': q.books_status()}
+    #status = { 'SEARCH': q.search_status(), 'BOOKS': q.books_status()}
+    status = None
     if client is None:
         ws.send_message_to_all(json.dumps(status))
     else:
@@ -30,10 +31,11 @@ def message_received(client, server, message):
     j = json.loads(message)
     print(j)
     if j['type'].upper() == 'SEARCH':
-        q.new_search(j['book'], j['extension'])
+        #q.new_search(j['book'], j['extension'])
+        pass
 
     if j['type'].upper() == 'BOOK':
-        q.new_dl(j['book'])
+        pass
 
     send_status(server)
 
@@ -42,7 +44,6 @@ def updated(force=False):
 
 
 if __name__ == '__main__':
-    q = qManager(updated)
     ws = WebsocketServer(8081, host='127.0.0.1')
     ws.set_fn_new_client(new_client)
     ws.set_fn_message_received(message_received)

bot.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python3.7
-import logging
-import shlex
-import readline
-import os
-from collections.abc import Iterable
-
-from bookclient import BookClient
-
-histfile = os.path.join(os.path.expanduser("~"), ".book_history")
-
-def usage():
-    print("USAGE:")
-    print("\t SEARCH <BOOK>")
-    print("\t BOOK <BOT COMMAND>")
-
-def cb(arg):
-    print("Called callback!")
-    if isinstance(arg, Iterable):
-        for item in arg:
-            print(item)
-    else:
-        print(arg)
-
-def main():
-    b = BookClient(cb)
-    while True:
-        line = input('> ').strip().lower()
-        try:
-            split = shlex.split(line)
-        except Exception as e:
-            print(e)
-            continue
-
-        if len(split) == 0:
-            continue
-
-        mode = split[0]
-        query = " ".join(split[1:])
-        b.request(mode, query)
-
-
-if __name__ == "__main__":
-    try:
-        readline.read_history_file(histfile)
-        # default history len is -1 (infinite), which may grow unruly
-        readline.set_history_length(10000)
-    except FileNotFoundError:
-        pass
-
-    try:
-        main()
-    except KeyboardInterrupt:
-        print("\nBye")
-    except EOFError:
-        print("\nBye")
-    readline.write_history_file(histfile)

front/js/app.js

@@ -8,7 +8,8 @@ app.controller('main', function($scope,$http,$interval) {
 	$scope.extension="";
 
 	$scope.activeTab = 'SEARCH';
-    ws = new WebSocket("wss://books.davidventura.com.ar/ws/");
+    //ws = new WebSocket("wss://books.davidventura.com.ar/ws/");
+    ws = new WebSocket("ws://david-dotopc.labs:8099/");
     ws.onmessage = function(event) {
         var j = JSON.parse(event.data);
         console.log(j);

ircclient.py

@@ -1,277 +0,0 @@
-#!/usr/bin/env python3
-import logging
-import os
-import queue
-import socket
-import time
-import utils
-import re
-
-from unzipper import unar
-from threading import Thread
-from collections import defaultdict
-
-logging.basicConfig(level=logging.DEBUG)
-
-MODE_SEARCH = 'search'
-MODE_BOOK = 'book'
-results_key = re.compile(r'_results_for[_ ]+(?P<key>.*?)\.', re.I)
-
-def query_to_job_key(query):
-    job = query
-    if job.startswith('!'):
-        # !Ook Brandon Sanderson - [Skyward 01] - Skyward (retail) (epub).rar  ::INFO:: 3.5MB
-        # !Horla-new Brandon Sanderson - Skyward (US) (epub).epub
-        job = ' '.join(job.split(' ')[1:]).strip()
-        # Brandon Sanderson - [Skyward 01] - Skyward (retail) (epub).rar  ::INFO:: 3.5MB
-        # Brandon Sanderson - Skyward (US) (epub).epub
-        job = re.sub(r'::.*$', '', job).strip()
-    return job.lower()
-
-def filename_to_job(fname):
-    match = results_key.search(fname)
-    if match: # list of results
-        return match.group('key').replace('_', ' ').lower()
-    # brandon_sanderson_-_skyward_(uk)_(epub).rar
-    return fname.replace('_', ' ').lower()
-
-def get_dcc_args(msg):
-    msg = msg.split(':')[2]
-    msg = msg.replace("\x01", "")
-    if not msg.startswith("DCC"):
-        return None
-    args = msg.replace("DCC SEND ", "").split(" ")
-    size = int(args.pop())
-    port = int(args.pop())
-    ip = utils.ip_from_decimal(int(args.pop()))
-    filename = "_".join(args).replace('"', '')
-    return ip, port, size, filename
-
-
-class IRCClient(Thread):
-    TIME_TO_FIRST_COMMAND = 30
-    HOST = "irc.irchighway.net"
-    PORT = 6667
-    CHANNEL = "#ebooks"
-    PATH = "/tmp/"
-    SEARCH_BOT = "searchook"
-    # ^ config
-    IGNORE = [
-        "NOTICE",
-        "PART",
-        "QUIT",
-        "332",
-        "333",
-        "372",
-        "353",
-        "366",
-        "251",
-        "252",
-        "254",
-        "255",
-        "265",
-        "266",
-        "396"]
-    joined_channel = False
-    connected = False
-    name = "bookbot" + utils.random_hash()
-    readbuffer = b''
-    time_joined = None
-    jobs = defaultdict(dict)
-    busy = False
-    running = True
-
-    def __init__(self, command_queue, results_queue):
-        super(IRCClient, self).__init__(daemon=True)
-        self.socket = None
-        self.command_queue = command_queue
-        self.results_queue = results_queue
-        self.send_queue = queue.Queue()
-
-        self.log = logging.getLogger(self.getName())
-        self.log.setLevel(logging.DEBUG)
-
-        nickstr = "NICK %s" % self.name
-        userstr = "USER %s %s bla :%s" % (self.name, self.HOST, self.name)
-
-        self.send_queue.put(nickstr)
-        self.send_queue.put(userstr)
-
-    def stop(self):
-        self.running = False
-    def run(self):
-        self.handle_connect()
-        while self.running:
-            if self.connected and self.joined_channel:
-                self.handle_commands()
-            self.process_send_queue()
-            try:
-                self.handle_read()
-            except socket.timeout:
-                continue
-            except socket.error as e:
-                self.log.error('socket error')
-                self.log.exception(e)
-                self.handle_connect()
-            except Exception as e:
-                self.log.exception(e)
-                break
-        self.handle_close()
-        self.log.info('Exiting RUN')
-
-    def handle_commands(self):
-        if self.command_queue.empty():
-            time.sleep(0.2)
-            return
-
-        elapsed = time.time() - self.time_joined
-        if elapsed < self.TIME_TO_FIRST_COMMAND:
-            self.log.info("commands to process, but we have to wait %d seconds", self.TIME_TO_FIRST_COMMAND - elapsed)
-            time.sleep(1)
-            return
-        command = self.command_queue.get()
-        self.log.info("command %s", command)
-        job = query_to_job_key(command['query'])
-        self.set_job_state(job, 'pending')
-
-        if command['mode'] == MODE_SEARCH:
-            self.send_queue.put("PRIVMSG %s :@%s %s " % (self.CHANNEL, self.SEARCH_BOT, command['query']))
-            return
-        elif command['mode'] == MODE_BOOK:
-            self.send_queue.put("PRIVMSG %s :%s " % (self.CHANNEL, command['query']))
-        else:
-            self.log.error('Invalid command')
-
-    def set_job_state(self, job, state):
-        self.log.info("Setting job [%s] to %s", job, state)
-        self.jobs[job].update({'state': state})
-        self.results_queue.put({'type': 'status', 'status': state, 'key': job})
-        if state == 'done':
-            del self.jobs[job]
-
-    def handle_connect(self):
-        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.socket.connect((self.HOST, self.PORT))
-        self.socket.settimeout(2)
-        self.log.info("connected")
-        self.connected = True
-
-    def handle_close(self):
-        self.log.info("closed")
-        self.connected = False
-        self.socket.close()
-
-    def get_data_from_irc(self):
-        data = self.socket.recv(4096)
-        if len(data) < 2:
-            return ""
-        # newline at the end?
-        if not (data[-1] == 10 and data[-2] == 13):
-            self.readbuffer += data
-            return ""
-
-        data = self.readbuffer + data
-        self.readbuffer = b''
-
-        # purge crap
-        data = (data.replace(b'\x95', b'').replace(b'0xc2', b'').decode('utf-8', 'ignore'))
-        return data
-
-    def handle_read(self):
-        lines = self.get_data_from_irc().splitlines()
-
-        for line in lines:
-            words = line.split(' ')
-            if len(words) < 2:
-                continue
-            msg_from = words[0]
-            comm = words[1]
-
-            if self.joined_channel:
-                # self.log.debug(line)
-                pass
-
-            if comm in self.IGNORE:
-                continue
-
-            if comm == "JOIN":
-                if self.name not in msg_from:  # msg "NICK joined the channel" not about me
-                    continue
-                self.log.info("Joined channel %s", self.CHANNEL)
-                self.time_joined = time.time()
-                self.joined_channel = True
-                continue
-
-            if comm == "PRIVMSG":
-                # private message not addressed to me
-                if words[2] != self.name:
-                    continue
-                self.log.info("privmsg: %s", line)
-                dcc_args = get_dcc_args(line)
-                if dcc_args is None:
-                    continue
-                ip, port, size, filename = dcc_args
-                job = filename_to_job(filename)
-                self.set_job_state(job, 'downloading')
-                self.busy = True
-                downloaded_filename = self.netcat(ip, port, size, filename, job)
-                self.set_job_state(job, 'unarchiving')
-                # TODO save state in redis on ip port size filename + output of handle files
-                files = unar(downloaded_filename, self.PATH)
-                self.busy = False
-                self.results_queue.put({'type': 'files', 'files': files})
-                self.set_job_state(job, 'done')
-
-            if comm == "PING" or msg_from == "PING":  # respond ping to avoid getting kicked
-                self.pong(line)
-                continue
-
-            if comm == "376":  # END MOTD
-                # MOTD complete, lets join the channel
-                self.join_channel(self.CHANNEL)
-                continue
-
-    def netcat(self, ip, port, size, filename, job_key):
-        filename = os.path.basename(filename).replace(" ", "_")
-        self.log.info('netcat: %s %d %d %s', ip, port, size, filename)
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.log.info("Receiving file")
-        s.connect((ip, port))
-
-        fname = os.path.join(self.PATH, filename)
-        f = open(fname, 'wb')
-        count = 0
-        last_perc = 0
-        while True:
-            data = s.recv(16384)
-            if len(data) == 0:
-                self.log.info("No data received - finished")
-                break
-            count += len(data)
-            f.write(data)
-            perc = int(100 * count / size)
-            if perc % 10 == 0 and perc != last_perc:
-                self.log.info("Download percentage: %d", perc)
-                self.set_job_state(job_key, "%d%%" % perc)
-                last_perc = perc
-            if count >= size:
-                break
-        s.close()
-        f.close()
-        return fname
-
-    def pong(self, data):
-        msg = data.replace("PING ", "")
-        self.send_queue.put("PONG %s" % msg)
-
-    def join_channel(self, channel):
-        self.send_queue.put("JOIN %s" % channel)
-
-    def process_send_queue(self):
-        if self.send_queue.empty():
-            return
-        data = self.send_queue.get()
-        if not data.startswith("PONG"):
-            self.log.info("Sending %s", data)
-        add = bytes(str(data), "utf-8") + bytes([13, 10])
-        self.socket.send(add)

qmanager.py

@@ -1,159 +0,0 @@
-#!/usr/bin/env python3
-"""Manage a list of book-jobs,
-return a description list for the website"""
-import os
-import shlex
-import re
-from time import time
-from bot import IRCClient
-
-
-class qManager:
-    """Main module class"""
-
-    tasks = []
-    last_id = 0
-    PATH = ""
-
-    def __init__(self, update_cb, path="/tmp/books/"):
-        """Init the manager. Set output path"""
-        if not os.path.isdir(path):
-            os.makedirs(path, exist_ok=True)
-        self.PATH = path
-        self.update_cb = update_cb
-
-    def new_dl(self, string):
-        """Download a book"""
-        irc_client = IRCClient(
-            string, "", "BOOK",
-            cb=self.update_cb,
-            logging=False,
-            path=self.PATH)
-        self.new_task(string, irc_client)
-
-    def new_search(self, keywords, fmt):
-        """Search for a book"""
-        irc_client = IRCClient(
-            keywords,
-            fmt,
-            "SEARCH",
-            cb=self.update_cb,
-            logging=False,
-            path=self.PATH)
-        self.new_task(keywords, irc_client)
-
-    def new_task(self, query, t):
-        """ Add a new 'task' to the list """
-        nt = task(query, t, self.last_id)
-        self.last_id += 1
-        self.tasks.append(nt)
-
-    def search_status(self):
-        return [ t for t in self.task_status() if t["TYPE"]=='SEARCH' ]
-
-    def books_status(self):
-        return [ t for t in self.task_status() if t["TYPE"]=='BOOK' ]
-
-    def task_status(self):
-        """ Return a dict with the status of each 'task' """
-        ret = []
-        for t in self.tasks:
-            elapsed = time() - t.START_TIME
-            ret.append({"ID": t.ID,
-                        "STATUS": t.get_status(),
-                        "PROGRESS": t.get_progress(),
-                        "OUT": t.get_output(),
-                        "ELAPSED": int(elapsed),
-                        "QUERY": t.QUERY,
-                        "EXTRA": t.get_extra(),
-                        "TYPE": t.get_type()})
-
-        return ret
-
-
-class task:
-    """ Task class. Only returns a status"""
-    CUR_STATUS = ""
-    OUTPUT = ""
-    ID = 0
-    CLIENT = None
-    START_TIME = None
-    QUERY = ""
-
-    def __init__(self, query, c, _id):
-        """Initialize the task"""
-        self.QUERY = query
-        self.CLIENT = c
-        self.ID = _id
-        self.START_TIME = time()
-
-    def get_progress(self):
-        """ Return progress."""
-        return self.CLIENT.PROGRESS
-
-    def get_extra(self):
-        """ Return extra output."""
-        return self.CLIENT.EXTRA_OUTPUT
-
-    def get_status(self):
-        """ Return status.  If timed out since last call, stop task """
-        if time() - self.START_TIME > 600:
-            self.CLIENT.do_timeout()
-        self.OUTPUT = self.CLIENT.OUTPUT
-        return self.CLIENT.STATUS
-
-    def get_type(self):
-        """ Return type """
-        return self.CLIENT.TYPE
-
-    def get_output(self):
-        """Return output:
-           If it's just a string, return it (Books).
-           If it's a list, parse it and return a representative dict.
-           Should cache this or something, doesn't make sense to
-           calculate on each call.
-        """
-        if isinstance(self.OUTPUT, str) or self.OUTPUT is None:
-            return self.OUTPUT
-        TAGS_R = re.compile(r'[\[(].*?[\])]|\.rar|v\d.*?\s')
-        books = self.OUTPUT
-        ret = []
-        for book in books:
-            book = book.replace("---", "")
-            groups = re.search(
-                r"(?P<BOT>^!\w+)(?P<BOOK>.*?)(?P<INFO>::INFO.*)?$", book)
-            if groups is None:
-                #print("[No groups] %s" % book)
-                return books
-            groups = groups.groupdict()
-
-            # remove tags from book
-            groups["BOOK"] = re.sub(TAGS_R, "", groups["BOOK"])
-            groups["TAGS"] = [r.strip("()[]")
-                              for r in re.findall(TAGS_R, book)]
-            groups["TEXT"] = book
-            ret.append(groups)
-        return ret
-
-# BASIC CLI INTERFACE
-if __name__ == "__main__":
-    q = qManager()
-    while True:
-        line = input()
-        words = shlex.split(line)
-        print(q.task_status())
-        if len(words) < 2:
-            continue
-        comm = words[0]
-        if comm.lower() == "quit":
-            break
-        if comm.lower() == "search":
-            if len(words) < 3:
-                continue
-            q.new_search(words[1], words[2])
-            continue
-        if comm.lower() == "dl":
-            q.new_dl(words[1])
-            continue
-
-        print("search <'multiple keywords'> <format>|dl <line>|quit")

setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup, find_packages
+setup(
+    name="bookworm",
+    version="0.1",
+    packages=find_packages(),
+)

unzipper.py

@@ -1,69 +0,0 @@
-"""Try and uncompress a source file to a dest dir"""
-import zipfile
-import os.path
-import subprocess
-
-USE_UNRAR = False
-
-
-def unar(source, dest_dir):
-    """Split input into zip or rar and parse accordingly.
-    Return source if it's not a zip or rar"""
-    print("uncompressing %s to %s" % (source, dest_dir))
-    if source.lower().endswith("zip"):
-        return unzip(source, dest_dir)
-    if source.lower().endswith("rar"):
-        return unrar(source, dest_dir)
-
-    print("NOT RAR? NOT ZIP? I'm panicking.")
-    print("I got %s" % source)
-    return [source]
-
-
-def unzip(source_filename, dest_dir):
-    """Unzip source_filename to dest_dir"""
-    print("Unzipping %s to %s" % (source_filename, dest_dir))
-    out = []
-    with zipfile.ZipFile(source_filename) as zfile:
-        for member in zfile.infolist():
-            # Path traversal defense copied from
-            # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789
-            words = member.filename.split('/')
-            path = dest_dir
-            for word in words[:-1]:
-                _, word = os.path.splitdrive(word)
-                _, word = os.path.split(word)
-                if word in (os.curdir, os.pardir, ''):
-                    continue
-                path = os.path.join(path, word)
-            target = os.path.join(path, member.filename.split('/')[-1])
-            out.append(target)
-            print("Extracting %s" % target)
-            zfile.extract(member, path)
-            print("Extracted %s" % target)
-    return out
-
-
-def unrar(source, dest_dir):
-    """Unzip source to dest_dir. Might use unar or unrar
-    print("Unraring %s to %s" % (source, dest_dir))
-    based on the flag USE_UNRAR"""
-    out = []
-    list_files = []
-    extract_files = []
-    if USE_UNRAR:
-        list_files = ["unrar", "lb", source]
-        extract_files = ["unrar", "x", "-o+", source, dest_dir]
-    else:
-        list_files = ["lsar", source]
-        extract_files = ["unar", "-f", "-o", dest_dir, source]
-
-    with subprocess.Popen(list_files, stdout=subprocess.PIPE) as proc:
-        out = proc.stdout.read().decode('utf-8').split("\n")
-        if out[0].endswith(": RAR"):
-            del out[0]  # header
-        if len(out[-1]) == 0:
-            del out[-1]
-    subprocess.run(extract_files, stdout=subprocess.DEVNULL)
-
-    return [os.path.join(dest_dir, file) for file in out]