#(c) 2004-2008 Thunderstone Media, LLC #Creative Commons Attribution-Noncommercial 3.0 United States License # #Python Developyment By: # #Ray Slakinski #August Trometer from __future__ import division from time import * import iPXSettings from iPXTools import * import gettext gettext.install('bittorrent', 'btlocale') def getTorrent(feedName, torrent, maxUploadRate, saveLocation, saveName): from BitTorrent.download import Feedback, Multitorrent from BitTorrent.defaultargs import get_defaults from BitTorrent.parseargs import printHelp from BitTorrent.zurllib import urlopen from BitTorrent.bencode import bdecode from BitTorrent.ConvertedMetainfo import ConvertedMetainfo from BitTorrent import configfile from BitTorrent import BTFailure from BitTorrent import version import re, threading uiname = 'bittorrent-console' defaults = get_defaults(uiname) config, args = configfile.parse_configuration_and_args(defaults,uiname, '', 0, 1) def fmtsize(n): return float(n) class DL(Feedback): def __init__(self, metainfo, config): self.doneflag = threading.Event() self.metainfo = metainfo self.config = config logIt('BT url: %s' % self.config['url']) logIt('BT save_as: %s' % self.config['save_as']) if self.config['max_upload_rate'] > 0: logIt('BT max_upload_rate: %s' % str(self.config['max_upload_rate'])) def run(self): import os try: config = self.config self.d = HeadlessDisplayer(self.doneflag) self.multitorrent = Multitorrent(self.config, self.doneflag, self.global_error) # raises BTFailure if bad metainfo = ConvertedMetainfo(bdecode(self.metainfo)) torrent_name = metainfo.name_fs if config['save_as']: if config['save_in']: raise BTFailure('You cannot specify both --save_as and ' '--save_in') saveas = config['save_as'] elif config['save_in']: saveas = os.path.join(config['save_in'], torrent_name) else: saveas = torrent_namef self.d.set_torrent_values(metainfo.name, os.path.abspath(saveas), metainfo.total_bytes, len(metainfo.hashes)) self.torrent = self.multitorrent.start_torrent(metainfo, self.config, self, saveas) except BTFailure, e: globals()['torrentStatus'] = 0 logIt(str(e)) return self.get_status() self.multitorrent.rawserver.listen_forever() self.d.display({'activity':'shutting down', 'fractionDone':0}) self.torrent.shutdown() def reread_config(self): try: newvalues = configfile.get_config(self.config, 'btdownloadcurses') except Exception, e: globals()['torrentStatus'] = 0 self.d.error('Error reading config: ' + str(e)) return self.config.update(newvalues) # The set_option call can potentially trigger something that kills # the torrent (when writing this the only possibility is a change in # max_files_open causing an IOError while closing files), and so # the self.failed() callback can run during this loop. for option, value in newvalues.iteritems(): self.multitorrent.set_option(option, value) for option, value in newvalues.iteritems(): self.torrent.set_option(option, value) def get_status(self): self.multitorrent.rawserver.add_task(self.get_status, self.config['display_interval']) status = self.torrent.get_status(self.config['spew']) if iPXSettings.DEBUG: #logIt(str(status)) #logIt(str(status['activity'])) logIt('lastTorrentBeat: %s' % str(globals()['lastTorrentBeat'])) if str(status['activity']) == 'shut down': self.doneflag.set() self.d.finished() elif str(status['activity']) == 'seeding': self.d.display({'activity':_("shutting down"), 'fractionDone':0}) self.torrent.shutdown() elif globals()['torrentStatus'] == 0: logIt(str(status)) self.d.display({'activity':_("shutting down"), 'fractionDone':0}) self.torrent.shutdown() elif globals()['lastTorrentBeat'] > iPXSettings.torrentMaxBeatTime: globals()['torrentStatus'] = 0 logIt(str(status)) logIt('Bittorrent is taking too long to download, aborting...') logIt('lastTorrentBeat: %s' % str(globals()['lastTorrentBeat'])) self.d.display({'activity':_("shutting down"), 'fractionDone':0}) self.torrent.shutdown() elif str(status['activity']) == 'downloading': globals()['torrentStatus'] = 1 self.d.display(status) elif str(status['activity']) == 'Initial startup': globals()['torrentStatus'] = 1 else: globals()['torrentStatus'] = 0 self.d.display({'activity':_("shutting down"), 'fractionDone':0}) self.torrent.shutdown() def global_error(self, level, text): self.d.error(text) def error(self, torrent, level, text): self.d.error(text) def failed(self, torrent, is_external): self.doneflag.set() def finished(self, torrent): self.d.finished() class HeadlessDisplayer(object): def __init__(self, doneflag): self.doneflag = doneflag self.done = False self.percentDone = '' self.timeEst = '' self.downRate = '---' self.upRate = '---' self.shareRating = '' self.seedStatus = '' self.peerStatus = '' self.errors = [] self.file = '' self.downloadTo = '' self.fileSize = '' self.numpieces = 0 def set_torrent_values(self, name, path, size, numpieces): self.file = name self.downloadTo = path self.fileSize = fmtsize(size) self.numpieces = numpieces def finished(self): self.done = True self.downRate = '---' self.display({'activity':'download succeeded', 'fractionDone':1}) def error(self, errormsg): logIt(str(errormsg)) def display(self, dict): from cStringIO import StringIO if dict.has_key('downRate'): if iPXSettings.DEBUG: logIt('downRate: %s' % str(dict['downRate'] / 1024)) logIt('upRate: %s' % str(dict['upRate'] / 1024)) logIt('lastDLSize: %s' % str(globals()['lastDLSize'])) logIt('fractionDone: %s / %s' % (str(dict['fractionDone'] * 100), str(dict['fractionDone']))) logIt('numSeeds: %s' % str(dict['numSeeds'])) logIt('numPeers: %s' % str(dict['numPeers'])) if (long(dict['numSeeds'] > 2)) or (dict['downRate'] > iPXSettings.torrentMinDownRate and globals()['lastTorrentBeat'] <= iPXSettings.torrentMaxBeatTime): globals()['lastTorrentBeat'] = 0 if globals()['lastDLSize'] < dict['fractionDone'] * 100: if iPXSettings.DEBUG: printMSG(';;1;;1;;%.2f;;%.2f;;%.2f' % (100.0, dict['fractionDone'] * 100, dict['downRate'] / 1024)) else: printMSG(';;1;;1;;%.2f;;%.2f' % (100.0, dict['fractionDone'] * 100)) globals()['lastDLSize'] = (dict['fractionDone'] * 100) + 1.0 else: globals()['lastTorrentBeat'] += 1 def print_spew(self, spew): s = StringIO() s.write('\n\n\n') for c in spew: s.write('%20s ' % c['ip']) if c['initiation'] == 'L': s.write('l') else: s.write('r') total, rate, interested, choked = c['upload'] s.write(' %10s %10s ' % (str(int(total/10485.76)/100), str(int(rate)))) if c['is_optimistic_unchoke']: s.write('*') else: s.write(' ') if interested: s.write('i') else: s.write(' ') if choked: s.write('c') else: s.write(' ') total, rate, interested, choked, snubbed = c['download'] s.write(' %10s %10s ' % (str(int(total/10485.76)/100), str(int(rate)))) if interested: s.write('i') else: s.write(' ') if choked: s.write('c') else: s.write(' ') if snubbed: s.write('s') else: s.write(' ') s.write('\n') print s.getvalue() status = 0 if re.search('=', saveName, re.IGNORECASE): saveNameSplit = saveName.split('=') if len(saveNameSplit[(len(saveNameSplit)-1)]) > 1: saveName = saveNameSplit[(len(saveNameSplit)-1)] saveName = stringCleaning(saveName) logIt('%s: %s [Torrent]' % (feedName, saveName)) printMSG('%s: %s [Torrent]' % (feedName, saveName)) try: checkDir(saveLocation) params = ['--url', torrent, '--max_upload_rate', maxUploadRate, '--minport', iPXSettings.torrentMinPort, '--maxport', iPXSettings.torrentMinPort, '--save_as', saveLocation + '/' + saveName] config, args = configfile.parse_configuration_and_args(defaults,uiname, params, 0, 1) if config['url']: h = urlopen(config['url']) metainfo = h.read() h.close() except Exception, msg: logIt('Torrent Download Failed') logIt('ERRORMSG: %s' % str(msg)) status = 0 try: dl = DL(metainfo, config) dl.run() if globals()['torrentStatus'] == 1: logIt('Completed Download: %s' % saveName) status = 1 else: logIt('Torrent Download Failed') status = 0 except Exception, msg: logIt('Torrent Download Failed') logIt('ERRORMSG: %s' % str(msg)) status = 0 return status, saveName def displayProgress(block_count, block_size, total_size): if globals()['lastDLSize'] < (float(block_count*block_size)/1024): printMSG(";;1;;1;;%.2f;;%.2f" % (float(total_size)/1024, float(block_count*block_size)/1024)) globals()['lastDLSize'] = float(block_count*block_size)/1024 + iPXSettings.lastDLStepSize def getFile(feedName, url, saveLocation, saveName, userName=None, password=''): import tempfile, shutil, re, urllib2, os saveName = stringCleaning(saveName) if not userName == None: import urlparse setOpener(urlparse.urlparse(url)[1], userName, password) else: setOpener() status = 1 if iPXSettings.useProxyServer and userName != None: try: status, url, saveName = getFileViaProxySSL(url, userName, password, True, False) printMSG('%s: %s' % (feedName, saveName)) status, tmpFileName, r = getFileViaProxySSL(url, userName, password, False, True) if not status == 200: status = 0 return 0, saveName except Exception, msg: logIt('Download Failed') logIt (str(msg)) return 0, saveName if os.path.isfile(tmpFileName): if not r.getheader('Content-Disposition') == None: if re.search('filename=', r.getheader('Content-Disposition'), re.IGNORECASE): textSplit = r.getheader('Content-Disposition') textSplit = textSplit.split(';') for text in textSplit: if re.search('filename=', text, re.IGNORECASE): logIt('Detected New Filename To Use:') newSaveNameSplit = text.split('=') newSaveName = newSaveNameSplit[len(newSaveNameSplit) -1] newSaveName = newSaveName.replace('"', '') logIt(str(newSaveName)) shutil.copy(tmpFileName, saveLocation + '/' + newSaveName) saveName = newSaveName elif re.search('=', saveName, re.IGNORECASE): saveNameSplit = saveName.split('=') if len(saveNameSplit[(len(saveNameSplit)-1)]) > 1: newSaveName = saveNameSplit[(len(saveNameSplit)-1)] shutil.move(tmpFileName, saveLocation + '/' + newSaveName) saveName = newSaveName else: shutil.move(tmpFileName, saveLocation + '/' + saveName) else: shutil.move(tmpFileName, saveLocation + '/' + saveName) else: logIt('%s: %s' % (feedName, url)) logIt('Saving to: %s/' % saveLocation) checkDir(saveLocation) saveNameParts = saveName.split('?') if len(saveNameParts) > 1: for part in saveNameParts: if part.find('.') > 0: if len(part[part.find('.'):])-1 == 3: saveName = part saveName = stringCleaning(saveName) printMSG('%s: %s' % (feedName, saveName)) try: url = url.strip() url = url.replace(' ', '%20') logIt(url) req = urllib2.Request(url) try: h = urllib2.urlopen(req) except IOError, e: logIt(str(e)) return 0, saveName #page = '' count = 0 n = 1024 # number of bytes to read at a time fileSize = 0 if h.info().has_key('Content-Length'): fileSize = float(h.info()['Content-Length']) / 1024 printMSG(';;1;;1;;100.00;;0.00') tmpFile = tempfile.mkstemp() tmpFileName = tmpFile[1] f = open(tmpFileName, 'ab') while True: a = h.read(n) f.write(a) if not a: break count += len(a) # len(a) may not be same as n for final read try: percentDone = ((float(count) /1024) / fileSize) * 100 except: percentDone = fileSize if percentDone >= globals()['lastDLSize'] + 1: globals()['lastDLSize'] = percentDone printMSG(';;1;;1;;100.00;;%.2f' % (percentDone)) printMSG(';;1;;1;;100.00;;100.00') f.close() os.close(tmpFile[0]) except Exception, msg: logIt('Download Failed') logIt (str(msg)) status = 0 try: if h.info().has_key('Content-Disposition'): if re.search('filename=', h.info()['Content-Disposition'], re.IGNORECASE): textSplit = h.info()['Content-Disposition'].split(';') for text in textSplit: if re.search('filename=', text, re.IGNORECASE): logIt('Detected New Filename To Use:') newSaveNameSplit = text.split('=') newSaveName = newSaveNameSplit[len(newSaveNameSplit) -1] newSaveName = newSaveName.replace('"', '') logIt(str(newSaveName)) shutil.copy(tmpFileName, saveLocation + '/' + newSaveName) saveName = newSaveName elif re.search('=', saveName, re.IGNORECASE): saveNameSplit = saveName.split('=') if len(saveNameSplit[(len(saveNameSplit)-1)]) > 1: newSaveName = saveNameSplit[(len(saveNameSplit)-1)] shutil.move(tmpFileName, saveLocation + '/' + newSaveName) saveName = newSaveName else: shutil.move(tmpFileName, saveLocation + '/' + saveName) else: shutil.move(tmpFileName, saveLocation + '/' + saveName) except Exception, msg: logIt('Filename/Move Error: %s' % str(msg)) logIt('Retrying move with no filename detection...') try: shutil.move(tmpFileName, saveLocation + '/' + saveName) except Exception, msg: logIt('Retry Error: %s' % str(msg)) if os.path.isfile(saveLocation + '/' + saveName): logIt('Completed Download: %s' % saveName) status = 1 else: logIt('Download Failed') status = 0 return status, saveName def downloadFile(url,fileType,saveDir,saveName,folderName,customGenre,convertToAAC,makeBookmarkable,userName=None,password=''): import shutil, re, os, sys iPXID = '' location = '' status = 1 importProg = '' feedName = folderName globals()['lastDLSize'] = 0.0 globals()['lastTorrentBeat'] = 0 globals()['torrentStatus'] = 1 if re.search('torrent', fileType, re.IGNORECASE): newSaveName = saveName[0:len(saveName) - 8] status, newSaveName = getTorrent(feedName, url, iPXSettings.torrentMaxUpRate, iPXSettings.tmpDownloadDir, newSaveName) if status == 1: checkDir(saveDir) tmpSize = os.path.getsize(iPXSettings.tmpDownloadDir + '/' + newSaveName) shutil.copy(iPXSettings.tmpDownloadDir + '/' + newSaveName, saveDir + '/' + newSaveName) os.unlink(iPXSettings.tmpDownloadDir + '/' + newSaveName) fileType = detectFileType(saveDir + '/' + newSaveName) # Audio Files if re.search('audio', fileType, re.IGNORECASE): if (iPXSettings.moveAudio > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, convertToAAC, makeBookmarkable) # Image Files elif re.search('image', fileType, re.IGNORECASE): if (iPXSettings.moveImages > 0): importProg = 'iPhoto' iPXID, importProg = updateiPhoto(saveDir, newSaveName, folderName, 0) # Video Files if (iPXSettings.moveVideo > 0): if sys.platform == 'darwin': if re.search('video/quicktime', fileType, re.IGNORECASE) or re.search('video/mpeg', fileType, re.IGNORECASE): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) elif sys.platform == 'win32': if iPXSettings.Prefs['exportApp'] == 1: if re.search('video', fileType, re.IGNORECASE): if not re.search('video/quicktime', fileType, re.IGNORECASE): importProg = 'WMP' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) else: if re.search('video/quicktime', fileType, re.IGNORECASE) or re.search('video/mpeg', fileType, re.IGNORECASE): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) # HTML Files are bad! elif re.search('html', fileType, re.IGNORECASE): if os.path.isfile(saveDir + '/' + newSaveName): os.unlink(saveDir + '/' + newSaveName) return newSaveName, '', '', '', 0 elif re.search('data', fileType, re.IGNORECASE): if (re.search('mp3$', newSaveName, re.IGNORECASE)): if (iPXSettings.moveAudio > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, convertToAAC, makeBookmarkable) elif (re.search('mov$', newSaveName, re.IGNORECASE)): if (iPXSettings.moveVideo > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) elif (re.search('aa$', newSaveName, IGNORECASE)): if (iPXSettings.moveVideo > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) # All Other Data Types else: pass saveName = newSaveName if len(iPXID) > 0: logIt('Returning with iPXID: %s' % str(iPXID)) return saveName, iPXID, importProg, fileType, status else: status, saveName = getFile(feedName, url, iPXSettings.tmpDownloadDir, saveName, userName, password) if status == 1: if len(fileType) <= 0 or re.search('text', fileType, re.IGNORECASE) or re.search('octet', fileType, re.IGNORECASE) or re.search('html', fileType, re.IGNORECASE) or re.search('data', fileType, re.IGNORECASE): fileType = detectFileType(iPXSettings.tmpDownloadDir + '/' + saveName) if not re.search('torrent', fileType, re.IGNORECASE): try: checkDir(saveDir) shutil.copy(iPXSettings.tmpDownloadDir + '/' + saveName, saveDir + '/' + saveName) os.unlink(iPXSettings.tmpDownloadDir + '/' + saveName) except Exception, msg: LogIt('ERRMSG: %s', msg) # Torrent Files if re.search('torrent', fileType, re.IGNORECASE): if os.path.isfile(saveDir + '/' + saveName): os.unlink(saveDir + '/' + saveName) # get rid of torrent extension if re.search('.torrent$', saveName, re.IGNORECASE): newSaveName = saveName[0:len(saveName) - 8] else: newSaveName = saveName status, newSaveName = getTorrent(feedName, url, 0, iPXSettings.tmpDownloadDir, newSaveName) if status == 1: checkDir(saveDir) shutil.copy(iPXSettings.tmpDownloadDir + '/' + newSaveName, saveDir + '/' + newSaveName) os.unlink(iPXSettings.tmpDownloadDir + '/' + newSaveName) fileType = detectFileType(saveDir + '/' + newSaveName) # Audio Files if re.search('audio', fileType, re.IGNORECASE): if (iPXSettings.moveAudio > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, convertToAAC, makeBookmarkable) # Image Files elif re.search('image', fileType, re.IGNORECASE): if (iPXSettings.moveImages > 0): importProg = 'iPhoto' iPXID, importProg = updateiPhoto(saveDir, newSaveName, folderName, 0) # Video Files if (iPXSettings.moveVideo > 0): if sys.platform == 'darwin': if re.search('video/quicktime', fileType, re.IGNORECASE) or re.search('video/mpeg', fileType, re.IGNORECASE): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) elif sys.platform == 'win32': if iPXSettings.Prefs['exportApp'] == 1: if re.search('video', fileType, re.IGNORECASE): if not re.search('video/quicktime', fileType, re.IGNORECASE): importProg = 'WMP' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) else: if re.search('video/quicktime', fileType, re.IGNORECASE) or re.search('video/mpeg', fileType, re.IGNORECASE): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) # HTML Files are bad! elif re.search('html', fileType, re.IGNORECASE): if os.path.isfile(saveDir + '/' + newSaveName): os.unlink(saveDir + '/' + newSaveName) return newSaveName, '', '', '', 0 elif re.search('data', fileType, re.IGNORECASE): if (re.search('mp3$', newSaveName, re.IGNORECASE)): if (iPXSettings.moveAudio > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, convertToAAC, makeBookmarkable) elif (re.search('mov$', newSaveName, re.IGNORECASE)) or (re.search('wmv$', newSaveName, re.IGNORECASE)): if (iPXSettings.moveVideo > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) elif (re.search('aa$', newSaveName, re.IGNORECASE)): if (iPXSettings.moveVideo > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, newSaveName, folderName, customGenre, False, False) # All Other Data Types else: pass # Audio Files elif re.search('audio', fileType, re.IGNORECASE): if (iPXSettings.moveAudio > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, convertToAAC, makeBookmarkable) # Image Files elif re.search('image', fileType, re.IGNORECASE): if (iPXSettings.moveImages > 0): importProg = 'iPhoto' iPXID, importProg = updateiPhoto(saveDir, saveName, folderName, 0) # Video Files if (iPXSettings.moveVideo > 0): if sys.platform == 'darwin': if re.search('video/quicktime', fileType, re.IGNORECASE) or re.search('video/mpeg', fileType, re.IGNORECASE): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, False, False) elif sys.platform == 'win32': if iPXSettings.Prefs['exportApp'] == 1: if re.search('video', fileType, re.IGNORECASE): if not re.search('video/quicktime', fileType, re.IGNORECASE): importProg = 'WMP' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, False, False) else: if re.search('video/quicktime', fileType, re.IGNORECASE): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, False, False) # HTML Files are bad! elif re.search('html', fileType, re.IGNORECASE): if os.path.isfile(saveDir + '/' + saveName): os.unlink(saveDir + '/' + saveName) return saveName, '', '', '', 0 elif re.search('data', fileType, re.IGNORECASE): if (re.search('mp3$', saveName, re.IGNORECASE)): if (iPXSettings.moveAudio > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, convertToAAC, makeBookmarkable) elif (re.search('mov$', saveName, re.IGNORECASE)): if (iPXSettings.moveVideo > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, False, False) elif (re.search('aa$', saveName, re.IGNORECASE)): if (iPXSettings.moveVideo > 0): importProg = 'iTunes' iPXID, importProg = updatePlaylist(saveDir, saveName, folderName, customGenre, False, False) # All Other Data Types else: pass logIt('Returning with iPXID: %s' % str(iPXID)) return saveName, iPXID, importProg, fileType, status