1334 lines
44 KiB
Python
1334 lines
44 KiB
Python
#(c) 2004-2008 Thunderstone Media, LLC
|
|
#Creative Commons Attribution-Noncommercial 3.0 United States License
|
|
#
|
|
#Python Developyment By:
|
|
#
|
|
#Ray Slakinski
|
|
#August Trometer
|
|
|
|
import iPXSettings
|
|
from iPXTools import *
|
|
from iPXDownloader import *
|
|
from time import *
|
|
from datetime import *
|
|
|
|
class Feeds:
|
|
def __init__(self):
|
|
self.feedList = []
|
|
self.__getEnabledFeeds(iPXSettings.FeedListPrefs['iPodderXFeeds'])
|
|
|
|
def retrFeeds(self):
|
|
import re
|
|
|
|
for feed in self.feedList:
|
|
if feed.has_key('feedURL'):
|
|
if len(feed['feedURL']) > 0:
|
|
if not feed.has_key('feedID'):
|
|
logIt('Feeds file could not provide feedID, generating new one')
|
|
feed['feedID'] = genHash(feed['feedURL'])
|
|
|
|
if iPXSettings.ranFromUI == False and checkForUI() == True:
|
|
logIt('Shutting down script - UI has started...')
|
|
break
|
|
elif feed.has_key('feedURL'):
|
|
if re.search('opml$', feed['feedURL'], re.IGNORECASE):
|
|
opmlFeeds = getOpmlFeeds(feed['feedURL'])
|
|
|
|
for entry in opmlFeeds:
|
|
feed['feedURL'] = opmlFeeds[entry]
|
|
feed['feedTitle'] = entry
|
|
feedData = FeedData(feed, None, False)
|
|
else:
|
|
if feed.has_key('ttl') and feed.has_key('lastChecked'):
|
|
currTime = mktime(gmtime())
|
|
lastChecked = mktime(strptime(feed['lastChecked'], '%a, %d %b %Y %H:%M:%S GMT'))
|
|
if (lastChecked + (float(feed['ttl']) * 60)) - currTime <= 0:
|
|
FeedData(feed)
|
|
else:
|
|
logIt('Skipping feed(%s) for TTL reasons.' % feed['feedURL'])
|
|
logIt('TTL Value = %s' % str((lastChecked + (float(feed['ttl']) * 60)) - currTime) + ' seconds.')
|
|
else:
|
|
FeedData(feed)
|
|
|
|
def retrFeed(self, feedID, retrType):
|
|
import re
|
|
|
|
for feed in self.feedList:
|
|
if not feed.has_key('feedID'):
|
|
logIt('Feeds file could not provide feedID, generating new one')
|
|
feed['feedID'] = genHash(feed['feedURL'])
|
|
|
|
if retrType > 0:
|
|
if feed['feedID'] == feedID:
|
|
if re.search('opml$', feed['feedURL'], re.IGNORECASE):
|
|
opmlFeeds = getOpmlFeeds(feed['feedURL'])
|
|
|
|
for entry in opmlFeeds:
|
|
feed['feedURL'] = opmlFeeds[entry]
|
|
feed['feedTitle'] = entry
|
|
FeedData(feed, None, False)
|
|
else:
|
|
FeedData(feed, None, False)
|
|
else:
|
|
if feed['feedURL'] == feedID:
|
|
if re.search('opml$', feed['feedURL'], re.IGNORECASE):
|
|
opmlFeeds = getOpmlFeeds(feed['feedURL'])
|
|
|
|
for entry in opmlFeeds:
|
|
feed['feedURL'] = opmlFeeds[entry]
|
|
feed['feedTitle'] = entry
|
|
FeedData(feed, None, False)
|
|
else:
|
|
FeedData(feed, None, False)
|
|
|
|
def retrEnclosure(self, feedID, entryGUID, encURL, hasFile=False):
|
|
getEncURL = entryGUID + ';;' + encURL
|
|
#logIt('FeedID: %s, getEncURL: %s' % (feedID, getEncURL))
|
|
for feed in self.feedList:
|
|
if feed['feedID'] == feedID:
|
|
FeedData(feed, getEncURL, False, hasFile)
|
|
|
|
def getEnclosuresList(self):
|
|
import os
|
|
|
|
encList = []
|
|
for feed in self.feedList:
|
|
if feed.has_key('feedID'):
|
|
hashFile = iPXSettings.rssPath + 'feedData/' + feed['feedID'] + '.ipxd'
|
|
if os.path.isfile(hashFile):
|
|
try:
|
|
EntriesData = readplist(hashFile)
|
|
if EntriesData.has_key('entries'):
|
|
for entry in EntriesData['entries']:
|
|
if entry.has_key('read'):
|
|
if entry['read'] == True:
|
|
if entry.has_key('flagged'):
|
|
if entry['flagged'] == False:
|
|
if entry.has_key('enclosures'):
|
|
for enc in entry['enclosures']:
|
|
if enc.has_key('path'):
|
|
if len(enc['path']) > 0:
|
|
encList.append(enc)
|
|
else:
|
|
if entry.has_key('enclosures'):
|
|
for enc in entry['enclosures']:
|
|
if enc.has_key('path'):
|
|
if len(enc['path']) > 0:
|
|
encList.append(enc)
|
|
else:
|
|
if entry.has_key('enclosures'):
|
|
for enc in entry['enclosures']:
|
|
if enc.has_key('path'):
|
|
if len(enc['path']) > 0:
|
|
encList.append(enc)
|
|
except Exception, msg:
|
|
logIt('getEnclosuresList Faied: %s' % msg)
|
|
|
|
return encList
|
|
|
|
def createQMCacheBridge(self):
|
|
import iPXQuotaManager, os
|
|
|
|
for feed in self.feedList:
|
|
if feed.has_key('feedID'):
|
|
hashFile = iPXSettings.rssPath + 'feedData/' + feed['feedID'] + '.ipxd'
|
|
if os.path.isfile(hashFile):
|
|
try:
|
|
EntriesData = readplist(hashFile)
|
|
if Entries.has_key('entries'):
|
|
for entry in EntriesData['entries']:
|
|
if entry.has_key('enclosures'):
|
|
for enc in entry['enclosures']:
|
|
if enc.has_key('path'):
|
|
encFile = os.path.join(iPXSettings.downloadDirectory + '/' + enc['path'] + '/' + enc['file'])
|
|
if os.path.isfile(encFile):
|
|
iPXQuotaManager.writeQuotaCache(feed['feedID'], entry['guid'], enc['path'] + '/' + enc['file'], enc['length'])
|
|
except Exception, msg:
|
|
logIt('getEnclosuresList Faied: %s' % msg)
|
|
|
|
return #encList
|
|
|
|
def __getEnabledFeeds(self, feedList):
|
|
newFeedList = []
|
|
self.feedList = []
|
|
|
|
append = self.feedList.append
|
|
for feed in feedList:
|
|
if feed['enabled']:
|
|
append(feed)
|
|
|
|
|
|
class FeedData:
|
|
def __init__(self, feed, getEncURL=None, useEtag=True, hasFile=False):
|
|
import re
|
|
|
|
self.feed = feed
|
|
self.status = 0
|
|
self.explicit = False
|
|
self.entries = {}
|
|
self.parsedFeed = {}
|
|
self.entriesData = []
|
|
|
|
printMSG(' ')
|
|
try:
|
|
if feed.has_key('customFeedTitle'):
|
|
if len(feed['customFeedTitle']) > 0:
|
|
printMSG('%s;;%s' % (HTML2Text(feed['customFeedTitle']), feed['feedID']))
|
|
else:
|
|
printMSG('%s;;%s' % (HTML2Text(feed['feedTitle']), feed['feedID']))
|
|
else:
|
|
printMSG('%s;;%s' % (HTML2Text(feed['feedTitle']), feed['feedID']))
|
|
except:
|
|
printMSG('%s;;%s' % (HTML2Text(feed['feedURL']), feed['feedID']))
|
|
|
|
status = 0
|
|
|
|
if not getEncURL == None:
|
|
if hasFile:
|
|
self.__getEntryEnclosure(getEncURL, hasFile)
|
|
else:
|
|
self.__getEntryEnclosure(getEncURL)
|
|
elif feed.has_key('etag') and iPXSettings.DEBUG == 0 and useEtag == True:
|
|
status = self.__getFeed(feed['feedURL'], feed['etag'], feed['feedID'])
|
|
elif feed.has_key('etag') and useEtag == False:
|
|
status = self.__getFeed(feed['feedURL'], 'None', feed['feedID'])
|
|
else:
|
|
status = self.__getFeed(feed['feedURL'], 'None', feed['feedID'])
|
|
|
|
if status >= 0:
|
|
feed['error'] = False
|
|
if status == 200 or status == 301 or status == 302:
|
|
if self.parsedFeed.has_key('feed'):
|
|
#Check feed level, which overrides entry level
|
|
if self.parsedFeed.feed.has_key('itunes_explicit'):
|
|
if re.search('yes', self.parsedFeed.feed.itunes_explicit, re.IGNORECASE):
|
|
self.explicit = True
|
|
elif self.parsedFeed.feed.has_key('media_adult'):
|
|
if re.search('true', self.parsedFeed.feed.media_adult, re.IGNORECASE):
|
|
self.explicit = True
|
|
feed['explicit'] = self.explicit
|
|
self.__getEntries()
|
|
elif (status != 500) and (status != 304) and (status != 302) and (status != 301) and (status != 0):
|
|
feed['error'] = True
|
|
logIt('Feed recieved a status of %d, expected 200 or 304' % status)
|
|
|
|
self.__saveFeedData(feed)
|
|
|
|
def __getEntries(self):
|
|
from sets import Set
|
|
import sys, os, re
|
|
|
|
newEntriesList = []
|
|
encList = []
|
|
tempList = []
|
|
hashFile = ''
|
|
numDownloads = 0
|
|
numDFiles = 0
|
|
totalDSize = 0
|
|
maxDownloads = 10000000
|
|
totalDSize = 0
|
|
|
|
if self.feed.has_key('downloadBehavior'):
|
|
if int(self.feed['downloadBehavior']) == 3:
|
|
maxDownloads = 0
|
|
elif int(self.feed['downloadBehavior']) == 2:
|
|
maxDownloads = 3
|
|
elif int(self.feed['downloadBehavior']) == 1:
|
|
maxDownloads = 1
|
|
self.feed['downloadBehavior'] = 0
|
|
|
|
hashFile = iPXSettings.rssPath + 'feedData/' + self.feed['feedID'] + '.ipxd'
|
|
|
|
if os.path.isfile(hashFile):
|
|
try:
|
|
EntriesData = readplist(hashFile)
|
|
if EntriesData.has_key('entries'):
|
|
entryOrig = EntriesData['entries']
|
|
else:
|
|
logIt('Hash file seems to be currupt... resetting file')
|
|
entryOrig = {}
|
|
except:
|
|
import plistlib
|
|
|
|
logIt('Hash file seems to be currupt... resetting file')
|
|
EntriesData = plistlib.Plist()
|
|
entryOrig = {}
|
|
else:
|
|
if sys.platform == 'darwin':
|
|
EntriesData = NSMutableDictionary.dictionary()
|
|
else:
|
|
import plistlib
|
|
|
|
EntriesData = plistlib.Plist()
|
|
entryOrig = {}
|
|
|
|
append = newEntriesList.append
|
|
for entry in entryOrig:
|
|
if not entry.has_key('guid'):
|
|
entry['guid'] = ''
|
|
|
|
if not entry.has_key('read'):
|
|
entry['read'] = True
|
|
|
|
if not entry['guid'] in encList:
|
|
if len(entry['guid']) > 1:
|
|
encList.append(entry['guid'])
|
|
|
|
append(entry)
|
|
|
|
if self.parsedFeed.has_key('entries'):
|
|
for entry in self.parsedFeed.entries:
|
|
if entry.has_key('modified_parsed'):
|
|
if not entry.modified_parsed == None:
|
|
pubDate = entry.modified_parsed
|
|
else:
|
|
pubDate = gmtime()
|
|
else:
|
|
pubDate = gmtime()
|
|
|
|
strCat = ''
|
|
catList = []
|
|
if entry.has_key('categories'):
|
|
for dict in entry.categories:
|
|
|
|
append = catList.append
|
|
for cat in dict:
|
|
if not type(cat) is NoneType:
|
|
if not cat == 'None':
|
|
try:
|
|
append(cat.encode('utf-8'))
|
|
except:
|
|
pass
|
|
categories = Set(catList)
|
|
for category in categories:
|
|
if not re.search('http://', category, re.IGNORECASE) or re.search('https://', category, re.IGNORECASE):
|
|
strCat = strCat + category + ', '
|
|
if len(strCat) > 0:
|
|
if strCat[len(strCat)-2] == ',':
|
|
strCat = strCat[:-2]
|
|
|
|
strAuthor = ''
|
|
if entry.has_key('author'):
|
|
strAuthor = entry.author
|
|
|
|
strLink = self.feed['feedURL']
|
|
if entry.has_key('link'):
|
|
strLink = entry.link
|
|
|
|
strCommentsLink = ''
|
|
if entry.has_key('comments'):
|
|
strCommentsLink = entry.comments
|
|
|
|
strSourceLink = ''
|
|
strSourceName = ''
|
|
if entry.has_key('source'):
|
|
if entry.source.has_key('url'):
|
|
strSourceLink = entry.source.url
|
|
if entry.source.has_key('value'):
|
|
strSourceName = entry.source.value
|
|
|
|
explicit = False
|
|
#Check feed level, which overrides entry level
|
|
if self.parsedFeed.feed.has_key('itunes_explicit'):
|
|
if re.search('yes', self.parsedFeed.feed.itunes_explicit, re.IGNORECASE):
|
|
explicit = True
|
|
elif self.parsedFeed.feed.has_key('media_adult'):
|
|
if re.search('true', self.parsedFeed.feed.media_adult, re.IGNORECASE):
|
|
explicit = True
|
|
#Check entry level if not set by feed level
|
|
if explicit == False:
|
|
if entry.has_key('itunes_explicit'):
|
|
if re.search('yes', entry.itunes_explicit, re.IGNORECASE):
|
|
explicit = True
|
|
elif entry.has_key('media_adult'):
|
|
if re.search('true', entry.media_adult, re.IGNORECASE):
|
|
explicit = True
|
|
|
|
uniDescription = unicode('No Entry Text Found In Feed', 'utf-8')
|
|
if entry.has_key('content'):
|
|
uniDescription = entry.content[0].value
|
|
elif entry.has_key('summary_detail'):
|
|
uniDescription = entry.summary_detail.value
|
|
elif entry.has_key('description'):
|
|
uniDescription = entry.description
|
|
try:
|
|
if len(uniDescription) == 0:
|
|
uniDescription = unicode('No Entry Text Found In Feed', 'utf-8')
|
|
except:
|
|
uniDescription = unicode('No Entry Text Found In Feed', 'utf-8')
|
|
|
|
uniDescription = uniDescription.strip()
|
|
uniDescription = uniDescription.strip(chr(0))
|
|
uniDescription = uniDescription.replace('\x00', '')
|
|
uniDescription = uniDescription.replace('\r', '')
|
|
|
|
if type(uniDescription) == StringType:
|
|
uniDescription = HTML2Text(uniDescription)
|
|
|
|
uniSummary = unicode('', 'utf-8')
|
|
if entry.has_key('summary_detail'):
|
|
uniDescription = entry.summary_detail.value
|
|
|
|
uniSummary = uniSummary.strip()
|
|
uniSummary = uniSummary.strip(chr(0))
|
|
uniSummary = uniSummary.replace('\x00', '')
|
|
uniSummary = uniSummary.replace('\r', '')
|
|
|
|
if type(uniSummary) == StringType:
|
|
uniSummary = HTML2Text(uniSummary)
|
|
|
|
if entry.has_key('guid'):
|
|
strGUID = entry.guid
|
|
elif entry.has_key('title'):
|
|
if len(entry.title) > 20:
|
|
strGUID = genHash(entry.title)
|
|
else:
|
|
strGUID = genHash(uniDescription)
|
|
else:
|
|
strGUID = genHash(uniDescription)
|
|
|
|
uniTitle = unicode('', 'utf-8')
|
|
if entry.has_key('title'):
|
|
uniTitle = HTML2Text(entry.title)
|
|
try:
|
|
uniTitle = tmpTitle.encode('utf-8')
|
|
except Exception, msg:
|
|
pass
|
|
|
|
strLink = self.feed['feedURL']
|
|
if entry.has_key('link'):
|
|
strLink = entry.link
|
|
|
|
encArray = []
|
|
if entry.has_key('enclosures') or entry.has_key('links') or entry.has_key('apple-wallpapers_image'):
|
|
if iPXSettings.showExplicit == True:
|
|
if numDownloads < maxDownloads:
|
|
encArray, numDownloaded, sizeDownloaded = self.__getEntryEnclosures(entry, True, self.feed['baseHost'])
|
|
else:
|
|
encArray, numDownloaded, sizeDownloaded = self.__getEntryEnclosures(entry, False)
|
|
elif explicit == True:
|
|
encArray, numDownloaded, sizeDownloaded = self.__getEntryEnclosures(entry, False)
|
|
else:
|
|
if numDownloads < maxDownloads:
|
|
encArray, numDownloaded, sizeDownloaded = self.__getEntryEnclosures(entry, True, self.feed['baseHost'])
|
|
else:
|
|
encArray, numDownloaded, sizeDownloaded = self.__getEntryEnclosures(entry, False)
|
|
if numDownloaded > 0:
|
|
numDownloaded = 0
|
|
numDownloads = numDownloads + 1
|
|
numDFiles = numDFiles + numDownloads
|
|
totalDSize = totalDSize + long(sizeDownloaded)
|
|
|
|
if self.feed.has_key('textToAudio'):
|
|
if not strGUID in encList:
|
|
if self.feed['textToAudio']:
|
|
tempEnc = self.downloadTextAsAudio(uniTitle, strftime('%A %B %d %Y', pubDate), uniDescription, strGUID)
|
|
|
|
if not tempEnc == None:
|
|
encArray.append(tempEnc)
|
|
|
|
newEntry = {'description': uniDescription,
|
|
'summary': uniSummary,
|
|
'link': strLink,
|
|
'title': uniTitle,
|
|
'read': False,
|
|
'enclosures': encArray,
|
|
'pubDate': strftime('%a, %d %b %Y %H:%M:%S GMT', pubDate),
|
|
#'datePublished' : strftime('%Y-%b-%dT%H:%M:%SZ', pubDate),
|
|
'category': strCat,
|
|
'commentsLink': strCommentsLink,
|
|
'feedTitle': HTML2Text(self.feed['feedTitle']),
|
|
'sourceTitle': strSourceName,
|
|
'sourceLink': HTML2Text(strSourceLink),
|
|
'explicit': explicit,
|
|
'author': strAuthor,
|
|
'guid': strGUID
|
|
}
|
|
|
|
tempList.append(newEntry['guid'])
|
|
|
|
if strGUID in encList:
|
|
for entry in newEntriesList:
|
|
if entry.has_key('guid'):
|
|
if entry['guid'] == newEntry['guid']:
|
|
if entry.has_key('origDescription'):
|
|
if not genHash(entry['origDescription']) == genHash(newEntry['description']):
|
|
entry['bakDescription'] = entry['origDescription']
|
|
entry['origDescription'] = newEntry['description']
|
|
entry['description'] = textDiff(entry['origDescription'], newEntry['description'])
|
|
entry['title'] = newEntry['title']
|
|
if genHash(HTML2Text(newEntry['description'])) != genHash(HTML2Text(entry['origDescription'])):
|
|
entry['read'] = False
|
|
if len(entry['enclosures']) < len(newEntry['enclosures']):
|
|
entry['enclosures'] = newEntry['enclosures']
|
|
entry['read'] = False
|
|
elif genHash(entry['description']) != genHash(newEntry['description']):
|
|
entry['bakDescription'] = entry['description']
|
|
entry['origDescription'] = newEntry['description']
|
|
entry['description'] = textDiff(entry['description'], newEntry['description'])
|
|
entry['title'] = newEntry['title']
|
|
if genHash(HTML2Text(newEntry['description'])) != genHash(HTML2Text(entry['origDescription'])):
|
|
entry['read'] = False
|
|
if len(entry['enclosures']) < len(newEntry['enclosures']):
|
|
entry['enclosures'] = newEntry['enclosures']
|
|
entry['read'] = False
|
|
elif genHash(entry['title']) != genHash(newEntry['title']):
|
|
if len(newEntry['title']) > 0:
|
|
entry['title'] = newEntry['title']
|
|
if len(entry['enclosures']) < len(newEntry['enclosures']):
|
|
entry['enclosures'] = newEntry['enclosures']
|
|
entry['read'] = False
|
|
elif len(entry['enclosures']) < len(newEntry['enclosures']):
|
|
entry['enclosures'] = newEntry['enclosures']
|
|
entry['read'] = False
|
|
else:
|
|
encList.append(newEntry['guid'])
|
|
newEntriesList.append(newEntry)
|
|
|
|
if numDFiles > 0:
|
|
self.__saveHashFile(EntriesData, newEntriesList, tempList, numDFiles, totalDSize, hashFile)
|
|
numDFiles = 0
|
|
totalDSize = 0.0
|
|
|
|
self.__saveHashFile(EntriesData, newEntriesList, tempList, numDownloads, totalDSize, hashFile)
|
|
|
|
def __saveHashFile(self, EntriesData, newEntriesList, tempList, numDownloads, totalDSize, hashFile, skipDelOld=False):
|
|
import os
|
|
|
|
logIt('Saving to hashFile: %s' % hashFile)
|
|
dellist = []
|
|
if skipDelOld == False:
|
|
#Lets delete old entries...
|
|
for entry in newEntriesList:
|
|
if not entry['guid'] in tempList:
|
|
flagged = False
|
|
if entry.has_key('flagged'):
|
|
flagged = entry['flagged']
|
|
hasFile = False
|
|
if entry.has_key('enclosures'):
|
|
if len(entry['enclosures']) > 0:
|
|
for enclosure in entry['enclosures']:
|
|
path = enclosure['path']
|
|
fullPath = iPXSettings.downloadDirectory + '/' + path
|
|
fileName = enclosure['file']
|
|
if os.path.isfile(fullPath + '/' + fileName):
|
|
hasFile = True
|
|
break
|
|
|
|
if hasFile == False:
|
|
if entry['read'] == True:
|
|
if flagged == False:
|
|
if len(entry['enclosures']) == 0:
|
|
try:
|
|
tmpEntryDate = strptime(entry['pubDate'], '%a, %d %b %Y %H:%M:%S %Z')
|
|
entryDate = date(tmpEntryDate[0], tmpEntryDate[1], tmpEntryDate[2])
|
|
dateDelta = date.today() - entryDate
|
|
strDateDelta = str(dateDelta).split()
|
|
if len(strDateDelta) > 1:
|
|
if iPXSettings.maxEntryAge > 0:
|
|
if int(strDateDelta[0]) > iPXSettings.maxEntryAge:
|
|
dellist.append(entry)
|
|
except:
|
|
pass
|
|
if len(dellist) > 0:
|
|
remove = newEntriesList.remove
|
|
for entry in dellist:
|
|
remove(entry)
|
|
|
|
if EntriesData.has_key('statistics'):
|
|
stats = EntriesData['statistics']
|
|
numDFiles = numDownloads
|
|
if stats.has_key('numDFiles'):
|
|
numDFiles = numDFiles + long(stats['numDFiles'])
|
|
totalDSizeMB = float(totalDSize) / 1048576.0
|
|
if stats.has_key('totalDSize'):
|
|
totalDSizeMB = float(totalDSizeMB) + float(stats['totalDSize'])
|
|
strTotalDSizeMB = '%.2f' % totalDSizeMB
|
|
firstCheck = strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime())
|
|
if stats.has_key('firstCheck'):
|
|
firstCheck = stats['firstCheck']
|
|
stats = {'firstCheck': firstCheck,
|
|
'lastCheck': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'numDFiles': str(numDFiles),
|
|
'totalDSize': strTotalDSizeMB}
|
|
else:
|
|
totalDSizeMB = float(totalDSize) / 1048576.0
|
|
strTotalDSizeMB = '%.2f' % totalDSizeMB
|
|
numDFiles = numDownloads
|
|
stats = {'firstCheck': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'lastCheck': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'numDFiles': str(numDFiles),
|
|
'totalDSize': strTotalDSizeMB}
|
|
|
|
EntriesData['statistics'] = stats
|
|
EntriesData['entries'] = newEntriesList
|
|
writeplist(EntriesData, hashFile)
|
|
|
|
def __getEntryEnclosure(self, getEncURL, hasFile=False):
|
|
import sys, os, re
|
|
|
|
logIt('getEncURL')
|
|
|
|
origEntriesList = []
|
|
hashFile = ''
|
|
totalDSize = 0
|
|
numDFiles = 1
|
|
|
|
hashFile = iPXSettings.rssPath + 'feedData/' + self.feed['feedID'] + '.ipxd'
|
|
|
|
if iPXSettings.ranHistCheck == False:
|
|
iPXSettings.histGUIDs = getHistory()
|
|
iPXSettings.ranHistCheck = True
|
|
|
|
if os.path.isfile(hashFile):
|
|
try:
|
|
EntriesData = readplist(hashFile)
|
|
if EntriesData.has_key('entries'):
|
|
entryOrig = EntriesData['entries']
|
|
else:
|
|
logIt('Hash file seems to be currupt... resetting file')
|
|
entryOrig = {}
|
|
except:
|
|
import plistlib
|
|
|
|
logIt('Hash file seems to be currupt... resetting file')
|
|
EntriesData = plistlib.Plist()
|
|
entryOrig = {}
|
|
else:
|
|
if sys.platform == 'darwin':
|
|
EntriesData = NSMutableDictionary.dictionary()
|
|
else:
|
|
import plistlib
|
|
|
|
EntriesData = plistlib.Plist()
|
|
entryOrig = {}
|
|
|
|
append = origEntriesList.append
|
|
for entry in entryOrig:
|
|
append(entry)
|
|
|
|
getEncURLSplit = getEncURL.split(';;')
|
|
guid = getEncURLSplit[0]
|
|
encURL = getEncURLSplit[1]
|
|
|
|
for entry in origEntriesList:
|
|
if entry['guid'] == guid:
|
|
for enclosure in entry['enclosures']:
|
|
if enclosure['enclosureURL'] == encURL:
|
|
downloadError = False
|
|
url = encURL.strip()
|
|
length = '0'
|
|
if enclosure.has_key('length'):
|
|
if len(enclosure['length']) > 0:
|
|
length = enclosure['length']
|
|
totalDSize = totalDSize + long(length)
|
|
fileType = enclosure['type']
|
|
guid = enclosure['guid']
|
|
feedTitle = HTML2Text(self.feed['feedTitle'])
|
|
feedTitle = feedTitle.replace('/', ' ')
|
|
if iPXSettings.organizeDownloads == 1:
|
|
customFolder = strftime('%m-%d-%Y',localtime())
|
|
else:
|
|
customFolder = feedTitle
|
|
if self.feed.has_key('customFolder'):
|
|
if len(self.feed['customFolder']) > 0:
|
|
customFolder = self.feed['customFolder']
|
|
customFolder = stringCleaning(customFolder)
|
|
|
|
if hasFile:
|
|
iPXID = ''
|
|
importProg = ''
|
|
fileType = ''
|
|
customGenre = ''
|
|
downloadStatus = enclosure['downloadStatus']
|
|
if self.feed.has_key('customGenre'):
|
|
customGenre = self.feed['customGenre']
|
|
path = enclosure['path']
|
|
fullPath = iPXSettings.downloadDirectory + '/' + path
|
|
fileName = enclosure['file']
|
|
if os.path.isfile(fullPath + '/' + fileName):
|
|
fileType = detectFileType(fullPath + '/' + fileName)
|
|
|
|
# Audio
|
|
if re.search('audio', fileType, re.IGNORECASE):
|
|
convertToAAC = False
|
|
if self.feed.has_key('convertToAAC'):
|
|
convertToAAC = self.feed['convertToAAC']
|
|
makeBookmarkable = False
|
|
if self.feed.has_key('makeBookmarkable'):
|
|
makeBookmarkable = self.feed['makeBookmarkable']
|
|
importProg = 'iTunes'
|
|
iPXID, importProg = updatePlaylist(fullPath, fileName, customFolder, customGenre, convertToAAC, makeBookmarkable)
|
|
|
|
# Image
|
|
elif re.search('image', fileType, re.IGNORECASE):
|
|
importProg = 'iPhoto'
|
|
iPXID, importProg = updateiPhoto(fullPath, fileName, customFolder, 0)
|
|
|
|
# Video Files
|
|
elif re.search('video/quicktime', fileType, re.IGNORECASE):
|
|
importProg = 'iTunes'
|
|
iPXID, importProg = updatePlaylist(fullPath, fileName, customFolder, '', False, False)
|
|
|
|
elif re.search('data', fileType, re.IGNORECASE):
|
|
if (re.search('mp3$', fileName, re.IGNORECASE)):
|
|
if (iPXSettings.moveAudio > 0):
|
|
importProg = 'iTunes'
|
|
iPXID, importProg = updatePlaylist(fullPath, fileName, customFolder, customGenre, convertToAAC, makeBookmarkable)
|
|
elif (re.search('mov$', fileName, re.IGNORECASE)):
|
|
if (iPXSettings.moveVideo > 0):
|
|
importProg = 'iTunes'
|
|
iPXID, importProg = updatePlaylist(fullPath, fileName, customFolder, customGenre, False, False)
|
|
|
|
elif (re.search('aa$', fileName, re.IGNORECASE)):
|
|
if (iPXSettings.moveVideo > 0):
|
|
importProg = 'iTunes'
|
|
iPXID, importProg = updatePlaylist(fullPath, fileName, customFolder, customGenre, False, False)
|
|
|
|
else:
|
|
fileNameSplit = url.split('/')
|
|
fileName = fileNameSplit[len(fileNameSplit) - 1]
|
|
fileName = latin1_to_ascii(fileName)
|
|
fileName = stringCleaning(fileName)
|
|
|
|
importProg = ''
|
|
fileLocation = ''
|
|
path = iPXSettings.downloadDirectory + '/' + customFolder
|
|
path = latin1_to_ascii(path)
|
|
|
|
downloadStatus = 0
|
|
fileName, iPXID, importProg, fileType, downloadStatus = self.iPXDownload(url, guid, path, fileName, length, fileType, True)
|
|
if downloadStatus == 1:
|
|
downloadError = False
|
|
if not guid in iPXSettings.histGUIDs:
|
|
saveHistory(guid)
|
|
iPXSettings.histGUIDs.append(guid)
|
|
else:
|
|
downloadError = True
|
|
|
|
totalDSize = long(length)
|
|
tempEnc = {'enclosureURL': url,
|
|
'length': length,
|
|
'type': fileType,
|
|
'file': fileName,
|
|
'path': latin1_to_ascii(customFolder),
|
|
'guid': guid,
|
|
'import': importProg,
|
|
'iPXID': iPXID,
|
|
'textToAudio': False,
|
|
'downloadStatus': downloadStatus,
|
|
'downloadDate': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'downloadError': downloadError}
|
|
|
|
enclosure.update(tempEnc)
|
|
|
|
tempList = []
|
|
self.__saveHashFile(EntriesData, origEntriesList, tempList, numDFiles, totalDSize, hashFile, True)
|
|
|
|
def __getEntryEnclosures(self, entry, doDownload, host=None):
|
|
from sets import Set
|
|
import re
|
|
|
|
returnEncList = []
|
|
numDownloaded = 0
|
|
sizeDownloaded = 0
|
|
|
|
keywords = []
|
|
if self.feed.has_key('keywords'):
|
|
keywords = self.feed['keywords'].split(',')
|
|
|
|
if iPXSettings.ranHistCheck == False:
|
|
iPXSettings.histGUIDs = getHistory()
|
|
iPXSettings.ranHistCheck = True
|
|
|
|
|
|
okDownload = True
|
|
|
|
if entry.has_key('guid'):
|
|
if not entry['guid'] == None:
|
|
if entry['guid'] in iPXSettings.histGUIDs:
|
|
okDownload = False
|
|
elif not host == None:
|
|
if host + entry['guid'] in iPXSettings.histGUIDs:
|
|
okDownload = False
|
|
else:
|
|
okDownload = True
|
|
else:
|
|
okDownload = True
|
|
else:
|
|
okDownload = True
|
|
|
|
strTitle = ''
|
|
uniDescription = unicode('', 'utf-8')
|
|
strCat = ''
|
|
if (len(keywords) > 0) and (doDownload == True):
|
|
if entry.has_key('title'):
|
|
strTitle = HTML2Text(entry.title)
|
|
if entry.has_key('summary_detail'):
|
|
uniDescription = entry.summary_detail.value
|
|
if len(uniDescription) == 0:
|
|
uniDescription = unicode('No Entry Text Found In Feed', 'utf-8')
|
|
if entry.has_key('content'):
|
|
uniDescription = entry.content[0].value
|
|
elif entry.has_key('content'):
|
|
uniDescription = entry.content[0].value
|
|
elif entry.has_key('description'):
|
|
uniDescription = entry.description
|
|
catList = []
|
|
if entry.has_key('categories'):
|
|
for dict in entry.categories:
|
|
for cat in dict:
|
|
try:
|
|
if not type(cat) is NoneType:
|
|
if not cat == 'None':
|
|
try:
|
|
catList.append(cat.encode('utf-8'))
|
|
except:
|
|
pass
|
|
except Exception, msg:
|
|
logIt('Type error: %s' % msg)
|
|
categories = Set(catList)
|
|
for category in categories:
|
|
if not re.search('http://', category, re.IGNORECASE) or re.search('https://', category, re.IGNORECASE):
|
|
strCat = strCat + category + ', '
|
|
if len(strCat) > 0:
|
|
if strCat[len(strCat)-2] == ',':
|
|
strCat = strCat[:-2]
|
|
|
|
if re.search('rss', self.parsedFeed.version, re.IGNORECASE):
|
|
if entry.has_key('enclosures'):
|
|
for enclosure in entry.enclosures:
|
|
if enclosure.has_key('url'):
|
|
if len(enclosure.url) > 0:
|
|
url = enclosure.url
|
|
url = url.strip()
|
|
url = prepURL(url)
|
|
|
|
fileNameSplit = url.split('/')
|
|
fileName = fileNameSplit[len(fileNameSplit) - 1]
|
|
fileName = latin1_to_ascii(fileName)
|
|
|
|
fileType = 'Unknown'
|
|
if enclosure.has_key('type'):
|
|
fileType = enclosure.type
|
|
|
|
length = '0'
|
|
if enclosure.has_key('length'):
|
|
if len(enclosure.length) > 0:
|
|
length = enclosure.length
|
|
try:
|
|
temp = long(length)
|
|
except:
|
|
length = '0'
|
|
|
|
if entry.has_key('guid'):
|
|
guid = entry.guid
|
|
else:
|
|
guid = enclosure.url
|
|
|
|
importProg = ''
|
|
fileLocation = ''
|
|
feedTitle = HTML2Text(self.feed['feedTitle'])
|
|
feedTitle = feedTitle.replace('/', '-')
|
|
if iPXSettings.organizeDownloads == 1:
|
|
customFolder = strftime('%m-%d-%Y',localtime())
|
|
else:
|
|
customFolder = feedTitle
|
|
if self.feed.has_key('customFolder'):
|
|
if len(self.feed['customFolder']) > 0:
|
|
customFolder = self.feed['customFolder']
|
|
customFolder = stringCleaning(customFolder)
|
|
path = iPXSettings.downloadDirectory + '/' + customFolder
|
|
path = latin1_to_ascii(path)
|
|
|
|
downloadStatus = 0
|
|
autoDownload = True
|
|
if self.feed.has_key('autoDownload'):
|
|
autoDownload = self.feed['autoDownload']
|
|
|
|
downloadError = False
|
|
if autoDownload:
|
|
encArray = []
|
|
if entry.has_key('enclosures') or entry.has_key('links'):
|
|
didFind = True
|
|
if len(keywords) > 0:
|
|
for keyword in keywords:
|
|
keywordSplit = keyword.split(' ')
|
|
for word in keywordSplit:
|
|
if len(word) > 1:
|
|
if (re.search(word.strip(), url, re.IGNORECASE) or re.search(word.strip(), strTitle, re.IGNORECASE) or re.search(word.strip(), uniDescription, re.IGNORECASE)or re.search(word.strip(), strCat, re.IGNORECASE)):
|
|
didFind = True
|
|
else:
|
|
didFind = False
|
|
break
|
|
|
|
if url in iPXSettings.histGUIDs:
|
|
doDownload = False
|
|
|
|
if didFind and okDownload and doDownload:
|
|
|
|
fileName, fileLocation, importProg, fileType, downloadStatus = self.iPXDownload(url, guid, path, fileName, length, fileType, doDownload)
|
|
if downloadStatus == 1:
|
|
numDownloaded = numDownloaded + 1
|
|
sizeDownloaded = long(sizeDownloaded) + long(length)
|
|
elif downloadStatus == 0:
|
|
downloadError = True
|
|
elif didFind and okDownload and not doDownload:
|
|
fileName, fileLocation, importProg, fileType, downloadStatus = self.iPXDownload(url, guid, path, fileName, length, fileType, doDownload)
|
|
|
|
tempEnc = {'enclosureURL': url,
|
|
'length': length,
|
|
'type': fileType,
|
|
'file': fileName,
|
|
'path': latin1_to_ascii(customFolder),
|
|
'guid': guid,
|
|
'import': importProg,
|
|
'iPXID': fileLocation,
|
|
'textToAudio': False,
|
|
'downloadError': downloadError,
|
|
'downloadDate': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'downloadStatus': downloadStatus}
|
|
|
|
returnEncList.append(tempEnc)
|
|
elif entry.has_key('apple-wallpapers_image'):
|
|
if len(entry['apple-wallpapers_image']) > 0:
|
|
url = entry['apple-wallpapers_image']
|
|
url = url.strip()
|
|
url = prepURL(url)
|
|
|
|
fileNameSplit = url.split('/')
|
|
fileName = fileNameSplit[len(fileNameSplit) - 1]
|
|
fileName = latin1_to_ascii(fileName)
|
|
|
|
fileType = 'image'
|
|
length = '0'
|
|
|
|
if entry.has_key('guid'):
|
|
guid = entry.guid
|
|
else:
|
|
guid = url
|
|
|
|
importProg = ''
|
|
fileLocation = ''
|
|
feedTitle = HTML2Text(self.feed['feedTitle'])
|
|
feedTitle = feedTitle.replace('/', '-')
|
|
if iPXSettings.organizeDownloads == 1:
|
|
customFolder = strftime('%m-%d-%Y',localtime())
|
|
else:
|
|
customFolder = feedTitle
|
|
if self.feed.has_key('customFolder'):
|
|
if len(self.feed['customFolder']) > 0:
|
|
customFolder = self.feed['customFolder']
|
|
customFolder = stringCleaning(customFolder)
|
|
path = iPXSettings.downloadDirectory + '/' + customFolder
|
|
path = latin1_to_ascii(path)
|
|
|
|
downloadStatus = 0
|
|
autoDownload = True
|
|
if self.feed.has_key('autoDownload'):
|
|
autoDownload = self.feed['autoDownload']
|
|
|
|
downloadError = False
|
|
if autoDownload:
|
|
encArray = []
|
|
if entry.has_key('enclosures') or entry.has_key('links') or entry.has_key('apple-wallpapers_image'):
|
|
didFind = True
|
|
if len(keywords) > 0:
|
|
for keyword in keywords:
|
|
keywordSplit = keyword.split(' ')
|
|
for word in keywordSplit:
|
|
if len(word) > 1:
|
|
if (re.search(word.strip(), url, re.IGNORECASE) or re.search(word.strip(), strTitle, re.IGNORECASE) or re.search(word.strip(), uniDescription, re.IGNORECASE)or re.search(word.strip(), strCat, re.IGNORECASE)):
|
|
didFind = True
|
|
else:
|
|
didFind = False
|
|
break
|
|
|
|
if url in iPXSettings.histGUIDs:
|
|
doDownload = False
|
|
|
|
if didFind and okDownload and doDownload:
|
|
fileName, fileLocation, importProg, fileType, downloadStatus = self.iPXDownload(url, guid, path, fileName, length, fileType, doDownload)
|
|
if downloadStatus == 1:
|
|
numDownloaded = numDownloaded + 1
|
|
sizeDownloaded = long(sizeDownloaded) + long(length)
|
|
elif downloadStatus == 0:
|
|
downloadError = True
|
|
elif didFind and okDownload and not doDownload:
|
|
fileName, fileLocation, importProg, fileType, downloadStatus = self.iPXDownload(url, guid, path, fileName, length, fileType, doDownload)
|
|
|
|
tempEnc = {'enclosureURL': url,
|
|
'length': length,
|
|
'type': fileType,
|
|
'file': fileName,
|
|
'path': latin1_to_ascii(customFolder),
|
|
'guid': guid,
|
|
'import': importProg,
|
|
'iPXID': fileLocation,
|
|
'textToAudio': False,
|
|
'downloadError': downloadError,
|
|
'downloadDate': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'downloadStatus': downloadStatus}
|
|
|
|
returnEncList.append(tempEnc)
|
|
elif re.search('atom', self.parsedFeed.version, re.IGNORECASE):
|
|
if entry.has_key('links'):
|
|
for link in entry.links:
|
|
if link.rel == 'enclosure':
|
|
if link.has_key('href'):
|
|
if len(link.href) > 0:
|
|
url = link.href
|
|
url = url.strip()
|
|
url = prepURL(url)
|
|
|
|
fileNameSplit = url.split('/')
|
|
fileName = fileNameSplit[len(fileNameSplit) - 1]
|
|
fileName = latin1_to_ascii(fileName)
|
|
|
|
fileType = ''
|
|
if link.has_key('type'):
|
|
fileType = link.type
|
|
|
|
length = '0'
|
|
if link.has_key('length'):
|
|
if len(link['length']) > 0:
|
|
length = link['length']
|
|
|
|
if entry.has_key('guid'):
|
|
guid = entry.guid
|
|
else:
|
|
guid = link.href
|
|
|
|
importProg = ''
|
|
fileLocation = ''
|
|
downloadError = False
|
|
feedTitle = HTML2Text(self.feed['feedTitle'])
|
|
feedTitle = feedTitle.replace('/', '-')
|
|
if iPXSettings.organizeDownloads == 1:
|
|
customFolder = strftime('%m-%d-%Y',localtime())
|
|
else:
|
|
customFolder = feedTitle
|
|
if self.feed.has_key('customFolder'):
|
|
if len(self.feed['customFolder']) > 0:
|
|
customFolder = self.feed['customFolder']
|
|
customFolder = stringCleaning(customFolder)
|
|
path = iPXSettings.downloadDirectory + '/' + customFolder
|
|
path = latin1_to_ascii(path)
|
|
|
|
if self.feed.has_key('autoDownload'):
|
|
if self.feed['autoDownload'] == True:
|
|
didFind = True
|
|
if len(keywords) > 0:
|
|
for keyword in keywords:
|
|
keywordSplit = keyword.split(' ')
|
|
for word in keywordSplit:
|
|
if len(word) > 1:
|
|
if (re.search(word, url, re.IGNORECASE) or re.search(word, strTitle, re.IGNORECASE) or re.search(word, uniDescription, re.IGNORECASE) or re.search(word, strCat, re.IGNORECASE)):
|
|
didFind = True
|
|
break
|
|
else:
|
|
didFind = False
|
|
|
|
if url in iPXSettings.histGUIDs:
|
|
doDownload = False
|
|
|
|
if didFind and okDownload:
|
|
fileName, fileLocation, importProg, fileType, downloadStatus = self.iPXDownload(url, guid, path, fileName,length, fileType, doDownload)
|
|
if downloadStatus == 1:
|
|
numDownloaded = numDownloaded + 1
|
|
sizeDownloaded = long(sizeDownloaded) + long(length)
|
|
elif downloadStatus == 0:
|
|
downloadError = True
|
|
|
|
tempEnc = {'enclosureURL': url,
|
|
'length': length,
|
|
'type': fileType,
|
|
'file': fileName,
|
|
'path': latin1_to_ascii(customFolder),
|
|
'guid': guid,
|
|
'import': importProg,
|
|
'textToAudio': False,
|
|
'downloadError': downloadError,
|
|
'downloadDate': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'iPXID': fileLocation}
|
|
|
|
returnEncList.append(tempEnc)
|
|
|
|
return returnEncList, numDownloaded, sizeDownloaded
|
|
|
|
def __saveFeedData(self, newFeed):
|
|
import sys
|
|
|
|
logIt('Saving feeds.plist')
|
|
try:
|
|
FeedListPrefs = readplist(iPXSettings.feedFile)
|
|
feedDetails = FeedListPrefs['iPodderXFeeds']
|
|
for feed in feedDetails:
|
|
if feed.has_key('feedURL'):
|
|
if feed['feedURL'] == newFeed['feedURL']:
|
|
feed.update(newFeed)
|
|
FeedListPrefs['iPodderXFeeds'] = feedDetails
|
|
#FeedListPrefs.write(iPXSettings.feedFile)
|
|
writeplist(FeedListPrefs, iPXSettings.feedFile)
|
|
except Exception, msg:
|
|
logIt(msg)
|
|
sys.exit(5)
|
|
|
|
def __getFeed(self, url, etagInfo, feedID):
|
|
import feedparser, urllib, urllib2, urlparse
|
|
|
|
parsedFeed = {}
|
|
etag = ''
|
|
|
|
self.feed['lastChecked'] = strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime())
|
|
self.feed['baseHost'] = 'http:/'
|
|
for part in urlparse.urlparse(url):
|
|
if not part == urlparse.urlparse(url)[0]:
|
|
if len(part) > 0:
|
|
if '/' in part:
|
|
splitPart = part.split('/')
|
|
for newPart in splitPart:
|
|
if newPart == splitPart[len(splitPart)-1]:
|
|
break
|
|
if len(newPart) > 0:
|
|
self.feed['baseHost'] = self.feed['baseHost'] + '/' + newPart
|
|
else:
|
|
self.feed['baseHost'] = self.feed['baseHost'] + '/' + part
|
|
self.feed['baseHost'] = self.feed['baseHost'] + '/'
|
|
logIt('etagInfo: %s' % etagInfo)
|
|
|
|
userName = None
|
|
if self.feed.has_key('username'):
|
|
userName = self.feed['username']
|
|
password = ''
|
|
if self.feed.has_key('password'):
|
|
password = decrypt(self.feed['password'])
|
|
|
|
if iPXSettings.useProxyServer and userName != None:
|
|
status, url, x = getFileViaProxySSL(url, userName, password, True, False)
|
|
status, feedData, r = getFileViaProxySSL(url, userName, password, False, False)
|
|
if status == 200:
|
|
parsedFeed = feedparser.parse(feedData)
|
|
if parsedFeed.has_key('entries'):
|
|
if len(parsedFeed.entries) > 0:
|
|
self.parsedFeed = parsedFeed
|
|
self.feed['etag'] = ''
|
|
if parsedFeed.feed.has_key('ttl'):
|
|
self.feed['ttl'] = parsedFeed.feed.ttl
|
|
else:
|
|
self.feed['ttl']
|
|
self.feed['feedTitle'] = HTML2Text(parsedFeed.feed.title)
|
|
if self.feed.has_key('customFeedTitle'):
|
|
if len(self.feed['customFeedTitle']) > 0:
|
|
self.feed['feedTitle'] = self.feed['customFeedTitle']
|
|
self.feed['siteURL'] = parsedFeed.feed.link
|
|
return 200
|
|
else:
|
|
return 0
|
|
else:
|
|
req = urllib2.Request(url)
|
|
req.add_header('Accept-encoding', 'gzip')
|
|
|
|
if not etagInfo == 'None':
|
|
req.add_header('If-None-Match', etagInfo)
|
|
if not userName == None:
|
|
setOpener(url, userName, password)
|
|
else:
|
|
setOpener()
|
|
|
|
try:
|
|
h = urllib2.urlopen(req)
|
|
except urllib2.HTTPError, e:
|
|
logIt('Feed recieved a status of %s' % e.code)
|
|
if e.code == 401:
|
|
printMSG('AUTHENTICATION_FAILED;;%s' % feedID)
|
|
if e.code == 407:
|
|
printMSG('PROXY_AUTHENTICATION_FAILED')
|
|
return -1
|
|
except Exception, msg:
|
|
logIt('Feed failed to connect %s' % msg)
|
|
return -1
|
|
|
|
try:
|
|
etag = h.info()['etag']
|
|
except:
|
|
etag = ''
|
|
|
|
try:
|
|
if h.info().has_key('Content-Encoding'):
|
|
if h.info()['Content-Encoding'] == 'gzip':
|
|
import gzip
|
|
from StringIO import StringIO
|
|
|
|
tempData = h.read()
|
|
|
|
feedData = gzip.GzipFile(fileobj=StringIO(tempData)).read()
|
|
else:
|
|
try:
|
|
feedData = h.read()
|
|
except Exception, msg:
|
|
logIt('FeedData ERRMSG: %s' % msg)
|
|
feedData = ''
|
|
if iPXSettings.SUPERDEBUG:
|
|
print feedData
|
|
parsedFeed = feedparser.parse(feedData)
|
|
if parsedFeed.has_key('entries'):
|
|
if len(parsedFeed.entries) > 0:
|
|
self.parsedFeed = parsedFeed
|
|
self.feed['etag'] = etag
|
|
if parsedFeed.feed.has_key('ttl'):
|
|
self.feed['ttl'] = parsedFeed.feed.ttl
|
|
else:
|
|
self.feed['ttl'] = 0
|
|
self.feed['feedTitle'] = HTML2Text(parsedFeed.feed.title)
|
|
if self.feed.has_key('customFeedTitle'):
|
|
if len(self.feed['customFeedTitle']) > 0:
|
|
self.feed['feedTitle'] = self.feed['customFeedTitle']
|
|
self.feed['siteURL'] = parsedFeed.feed.link
|
|
parsedFeed.status = 200
|
|
|
|
return int(parsedFeed.status)
|
|
except:
|
|
return -1
|
|
|
|
def iPXDownload(self, encURL, encGUID, downloadPath, fileName, size, fileType, doDownload):
|
|
logIt('Downloading: %s' % encURL)
|
|
status = 1
|
|
fileLocation = ''
|
|
importProg = ''
|
|
head = None
|
|
|
|
userName = None
|
|
password = ''
|
|
if self.feed.has_key('username'):
|
|
userName = self.feed['username']
|
|
if self.feed.has_key('password'):
|
|
password = decrypt(self.feed['password'])
|
|
|
|
#head = HTTPHead(encURL, userName, password)
|
|
if head != None:
|
|
if len(head.getName()) > 0:
|
|
fileName = head.getName()
|
|
|
|
if head.getType() != None:
|
|
fileType = head.getType()
|
|
|
|
if head.getLength() != None:
|
|
size = long(head.getLength())
|
|
|
|
if iPXSettings.quotaEnabled > 0 and doDownload == True:
|
|
import iPXQuotaManager
|
|
if iPXSettings.ranQuotaTest == False:
|
|
printMSG('Compiling SmartSpace Information...')
|
|
iPXSettings.delList, iPXSettings.quotaSizeTaken = iPXQuotaManager.getAllowedDelList()
|
|
iPXSettings.ranQuotaTest = True
|
|
iPXQuotaManager.getToQuota(size)
|
|
|
|
convertToAAC = False
|
|
if self.feed.has_key('convertToAAC'):
|
|
convertToAAC = self.feed['convertToAAC']
|
|
makeBookmarkable = False
|
|
if self.feed.has_key('makeBookmarkable'):
|
|
makeBookmarkable = self.feed['makeBookmarkable']
|
|
|
|
if doDownload:
|
|
customGenre = ''
|
|
if self.feed.has_key('customGenre'):
|
|
customGenre = self.feed['customGenre']
|
|
if iPXSettings.organizeDownloads == 1:
|
|
customFolder = strftime('%m-%d-%Y',localtime())
|
|
else:
|
|
customFolder = self.feed['feedTitle']
|
|
if self.feed.has_key('customFolder'):
|
|
if len(self.feed['customFolder']) > 0:
|
|
customFolder = self.feed['customFolder']
|
|
customFolder = latin1_to_ascii(customFolder)
|
|
fileName, fileLocation, importProg, fileType, status = downloadFile(encURL,fileType,downloadPath,fileName,customFolder,customGenre,convertToAAC,makeBookmarkable,userName,password)
|
|
if not encGUID in iPXSettings.histGUIDs and status == 1:
|
|
saveHistory(encGUID)
|
|
if iPXSettings.anonFeedback and status == 1:
|
|
doPing(encURL, self.feed['feedURL'])
|
|
elif not encGUID in iPXSettings.histGUIDs:
|
|
saveHistory(encGUID)
|
|
|
|
return fileName, fileLocation, importProg, fileType, status
|
|
|
|
def downloadTextAsAudio(self, strTitle, pubDate, text, strGUID):
|
|
import sys, os
|
|
|
|
customFolder = self.feed['feedTitle']
|
|
feedTitle = customFolder
|
|
|
|
if len(strTitle) == 0:
|
|
strTitle = customFolder + ': '
|
|
x = 0
|
|
textSplit = text.split(' ')
|
|
for word in textSplit:
|
|
tmpWord = word + ' '
|
|
strTitle = strTitle + tmpWord
|
|
if x == 5:
|
|
break
|
|
else:
|
|
x = x + 1
|
|
strTitle = strTitle.strip()
|
|
|
|
text = 'From ' + feedTitle + ', posted on ' + pubDate + '. '+ text
|
|
|
|
if self.feed.has_key('customFolder'):
|
|
if len(self.feed['customFolder']) > 0:
|
|
customFolder = self.feed['customFolder']
|
|
customFolder = stringCleaning(customFolder)
|
|
|
|
if sys.platform == 'darwin':
|
|
saveDir = iPXSettings.downloadDirectory + '/' + customFolder
|
|
elif sys.platform == 'win32':
|
|
saveDir = iPXSettings.downloadDirectory + '\\' + customFolder
|
|
checkDir(saveDir)
|
|
|
|
saveName = HTML2Text(strTitle)
|
|
saveName = pubDate + ' - ' + saveName
|
|
saveName = stringCleaning(saveName)
|
|
if sys.platform == 'darwin':
|
|
saveName = saveName + '.aiff'
|
|
elif sys.platform == 'win32':
|
|
saveName = saveName + '.wav'
|
|
|
|
if text2Audio(text, saveDir, saveName) == True:
|
|
length = 0
|
|
if os.path.isfile(saveDir + '/' + saveName):
|
|
length = os.path.getsize(saveDir + '/' + saveName)
|
|
|
|
if iPXSettings.quotaEnabled > 0:
|
|
import iPXQuotaManager
|
|
iPXQuotaManager.getToQuota(length)
|
|
importProg = ''
|
|
fileLocation = ''
|
|
if (iPXSettings.moveAudio > 0):
|
|
customGenre = 'iPodderX NewsCaster'
|
|
if self.feed.has_key('customGenre'):
|
|
customGenre = self.feed['customGenre']
|
|
convertToAAC = True
|
|
if self.feed.has_key('convertToAAC'):
|
|
convertToAAC = self.feed['convertToAAC']
|
|
makeBookmarkable = False
|
|
if self.feed.has_key('makeBookmarkable'):
|
|
makeBookmarkable = self.feed['makeBookmarkable']
|
|
|
|
fileLocation, importProg = updatePlaylist(saveDir, saveName, customFolder, customGenre, convertToAAC, makeBookmarkable)
|
|
|
|
tempEnc = {'length': str(length),
|
|
'type': 'audio',
|
|
'file': saveName,
|
|
'path': saveDir,
|
|
'guid': strGUID,
|
|
'import': importProg,
|
|
'iPXID': fileLocation,
|
|
'textToAudio': True,
|
|
'downloadError': False,
|
|
'downloadDate': strftime('%a, %d %b %Y %H:%M:%S GMT',gmtime()),
|
|
'downloadStatus': '1'}
|
|
|
|
return tempEnc
|
|
else:
|
|
return None
|
|
|