Resume downloads with PycURL RESUME_FROM
If you’ve ever played with urllib, you might have noticed its reasonably slow download speeds and wanted something a little bit more. That’s where cURL comes into play. It’s a very powerful library and is used in applications such as Gwibber. Seeing that this ships with Ubuntu, it is a good fit for Keryx’s new backend library Unwrapt.
I spent yesterday working on this new downloading code and wanted to share my code as I found the documentation (in reality, the entire library too) is primarily in C style and confusing examples. This is an easy to use class that helps to take care of some of the mundane details like following redirects (which is important! if you don’t enable this you won’t be able to reach a lot of files). I plan on extending this to support proxies soon as well as a GTK progress function.
# Unwrapt - cross-platform package system emulator
# Copyright (C) 2010 Chris Oliver
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
import math
import pycurl
import os
import time
import sys
class EasyCurl:
def __init__(self, proxy=None):
self.pco = pycurl.Curl()
self.pco.setopt(pycurl.FOLLOWLOCATION, 1)
self.pco.setopt(pycurl.MAXREDIRS, 5)
#self.pco.setopt(pycurl.TIMEOUT, 5*3600)
self.pco.setopt(pycurl.CONNECTTIMEOUT, 30)
self.pco.setopt(pycurl.AUTOREFERER, 1)
#TODO: Proxy!
def perform(self, url, filename=None, resume=True, progress=None):
# Generate filename if not given
if not filename:
filename = url.strip("/").split("/")[-1].strip()
self.filename = filename
# Get resume information
self.existing = self.start_existing = 0
if resume and os.path.exists(filename):
self.existing = self.start_existing = os.path.getsize(filename)
self.pco.setopt(pycurl.RESUME_FROM, self.existing)
# Configure progress hook
if progress:
self.pco.setopt(pycurl.NOPROGRESS, 0)
self.pco.setopt(pycurl.PROGRESSFUNCTION, progress)
# Configure url and destination
self.pco.setopt(pycurl.URL, url)
self.pco.setopt(pycurl.WRITEDATA, open(filename, "ab"))
# Start
self.pco.perform()
sys.stdout.write("n")
def textprogress(self, download_t, download_d, upload_t, upload_d):
downloaded = download_d self.existing
total = download_t self.start_existing
try: frac = float(downloaded)/float(total)
except: frac = 0
bar = "=" * int(25*frac)
sys.stdout.write("r%-25.25s %3i%% |%-25.25s| %5sB of %5sB" %
(self.filename,
frac*100,
bar,
format_number(downloaded),
format_number(total)))
# Borrowed from the urlgrabber source
def format_number(number, SI=0, space=' '):
"""Turn numbers into human-readable metric-like numbers"""
symbols = ['', # (none)
'k', # kilo
'M', # mega
'G', # giga
'T', # tera
'P', # peta
'E', # exa
'Z', # zetta
'Y'] # yotta
if SI: step = 1000.0
else: step = 1024.0
thresh = 999
depth = 0
max_depth = len(symbols) - 1
# we want numbers between 0 and thresh, but don't exceed the length
# of our list. In that event, the formatting will be screwed up,
# but it'll still show the right number.
while number > thresh and depth < max_depth:
depth = depth 1
number = number / step
if type(number) == type(1) or type(number) == type(1L):
# it's an int or a long, which means it didn't get divided,
# which means it's already short enough
format = '%i%s%s'
elif number < 9.95:
# must use 9.95 for proper sizing. For example, 9.99 will be
# rounded to 10.0 with the .1f format string (which is too long)
format = '%.1f%s%s'
else:
format = '%.0f%s%s'
return(format % (float(number or 0), space, symbols[depth]))
if __name__ == "__main__":
url = "http://launchpad.net/keryx/stable/0.92/ download/keryx_0.92.4.zip"
ec = EasyCurl()
ec.perform(url, progress=ec.textprogress)