# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
#
# Copyright 2011 Willem Jansen
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import pdb
import base64
import httplib
import re
import urllib
import urllib2
import xml.dom.minidom
import collections
import mimetypes
import time
import duplicity.backend
from duplicity import globals
from duplicity import log
from duplicity.errors import * address@hidden
from duplicity import urlparse_2_5 as urlparser
class RapidshareBackend(duplicity.backend.Backend):
"""Backend for accessing Rapidshare - contributed in 2011 by Willem Jansen """
listbody = """\
"""
"""Connect to remote store using RapidShare API"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.parsed_url = parsed_url
# For development purposes a free account allows for testing. Does not make really make sense for productive use
self.account_status = "free"
if parsed_url.path:
foldpath = re.compile('/+')
self.directory = foldpath.sub('/', parsed_url.path + '/' )
else:
self.directory = '/'
# set username as work-around: otherwise username="None"
self.parsed_url.username=parsed_url.path.split('@')[0].split('//')[1]
tmp = self.directory.split('/')[2:]
tmp.pop()
self.directory = '/'.join(tmp)
log.Info("Using Rapidshare host %s" % ('api.rapidshare.com',))
log.Info("Using Rapidshare port %s" % (443,))
log.Info("Using Rapidshare directory %s" % (self.directory,))
if parsed_url.scheme == 'rs':
self.api_conn = httplib.HTTPSConnection('api.rapidshare.com', 443)
else:
raise BackendException("Unknown URI scheme: %s" % (parsed_url.scheme))
for n in range(1, globals.num_retries+1):
#Identify FolderID
#Get Rapidshare directory structure
params = urllib.urlencode({'sub': 'listrealfolders', 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
if response.status == 401:
continue
# parse response and see if directory exists
response_lines = response.read().split('\n')
response_lines.pop()
rs_directory = collections.namedtuple('Folder', 'RealFolder_ID, Parent_RealFolder_ID, Name, BROWSE_ACL, UPLOAD_ACL, DOWNLOAD_ACL')
rs_dir_list = map(rs_directory._make, map(lambda x: x.split(',') ,response_lines))
# print(rs_dir_list)
#identify RealFolderID, if directory dos not exist, create it
# pdb.set_trace()
self.FolderID = 0
for current_dir in self.directory.split("/"):
item = filter(lambda x: (x.Parent_RealFolder_ID==str(self.FolderID) and x.Name==current_dir), rs_dir_list)
if item == []:
# if the directory does not exist, create it.
log.Info("Directory '%s' being created." % current_dir)
params = urllib.urlencode({'sub': 'addrealfolder', 'name': current_dir, 'parent': self.FolderID, 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
self.FolderID = self.api_conn.getresponse().read()
else:
self.FolderID = item[0].RealFolder_ID
if self.FolderID == -1:
raise BackendException("Unable to create directory: %s" % (self.directory))
else:
return
log.Warn("Rapidshare backend giving up after %d attempts to get remote directory information" % (globals.num_retries))
raise BackendException((response.status, response.reason))
def close(self):
self.api_conn.close()
def list(self):
"""List files in directory"""
log.Info("Listing directory %s on RS server" % (self.directory,))
# Get directory listing
for n in range(1, globals.num_retries+1):
params = urllib.urlencode({'sub': 'listfiles', 'realfolder': self.FolderID, 'fields':'fileid,filename', 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
if response.status == 401:
continue
directory_listing_string = response.read()
result = []
directory_listing = directory_listing_string.split('\n')
directory_listing.pop()
for i in directory_listing:
result.append(i.split(',')[1])
# pdb.set_trace()
return result
log.Warn("Rapidshare backend giving up after %d attempts to list directory %s" % (globals.num_retries, self.directory))
raise BackendException((response.status, response.reason))
def get(self, remote_filename, local_path):
"""Get remote filename, saving it to local_path"""
target_file = local_path.open("wb")
for n in range(1, globals.num_retries+1):
log.Info("Retrieving %s from Rapidshare server" % ((self.directory + '/' + remote_filename)))
# get Filelist
params = urllib.urlencode({'sub': 'listfiles', 'realfolder': self.FolderID, 'fields':'fileid,filename', 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
if response.status == 401:
continue
tmp_list = response.read().split('\n')
tmp_list.pop()
directory_listing = map(lambda x: x.split(','), tmp_list)
# Identify File
# pdb.set_trace()
FileID = filter(lambda x: (x[1]==remote_filename), directory_listing)[0]
# Obtain Download-information
if self.account_status == "free":
params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'try': '1'})
else:
params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'try': '1', 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
tmp = response.read()
if (response.status == 401 or tmp[:5]=="ERROR"):
log.Info("RS GET attempt #%d failed: %s %s" % (n, response.status, response.reason))
continue
# Respone has the format "DL:$hostname,$dlauth,$countdown"
download_coordinates = tmp[3:].split(',')
# Download file
if self.account_status == "free":
log.Info("Waiting time for Rapidshare-download: %s seconds" % (float(download_coordinates[2]),))
time.sleep(float(download_coordinates[2]))
connection = httplib.HTTPConnection(download_coordinates[0], 80)
params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'dlauth': download_coordinates[1]})
else:
connection = httplib.HTTPSConnection(download_coordinates[0], 443)
params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'login': self.parsed_url.username, 'password': self.get_password()})
connection.request("POST", "/cgi-bin/rsapi.cgi", params)
response = connection.getresponse()
if response.status == 200:
target_file.write(response.read())
assert not target_file.close()
local_path.setdata()
return
log.Info("RS GET attempt #%d failed: %s %s" % (n, response.status, response.reason))
log.Warn("RS backend giving up after %d attempts to GET %s" % (globals.num_retries, url))
raise BackendException((response.status, response.reason))
def encode_multipart_formdata(self, fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
"""
BOUNDARY = '----------632865735RS4EVER5675865'
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (files[0], files[1]))
L.append('Content-Type: %s' % self.get_content_type(files[1]))
L.append('')
L.append(files[2])
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def get_content_type(self, filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def put(self, source_path, remote_filename = None):
"""Transfer source_path to remote_filename"""
# pdb.set_trace()
if not remote_filename:
remote_filename = source_path.get_filename()
# url = self.directory + remote_filename
source_file = source_path.open("rb")
for n in range(1, globals.num_retries+1):
log.Info("Saving %s on RS server" % (self.directory + '/' + remote_filename))
# Identify next free uploadserver
params = urllib.urlencode({'sub': 'nextuploadserver'})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
if response.status == 401:
continue
rs_uploadserverID = response.read()
#upload file
# def post_multipart(host, selector, fields, files):
# """
# Post fields and files to an http host as multipart/form-data.
# fields is a sequence of (name, value) elements for regular form fields.
# files is a sequence of (name, filename, value) elements for data to be uploaded as files
# Return the server's response page.
# """
fields = ('sub', 'upload'), ('login', self.parsed_url.username), ('password', self.get_password()), ('folder', str(self.FolderID))
files = 'filecontent', remote_filename, str(source_file.read())
content_type, body = self.encode_multipart_formdata(fields, files)
connection = httplib.HTTPConnection('rs'+rs_uploadserverID+'.rapidshare.com', 80)
#connection = httplib.HTTPSConnection('rs'+rs_uploadserverID+'.rapidshare.com', 443)
headers = {
'User-Agent': 'Firefox',
'Content-Type': content_type
}
connection.request('POST', "/cgi-bin/rsapi.cgi", body, headers)
response = connection.getresponse()
if response.status == 401:
continue
elif ((response.status == 200) or (response.status == 201)):
response.read()
assert not source_file.close()
return
log.Info("RS upload attempt #%d failed: %s %s" % (n, response.status, response.reason))
log.Warn("RS backend giving up after %d attempts to upload %s" % (globals.num_retries, (self.directory + '/' + remote_filename)))
raise BackendException((response.status, response.reason))
def delete(self, filename_list):
"""Delete files in filename_list"""
# pdb.set_trace()
for n in range(1, globals.num_retries+1):
log.Info("Deleting %s from RS server" % ((self.directory + ':' +','.join(filename_list)) ,))
# list files and identify respective IDs
params = urllib.urlencode({'sub': 'listfiles', 'realfolder': self.FolderID, 'fields':'fileid,filename', 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
if response.status == 401:
continue
directory_listing_string = response.read()
directory_listing = directory_listing_string.split('\n')
directory_listing.pop()
dir_list = map (lambda x: x.split(','), directory_listing)
# Identify fileIDs
IDlist = []
for filename in filename_list:
IDlist.append(filter(lambda x: (x[1]==filename), dir_list)[0][0])
# Delete Files by their ID
# pdb.set_trace()
params = urllib.urlencode({'sub': 'deletefiles', 'files': ','.join(IDlist), 'login': self.parsed_url.username, 'password': self.get_password()})
self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params)
response = self.api_conn.getresponse()
if response.status == 200:
info = response.read()
break
log.Info("RS DELETE attempt #%d failed: %s %s" % (n, response.status, response.reason))
if n == globals.num_retries +1:
log.Warn("RS backend giving up after %d attempts to DELETE %s" % (globals.num_retries, url))
raise BackendException((response.status, response.reason))
duplicity.backend.register_backend("rs", RapidshareBackend)