# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright 2011 Willem Jansen # # This file is part of duplicity. # # Duplicity is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2 of the License, or (at your # option) any later version. # # Duplicity is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with duplicity; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import pdb import base64 import httplib import re import urllib import urllib2 import xml.dom.minidom import collections import mimetypes import time import duplicity.backend from duplicity import globals from duplicity import log from duplicity.errors import * address@hidden from duplicity import urlparse_2_5 as urlparser class RapidshareBackend(duplicity.backend.Backend): """Backend for accessing Rapidshare - contributed in 2011 by Willem Jansen """ listbody = """\ """ """Connect to remote store using RapidShare API""" def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) self.parsed_url = parsed_url # For development purposes a free account allows for testing. Does not make really make sense for productive use self.account_status = "free" if parsed_url.path: foldpath = re.compile('/+') self.directory = foldpath.sub('/', parsed_url.path + '/' ) else: self.directory = '/' # set username as work-around: otherwise username="None" self.parsed_url.username=parsed_url.path.split('@')[0].split('//')[1] tmp = self.directory.split('/')[2:] tmp.pop() self.directory = '/'.join(tmp) log.Info("Using Rapidshare host %s" % ('api.rapidshare.com',)) log.Info("Using Rapidshare port %s" % (443,)) log.Info("Using Rapidshare directory %s" % (self.directory,)) if parsed_url.scheme == 'rs': self.api_conn = httplib.HTTPSConnection('api.rapidshare.com', 443) else: raise BackendException("Unknown URI scheme: %s" % (parsed_url.scheme)) for n in range(1, globals.num_retries+1): #Identify FolderID #Get Rapidshare directory structure params = urllib.urlencode({'sub': 'listrealfolders', 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() if response.status == 401: continue # parse response and see if directory exists response_lines = response.read().split('\n') response_lines.pop() rs_directory = collections.namedtuple('Folder', 'RealFolder_ID, Parent_RealFolder_ID, Name, BROWSE_ACL, UPLOAD_ACL, DOWNLOAD_ACL') rs_dir_list = map(rs_directory._make, map(lambda x: x.split(',') ,response_lines)) # print(rs_dir_list) #identify RealFolderID, if directory dos not exist, create it # pdb.set_trace() self.FolderID = 0 for current_dir in self.directory.split("/"): item = filter(lambda x: (x.Parent_RealFolder_ID==str(self.FolderID) and x.Name==current_dir), rs_dir_list) if item == []: # if the directory does not exist, create it. log.Info("Directory '%s' being created." % current_dir) params = urllib.urlencode({'sub': 'addrealfolder', 'name': current_dir, 'parent': self.FolderID, 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) self.FolderID = self.api_conn.getresponse().read() else: self.FolderID = item[0].RealFolder_ID if self.FolderID == -1: raise BackendException("Unable to create directory: %s" % (self.directory)) else: return log.Warn("Rapidshare backend giving up after %d attempts to get remote directory information" % (globals.num_retries)) raise BackendException((response.status, response.reason)) def close(self): self.api_conn.close() def list(self): """List files in directory""" log.Info("Listing directory %s on RS server" % (self.directory,)) # Get directory listing for n in range(1, globals.num_retries+1): params = urllib.urlencode({'sub': 'listfiles', 'realfolder': self.FolderID, 'fields':'fileid,filename', 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() if response.status == 401: continue directory_listing_string = response.read() result = [] directory_listing = directory_listing_string.split('\n') directory_listing.pop() for i in directory_listing: result.append(i.split(',')[1]) # pdb.set_trace() return result log.Warn("Rapidshare backend giving up after %d attempts to list directory %s" % (globals.num_retries, self.directory)) raise BackendException((response.status, response.reason)) def get(self, remote_filename, local_path): """Get remote filename, saving it to local_path""" target_file = local_path.open("wb") for n in range(1, globals.num_retries+1): log.Info("Retrieving %s from Rapidshare server" % ((self.directory + '/' + remote_filename))) # get Filelist params = urllib.urlencode({'sub': 'listfiles', 'realfolder': self.FolderID, 'fields':'fileid,filename', 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() if response.status == 401: continue tmp_list = response.read().split('\n') tmp_list.pop() directory_listing = map(lambda x: x.split(','), tmp_list) # Identify File # pdb.set_trace() FileID = filter(lambda x: (x[1]==remote_filename), directory_listing)[0] # Obtain Download-information if self.account_status == "free": params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'try': '1'}) else: params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'try': '1', 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() tmp = response.read() if (response.status == 401 or tmp[:5]=="ERROR"): log.Info("RS GET attempt #%d failed: %s %s" % (n, response.status, response.reason)) continue # Respone has the format "DL:$hostname,$dlauth,$countdown" download_coordinates = tmp[3:].split(',') # Download file if self.account_status == "free": log.Info("Waiting time for Rapidshare-download: %s seconds" % (float(download_coordinates[2]),)) time.sleep(float(download_coordinates[2])) connection = httplib.HTTPConnection(download_coordinates[0], 80) params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'dlauth': download_coordinates[1]}) else: connection = httplib.HTTPSConnection(download_coordinates[0], 443) params = urllib.urlencode({'sub': 'download', 'fileid': FileID[0], 'filename':FileID[1], 'login': self.parsed_url.username, 'password': self.get_password()}) connection.request("POST", "/cgi-bin/rsapi.cgi", params) response = connection.getresponse() if response.status == 200: target_file.write(response.read()) assert not target_file.close() local_path.setdata() return log.Info("RS GET attempt #%d failed: %s %s" % (n, response.status, response.reason)) log.Warn("RS backend giving up after %d attempts to GET %s" % (globals.num_retries, url)) raise BackendException((response.status, response.reason)) def encode_multipart_formdata(self, fields, files): """ fields is a sequence of (name, value) elements for regular form fields. files is a sequence of (name, filename, value) elements for data to be uploaded as files Return (content_type, body) ready for httplib.HTTP instance """ BOUNDARY = '----------632865735RS4EVER5675865' CRLF = '\r\n' L = [] for (key, value) in fields: L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"' % key) L.append('') L.append(value) L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (files[0], files[1])) L.append('Content-Type: %s' % self.get_content_type(files[1])) L.append('') L.append(files[2]) L.append('--' + BOUNDARY + '--') L.append('') body = CRLF.join(L) content_type = 'multipart/form-data; boundary=%s' % BOUNDARY return content_type, body def get_content_type(self, filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' def put(self, source_path, remote_filename = None): """Transfer source_path to remote_filename""" # pdb.set_trace() if not remote_filename: remote_filename = source_path.get_filename() # url = self.directory + remote_filename source_file = source_path.open("rb") for n in range(1, globals.num_retries+1): log.Info("Saving %s on RS server" % (self.directory + '/' + remote_filename)) # Identify next free uploadserver params = urllib.urlencode({'sub': 'nextuploadserver'}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() if response.status == 401: continue rs_uploadserverID = response.read() #upload file # def post_multipart(host, selector, fields, files): # """ # Post fields and files to an http host as multipart/form-data. # fields is a sequence of (name, value) elements for regular form fields. # files is a sequence of (name, filename, value) elements for data to be uploaded as files # Return the server's response page. # """ fields = ('sub', 'upload'), ('login', self.parsed_url.username), ('password', self.get_password()), ('folder', str(self.FolderID)) files = 'filecontent', remote_filename, str(source_file.read()) content_type, body = self.encode_multipart_formdata(fields, files) connection = httplib.HTTPConnection('rs'+rs_uploadserverID+'.rapidshare.com', 80) #connection = httplib.HTTPSConnection('rs'+rs_uploadserverID+'.rapidshare.com', 443) headers = { 'User-Agent': 'Firefox', 'Content-Type': content_type } connection.request('POST', "/cgi-bin/rsapi.cgi", body, headers) response = connection.getresponse() if response.status == 401: continue elif ((response.status == 200) or (response.status == 201)): response.read() assert not source_file.close() return log.Info("RS upload attempt #%d failed: %s %s" % (n, response.status, response.reason)) log.Warn("RS backend giving up after %d attempts to upload %s" % (globals.num_retries, (self.directory + '/' + remote_filename))) raise BackendException((response.status, response.reason)) def delete(self, filename_list): """Delete files in filename_list""" # pdb.set_trace() for n in range(1, globals.num_retries+1): log.Info("Deleting %s from RS server" % ((self.directory + ':' +','.join(filename_list)) ,)) # list files and identify respective IDs params = urllib.urlencode({'sub': 'listfiles', 'realfolder': self.FolderID, 'fields':'fileid,filename', 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() if response.status == 401: continue directory_listing_string = response.read() directory_listing = directory_listing_string.split('\n') directory_listing.pop() dir_list = map (lambda x: x.split(','), directory_listing) # Identify fileIDs IDlist = [] for filename in filename_list: IDlist.append(filter(lambda x: (x[1]==filename), dir_list)[0][0]) # Delete Files by their ID # pdb.set_trace() params = urllib.urlencode({'sub': 'deletefiles', 'files': ','.join(IDlist), 'login': self.parsed_url.username, 'password': self.get_password()}) self.api_conn.request("POST", "/cgi-bin/rsapi.cgi", params) response = self.api_conn.getresponse() if response.status == 200: info = response.read() break log.Info("RS DELETE attempt #%d failed: %s %s" % (n, response.status, response.reason)) if n == globals.num_retries +1: log.Warn("RS backend giving up after %d attempts to DELETE %s" % (globals.num_retries, url)) raise BackendException((response.status, response.reason)) duplicity.backend.register_backend("rs", RapidshareBackend)