Re: [Duplicity-talk] WebDAV backend

[Jörg Mühlberger]

> > Hello!
> > is anybody using duplicity with a WebDAV backend? I found this patch
> > http://savannah.nongnu.org/patch/?func=detailitem&item_id=5185
> > but it doesn't work on my Debian 3.1 machine.
> >     
>
> What do you mean? Does the patch not apply?
>   
The patch itself is working, but I get errors of other subroutines that 
this backend needs to start to make a connection via http.
So if I run the following command:

:~# duplicity  /home/me/  webdav://address@hidden/

I get the following error message.

Traceback (most recent call last):
 File "/usr/bin/duplicity", line 358, in ?
   if __name__ == "__main__": main()
 File "/usr/bin/duplicity", line 334, in main
   globals.archive_dir).set_values()
 File "/usr/lib/python2.3/site-packages/duplicity/collections.py", line 
355, in set_values
   backend_filename_list = self.backend.list()
 File "/usr/lib/python2.3/site-packages/duplicity/backends.py", line 
495, in list
   dom = xml.dom.minidom.parseString(document)
 File "/usr/lib/python2.3/xml/dom/minidom.py", line 1929, in parseString
   return expatbuilder.parseString(string)
 File "/usr/lib/python2.3/xml/dom/expatbuilder.py", line 940, in 
parseString
   return builder.parseString(string)
 File "/usr/lib/python2.3/xml/dom/expatbuilder.py", line 223, in 
parseString
   parser.Parse(string, True)
xml.parsers.expat.ExpatError: syntax error: line 1, column 49


I have attached my backends.py for your information.
If there is anything else that I can help you with, to trace this error, 
just let me know!


Thanks in advance

Jörg

# Copyright 2002 Ben Escoto
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

"""Provides functions and classes for getting/sending files to destination"""

import os, types, ftplib, tempfile
import log, path, dup_temp, file_naming
import time, base64, getpass
import xml.dom.minidom, httplib
from urllib import unquote

class BackendException(Exception): pass
class ParsingException(Exception): pass

def get_backend(url_string):
        """Return Backend object from url string, or None if not a url string

        url strings are like
        scp://foobar:address@hidden:124/usr/local.  If a protocol
        is unsupported a fatal error will be raised.

        """
        global protocol_class_dict
        try: pu = ParsedUrl(url_string)
        except ParsingException: return None

        try: backend_class = protocol_class_dict[pu.protocol]
        except KeyError: log.FatalError("Unknown protocol '%s'" % 
(pu.protocol,))
        return backend_class(pu)

class ParsedUrl:
        """Contains information gleaned from a generic url"""
        protocol = None # set to string like "ftp" indicating protocol
        suffix = None # Set to everything after protocol://

        server = None # First part of suffix (part before '/')
        path = None # Second part of suffix (part after '/')

        host = None # Set to host, if can be extracted
        user = None # Set to user, as in ftp://address@hidden/whatever
        port = None # Set to port, like scp://host:port/foo

        def __init__(self, url_string):
                """Create ParsedUrl object, process url_string"""
                self.url_string = url_string
                self.set_protocol_suffix()
                self.set_server_path()
                self.set_host_user_port()

        def bad_url(self, message = None):
                """Report a bad url, using message if given"""
                if message:
                        err_string = "Bad URL string '%s': %s" % 
(self.url_string, message)
                else: err_string = "Bad URL string '%s'" % (self.url_string,)
                raise ParsingException(err_string)

        def set_protocol_suffix(self):
                """Parse self.url_string, setting self.protocol and 
self.suffix"""
                colon_position = self.url_string.find(":")
                if colon_position < 1: self.bad_url("No colon (:) found")
                self.protocol = self.url_string[:colon_position]
                if self.url_string[colon_position+1:colon_position+3] != '//':
                        self.bad_url("first colon not followed by '//'")
                self.suffix = self.url_string[colon_position+3:]

        def set_server_path(self):
                """Set self.server and self.path from self.suffix"""
                comps = self.suffix.split('/')
                assert len(comps) > 0
                self.server = comps[0]
                if len(comps) > 1:
                        self.path = '/'.join(comps[1:])

        def set_host_user_port(self):
                """Set self.host, self.user, and self.port from self.server"""
                if not self.server: return

                # Extract port
                port_comps = self.server.split(":")
                if len(port_comps) >= 2:
                        try: self.port = int(port_comps[-1])
                        except ValueError: user_host = self.server
                        else: user_host = ":".join(port_comps[:-1])
                else: user_host = self.server

                # Set user and host
                user_comps = user_host.split("@")
                if len(user_comps) >= 2:
                        self.user = user_comps[0]
                        self.host = "@".join(user_comps[1:])
                else: self.host = user_host


class Backend:
        """Represent a connection to the destination device/computer

        Classes that subclass this should implement the put, get, list,
        and delete methods.

        """
        def init(self, parsed_url): pass

        def put(self, source_path, remote_filename = None):
                """Transfer source_path (Path object) to remote_filename 
(string)

                If remote_filename is None, get the filename from the last
                path component of pathname.

                """
                if not remote_filename: remote_filename = 
source_path.get_filename()
                pass

        def get(self, remote_filename, local_path):
                """Retrieve remote_filename and place in local_path"""
                local_path.setdata()
                pass
        
        def list(self):
                """Return list of filenames (strings) present in backend"""
                pass

        def delete(self, filename_list):
                """Delete each filename in filename_list, in order if 
possible"""
                pass

        def run_command(self, commandline):
                """Run given commandline with logging and error detection"""
                log.Log("Running '%s'" % commandline, 5)
                if os.system(commandline):
                        raise BackendException("Error running '%s'" % 
commandline)

        def popen(self, commandline):
                """Run command and return stdout results"""
                log.Log("Reading results of '%s'" % commandline, 5)
                fout = os.popen(commandline)
                results = fout.read()
                if fout.close():
                        raise BackendException("Error running '%s'" % 
commandline)
                return results

        def get_fileobj_read(self, filename, parseresults = None):
                """Return fileobject opened for reading of filename on backend

                The file will be downloaded first into a temp file.  When the
                returned fileobj is closed, the temp file will be deleted.

                """
                if not parseresults:
                        parseresults = file_naming.parse(filename)
                        assert parseresults, "Filename not correctly parsed"
                tdp = dup_temp.new_tempduppath(parseresults)
                self.get(filename, tdp)
                tdp.setdata()
                return tdp.filtered_open_with_delete("rb")

        def get_fileobj_write(self, filename, parseresults = None,
                                                  sizelist = None):
                """Return fileobj opened for writing, write to backend on close

                The file will be encoded as specified in parseresults (or as
                read from the filename), and stored in a temp file until it
                can be copied over and deleted.

                If sizelist is not None, it should be set to an empty list.
                The number of bytes will be inserted into the list.

                """
                if not parseresults:
                        parseresults = file_naming.parse(filename)
                        assert parseresults, "Filename %s not correctly parsed" 
% filename
                tdp = dup_temp.new_tempduppath(parseresults)

                def close_file_hook():
                        """This is called when returned fileobj is closed"""
                        self.put(tdp, filename)
                        if sizelist is not None:
                                tdp.setdata()
                                sizelist.append(tdp.getsize())
                        tdp.delete()

                fh = dup_temp.FileobjHooked(tdp.filtered_open("wb"))
                fh.addhook(close_file_hook)
                return fh

        def get_data(self, filename, parseresults = None):
                """Retrieve a file from backend, process it, return contents"""
                fin = self.get_fileobj_read(filename, parseresults)
                buf = fin.read()
                assert not fin.close()
                return buf

        def put_data(self, buffer, filename, parseresults = None):
                """Put buffer into filename on backend after processing"""
                fout = self.get_fileobj_write(filename, parseresults)
                fout.write(buffer)
                assert not fout.close()

        def close(self):
                """This is called when a connection is no longer needed"""
                pass


class LocalBackend(Backend):
        """Use this backend when saving to local disk

        Urls look like file://testfiles/output.  Relative to root can be
        gotten with extra slash (file:///usr/local).

        """
        def __init__(self, parsed_url):
                self.remote_pathdir = path.Path(parsed_url.suffix)

        def put(self, source_path, remote_filename = None, rename = None):
                """If rename is set, try that first, copying if doesn't work"""
                if not remote_filename: remote_filename = 
source_path.get_filename()
                target_path = self.remote_pathdir.append(remote_filename)
                log.Log("Writing %s" % target_path.name, 6)
                if rename:
                        try: source_path.rename(target_path)
                        except OSError: pass
                        else: return
                target_path.writefileobj(source_path.open("rb"))

        def get(self, filename, local_path):
                """Get file and put in local_path (Path object)"""
                source_path = self.remote_pathdir.append(filename)
                local_path.writefileobj(source_path.open("rb"))

        def list(self):
                """List files in that directory"""
                return self.remote_pathdir.listdir()

        def delete(self, filename_list):
                """Delete all files in filename list"""
                assert type(filename_list) is not types.StringType
                try:
                        for filename in filename_list:
                                self.remote_pathdir.append(filename).delete()
                except OSError, e: raise BackendException(str(e))


# The following can be redefined to use different shell commands from
# ssh or scp or to add more arguments.  However, the replacements must
# have the same syntax.  Also these strings will be executed by the
# shell, so shouldn't have strange characters in them.
ssh_command = "ssh"
scp_command = "scp"

class scpBackend(Backend):
        """This backend copies files using scp.  List not supported"""
        def __init__(self, parsed_url):
                """scpBackend initializer"""
                self.host_string = parsed_url.server # of form 
address@hidden:port
                self.remote_dir = parsed_url.path # can be empty string
                if self.remote_dir: self.remote_prefix = self.remote_dir + "/"
                else: self.remote_prefix = ""

        def put(self, source_path, remote_filename = None):
                """Use scp to copy source_dir/filename to remote computer"""
                if not remote_filename: remote_filename = 
source_path.get_filename()
                commandline = "%s %s %s:%s%s" % \
                                          (scp_command, source_path.name, 
self.host_string,
                                           self.remote_prefix, remote_filename)
                self.run_command(commandline)

        def get(self, remote_filename, local_path):
                """Use scp to get a remote file"""
                commandline = "%s %s:%s%s %s" % \
                                          (scp_command, self.host_string, 
self.remote_prefix,
                                           remote_filename, local_path.name)
                self.run_command(commandline)
                local_path.setdata()
                if not local_path.exists():
                        raise BackendException("File %s not found" % 
local_path.name)
                
        def list(self):
                """List files available for scp

                Note that this command can get confused when dealing with
                files with newlines in them, as the embedded newlines cannot
                be distinguished from the file boundaries.

                """
                commandline = ("%s %s ls %s" %
                                           (ssh_command, self.host_string, 
self.remote_dir))
                return filter(lambda x: x, self.popen(commandline).split("\n"))

        def delete(self, filename_list):
                """Runs ssh rm to delete files.  Files must not require 
quoting"""
                assert len(filename_list) > 0
                pathlist = map(lambda fn: self.remote_prefix + fn, 
filename_list)
                del_prefix = "%s %s rm " % (ssh_command, self.host_string)

                # Delete in groups of 10 to avoid overflowing command line
                for i in range(0, len(pathlist), 10):
                        commandline = del_prefix + " ".join(pathlist[i:i+10])
                        self.run_command(commandline)


class sftpBackend(Backend):
        """This backend uses sftp to perform file operations"""
        pass # Do this later


class ftpBackend(Backend):
        """Connect to remote store using File Transfer Protocol"""
        SLEEP = 10 # time in seconds before we try to reconnect on temporary 
errors
        def __init__(self, parsed_url):
                """Create a new ftp backend object, log in to host"""
                self.parsed_url = parsed_url
                self.ftp = ftplib.FTP()
                if parsed_url.port is None: self.error_wrap('connect', 
parsed_url.host)
                else: self.error_wrap('connect', parsed_url.host, 
parsed_url.port)

                if parsed_url.user is not None:
                        self.error_wrap('login', parsed_url.user, 
self.get_password())
                else: self.error_wrap('login')
                self.ftp.cwd(parsed_url.path)

        def error_wrap(self, command, *args):
                """Run self.ftp.command(*args), but raise BackendException on 
error"""
                try: return ftplib.FTP.__dict__[command](self.ftp, *args)
                except ftplib.error_temp, e:
                        log.Log("Temporary error '%s'. Trying to reconnect in 
%d seconds." %
                                (str(e), self.SLEEP), 3)
                        time.sleep(self.SLEEP)
                        self.__init__(self.parsed_url)
                        self.error_wrap(command, *args)
                except ftplib.all_errors, e: raise BackendException(e)

        def get_password(self):
                """Get ftp password using environment if possible"""
                try: return os.environ['FTP_PASSWORD']
                except KeyError:
                        log.Log("FTP_PASSWORD not set, using empty ftp 
password", 3)
                        return ''

        def put(self, source_path, remote_filename = None):
                """Transfer source_path to remote_filename"""
                if not remote_filename: remote_filename = 
source_path.get_filename()
                source_file = source_path.open("rb")
                log.Log("Saving %s on FTP server" % (remote_filename,), 5)
                self.error_wrap('storbinary', "STOR "+remote_filename, 
source_file)
                assert not source_file.close()

        def get(self, remote_filename, local_path):
                """Get remote filename, saving it to local_path"""
                target_file = local_path.open("wb")
                log.Log("Retrieving %s from FTP server" % (remote_filename,), 5)
                self.error_wrap('retrbinary', "RETR "+remote_filename,
                                                target_file.write)
                assert not target_file.close()
                local_path.setdata()

        def list(self):
                """List files in directory"""
                log.Log("Listing files on FTP server", 5)
                return self.error_wrap('nlst')

        def delete(self, filename_list):
                """Delete files in filename_list"""
                for filename in filename_list:
                        log.Log("Deleting %s from FTP server" % (filename,), 5)
                        self.error_wrap('delete', filename)

        def close(self):
                """Shut down connection"""
                self.error_wrap('quit')


class rsyncBackend(Backend):
        """Connect to remote store using rsync

        rsync backend contributed by Sebastian Wilhelmi <address@hidden>

        """
        def __init__(self, parsed_url):
                """rsyncBackend initializer"""
                self.url_string = parsed_url.url_string
                if self.url_string[-1] != '/':
                        self.url_string += '/'

        def put(self, source_path, remote_filename = None):
                """Use rsync to copy source_dir/filename to remote computer"""
                if not remote_filename: remote_filename = 
source_path.get_filename()
                remote_path = os.path.join (self.url_string, remote_filename)
                commandline = "rsync %s %s" % (source_path.name, remote_path)
                self.run_command(commandline)

        def get(self, remote_filename, local_path):
                """Use rsync to get a remote file"""
                remote_path = os.path.join (self.url_string, remote_filename)
                commandline = "rsync %s %s" % (remote_path, local_path.name)
                self.run_command(commandline)
                local_path.setdata()
                if not local_path.exists():
                        raise BackendException("File %s not found" % 
local_path.name)
                
        def list(self):
                """List files"""
                def split (str):
                        line = str.split ()
                        if len (line) > 4 and line[4] != '.':
                                return line[4]
                        else:
                                return None
                commandline = "rsync %s" % self.url_string
                return filter (lambda x: x, map (split, 
self.popen(commandline).split('\n')))

        def delete(self, filename_list):
                """Delete files."""
                delete_list = filename_list
                dont_delete_list = []
                for file in self.list ():
                        if file in delete_list:
                                delete_list.remove (file)
                        else:
                                dont_delete_list.append (file)
                if len (delete_list) > 0:
                        raise BackendException("Files %s not found" % str 
(delete_list))

                dir = tempfile.mktemp ()
                exclude_name = tempfile.mktemp ()
                exclude = open (exclude_name, 'w')
                to_delete = [exclude_name]
                os.mkdir (dir)
                for file in dont_delete_list:
                                path = os.path.join (dir, file)
                                to_delete.append (path)
                                f = open (path, 'w')
                                f.close ()
                                print >>exclude, file
                exclude.close ()
                commandline = ("rsync --recursive --delete --exclude-from=%s 
%s/ %s" %
                               (exclude_name, dir, self.url_string))
                self.run_command(commandline)
                for file in to_delete:
                        os.unlink (file)
                os.rmdir (dir)
                
class webdavBackend(Backend):
        directory = None
        conn = None
        headers = {}
        
        """Connect to remote store using WebDAV Protocol"""
        SLEEP = 10 # time in seconds before we try to reconnect on temporary 
errors
        def __init__(self, parsed_url):
                self.parsed_url = parsed_url
                
                self.directory = '/' + parsed_url.path.rstrip('/') + '/'
                log.Log("Using WebDAV host %s" % (parsed_url.host,), 5)
                log.Log("Using WebDAV direcotry %s" % (self.directory,), 5)
                
                self.conn = httplib.HTTPConnection( parsed_url.host )
                
                try: password = os.environ['FTP_PASSWORD']
                except KeyError:
                        password = getpass.getpass('Password for 
'+parsed_url.user+'@'+parsed_url.host+': ')
                
                self.headers['Authorization'] = 'Basic ' + base64.encodestring( 
parsed_url.user+':'+ password)

        def _getText(self,nodelist):
                rc = ""
                for node in nodelist:
                        if node.nodeType == node.TEXT_NODE:
                                rc = rc + node.data
                return rc

        def close(self):
                self.conn.close()
                
        def list(self):
                """List files in directory"""
                log.Log("Listing directory %s on WebDAV server" % 
(self.directory,), 5)
                self.conn.request( "PROPFIND", self.directory, None, 
self.headers )
                response = self.conn.getresponse()

                document = response.read()
                dom = xml.dom.minidom.parseString(document)

                result = []
                for href in dom.getElementsByTagName( '__NS1:href' ):
                        filename = 
unquote(self._getText(href.childNodes).strip())
                        if filename.startswith( self.directory ):
                                filename = filename.replace(self.directory,'',1)
                                result.append(filename)
                return result

        def get(self, remote_filename, local_path):
                """Get remote filename, saving it to local_path"""
                url = self.directory + remote_filename
                log.Log("Retrieving %s from FTP server" % ( url ,), 5)
                target_file = local_path.open("wb")
                self.conn.request( "GET", url, None, self.headers )
                response = self.conn.getresponse()
                target_file.write( response.read() )
                assert not target_file.close()
                local_path.setdata()

        def put(self, source_path, remote_filename = None):
                """Transfer source_path to remote_filename"""
                if not remote_filename: 
                        remote_filename = source_path.get_filename()
                url = self.directory + remote_filename
                source_file = source_path.open("rb")
                log.Log("Saving %s on WebDAV server" % (url,), 5)
                self.conn.request( "PUT", url, source_file.read(), self.headers 
)
                response = self.conn.getresponse()
                response.read()
                assert not source_file.close()

        def delete(self, filename_list):
                """Delete files in filename_list"""
                for filename in filename_list:
                        url = self.directory + filename
                        log.Log("Deleting %s from WebDAV server" % (url,), 5)
                        self.conn.request( "DELETE", url, None, self.headers )
                        response = self.conn.getresponse()
                        response.read()

# Dictionary relating protocol strings to backend_object classes.
protocol_class_dict = {"scp": scpBackend,
                                           "ssh": scpBackend,
                                           "file": LocalBackend,
                                           "ftp": ftpBackend,
                                           "rsync": rsyncBackend,
                                           "webdav": webdavBackend}