Mailman 3 [Twisted-Python] Conch SFTP Questions - Twisted

22 Sep 2020

      Hey folks, I've cobbled together an SFTP client based on bits and pieces
I've found around the web. The issue is that it appears to be almost one
shot. I will need to send many files (the number not known ahead of time).
It's not clear to me when the connection is closed or how many factories
I'm creating. All the code I've grabbed looks like it's creating a new
factory for every SFTP file I send. Here's some of the code I have. It's
fairly straight forward in that it creates a directory if it doesn't exist
and then writes a file.

@attr.s(frozen=True)
class FileInfo(object):
    """
    Class that tells SFTP details about the file to send.
    """
    directory = attr.ib(converter=str)  # type: str
    name = attr.ib(converter=str)  # type: str
    data = attr.ib()  # type: str
    chunk_size = attr.ib(converter=int, default=CHUNK_SIZE)  # type: int

    def to_path(self):
        """
        Turns the folder and file name into a file path.
        """
        return self.directory + "/" + self.name

@attr.s(frozen=True)
class SFTPClientOptions(object):
    """
    Client options for sending SFTP files.

    :param host: the host of the SFTP server
    :param port: the port ofo the SFTP server
    :param fingerprint: the expected fingerprint of the host
    :param user: the user to login as
    :param identity: the identity file, optional and like the "-i"
command line option
    :param password: an optional password
    """
    host = attr.ib(converter=str)  # type: str
    port = attr.ib(converter=int)  # type: int
    fingerprint = attr.ib(converter=str)  # type: str
    user = attr.ib(converter=str)  # type: str
    identity = attr.ib(converter=optional(str), default=None)  # type:
Optional[str]
    password = attr.ib(converter=optional(str), default=None)  # type:
Optional[str]

@inlineCallbacks
def sftp_send(client_options, file_info):
    # type: (SFTPClientOptions, FileInfo)->Deferred
    """
    Primary function to send an file over SFTP. You can send a
password, identity, or both.
    :param client_options: the client connection options
    :param file_info: contains the file info to write
    :return: A deferred that signals "OK" if successful.
    """
    options = ClientOptions()
    options["host"] = client_options.host
    options["port"] = client_options.port
    options["password"] = client_options.password
    options["fingerprint"] = client_options.fingerprint

    if client_options.identity:
        options.identitys = [client_options.identity]

    conn = SFTPConnection()
    auth = SFTPUserAuthClient(client_options.user, options, conn)
    yield connect(client_options.host, client_options.port, options,
_verify_host_key, auth)

    sftpClient = yield conn.getSftpClientDeferred()
    yield _send_file(sftpClient, file_info)

    returnValue("OK")

def _verify_host_key(transport, host, pubKey, fingerprint):
    """
    Verify a host's key. Based on what is specified in options.

    @param host: Due to a bug in L{SSHClientTransport.verifyHostKey}, this is
    always the dotted-quad IP address of the host being connected to.
    @type host: L{str}

    @param transport: the client transport which is attempting to connect to
    the given host.
    @type transport: L{SSHClientTransport}

    @param fingerprint: the fingerprint of the given public key, in
    xx:xx:xx:... format.

    @param pubKey: The public key of the server being connected to.
    @type pubKey: L{str}

    @return: a L{Deferred} which is success or error
    """
    expected = transport.factory.options.get("fingerprint", "no_fingerprint")
    if fingerprint == expected:
        return succeed(1)

    log.error(
        "SSH Host Key fingerprint of ({fp}) does not match the
expected value of ({expected}).",
        fp=fingerprint, expected=expected)

    return fail(ConchError("Host fingerprint is unexpected."))

class SFTPSession(SSHChannel):
    """
    Creates an SFTP session.
    """
    name = "session"

    @inlineCallbacks
    def channelOpen(self, whatever):
        """
        Called when the channel is opened.  "whatever" is any data that the
        other side sent us when opening the channel.

        @type whatever: L{bytes}
        """
        yield self.conn.sendRequest(self, "subsystem", NS("sftp"),
wantReply=True)

        client = FileTransferClient()
        client.makeConnection(self)
        self.dataReceived = client.dataReceived
        self.conn.notifyClientIsReady(client)

class SFTPConnection(SSHConnection):
    def __init__(self):
        """
        Adds a deferred here so client can add a callback when the
SFTP client is ready.
        """
        SSHConnection.__init__(self)
        self._sftpClient = Deferred()

    def serviceStarted(self):
        """
        Opens an SFTP session when the SSH connection has been started.
        """
        self.openChannel(SFTPSession())

    def notifyClientIsReady(self, client):
        """
        Trigger callbacks associated with our SFTP client deferred. It's ready!
        """
        self._sftpClient.callback(client)

    def getSftpClientDeferred(self):
        return self._sftpClient

class SFTPUserAuthClient(SSHUserAuthClient):
    """
    Twisted Conch doesn't have a way of getting a password. By default
it gets it from stdin. This allows it
    to be retrieved from options instead.
    """
    def getPassword(self, prompt = None):
        """
        Get the password from the client options, is specified.
        """
        if "password" in self.options:
            return succeed(self.options["password"])

        return SSHUserAuthClient.getPassword(self, prompt)

@inlineCallbacks
def _send_file(client, file_info):
    # type: (FileTransferClient, FileInfo) -> Deferred
    """
    Creates a directory if required and then creates the file.
    :param client: the SFTP client to use
    :param file_info: contains file name, directory, and data
    """
    try:
        yield client.makeDirectory(file_info.directory, {})

    except SFTPError as e:
        # In testing on various system, either a 4 or an 11 will
indicate the directory
        # already exist. We are fine with that and want to continue if
it does. If we misinterpreted
        # error code here we are probably still ok since we will just
get the more systemic error
        # again on the next call to openFile.
        if e.code != 4 and e.code != 11:
            raise e

    f = yield client.openFile(file_info.to_path(), FXF_WRITE |
FXF_CREAT | FXF_TRUNC, {})

    try:
        yield _write_chunks(f, file_info.data, file_info.chunk_size)

    finally:
        yield f.close()

@inlineCallbacks
def _write_chunks(f, data, chunk_size):
    # type: (ClientFile, str, int) -> Deferred
    """
    Convenience function to write data in chunks

    :param f: the file to write to
    :param data: the data to write
    :param chunk_size: the chunk size
    """
    for offset in range(0, len(data), chunk_size):
        chunk = data[offset: offset + chunk_size]
        yield f.writeChunk(offset, chunk)

It gets called like this:

return sftp.sftp_send(
    client_options=SFTPClientOptions(
        host=self.options.host,
        port=self.options.port,
        user=self.options.user,
        fingerprint=self.options.fingerprint,
        identity=getattr(self.options, "identity", None),
        password=self._getPassword()),
    file_info=sftp.FileInfo(
        directory=self.options.directory,
        name=fileName,
        data=data,
        chunk_size=getattr(self.options, "chunkSize", sftp.CHUNK_SIZE)))

But I supposed I'd like to see something more like this:

sftpClient = self.getSftpClient(
    client_options=SFTPClientOptions(
        host=self.options.host,
        port=self.options.port,
        user=self.options.user,
        fingerprint=self.options.fingerprint,
        identity=getattr(self.options, "identity", None),
        password=self._getPassword()))

return sftpClient.send(
    file_info=sftp.FileInfo(
        directory=self.options.directory,
        name=fileName,
        data=data,
        chunk_size=getattr(self.options, "chunkSize", sftp.CHUNK_SIZE)))

Where sftpClient reuses the existing SSH connection if it is active (rather
than logging in each time). But maybe the sftp service doesn't multiplex so
I have to create a new SSHClientFactory every time I want to send a
distinct file?

Sorry for all the questions, new to twisted and a bit confused. Thanks!

Robert

[Twisted-Python] Conch SFTP Questions

Robert DiFalco

Adi Roiban

Robert DiFalco

Glyph

Robert DiFalco

tags

participants (3)