Import Paramiko to AWS Lambda

PRECONDITION

Today let’s see how to import the paramiko library into AWS Lambda with this errors: from_buffer() cannot return the address of the raw string within a str or unicode or bytearray object and unable to import function etc.

PROCEDURE

For security i use an AWS instances for install the dependency, because there is python 2.7 installed and windows could create problems.

Create an instances with Amazon Linux 2 AMI and an Public IP/Elastic IP. Use ssh to enter in the machine with our private key.

Create and enter in a new directory and follow this instructions:

Copy to Clipboard

Now zip this folder

Copy to Clipboard

and copy this zip in our local pc by scp.

Remember to insert your python file in the zip where you import the python paramiko library.

This is an example to use the paramiko library for get and put a file in two different server by sftp:

Copy to Clipboard

import paramiko
import boto3

HOST_GET = "SERVER IP SOURCE"
PORT_GET = 22
USERNAME_GET = "user1"
PASSWORD_GET = "testJoel"
REMOTE_FILE_PATH_GET = "/files/file.txt"
LOCAL_FILE_PATH_GET = '/tmp/filetr.txt'

HOST_PUT = "SERVER IP RECIPIENT"
PORT_PUT = 22
USERNAME_PUT = "user1"
PASSWORD_PUT = "testJoel"
REMOTE_FILE_PATH_PUT = "/filesTransfered/file.txt"
LOCAL_FILE_PATH_PUT = '/tmp/filetr.txt'

def worker_handler(event, context):
    client = paramiko.SSHClient()
    print("CONNECTION GET HOST" + HOST_GET + "TO PORT: " + str(PORT_GET))
    t = paramiko.Transport(HOST_GET, PORT_GET)
    t.connect(username=USERNAME_GET,password=PASSWORD_GET)
    sftp = paramiko.SFTPClient.from_transport(t)

print("GET FILE")
    sftp.get(REMOTE_FILE_PATH_GET, LOCAL_FILE_PATH_GET)
    print("CLOSE CONNECTION HOST" + HOST_GET + "TO PORT: " + str(PORT_GET))
    sftp.close()
    t.close()
    
    # SFTP PUT
    print("CONNECTION PUT HOST" + HOST_PUT + "TO PORT: " + str(PORT_PUT))
    t = paramiko.Transport(HOST_PUT, PORT_PUT)
    t.connect(username=USERNAME_PUT,password=PASSWORD_PUT)
    sftp = paramiko.SFTPClient.from_transport(t)

print("PUT FILE")
    sftp.put(LOCAL_FILE_PATH_PUT,REMOTE_FILE_PATH_PUT)
    print("CLOSE CONNECTION HOST" + HOST_PUT + "TO PORT: " + str(PORT_PUT))

sftp.close()
    t.close()

In this other example we save the file downloaded in a S3 bucket and save it by aws cli in the instances (identified by tag):

Copy to Clipboard

import paramiko
import boto3
import time

HOST_GET = "SERVER IP SENDER"
PORT_GET = 22
USERNAME_GET = "user1"
PASSWORD_GET = "testJoel"
REMOTE_FILE_PATH_GET = "/files/file.txt"
LOCAL_FILE_PATH_GET = '/tmp/filetr.txt'

S3_BUCKET_NAME = "sftpfilewrite"

TAG = "aws:autoscaling:groupName"
VALUE = "CRM-VTE-AS"
COMMAND = "aws s3 cp s3://sftpfilewrite/file/filetr.txt /home/vtecrm/importfile"
S3_KEY_FILE = "file/filetr.txt"

print("GET FILE")
    sftp.get(REMOTE_FILE_PATH_GET, LOCAL_FILE_PATH_GET)
    print("CLOSE CONNECTION HOST" + HOST_GET + "TO PORT: " + str(PORT_GET))
    sftp.close()
    t.close()

print("START UPLOAD")
    s3 = boto3.client('s3')
    print("SLEEP FOR FILE RECEPTION")
    time.sleep(10)
    s3.upload_file(LOCAL_FILE_PATH_GET, S3_BUCKET_NAME, S3_KEY_FILE)
    client = boto3.client('ec2')
    print("START DESCRIBE INSTANCES")
    result2 = client.describe_instances(Filters=[{'Name': 'tag:' + TAG, 'Values':[VALUE]}])
    instanceID = ""
    for reservation in result2["Reservations"]:
        for instance in reservation["Instances"]:
            instanceID= instance["InstanceId"]

print("DESCRIBEID" + str(instanceID))
    ssm_client = boto3.client('ssm')
    response = ssm_client.send_command(InstanceIds=[instanceID],DocumentName="AWS-RunShellScript", Parameters={'commands': [COMMAND]})
    response = ssm_client.send_command(InstanceIds=[instanceID],DocumentName="AWS-RunShellScript", Parameters={'commands': [COMMAND]})
    print("SLEEP FOR CHECK SUCCESS COMMAND")
    time.sleep(10)

command_id = response['Command']['CommandId']
    result = str(ssm_client.list_commands(CommandId=command_id,InstanceId=instanceID))
    print("RESPONSE" + str(result))

Import the zip in the lambda, use python 2.7, specify the name and file of where you import your code:

The policy for the lambda function:

EC2 filtered by tag

Copy to Clipboard

SSM

Copy to Clipboard

For the role to attach at the instance:

Copy to Clipboard

SSM

Copy to Clipboard

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "ssm:DescribeAssociation",
                "ssm:GetDeployablePatchSnapshotForInstance",
                "ssm:GetDocument",
                "ssm:DescribeDocument",
                "ssm:GetManifest",
                "ssm:GetParameters",
                "ssm:ListAssociations",
                "ssm:ListInstanceAssociations",
                "ssm:PutInventory",
                "ssm:PutComplianceItems",
                "ssm:PutConfigurePackageResult",
                "ssm:UpdateAssociationStatus",
                "ssm:UpdateInstanceAssociationStatus",
                "ssm:UpdateInstanceInformation"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ssmmessages:CreateControlChannel",
                "ssmmessages:CreateDataChannel",
                "ssmmessages:OpenControlChannel",
                "ssmmessages:OpenDataChannel"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2messages:AcknowledgeMessage",
                "ec2messages:DeleteMessage",
                "ec2messages:FailMessage",
                "ec2messages:GetEndpoint",
                "ec2messages:GetMessages",
                "ec2messages:SendReply"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "cloudwatch:PutMetricData"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:DescribeInstanceStatus"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "ds:CreateComputer",
                "ds:DescribeDirectories"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:DescribeLogGroups",
                "logs:DescribeLogStreams",
                "logs:PutLogEvents"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": "s3:GetObject",
            "Resource": [
                "arn:aws:s3:::aws-ssm-eu-west-1/*",
                "arn:aws:s3:::aws-windows-downloads-eu-west-1/*",
                "arn:aws:s3:::amazon-ssm-eu-west-1/*",
                "arn:aws:s3:::amazon-ssm-packages-eu-west-1/*",
                "arn:aws:s3:::eu-west-1-birdwatcher-prod/*",
                "arn:aws:s3:::patch-baseline-snapshot-eu-west-1/*"
            ]
        }
    ]
}