How to read image file from S3 bucket directly into memory?

Question

I have the following code

import matplotlib.pyplot as plt

import matplotlib.image as mpimg

import numpy as np

import boto3

s3 = boto3.resource('s3', region_name='us-east-2')

bucket = s3.Bucket('sentinel-s2-l1c')

object = bucket.Object('tiles/10/S/DG/2015/12/7/0/B01.jp2')

object.download_file('B01.jp2')

img=mpimg.imread('B01.jp2')

imgplot = plt.imshow(img)

plt.show(imgplot)

and it works. But the problem it downloads the file into the current directory first. Is it possible to read the file and decode it as an image directly in RAM?

Answer 1

You can use Python's NamedTemporaryFile and this code will create temporary files that will be deleted when the file gets closed.

import matplotlib.pyplot as plt

import matplotlib.image as mpimg

import numpy as np

import boto3

import tempfile

s3 = boto3.resource('s3', region_name='us-east-2')

bucket = s3.Bucket('sentinel-s2-l1c')

object = bucket.Object('tiles/10/S/DG/2015/12/7/0/B01.jp2')

tmp = tempfile.NamedTemporaryFile()

with open(tmp.name, 'wb') as f:

    object.download_fileobj(f)

    img=mpimg.imread(tmp.name)

    # ...Do jobs using img

Comments

import numpy as np

import boto3

import cv2

s3 = boto3.client("s3")

bucket_name = "bucket_name"

# fetching object from bucket

file_obj = s3.get_object(Bucket=bucket_name, Key='path/to/image' )

# reading the file content in bytes

file_content = file_obj["Body"].read()

# creating 1D array from bytes data range between[0,255]

np_array = np.frombuffer(file_content, np.uint8)

# decoding array

image_np = cv2.imdecode(np_array, cv2.IMREAD_COLOR)

gray = cv2.cvtColor(image_np, cv2.COLOR_BGR2GRAY)

# now you can save the read image from s3 to local machine if you want or you can use the image contents from gray or from image_np

cv2.imwrite(r"path/to/new_image", gray)