How to copy/move all objects in Amazon S3 from one prefix to other using the AWS SDK for Node.js

Question

How do I copy all objects from one prefix to others? I have tried all possible ways to copy all objects in one shot from one prefix to other, but the only way that seems to work is by looping over a list of objects and copying them one by one. This is really inefficient. If I have hundreds of files in a folder, will I have to make 100 calls?

var params = {

         Bucket: bucket,

         CopySource: bucket+'/'+oldDirName+'/filename.txt',

         Key: newDirName+'/filename.txt',

 };

s3.copyObject(params, function(err, data) {

  if (err) {

      callback.apply(this, [{

          type: "error",

          message: "Error while renaming Directory",

          data: err

      }]);

  } else {

      callback.apply(this, [{

          type: "success",

          message: "Directory renamed successfully",

          data: data

      }]);

  }

});

Answer 1

You will need to make one AWS.S3.listObjects() to list your objects with a specific prefix. But you are correct in that you will need to make one call for every object that you want to copy from one bucket/prefix to the same or another bucket/prefix.

You can also use a utility library like async to manage your requests.

var AWS = require('aws-sdk');

var async = require('async');

var bucketName = 'foo';

var oldPrefix = 'abc/';

var newPrefix = 'xyz/';

var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});

var done = function(err, data) {

  if (err) console.log(err);

  else console.log(data);

};

s3.listObjects({Prefix: oldPrefix}, function(err, data) {

  if (data.Contents.length) {

    async.each(data.Contents, function(file, cb) {

      var params = {

        Bucket: bucketName,

        CopySource: bucketName + '/' + file.Key,

        Key: file.Key.replace(oldPrefix, newPrefix)

      };

      s3.copyObject(params, function(copyErr, copyData){

        if (copyErr) {

          console.log(err);

        }

        else {

          console.log('Copied: ', params.Key);

          cb();

        }

      });

    }, done);

  }

});

To learn more about AWS, you can read AWS Tutorial and if you wish to understand the concept in detail, you can also enroll in AWS Training.