node.js – 如何从Lambda函数解析AWS S3文件

我想使用通过在S3存储桶中创建新对象而触发的Lambda函数来读取文件并提取一些数据并将其写入放置在另一个S3存储桶中的文件.

到目前为止,我的功能正常,将文件从一个S3存储桶复制到另一个存储桶,但我无法弄清楚如何添加一个函数来处理文件并将结果写入最终的S3目的地.

这些文件是简单的文本文件,我需要从文件中的每一行中提取数据.

如果我正在使用的Node.js代码添加了一个额外的函数来处理文件 – 请参阅注释？我在哪里寻求帮助.

// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');


// get reference to S3 client 
var s3 = new AWs.S3();

exports.handler = function(event,context) {
    // Read options from the event.
    console.log("Reading options from event:\n",util.inspect(event,{depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    // Object key may have spaces or unicode non-ASCII characters.
    var srcKey    =
    decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g," "));  
    var dstBucket = "inputBucket";
    var dstKey    = srcKey + ".txt";

    // Sanity check: validate that source and desTination are different buckets.
    if (srcBucket == dstBucket) {
        console.error("DesTination bucket must not match source bucket.");
        return;
    }

    // Infer the file type.
    var typeMatch = srcKey.match(/\.([^.]*)$/);
    if (!typeMatch) {
        console.error('unable to infer file type for key ' + srcKey);
        return;
    }
    var imageType = typeMatch[1];
    if (imageType != "txt") {
        console.log('skipping non-image ' + srcKey);
        return;
    }

    // Download the image from S3,transform,and upload to a different S3 bucket.
    async.waterfall([
        function download(next) {
            // Download the file from S3 into a buffer.
            s3.getObject({
                    Bucket: srcBucket,Key: srcKey
                },next);
            },function transform(response,next) {
            // Read the file we have just downloaded 
            // ? response.body ?
            var rl = require('readline').createInterface({
                input: require('fs').createReadStream('file.in')
            });

            // Process each line here wriTing the result to an output buffer?
            rl.on('line',function (linE) {
                 console.log('Line from file:',linE);
                //Do something with the line... 

                //Create some output String 'outputline'

                //Write 'outputline' to an output buffer 'outbuff'
                // ??

            });
            // Now pass the output buffer to the next function
            // so it can be uploaded to another S3 bucket 
            // ?? 
            next;
        }
        function upload(response,next) {
            // Stream the file to a different S3 bucket.
            s3.putObject({
                    Bucket: dstBucket,Key: dstKey,Body: response.body,ContentType: response.contentType
                },next);
            }
        ],function (err) {
            if (err) {
                console.error(
                    'Unable to process ' + srcBucket + '/' + srcKey +
                    ' and upload to ' + dstBucket + '/' + dstKey +
                    ' due to an error: ' + err
                );
            } else {
                console.log(
                    'successfully processed ' + srcBucket + '/' + srcKey +
                    ' and uploaded to ' + dstBucket + '/' + dstKey
                );
            }

            context.done();
        }
    );
};