Skip to content

Commit

Permalink
Fix ManagedUpload never finishes when stream size == part size
Browse files Browse the repository at this point in the history
When uploading a stream with an unknown initial length and a total length
exactly equal to a multiple of the part size and a queueSize of 0, the
upload callback is never finished.

This is caused by only checking whether the upload finished after uploading
the last part, but not when the end event is received.

In our case the end event is received after the last part has been uploaded,
but there is also no more data to be uploaded, resulting in a hanging upload.
  • Loading branch information
johanneswuerbach committed Mar 3, 2017
1 parent 975b315 commit b982ac8
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 1 deletion.
6 changes: 5 additions & 1 deletion lib/s3/managed_upload.js
Expand Up @@ -173,7 +173,11 @@ AWS.S3.ManagedUpload = AWS.util.inherit({
on('end', function() {
self.isDoneChunking = true;
self.numParts = self.totalPartNumbers;
self.fillQueue.call(self);
if (self.isDoneChunking && self.totalPartNumbers >= 1 && self.doneParts === self.numParts) {
self.finishMultiPart();
} else {
self.fillQueue.call(self);
}
});
}
}
Expand Down
54 changes: 54 additions & 0 deletions test/s3/managed_upload.spec.coffee
Expand Up @@ -368,6 +368,60 @@ describe 'AWS.S3.ManagedUpload', ->
expect(e.message).to.equal('message')
done()

it 'can send a stream that is exactly equal to part size', (done) ->
partSize = 5 * 1024 * 1024
require('crypto').randomBytes partSize, (err, buf) ->
return done(err) if err

stream = AWS.util.buffer.toStream buf
reqs = helpers.mockResponses [
{ data: UploadId: 'uploadId' }
{ data: ETag: 'ETAG1' }
]
upload = new AWS.S3.ManagedUpload({
partSize: partSize,
queueSize: 1,
params: { Body: stream }
})
upload.send (err) ->
return done(err) if err

expect(helpers.operationsForRequests(reqs)).to.eql [
's3.createMultipartUpload',
's3.uploadPart',
's3.completeMultipartUpload'
]
done()

it 'can send a stream that is exactly divisible by part size', (done) ->
partSize = 5 * 1024 * 1024
streamSize = 2 * partSize
require('crypto').randomBytes streamSize, (err, buf) ->
return done(err) if err

stream = AWS.util.buffer.toStream buf
reqs = helpers.mockResponses [
{ data: UploadId: 'uploadId' }
{ data: ETag: 'ETAG1' }
{ data: ETag: 'ETAG2' }
{ data: ETag: 'FINAL_ETAG', Location: 'FINAL_LOCATION' }
]
upload = new AWS.S3.ManagedUpload({
partSize: partSize,
queueSize: 1,
params: { Body: stream }
})
upload.send (err) ->
return done(err) if err

expect(helpers.operationsForRequests(reqs)).to.eql [
's3.createMultipartUpload'
's3.uploadPart'
's3.uploadPart'
's3.completeMultipartUpload'
]
done()

if typeof Promise == 'function'
describe 'promise', ->
thenFunction = (d) ->
Expand Down

0 comments on commit b982ac8

Please sign in to comment.