Reply proper errors on short reads quickly as UnexpectedShortReadException

Minio/Errors/DataSizeMismatchException.cs → Minio/Errors/UnexpectedShortReadException.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * Minio .NET Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,16 +21,16 @@ using System.Text;
 
 namespace Minio.Errors
 {
-    public class DataSizeMismatchException : ClientException
+    public class UnexpectedShortReadException : ClientException
     {
-        public DataSizeMismatchException(string bucket, string key, long userSpecifiedSize, long actualReadSize)
+        public UnexpectedShortReadException(string bucket, string key, long userSpecifiedSize, long actualReadSize)
         {
             this.Bucket = bucket;
             this.Key = key;
             this.UserSpecifiedSize = userSpecifiedSize;
             this.ActualReadSize = actualReadSize;
         }
-        public DataSizeMismatchException()
+        public UnexpectedShortReadException()
             : this(null, null, 0, 0)
         {
 

Minio/Minio.csproj

@@ -22,6 +22,7 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
     <UseVSHostingProcess>false</UseVSHostingProcess>
+    <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
     <DebugType>pdbonly</DebugType>
@@ -30,6 +31,7 @@
     <DefineConstants>TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="RestSharp">
@@ -61,7 +63,7 @@
     <Compile Include="Errors\InternalClientException.cs" />
     <Compile Include="Errors\BucketNotFoundException.cs" />
     <Compile Include="Errors\BucketExistsException.cs" />
-    <Compile Include="Errors\DataSizeMismatchException.cs" />
+    <Compile Include="Errors\UnexpectedShortReadException.cs" />
     <Compile Include="Regions.cs" />
     <Compile Include="Errors\ClientException.cs" />
     <Compile Include="Xml\AccessControlPolicy.cs" />

Minio/MinioClient.cs

@@ -29,7 +29,8 @@ namespace Minio
 {
     public class MinioClient
     {
-        private static int PART_SIZE = 5 * 1024 * 1024;
+        private static long minimumPartSize = 5 * 1024L * 1024L;
+        private static long maximumPartSize = 5 * 1024L * 1024L * 1024L;
 
         private RestClient client;
         private string region;
@@ -445,16 +446,12 @@ namespace Minio
         /// <param name="data">Stream of bytes to send</param>
         public void PutObject(string bucket, string key, long size, string contentType, Stream data)
         {
-            if (size <= MinioClient.PART_SIZE)
+            if (size <= MinioClient.minimumPartSize)
             {
                 var bytes = ReadFull(data, (int)size);
-                if (data.ReadByte() > 0)
-                {
-                    throw new DataSizeMismatchException();
-                }
                 if (bytes.Length != (int)size)
                 {
-                    throw new DataSizeMismatchException(bucket, key, size, bytes.Length);
+                    throw new UnexpectedShortReadException(bucket, key, size, bytes.Length);
                 }
                 this.DoPutObject(bucket, key, null, 0, contentType, bytes);
             }
@@ -489,12 +486,19 @@ namespace Minio
                 while (totalWritten < size)
                 {
                     partNumber++;
-                    var currentPartSize = (int)Math.Min((long)partSize, (size - totalWritten));
-                    byte[] dataToCopy = ReadFull(data, currentPartSize);
+                    byte[] dataToCopy = ReadFull(data, (int)partSize);
                     if (dataToCopy == null)
                     {
                         break;
                     }
+                    if (dataToCopy.Length < partSize)
+                    {
+                        var expectedSize = size - totalWritten;
+                        if (expectedSize != dataToCopy.Length)
+                        {
+                            throw new UnexpectedShortReadException(bucket, key, expectedSize, dataToCopy.Length);
+                        }
+                    }
                     System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
                     byte[] hash = md5.ComputeHash(dataToCopy);
                     string etag = BitConverter.ToString(hash).Replace("-", string.Empty).ToLower();
@@ -506,17 +510,6 @@ namespace Minio
                     totalWritten += dataToCopy.Length;
                 }
 
-                // test if any more data is on the stream
-                if (data.ReadByte() != -1)
-                {
-                    throw new DataSizeMismatchException(bucket, key, size, totalWritten+1);
-                }
-
-                if (totalWritten != size)
-                {
-                    throw new DataSizeMismatchException(bucket, key, size, totalWritten);
-                }
-
                 foreach (int curPartNumber in etags.Keys)
                 {
                     if (curPartNumber > partNumber)
@@ -590,11 +583,19 @@ namespace Minio
             throw ParseError(response);
         }
 
-        private int CalculatePartSize(long size)
+        private long CalculatePartSize(long size)
         {
-            int minimumPartSize = PART_SIZE; // 5MB
-            int partSize = (int)(size / 9999); // using 10000 may cause part size to become too small, and not fit the entire object in
-            return Math.Max(minimumPartSize, partSize);
+            // make sure to have enough buffer for last part, use 9999 instead of 10000
+            long partSize = (size / 9999);
+            if (partSize > MinioClient.minimumPartSize)
+            {
+                if (partSize > MinioClient.maximumPartSize)
+                {
+                    return MinioClient.maximumPartSize;
+                }
+                return partSize;
+            }
+            return MinioClient.minimumPartSize;
         }
 
         private IEnumerable<Part> ListParts(string bucket, string key, string uploadId)