You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

2239 lines
90 KiB

  1. // Copyright (c) 2015-2021 MinIO, Inc.
  2. //
  3. // This file is part of MinIO Object Storage stack
  4. //
  5. // This program is free software: you can redistribute it and/or modify
  6. // it under the terms of the GNU Affero General Public License as published by
  7. // the Free Software Foundation, either version 3 of the License, or
  8. // (at your option) any later version.
  9. //
  10. // This program is distributed in the hope that it will be useful
  11. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. // GNU Affero General Public License for more details.
  14. //
  15. // You should have received a copy of the GNU Affero General Public License
  16. // along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. package cmd
  18. import (
  19. "bytes"
  20. "context"
  21. "fmt"
  22. "math/rand"
  23. "reflect"
  24. "runtime"
  25. "strings"
  26. "testing"
  27. "github.com/dustin/go-humanize"
  28. "github.com/minio/minio/internal/config/storageclass"
  29. "github.com/minio/minio/internal/hash"
  30. "github.com/minio/minio/internal/ioutil"
  31. )
  32. // Wrapper for calling NewMultipartUpload tests for both Erasure multiple disks and single node setup.
  33. func TestObjectNewMultipartUpload(t *testing.T) {
  34. if runtime.GOOS == globalWindowsOSName {
  35. t.Skip()
  36. }
  37. ExecObjectLayerTest(t, testObjectNewMultipartUpload)
  38. }
  39. // Tests validate creation of new multipart upload instance.
  40. func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) {
  41. bucket := "minio-bucket"
  42. object := "minio-object"
  43. opts := ObjectOptions{}
  44. _, err := obj.NewMultipartUpload(context.Background(), "--", object, opts)
  45. if err == nil {
  46. t.Fatalf("%s: Expected to fail since bucket name is invalid.", instanceType)
  47. }
  48. errMsg := "Bucket not found: minio-bucket"
  49. // operation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist.
  50. _, err = obj.NewMultipartUpload(context.Background(), bucket, object, opts)
  51. if err == nil {
  52. t.Fatalf("%s: Expected to fail since the NewMultipartUpload is initialized on a non-existent bucket.", instanceType)
  53. }
  54. if errMsg != err.Error() {
  55. t.Errorf("%s, Expected to fail with Error \"%s\", but instead found \"%s\".", instanceType, errMsg, err.Error())
  56. }
  57. // Create bucket before initiating NewMultipartUpload.
  58. err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
  59. if err != nil {
  60. // failed to create newbucket, abort.
  61. t.Fatalf("%s : %s", instanceType, err.Error())
  62. }
  63. res, err := obj.NewMultipartUpload(context.Background(), bucket, "\\", opts)
  64. if err != nil {
  65. t.Fatalf("%s : %s", instanceType, err.Error())
  66. }
  67. err = obj.AbortMultipartUpload(context.Background(), bucket, "\\", res.UploadID, opts)
  68. if err != nil {
  69. switch err.(type) {
  70. case InvalidUploadID:
  71. t.Fatalf("%s: New Multipart upload failed to create uuid file.", instanceType)
  72. default:
  73. t.Fatal(err.Error())
  74. }
  75. }
  76. }
  77. // Wrapper for calling AbortMultipartUpload tests for both Erasure multiple disks and single node setup.
  78. func TestObjectAbortMultipartUpload(t *testing.T) {
  79. ExecObjectLayerTest(t, testObjectAbortMultipartUpload)
  80. }
  81. // Tests validate creation of abort multipart upload instance.
  82. func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) {
  83. bucket := "minio-bucket"
  84. object := "minio-object"
  85. opts := ObjectOptions{}
  86. // Create bucket before initiating NewMultipartUpload.
  87. err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
  88. if err != nil {
  89. // failed to create newbucket, abort.
  90. t.Fatalf("%s : %s", instanceType, err.Error())
  91. }
  92. res, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts)
  93. if err != nil {
  94. t.Fatalf("%s : %s", instanceType, err.Error())
  95. }
  96. uploadID := res.UploadID
  97. abortTestCases := []struct {
  98. bucketName string
  99. objName string
  100. uploadID string
  101. expectedErrType error
  102. }{
  103. {"--", object, uploadID, BucketNameInvalid{}},
  104. {"foo", object, uploadID, BucketNotFound{}},
  105. {bucket, object, "foo-foo", InvalidUploadID{}},
  106. {bucket, object, uploadID, nil},
  107. }
  108. if runtime.GOOS != globalWindowsOSName {
  109. abortTestCases = append(abortTestCases, struct {
  110. bucketName string
  111. objName string
  112. uploadID string
  113. expectedErrType error
  114. }{bucket, "\\", uploadID, InvalidUploadID{}})
  115. }
  116. // Iterating over creatPartCases to generate multipart chunks.
  117. for i, testCase := range abortTestCases {
  118. err = obj.AbortMultipartUpload(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, opts)
  119. if testCase.expectedErrType == nil && err != nil {
  120. t.Errorf("Test %d, unexpected err is received: %v, expected:%v\n", i+1, err, testCase.expectedErrType)
  121. }
  122. if testCase.expectedErrType != nil && !isSameType(err, testCase.expectedErrType) {
  123. t.Errorf("Test %d, unexpected err is received: %v, expected:%v\n", i+1, err, testCase.expectedErrType)
  124. }
  125. }
  126. }
  127. // Wrapper for calling isUploadIDExists tests for both Erasure multiple disks and single node setup.
  128. func TestObjectAPIIsUploadIDExists(t *testing.T) {
  129. ExecObjectLayerTest(t, testObjectAPIIsUploadIDExists)
  130. }
  131. // Tests validates the validator for existence of uploadID.
  132. func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestErrHandler) {
  133. bucket := "minio-bucket"
  134. object := "minio-object"
  135. // Create bucket before initiating NewMultipartUpload.
  136. err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
  137. if err != nil {
  138. // Failed to create newbucket, abort.
  139. t.Fatalf("%s : %s", instanceType, err.Error())
  140. }
  141. _, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
  142. if err != nil {
  143. t.Fatalf("%s : %s", instanceType, err.Error())
  144. }
  145. opts := ObjectOptions{}
  146. err = obj.AbortMultipartUpload(context.Background(), bucket, object, "abc", opts)
  147. switch err.(type) {
  148. case InvalidUploadID:
  149. default:
  150. t.Fatalf("%s: Expected uploadIDPath to exist.", instanceType)
  151. }
  152. }
  153. // Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup.
  154. func TestObjectAPIPutObjectPart(t *testing.T) {
  155. ExecExtendedObjectLayerTest(t, testObjectAPIPutObjectPart)
  156. }
  157. // Tests validate correctness of PutObjectPart.
  158. func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrHandler) {
  159. // Generating cases for which the PutObjectPart fails.
  160. bucket := "minio-bucket"
  161. object := "minio-object"
  162. opts := ObjectOptions{}
  163. // Create bucket before initiating NewMultipartUpload.
  164. err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
  165. if err != nil {
  166. // Failed to create newbucket, abort.
  167. t.Fatalf("%s : %s", instanceType, err.Error())
  168. }
  169. // Initiate Multipart Upload on the above created bucket.
  170. res, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts)
  171. if err != nil {
  172. // Failed to create NewMultipartUpload, abort.
  173. t.Fatalf("%s : %s", instanceType, err.Error())
  174. }
  175. err = obj.MakeBucket(context.Background(), "abc", MakeBucketOptions{})
  176. if err != nil {
  177. // Failed to create newbucket, abort.
  178. t.Fatalf("%s : %s", instanceType, err.Error())
  179. }
  180. resN, err := obj.NewMultipartUpload(context.Background(), "abc", "def", opts)
  181. if err != nil {
  182. // Failed to create NewMultipartUpload, abort.
  183. t.Fatalf("%s : %s", instanceType, err.Error())
  184. }
  185. uploadID := res.UploadID
  186. // Creating a dummy bucket for tests.
  187. err = obj.MakeBucket(context.Background(), "unused-bucket", MakeBucketOptions{})
  188. if err != nil {
  189. // Failed to create newbucket, abort.
  190. t.Fatalf("%s : %s", instanceType, err.Error())
  191. }
  192. obj.DeleteBucket(context.Background(), "abc", DeleteBucketOptions{})
  193. // Collection of non-exhaustive PutObjectPart test cases, valid errors
  194. // and success responses.
  195. testCases := []struct {
  196. bucketName string
  197. objName string
  198. uploadID string
  199. PartID int
  200. inputReaderData string
  201. inputMd5 string
  202. inputSHA256 string
  203. inputDataSize int64
  204. // flag indicating whether the test should pass.
  205. shouldPass bool
  206. // expected error output.
  207. expectedMd5 string
  208. expectedError error
  209. }{
  210. // Test case 1-4.
  211. // Cases with invalid bucket name.
  212. {bucketName: ".test", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: .test")},
  213. {bucketName: "------", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: ------")},
  214. {
  215. bucketName: "$this-is-not-valid-too", objName: "obj", PartID: 1,
  216. expectedError: fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too"),
  217. },
  218. {bucketName: "a", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: a")},
  219. // Test case - 5.
  220. // Case with invalid object names.
  221. {bucketName: bucket, PartID: 1, expectedError: fmt.Errorf("%s", "Object name invalid: minio-bucket/")},
  222. // Test case - 6.
  223. // Valid object and bucket names but non-existent bucket.
  224. {bucketName: "abc", objName: "def", uploadID: resN.UploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: abc")},
  225. // Test Case - 7.
  226. // Existing bucket, but using a bucket on which NewMultipartUpload is not Initiated.
  227. {bucketName: "unused-bucket", objName: "def", uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")},
  228. // Test Case - 8.
  229. // Existing bucket, object name different from which NewMultipartUpload is constructed from.
  230. // Expecting "Invalid upload id".
  231. {bucketName: bucket, objName: "def", uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")},
  232. // Test Case - 9.
  233. // Existing bucket, bucket and object name are the ones from which NewMultipartUpload is constructed from.
  234. // But the uploadID is invalid.
  235. // Expecting "Invalid upload id".
  236. {bucketName: bucket, objName: object, uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")},
  237. // Test Case - 10.
  238. // Case with valid UploadID, existing bucket name.
  239. // But using the bucket name from which NewMultipartUpload is not constructed from.
  240. {bucketName: "unused-bucket", objName: object, uploadID: uploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id "+uploadID)},
  241. // Test Case - 11.
  242. // Case with valid UploadID, existing bucket name.
  243. // But using the object name from which NewMultipartUpload is not constructed from.
  244. {bucketName: bucket, objName: "none-object", uploadID: uploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id "+uploadID)},
  245. // Test case - 12.
  246. // Input to replicate Md5 mismatch.
  247. {
  248. bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputMd5: "d41d8cd98f00b204e9800998ecf8427f",
  249. expectedError: hash.BadDigest{ExpectedMD5: "d41d8cd98f00b204e9800998ecf8427f", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"},
  250. },
  251. // Test case - 13.
  252. // When incorrect sha256 is provided.
  253. {
  254. bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854",
  255. expectedError: hash.SHA256Mismatch{
  256. ExpectedSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854",
  257. CalculatedSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
  258. },
  259. },
  260. // Test case - 14.
  261. // Input with size more than the size of actual data inside the reader.
  262. {
  263. bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputReaderData: "abcd", inputMd5: "e2fc714c4727ee9395f324cd2e7f3335", inputDataSize: int64(len("abcd") + 1),
  264. expectedError: hash.BadDigest{ExpectedMD5: "e2fc714c4727ee9395f324cd2e7f3335", CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f"},
  265. },
  266. // Test case - 15.
  267. // Input with size less than the size of actual data inside the reader.
  268. {
  269. bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputReaderData: "abcd", inputMd5: "900150983cd24fb0d6963f7d28e17f73", inputDataSize: int64(len("abcd") - 1),
  270. expectedError: ioutil.ErrOverread,
  271. },
  272. // Test case - 16-19.
  273. // Validating for success cases.
  274. {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputReaderData: "abcd", inputMd5: "e2fc714c4727ee9395f324cd2e7f331f", inputSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", inputDataSize: int64(len("abcd")), shouldPass: true},
  275. {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 2, inputReaderData: "efgh", inputMd5: "1f7690ebdd9b4caf8fab49ca1757bf27", inputSHA256: "e5e088a0b66163a0a26a5e053d2a4496dc16ab6e0e3dd1adf2d16aa84a078c9d", inputDataSize: int64(len("efgh")), shouldPass: true},
  276. {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 3, inputReaderData: "ijkl", inputMd5: "09a0877d04abf8759f99adec02baf579", inputSHA256: "005c19658919186b85618c5870463eec8d9b8c1a9d00208a5352891ba5bbe086", inputDataSize: int64(len("abcd")), shouldPass: true},
  277. {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 4, inputReaderData: "mnop", inputMd5: "e132e96a5ddad6da8b07bba6f6131fef", inputSHA256: "f1afc31479522d6cff1ed068f93998f05a8cd3b22f5c37d7f307084f62d1d270", inputDataSize: int64(len("abcd")), shouldPass: true},
  278. }
  279. // Validate all the test cases.
  280. for i, testCase := range testCases {
  281. actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, testCase.inputSHA256), opts)
  282. // All are test cases above are expected to fail.
  283. if actualErr != nil && testCase.shouldPass {
  284. t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
  285. }
  286. if actualErr == nil && !testCase.shouldPass {
  287. t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error())
  288. }
  289. // Failed as expected, but does it fail for the expected reason.
  290. if actualErr != nil && !testCase.shouldPass {
  291. if testCase.expectedError.Error() != actualErr.Error() {
  292. t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
  293. }
  294. }
  295. // Test passes as expected, but the output values are verified for correctness here.
  296. if actualErr == nil && testCase.shouldPass {
  297. // Asserting whether the md5 output is correct.
  298. if testCase.inputMd5 != actualInfo.ETag {
  299. t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualInfo.ETag)
  300. }
  301. }
  302. }
  303. }
  304. // Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup.
  305. func TestListMultipartUploads(t *testing.T) {
  306. ExecExtendedObjectLayerTest(t, testListMultipartUploads)
  307. }
  308. // testListMultipartUploads - Tests validate listing of multipart uploads.
  309. func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHandler) {
  310. bucketNames := []string{"minio-bucket", "minio-2-bucket", "minio-3-bucket"}
  311. objectNames := []string{"minio-object-1.txt", "minio-object.txt", "neymar-1.jpeg", "neymar.jpeg", "parrot-1.png", "parrot.png"}
  312. uploadIDs := []string{}
  313. opts := ObjectOptions{}
  314. // bucketnames[0].
  315. // objectNames[0].
  316. // uploadIds [0].
  317. // Create bucket before initiating NewMultipartUpload.
  318. err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{})
  319. if err != nil {
  320. // Failed to create newbucket, abort.
  321. t.Fatalf("%s : %s", instanceType, err.Error())
  322. }
  323. // Initiate Multipart Upload on the above created bucket.
  324. res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
  325. if err != nil {
  326. // Failed to create NewMultipartUpload, abort.
  327. t.Fatalf("%s : %s", instanceType, err.Error())
  328. }
  329. uploadIDs = append(uploadIDs, res.UploadID)
  330. // bucketnames[1].
  331. // objectNames[0].
  332. // uploadIds [1-3].
  333. // Bucket to test for multiple upload Id's for a given object.
  334. err = obj.MakeBucket(context.Background(), bucketNames[1], MakeBucketOptions{})
  335. if err != nil {
  336. // Failed to create newbucket, abort.
  337. t.Fatalf("%s : %s", instanceType, err.Error())
  338. }
  339. for i := 0; i < 3; i++ {
  340. // Initiate Multipart Upload on bucketNames[1] for the same object 3 times.
  341. // Used to test the listing for the case of multiple uploadID's for a given object.
  342. res, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], opts)
  343. if err != nil {
  344. // Failed to create NewMultipartUpload, abort.
  345. t.Fatalf("%s : %s", instanceType, err.Error())
  346. }
  347. uploadIDs = append(uploadIDs, res.UploadID)
  348. }
  349. // Bucket to test for multiple objects, each with unique UUID.
  350. // bucketnames[2].
  351. // objectNames[0-2].
  352. // uploadIds [4-9].
  353. err = obj.MakeBucket(context.Background(), bucketNames[2], MakeBucketOptions{})
  354. if err != nil {
  355. // Failed to create newbucket, abort.
  356. t.Fatalf("%s : %s", instanceType, err.Error())
  357. }
  358. // Initiate Multipart Upload on bucketNames[2].
  359. // Used to test the listing for the case of multiple objects for a given bucket.
  360. for i := 0; i < 6; i++ {
  361. res, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], opts)
  362. if err != nil {
  363. // Failed to create NewMultipartUpload, abort.
  364. t.Fatalf("%s : %s", instanceType, err.Error())
  365. }
  366. // uploadIds [4-9].
  367. uploadIDs = append(uploadIDs, res.UploadID)
  368. }
  369. // Create multipart parts.
  370. // Need parts to be uploaded before MultipartLists can be called and tested.
  371. createPartCases := []struct {
  372. bucketName string
  373. objName string
  374. uploadID string
  375. PartID int
  376. inputReaderData string
  377. inputMd5 string
  378. inputDataSize int64
  379. expectedMd5 string
  380. }{
  381. // Case 1-4.
  382. // Creating sequence of parts for same uploadID.
  383. // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
  384. {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  385. {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
  386. {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
  387. {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
  388. // Cases 5-7.
  389. // Create parts with 3 uploadID's for the same object.
  390. // Testing for listing of all the uploadID's for given object.
  391. // Insertion with 3 different uploadID's are done for same bucket and object.
  392. {bucketNames[1], objectNames[0], uploadIDs[1], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  393. {bucketNames[1], objectNames[0], uploadIDs[2], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  394. {bucketNames[1], objectNames[0], uploadIDs[3], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  395. // Case 8-13.
  396. // Generating parts for different objects.
  397. {bucketNames[2], objectNames[0], uploadIDs[4], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  398. {bucketNames[2], objectNames[1], uploadIDs[5], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  399. {bucketNames[2], objectNames[2], uploadIDs[6], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  400. {bucketNames[2], objectNames[3], uploadIDs[7], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  401. {bucketNames[2], objectNames[4], uploadIDs[8], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  402. {bucketNames[2], objectNames[5], uploadIDs[9], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  403. }
  404. sha256sum := ""
  405. // Iterating over creatPartCases to generate multipart chunks.
  406. for _, testCase := range createPartCases {
  407. _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts)
  408. if err != nil {
  409. t.Fatalf("%s : %s", instanceType, err.Error())
  410. }
  411. }
  412. // Expected Results set for asserting ListObjectMultipart test.
  413. listMultipartResults := []ListMultipartsInfo{
  414. // listMultipartResults - 1.
  415. // Used to check that the result produces only one output for the 4 parts uploaded in cases 1-4 of createPartCases above.
  416. // ListMultipartUploads doesn't list the parts.
  417. {
  418. MaxUploads: 100,
  419. Uploads: []MultipartInfo{
  420. {
  421. Object: objectNames[0],
  422. UploadID: uploadIDs[0],
  423. },
  424. },
  425. },
  426. // listMultipartResults - 2.
  427. // Used to check that the result produces if keyMarker is set to the only available object.
  428. // `KeyMarker` is set.
  429. // ListMultipartUploads doesn't list the parts.
  430. {
  431. MaxUploads: 100,
  432. KeyMarker: "minio-object-1.txt",
  433. },
  434. // listMultipartResults - 3.
  435. // `KeyMarker` is set, no MultipartInfo expected.
  436. // ListMultipartUploads doesn't list the parts.
  437. // `Maxupload` value is asserted.
  438. {
  439. MaxUploads: 100,
  440. KeyMarker: "orange",
  441. },
  442. // listMultipartResults - 4.
  443. // `KeyMarker` is set, no MultipartInfo expected.
  444. // Maxupload value is asserted.
  445. {
  446. MaxUploads: 1,
  447. KeyMarker: "orange",
  448. },
  449. // listMultipartResults - 5.
  450. // `KeyMarker` is set. It contains part of the objectname as `KeyPrefix`.
  451. // Expecting the result to contain one MultipartInfo entry and Istruncated to be false.
  452. {
  453. MaxUploads: 10,
  454. KeyMarker: "min",
  455. IsTruncated: false,
  456. Uploads: []MultipartInfo{
  457. {
  458. Object: objectNames[0],
  459. UploadID: uploadIDs[0],
  460. },
  461. },
  462. },
  463. // listMultipartResults - 6.
  464. // `KeyMarker` is set. It contains part of the objectname as `KeyPrefix`.
  465. // `MaxUploads` is set equal to the number of meta data entries in the result, the result contains only one entry.
  466. // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false.
  467. {
  468. MaxUploads: 1,
  469. KeyMarker: "min",
  470. IsTruncated: false,
  471. Uploads: []MultipartInfo{
  472. {
  473. Object: objectNames[0],
  474. UploadID: uploadIDs[0],
  475. },
  476. },
  477. },
  478. // listMultipartResults - 7.
  479. // `KeyMarker` is set. It contains part of the objectname as `KeyPrefix`.
  480. // Testing for the case with `MaxUploads` set to 0.
  481. // Expecting the result to contain no MultipartInfo entry since `MaxUploads` is set to 0.
  482. // Expecting `IsTruncated` to be true.
  483. {
  484. MaxUploads: 0,
  485. KeyMarker: "min",
  486. IsTruncated: true,
  487. },
  488. // listMultipartResults - 8.
  489. // `KeyMarker` is set. It contains part of the objectname as KeyPrefix.
  490. // Testing for the case with `MaxUploads` set to 0.
  491. // Expecting the result to contain no MultipartInfo entry since `MaxUploads` is set to 0.
  492. // Expecting `isTruncated` to be true.
  493. {
  494. MaxUploads: 0,
  495. KeyMarker: "min",
  496. IsTruncated: true,
  497. },
  498. // listMultipartResults - 9.
  499. // `KeyMarker` is set. It contains part of the objectname as KeyPrefix.
  500. // `KeyMarker` is set equal to the object name in the result.
  501. // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false.
  502. {
  503. MaxUploads: 2,
  504. KeyMarker: "minio-object",
  505. IsTruncated: false,
  506. Uploads: []MultipartInfo{
  507. {
  508. Object: objectNames[0],
  509. UploadID: uploadIDs[0],
  510. },
  511. },
  512. },
  513. // listMultipartResults - 10.
  514. // Prefix is set. It is set equal to the object name.
  515. // MaxUploads is set more than number of meta data entries in the result.
  516. // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false.
  517. {
  518. MaxUploads: 2,
  519. Prefix: "minio-object-1.txt",
  520. IsTruncated: false,
  521. Uploads: []MultipartInfo{
  522. {
  523. Object: objectNames[0],
  524. UploadID: uploadIDs[0],
  525. },
  526. },
  527. },
  528. // listMultipartResults - 11.
  529. // Setting `Prefix` to contain the object name as its prefix.
  530. // MaxUploads is set more than number of meta data entries in the result.
  531. // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false.
  532. {
  533. MaxUploads: 2,
  534. Prefix: "min",
  535. IsTruncated: false,
  536. Uploads: []MultipartInfo{
  537. {
  538. Object: objectNames[0],
  539. UploadID: uploadIDs[0],
  540. },
  541. },
  542. },
  543. // listMultipartResults - 12.
  544. // Setting `Prefix` to contain the object name as its prefix.
  545. // MaxUploads is set equal to number of meta data entries in the result.
  546. // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false.
  547. {
  548. MaxUploads: 1,
  549. Prefix: "min",
  550. IsTruncated: false,
  551. Uploads: []MultipartInfo{
  552. {
  553. Object: objectNames[0],
  554. UploadID: uploadIDs[0],
  555. },
  556. },
  557. },
  558. // listMultipartResults - 13.
  559. // `Prefix` is set. It doesn't contain object name as its preifx.
  560. // MaxUploads is set more than number of meta data entries in the result.
  561. // Expecting no `Uploads` metadata.
  562. {
  563. MaxUploads: 2,
  564. Prefix: "orange",
  565. IsTruncated: false,
  566. },
  567. // listMultipartResults - 14.
  568. // `Prefix` is set. It doesn't contain object name as its preifx.
  569. // MaxUploads is set more than number of meta data entries in the result.
  570. // Expecting the result to contain 0 uploads and isTruncated to false.
  571. {
  572. MaxUploads: 2,
  573. Prefix: "Asia",
  574. IsTruncated: false,
  575. },
  576. // listMultipartResults - 15.
  577. // Setting `Delimiter`.
  578. // MaxUploads is set more than number of meta data entries in the result.
  579. // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false.
  580. {
  581. MaxUploads: 2,
  582. Delimiter: SlashSeparator,
  583. Prefix: "",
  584. IsTruncated: false,
  585. Uploads: []MultipartInfo{
  586. {
  587. Object: objectNames[0],
  588. UploadID: uploadIDs[0],
  589. },
  590. },
  591. },
  592. // listMultipartResults - 16.
  593. // Testing for listing of 3 uploadID's for a given object.
  594. // Will be used to list on bucketNames[1].
  595. {
  596. MaxUploads: 100,
  597. Uploads: []MultipartInfo{
  598. {
  599. Object: objectNames[0],
  600. UploadID: uploadIDs[1],
  601. },
  602. {
  603. Object: objectNames[0],
  604. UploadID: uploadIDs[2],
  605. },
  606. {
  607. Object: objectNames[0],
  608. UploadID: uploadIDs[3],
  609. },
  610. },
  611. },
  612. // listMultipartResults - 17.
  613. // Testing for listing of 3 uploadID's (uploadIDs[1-3]) for a given object with uploadID Marker set.
  614. // uploadIDs[1] is set as UploadMarker, Expecting it to be skipped in the result.
  615. // uploadIDs[2] and uploadIDs[3] are expected to be in the result.
  616. // Istruncted is expected to be false.
  617. // Will be used to list on bucketNames[1].
  618. {
  619. MaxUploads: 100,
  620. KeyMarker: "minio-object-1.txt",
  621. UploadIDMarker: uploadIDs[1],
  622. IsTruncated: false,
  623. Uploads: []MultipartInfo{
  624. {
  625. Object: objectNames[0],
  626. UploadID: uploadIDs[2],
  627. },
  628. {
  629. Object: objectNames[0],
  630. UploadID: uploadIDs[3],
  631. },
  632. },
  633. },
  634. // listMultipartResults - 18.
  635. // Testing for listing of 3 uploadID's (uploadIDs[1-3]) for a given object with uploadID Marker set.
  636. // uploadIDs[2] is set as UploadMarker, Expecting it to be skipped in the result.
  637. // Only uploadIDs[3] are expected to be in the result.
  638. // Istruncted is expected to be false.
  639. // Will be used to list on bucketNames[1].
  640. {
  641. MaxUploads: 100,
  642. KeyMarker: "minio-object-1.txt",
  643. UploadIDMarker: uploadIDs[2],
  644. IsTruncated: false,
  645. Uploads: []MultipartInfo{
  646. {
  647. Object: objectNames[0],
  648. UploadID: uploadIDs[3],
  649. },
  650. },
  651. },
  652. // listMultipartResults - 19.
  653. // Testing for listing of 3 uploadID's for a given object, setting maxKeys to be 2.
  654. // There are 3 MultipartInfo in the result (uploadIDs[1-3]), it should be truncated to 2.
  655. // Since there is only single object for bucketNames[1], the NextKeyMarker is set to its name.
  656. // The last entry in the result, uploadIDs[2], that is should be set as NextUploadIDMarker.
  657. // Will be used to list on bucketNames[1].
  658. {
  659. MaxUploads: 2,
  660. IsTruncated: true,
  661. NextKeyMarker: objectNames[0],
  662. NextUploadIDMarker: uploadIDs[2],
  663. Uploads: []MultipartInfo{
  664. {
  665. Object: objectNames[0],
  666. UploadID: uploadIDs[1],
  667. },
  668. {
  669. Object: objectNames[0],
  670. UploadID: uploadIDs[2],
  671. },
  672. },
  673. },
  674. // listMultipartResults - 20.
  675. // Testing for listing of 3 uploadID's for a given object, setting maxKeys to be 1.
  676. // There are 3 MultipartInfo in the result (uploadIDs[1-3]), it should be truncated to 1.
  677. // The last entry in the result, uploadIDs[1], that is should be set as NextUploadIDMarker.
  678. // Will be used to list on bucketNames[1].
  679. {
  680. MaxUploads: 1,
  681. IsTruncated: true,
  682. NextKeyMarker: objectNames[0],
  683. NextUploadIDMarker: uploadIDs[1],
  684. Uploads: []MultipartInfo{
  685. {
  686. Object: objectNames[0],
  687. UploadID: uploadIDs[1],
  688. },
  689. },
  690. },
  691. // listMultipartResults - 21.
  692. // Testing for listing of 3 uploadID's for a given object, setting maxKeys to be 3.
  693. // There are 3 MultipartInfo in the result (uploadIDs[1-3]), hence no truncation is expected.
  694. // Since all the MultipartInfo is listed, expecting no values for NextUploadIDMarker and NextKeyMarker.
  695. // Will be used to list on bucketNames[1].
  696. {
  697. MaxUploads: 3,
  698. IsTruncated: false,
  699. Uploads: []MultipartInfo{
  700. {
  701. Object: objectNames[0],
  702. UploadID: uploadIDs[1],
  703. },
  704. {
  705. Object: objectNames[0],
  706. UploadID: uploadIDs[2],
  707. },
  708. {
  709. Object: objectNames[0],
  710. UploadID: uploadIDs[3],
  711. },
  712. },
  713. },
  714. // listMultipartResults - 22.
  715. // Testing for listing of 3 uploadID's for a given object, setting `prefix` to be "min".
  716. // Will be used to list on bucketNames[1].
  717. {
  718. MaxUploads: 10,
  719. IsTruncated: false,
  720. Prefix: "min",
  721. Uploads: []MultipartInfo{
  722. {
  723. Object: objectNames[0],
  724. UploadID: uploadIDs[1],
  725. },
  726. {
  727. Object: objectNames[0],
  728. UploadID: uploadIDs[2],
  729. },
  730. {
  731. Object: objectNames[0],
  732. UploadID: uploadIDs[3],
  733. },
  734. },
  735. },
  736. // listMultipartResults - 23.
  737. // Testing for listing of 3 uploadID's for a given object
  738. // setting `prefix` to be "orange".
  739. // Will be used to list on bucketNames[1].
  740. {
  741. MaxUploads: 10,
  742. IsTruncated: false,
  743. Prefix: "orange",
  744. },
  745. // listMultipartResults - 24.
  746. // Testing for listing of 3 uploadID's for a given object.
  747. // setting `prefix` to be "Asia".
  748. // Will be used to list on bucketNames[1].
  749. {
  750. MaxUploads: 10,
  751. IsTruncated: false,
  752. Prefix: "Asia",
  753. },
  754. // listMultipartResults - 25.
  755. // Testing for listing of 3 uploadID's for a given object.
  756. // setting `prefix` and uploadIDMarker.
  757. // Will be used to list on bucketNames[1].
  758. {
  759. MaxUploads: 10,
  760. KeyMarker: "minio-object-1.txt",
  761. IsTruncated: false,
  762. Prefix: "min",
  763. UploadIDMarker: uploadIDs[1],
  764. Uploads: []MultipartInfo{
  765. {
  766. Object: objectNames[0],
  767. UploadID: uploadIDs[2],
  768. },
  769. {
  770. Object: objectNames[0],
  771. UploadID: uploadIDs[3],
  772. },
  773. },
  774. },
  775. // Operations on bucket 2.
  776. // listMultipartResults - 26.
  777. // checking listing everything.
  778. {
  779. MaxUploads: 100,
  780. IsTruncated: false,
  781. Uploads: []MultipartInfo{
  782. {
  783. Object: objectNames[0],
  784. UploadID: uploadIDs[4],
  785. },
  786. {
  787. Object: objectNames[1],
  788. UploadID: uploadIDs[5],
  789. },
  790. {
  791. Object: objectNames[2],
  792. UploadID: uploadIDs[6],
  793. },
  794. {
  795. Object: objectNames[3],
  796. UploadID: uploadIDs[7],
  797. },
  798. {
  799. Object: objectNames[4],
  800. UploadID: uploadIDs[8],
  801. },
  802. {
  803. Object: objectNames[5],
  804. UploadID: uploadIDs[9],
  805. },
  806. },
  807. },
  808. // listMultipartResults - 27.
  809. // listing with `prefix` "min".
  810. {
  811. MaxUploads: 100,
  812. IsTruncated: false,
  813. Prefix: "min",
  814. Uploads: []MultipartInfo{
  815. {
  816. Object: objectNames[0],
  817. UploadID: uploadIDs[4],
  818. },
  819. {
  820. Object: objectNames[1],
  821. UploadID: uploadIDs[5],
  822. },
  823. },
  824. },
  825. // listMultipartResults - 28.
  826. // listing with `prefix` "ney".
  827. {
  828. MaxUploads: 100,
  829. IsTruncated: false,
  830. Prefix: "ney",
  831. Uploads: []MultipartInfo{
  832. {
  833. Object: objectNames[2],
  834. UploadID: uploadIDs[6],
  835. },
  836. {
  837. Object: objectNames[3],
  838. UploadID: uploadIDs[7],
  839. },
  840. },
  841. },
  842. // listMultipartResults - 29.
  843. // listing with `prefix` "parrot".
  844. {
  845. MaxUploads: 100,
  846. IsTruncated: false,
  847. Prefix: "parrot",
  848. Uploads: []MultipartInfo{
  849. {
  850. Object: objectNames[4],
  851. UploadID: uploadIDs[8],
  852. },
  853. {
  854. Object: objectNames[5],
  855. UploadID: uploadIDs[9],
  856. },
  857. },
  858. },
  859. // listMultipartResults - 30.
  860. // listing with `prefix` "neymar.jpeg".
  861. // prefix set to object name.
  862. {
  863. MaxUploads: 100,
  864. IsTruncated: false,
  865. Prefix: "neymar.jpeg",
  866. Uploads: []MultipartInfo{
  867. {
  868. Object: objectNames[3],
  869. UploadID: uploadIDs[7],
  870. },
  871. },
  872. },
  873. // listMultipartResults - 31.
  874. // checking listing with marker set to 3.
  875. // `NextUploadIDMarker` is expected to be set on last uploadID in the result.
  876. // `NextKeyMarker` is expected to be set on the last object key in the list.
  877. {
  878. MaxUploads: 3,
  879. IsTruncated: true,
  880. NextUploadIDMarker: uploadIDs[6],
  881. NextKeyMarker: objectNames[2],
  882. Uploads: []MultipartInfo{
  883. {
  884. Object: objectNames[0],
  885. UploadID: uploadIDs[4],
  886. },
  887. {
  888. Object: objectNames[1],
  889. UploadID: uploadIDs[5],
  890. },
  891. {
  892. Object: objectNames[2],
  893. UploadID: uploadIDs[6],
  894. },
  895. },
  896. },
  897. // listMultipartResults - 32.
  898. // checking listing with marker set to no of objects in the list.
  899. // `NextUploadIDMarker` is expected to be empty since all results are listed.
  900. // `NextKeyMarker` is expected to be empty since all results are listed.
  901. {
  902. MaxUploads: 6,
  903. IsTruncated: false,
  904. Uploads: []MultipartInfo{
  905. {
  906. Object: objectNames[0],
  907. UploadID: uploadIDs[4],
  908. },
  909. {
  910. Object: objectNames[1],
  911. UploadID: uploadIDs[5],
  912. },
  913. {
  914. Object: objectNames[2],
  915. UploadID: uploadIDs[6],
  916. },
  917. {
  918. Object: objectNames[3],
  919. UploadID: uploadIDs[7],
  920. },
  921. {
  922. Object: objectNames[4],
  923. UploadID: uploadIDs[8],
  924. },
  925. {
  926. Object: objectNames[5],
  927. UploadID: uploadIDs[9],
  928. },
  929. },
  930. },
  931. // listMultipartResults - 33.
  932. // checking listing with `UploadIDMarker` set.
  933. {
  934. MaxUploads: 10,
  935. IsTruncated: false,
  936. UploadIDMarker: uploadIDs[6],
  937. Uploads: []MultipartInfo{
  938. {
  939. Object: objectNames[3],
  940. UploadID: uploadIDs[7],
  941. },
  942. {
  943. Object: objectNames[4],
  944. UploadID: uploadIDs[8],
  945. },
  946. {
  947. Object: objectNames[5],
  948. UploadID: uploadIDs[9],
  949. },
  950. },
  951. },
  952. // listMultipartResults - 34.
  953. // checking listing with `KeyMarker` set.
  954. {
  955. MaxUploads: 10,
  956. IsTruncated: false,
  957. KeyMarker: objectNames[3],
  958. Uploads: []MultipartInfo{
  959. {
  960. Object: objectNames[4],
  961. UploadID: uploadIDs[8],
  962. },
  963. {
  964. Object: objectNames[5],
  965. UploadID: uploadIDs[9],
  966. },
  967. },
  968. },
  969. // listMultipartResults - 35.
  970. // Checking listing with `Prefix` and `KeyMarker`.
  971. // No upload MultipartInfo in the result expected since KeyMarker is set to last Key in the result.
  972. {
  973. MaxUploads: 10,
  974. IsTruncated: false,
  975. Prefix: "minio-object",
  976. KeyMarker: objectNames[1],
  977. },
  978. // listMultipartResults - 36.
  979. // checking listing with `Prefix` and `UploadIDMarker` set.
  980. {
  981. MaxUploads: 10,
  982. IsTruncated: false,
  983. Prefix: "minio",
  984. UploadIDMarker: uploadIDs[4],
  985. Uploads: []MultipartInfo{
  986. {
  987. Object: objectNames[1],
  988. UploadID: uploadIDs[5],
  989. },
  990. },
  991. },
  992. // listMultipartResults - 37.
  993. // Checking listing with `KeyMarker` and `UploadIDMarker` set.
  994. {
  995. MaxUploads: 10,
  996. IsTruncated: false,
  997. KeyMarker: "minio-object.txt",
  998. UploadIDMarker: uploadIDs[5],
  999. },
  1000. }
  1001. // Collection of non-exhaustive ListMultipartUploads test cases, valid errors
  1002. // and success responses.
  1003. testCases := []struct {
  1004. // Inputs to ListMultipartUploads.
  1005. bucket string
  1006. prefix string
  1007. keyMarker string
  1008. uploadIDMarker string
  1009. delimiter string
  1010. maxUploads int
  1011. // Expected output of ListMultipartUploads.
  1012. expectedResult ListMultipartsInfo
  1013. expectedErr error
  1014. // Flag indicating whether the test is expected to pass or not.
  1015. shouldPass bool
  1016. }{
  1017. // Test cases with invalid bucket names ( Test number 1-4 ).
  1018. {".test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
  1019. {"Test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
  1020. {"---", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
  1021. {"ad", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
  1022. // Valid bucket names, but they do not exist (Test number 5-7).
  1023. {"volatile-bucket-1", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
  1024. {"volatile-bucket-2", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
  1025. {"volatile-bucket-3", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
  1026. // Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9).
  1027. {bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true},
  1028. {bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true},
  1029. // If marker is *after* the last possible object from the prefix it should return an empty list.
  1030. {
  1031. bucketNames[0], "Asia", "europe-object", "", "", 0,
  1032. ListMultipartsInfo{KeyMarker: "europe-object", Prefix: "Asia", IsTruncated: false},
  1033. nil, true,
  1034. },
  1035. // Setting an invalid combination of uploadIDMarker and Marker (Test number 11-12).
  1036. {
  1037. bucketNames[0], "asia", "asia/europe/", "abc", "", 0,
  1038. ListMultipartsInfo{},
  1039. fmt.Errorf("Invalid combination of uploadID marker '%s' and marker '%s'", "abc", "asia/europe/"), false,
  1040. },
  1041. {
  1042. // Contains a base64 padding character
  1043. bucketNames[0], "asia", "asia/europe", "abc=", "", 0,
  1044. ListMultipartsInfo{},
  1045. fmt.Errorf("Malformed upload id %s", "abc="), false,
  1046. },
  1047. // Setting up valid case of ListMultiPartUploads.
  1048. // Test case with multiple parts for a single uploadID (Test number 13).
  1049. {bucketNames[0], "", "", "", "", 100, listMultipartResults[0], nil, true},
  1050. // Test with a KeyMarker (Test number 14-17).
  1051. {bucketNames[0], "", "minio-object-1.txt", "", "", 100, listMultipartResults[1], nil, true},
  1052. {bucketNames[0], "", "orange", "", "", 100, listMultipartResults[2], nil, true},
  1053. {bucketNames[0], "", "orange", "", "", 1, listMultipartResults[3], nil, true},
  1054. {bucketNames[0], "", "min", "", "", 10, listMultipartResults[4], nil, true},
  1055. // Test case with keyMarker set equal to number of parts in the result. (Test number 18).
  1056. {bucketNames[0], "", "min", "", "", 1, listMultipartResults[5], nil, true},
  1057. // Test case with keyMarker set to 0. (Test number 19).
  1058. {bucketNames[0], "", "min", "", "", 0, listMultipartResults[6], nil, true},
  1059. // Test case with keyMarker less than 0. (Test number 20).
  1060. // {bucketNames[0], "", "min", "", "", -1, listMultipartResults[7], nil, true},
  1061. // The result contains only one entry. The KeyPrefix is set to the object name in the result.
  1062. // Expecting the result to skip the KeyPrefix entry in the result (Test number 21).
  1063. {bucketNames[0], "", "minio-object", "", "", 2, listMultipartResults[8], nil, true},
  1064. // Test case containing prefix values.
  1065. // Setting prefix to be equal to object name.(Test number 22).
  1066. {bucketNames[0], "minio-object-1.txt", "", "", "", 2, listMultipartResults[9], nil, true},
  1067. // Setting `prefix` to contain the object name as its prefix (Test number 23).
  1068. {bucketNames[0], "min", "", "", "", 2, listMultipartResults[10], nil, true},
  1069. // Setting `prefix` to contain the object name as its prefix (Test number 24).
  1070. {bucketNames[0], "min", "", "", "", 1, listMultipartResults[11], nil, true},
  1071. // Setting `prefix` to not to contain the object name as its prefix (Test number 25-26).
  1072. {bucketNames[0], "orange", "", "", "", 2, listMultipartResults[12], nil, true},
  1073. {bucketNames[0], "Asia", "", "", "", 2, listMultipartResults[13], nil, true},
  1074. // setting delimiter (Test number 27).
  1075. {bucketNames[0], "", "", "", SlashSeparator, 2, listMultipartResults[14], nil, true},
  1076. // Test case with multiple uploadID listing for given object (Test number 28).
  1077. {bucketNames[1], "", "", "", "", 100, listMultipartResults[15], nil, true},
  1078. // Test case with multiple uploadID listing for given object, but uploadID marker set.
  1079. // Testing whether the marker entry is skipped (Test number 29-30).
  1080. {bucketNames[1], "", "minio-object-1.txt", uploadIDs[1], "", 100, listMultipartResults[16], nil, true},
  1081. {bucketNames[1], "", "minio-object-1.txt", uploadIDs[2], "", 100, listMultipartResults[17], nil, true},
  1082. // Test cases with multiple uploadID listing for a given object (Test number 31-32).
  1083. // MaxKeys set to values lesser than the number of entries in the MultipartInfo.
  1084. // IsTruncated is expected to be true.
  1085. {bucketNames[1], "", "", "", "", 2, listMultipartResults[18], nil, true},
  1086. {bucketNames[1], "", "", "", "", 1, listMultipartResults[19], nil, true},
  1087. // MaxKeys set to the value which is equal to no of entries in the MultipartInfo (Test number 33).
  1088. // In case of bucketNames[1], there are 3 entries.
  1089. // Since all available entries are listed, IsTruncated is expected to be false
  1090. // and NextMarkers are expected to empty.
  1091. {bucketNames[1], "", "", "", "", 3, listMultipartResults[20], nil, true},
  1092. // Adding prefix (Test number 34-36).
  1093. {bucketNames[1], "min", "", "", "", 10, listMultipartResults[21], nil, true},
  1094. {bucketNames[1], "orange", "", "", "", 10, listMultipartResults[22], nil, true},
  1095. {bucketNames[1], "Asia", "", "", "", 10, listMultipartResults[23], nil, true},
  1096. // Test case with `Prefix` and `UploadIDMarker` (Test number 37).
  1097. {bucketNames[1], "min", "minio-object-1.txt", uploadIDs[1], "", 10, listMultipartResults[24], nil, true},
  1098. // Test case for bucket with multiple objects in it.
  1099. // Bucket used : `bucketNames[2]`.
  1100. // Objects used: `objectNames[1-5]`.
  1101. // UploadId's used: uploadIds[4-8].
  1102. // (Test number 39).
  1103. {bucketNames[2], "", "", "", "", 100, listMultipartResults[25], nil, true},
  1104. // Test cases with prefixes.
  1105. // Testing listing with prefix set to "min" (Test number 40) .
  1106. {bucketNames[2], "min", "", "", "", 100, listMultipartResults[26], nil, true},
  1107. // Testing listing with prefix set to "ney" (Test number 41).
  1108. {bucketNames[2], "ney", "", "", "", 100, listMultipartResults[27], nil, true},
  1109. // Testing listing with prefix set to "par" (Test number 42).
  1110. {bucketNames[2], "parrot", "", "", "", 100, listMultipartResults[28], nil, true},
  1111. // Testing listing with prefix set to object name "neymar.jpeg" (Test number 43).
  1112. {bucketNames[2], "neymar.jpeg", "", "", "", 100, listMultipartResults[29], nil, true},
  1113. // Testing listing with `MaxUploads` set to 3 (Test number 44).
  1114. {bucketNames[2], "", "", "", "", 3, listMultipartResults[30], nil, true},
  1115. // In case of bucketNames[2], there are 6 entries (Test number 45).
  1116. // Since all available entries are listed, IsTruncated is expected to be false
  1117. // and NextMarkers are expected to empty.
  1118. {bucketNames[2], "", "", "", "", 6, listMultipartResults[31], nil, true},
  1119. // Test case with `KeyMarker` (Test number 47).
  1120. {bucketNames[2], "", objectNames[3], "", "", 10, listMultipartResults[33], nil, true},
  1121. // Test case with `prefix` and `KeyMarker` (Test number 48).
  1122. {bucketNames[2], "minio-object", objectNames[1], "", "", 10, listMultipartResults[34], nil, true},
  1123. }
  1124. for i, testCase := range testCases {
  1125. // fmt.Println(i+1, testCase) // uncomment to peek into the test cases.
  1126. actualResult, actualErr := obj.ListMultipartUploads(context.Background(), testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
  1127. if actualErr != nil && testCase.shouldPass {
  1128. t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
  1129. }
  1130. if actualErr == nil && !testCase.shouldPass {
  1131. t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error())
  1132. }
  1133. // Failed as expected, but does it fail for the expected reason.
  1134. if actualErr != nil && !testCase.shouldPass {
  1135. if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
  1136. t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr.Error(), actualErr.Error())
  1137. }
  1138. }
  1139. // Passes as expected, but asserting the results.
  1140. if actualErr == nil && testCase.shouldPass {
  1141. expectedResult := testCase.expectedResult
  1142. // Asserting the MaxUploads.
  1143. if actualResult.MaxUploads != expectedResult.MaxUploads {
  1144. t.Errorf("Test %d: %s: Expected the MaxUploads to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxUploads, actualResult.MaxUploads)
  1145. }
  1146. // Asserting Prefix.
  1147. if actualResult.Prefix != expectedResult.Prefix {
  1148. t.Errorf("Test %d: %s: Expected Prefix to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Prefix, actualResult.Prefix)
  1149. }
  1150. // Asserting Delimiter.
  1151. if actualResult.Delimiter != expectedResult.Delimiter {
  1152. t.Errorf("Test %d: %s: Expected Delimiter to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Delimiter, actualResult.Delimiter)
  1153. }
  1154. // Asserting the keyMarker.
  1155. if actualResult.KeyMarker != expectedResult.KeyMarker {
  1156. t.Errorf("Test %d: %s: Expected keyMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.KeyMarker, actualResult.KeyMarker)
  1157. }
  1158. }
  1159. }
  1160. }
  1161. // Wrapper for calling TestListObjectPartsStale tests for both Erasure multiple disks and single node setup.
  1162. func TestListObjectPartsStale(t *testing.T) {
  1163. ExecObjectLayerDiskAlteredTest(t, testListObjectPartsStale)
  1164. }
  1165. // testListObjectPartsStale - Tests validate listing of object parts when parts are stale
  1166. func testListObjectPartsStale(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
  1167. bucketNames := []string{"minio-bucket", "minio-2-bucket"}
  1168. objectNames := []string{"minio-object-1.txt"}
  1169. uploadIDs := []string{}
  1170. globalStorageClass.Update(storageclass.Config{
  1171. RRS: storageclass.StorageClass{
  1172. Parity: 2,
  1173. },
  1174. Standard: storageclass.StorageClass{
  1175. Parity: 4,
  1176. },
  1177. })
  1178. // bucketnames[0].
  1179. // objectNames[0].
  1180. // uploadIds [0].
  1181. // Create bucket before initiating NewMultipartUpload.
  1182. err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{})
  1183. if err != nil {
  1184. // Failed to create newbucket, abort.
  1185. t.Fatalf("%s : %s", instanceType, err.Error())
  1186. }
  1187. opts := ObjectOptions{}
  1188. // Initiate Multipart Upload on the above created bucket.
  1189. res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
  1190. if err != nil {
  1191. // Failed to create NewMultipartUpload, abort.
  1192. t.Fatalf("%s : %s", instanceType, err.Error())
  1193. }
  1194. z := obj.(*erasureServerPools)
  1195. er := z.serverPools[0].sets[0]
  1196. uploadIDs = append(uploadIDs, res.UploadID)
  1197. // Create multipart parts.
  1198. // Need parts to be uploaded before MultipartLists can be called and tested.
  1199. createPartCases := []struct {
  1200. bucketName string
  1201. objName string
  1202. uploadID string
  1203. PartID int
  1204. inputReaderData string
  1205. inputMd5 string
  1206. inputDataSize int64
  1207. expectedMd5 string
  1208. }{
  1209. // Case 1-4.
  1210. // Creating sequence of parts for same uploadID.
  1211. // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
  1212. {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  1213. {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
  1214. {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
  1215. {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
  1216. }
  1217. sha256sum := ""
  1218. // Iterating over creatPartCases to generate multipart chunks.
  1219. for _, testCase := range createPartCases {
  1220. _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts)
  1221. if err != nil {
  1222. t.Fatalf("%s : %s", instanceType, err.Error())
  1223. }
  1224. }
  1225. erasureDisks := er.getDisks()
  1226. uploadIDPath := er.getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0])
  1227. dataDirs, err := erasureDisks[0].ListDir(context.Background(), minioMetaMultipartBucket, minioMetaMultipartBucket, uploadIDPath, -1)
  1228. if err != nil {
  1229. t.Fatalf("%s : %s", instanceType, err.Error())
  1230. }
  1231. var dataDir string
  1232. for _, folder := range dataDirs {
  1233. if strings.HasSuffix(folder, SlashSeparator) {
  1234. dataDir = folder
  1235. break
  1236. }
  1237. }
  1238. toDel := (len(erasureDisks) / 2) + 1
  1239. for _, disk := range erasureDisks[:toDel] {
  1240. disk.DeleteBulk(context.Background(), minioMetaMultipartBucket, []string{pathJoin(uploadIDPath, dataDir, "part.2")}...)
  1241. }
  1242. partInfos := []ListPartsInfo{
  1243. // partinfos - 0.
  1244. {
  1245. Bucket: bucketNames[0],
  1246. Object: objectNames[0],
  1247. MaxParts: 10,
  1248. UploadID: uploadIDs[0],
  1249. Parts: []PartInfo{
  1250. {
  1251. PartNumber: 1,
  1252. Size: 4,
  1253. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1254. },
  1255. {
  1256. PartNumber: 3,
  1257. Size: 4,
  1258. ETag: "09a0877d04abf8759f99adec02baf579",
  1259. },
  1260. {
  1261. PartNumber: 4,
  1262. Size: 4,
  1263. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1264. },
  1265. },
  1266. },
  1267. // partinfos - 1.
  1268. {
  1269. Bucket: bucketNames[0],
  1270. Object: objectNames[0],
  1271. MaxParts: 3,
  1272. UploadID: uploadIDs[0],
  1273. Parts: []PartInfo{
  1274. {
  1275. PartNumber: 1,
  1276. Size: 4,
  1277. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1278. },
  1279. {
  1280. PartNumber: 3,
  1281. Size: 4,
  1282. ETag: "09a0877d04abf8759f99adec02baf579",
  1283. },
  1284. {
  1285. PartNumber: 4,
  1286. Size: 4,
  1287. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1288. },
  1289. },
  1290. },
  1291. // partinfos - 2.
  1292. {
  1293. Bucket: bucketNames[0],
  1294. Object: objectNames[0],
  1295. MaxParts: 2,
  1296. NextPartNumberMarker: 3,
  1297. IsTruncated: true,
  1298. UploadID: uploadIDs[0],
  1299. Parts: []PartInfo{
  1300. {
  1301. PartNumber: 1,
  1302. Size: 4,
  1303. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1304. },
  1305. {
  1306. PartNumber: 3,
  1307. Size: 4,
  1308. ETag: "09a0877d04abf8759f99adec02baf579",
  1309. },
  1310. },
  1311. },
  1312. // partinfos - 3.
  1313. {
  1314. Bucket: bucketNames[0],
  1315. Object: objectNames[0],
  1316. MaxParts: 2,
  1317. IsTruncated: false,
  1318. UploadID: uploadIDs[0],
  1319. PartNumberMarker: 3,
  1320. Parts: []PartInfo{
  1321. {
  1322. PartNumber: 4,
  1323. Size: 4,
  1324. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1325. },
  1326. },
  1327. },
  1328. // partinfos - 4.
  1329. {
  1330. Bucket: bucketNames[0],
  1331. Object: objectNames[0],
  1332. MaxParts: 2,
  1333. IsTruncated: false,
  1334. UploadID: uploadIDs[0],
  1335. PartNumberMarker: 4,
  1336. },
  1337. // partinfos - 5.
  1338. {
  1339. Bucket: bucketNames[0],
  1340. Object: objectNames[0],
  1341. MaxParts: 2,
  1342. IsTruncated: false,
  1343. UploadID: uploadIDs[0],
  1344. PartNumberMarker: 100,
  1345. },
  1346. }
  1347. // Collection of non-exhaustive ListObjectParts test cases, valid errors
  1348. // and success responses.
  1349. testCases := []struct {
  1350. bucket string
  1351. object string
  1352. uploadID string
  1353. partNumberMarker int
  1354. maxParts int
  1355. // Expected output of ListPartsInfo.
  1356. expectedResult ListPartsInfo
  1357. expectedErr error
  1358. // Flag indicating whether the test is expected to pass or not.
  1359. shouldPass bool
  1360. }{
  1361. // Test cases with invalid bucket names (Test number 1-4).
  1362. {".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
  1363. {"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
  1364. {"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
  1365. {"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
  1366. // Test cases for listing uploadID with single part.
  1367. // Valid bucket names, but they do not exist (Test number 5-7).
  1368. {"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
  1369. {"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
  1370. {"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
  1371. // Test case for Asserting for invalid objectName (Test number 8).
  1372. {bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
  1373. // Asserting for Invalid UploadID (Test number 9).
  1374. {bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false},
  1375. // Test case for uploadID with multiple parts (Test number 12).
  1376. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true},
  1377. // Test case with maxParts set to less than number of parts (Test number 13).
  1378. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true},
  1379. // Test case with partNumberMarker set (Test number 14).
  1380. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 2, partInfos[2], nil, true},
  1381. // Test case with partNumberMarker set (Test number 15).
  1382. {bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[3], nil, true},
  1383. // Test case with partNumberMarker set (Test number 16).
  1384. {bucketNames[0], objectNames[0], uploadIDs[0], 4, 2, partInfos[4], nil, true},
  1385. // Test case with partNumberMarker set (Test number 17).
  1386. {bucketNames[0], objectNames[0], uploadIDs[0], 100, 2, partInfos[5], nil, true},
  1387. }
  1388. for i, testCase := range testCases {
  1389. actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, ObjectOptions{})
  1390. if actualErr != nil && testCase.shouldPass {
  1391. t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
  1392. }
  1393. if actualErr == nil && !testCase.shouldPass {
  1394. t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error())
  1395. }
  1396. // Failed as expected, but does it fail for the expected reason.
  1397. if actualErr != nil && !testCase.shouldPass {
  1398. if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
  1399. t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr, actualErr)
  1400. }
  1401. }
  1402. // Passes as expected, but asserting the results.
  1403. if actualErr == nil && testCase.shouldPass {
  1404. expectedResult := testCase.expectedResult
  1405. // Asserting the MaxParts.
  1406. if actualResult.MaxParts != expectedResult.MaxParts {
  1407. t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts)
  1408. }
  1409. // Asserting Object Name.
  1410. if actualResult.Object != expectedResult.Object {
  1411. t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object)
  1412. }
  1413. // Asserting UploadID.
  1414. if actualResult.UploadID != expectedResult.UploadID {
  1415. t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID)
  1416. }
  1417. // Asserting NextPartNumberMarker.
  1418. if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
  1419. t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
  1420. }
  1421. // Asserting PartNumberMarker.
  1422. if actualResult.PartNumberMarker != expectedResult.PartNumberMarker {
  1423. t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker)
  1424. }
  1425. // Asserting the BucketName.
  1426. if actualResult.Bucket != expectedResult.Bucket {
  1427. t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
  1428. }
  1429. // Asserting IsTruncated.
  1430. if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
  1431. t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
  1432. }
  1433. // Asserting the number of Parts.
  1434. if len(expectedResult.Parts) != len(actualResult.Parts) {
  1435. t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
  1436. } else {
  1437. // Iterating over the partInfos and asserting the fields.
  1438. for j, actualMetaData := range actualResult.Parts {
  1439. // Asserting the PartNumber in the PartInfo.
  1440. if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
  1441. t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
  1442. }
  1443. // Asserting the Size in the PartInfo.
  1444. if actualMetaData.Size != expectedResult.Parts[j].Size {
  1445. t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
  1446. }
  1447. // Asserting the ETag in the PartInfo.
  1448. if actualMetaData.ETag != expectedResult.Parts[j].ETag {
  1449. t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
  1450. }
  1451. }
  1452. }
  1453. }
  1454. }
  1455. }
  1456. // Wrapper for calling TestListObjectPartsDiskNotFound tests for both Erasure multiple disks and single node setup.
  1457. func TestListObjectPartsDiskNotFound(t *testing.T) {
  1458. ExecObjectLayerDiskAlteredTest(t, testListObjectPartsDiskNotFound)
  1459. }
  1460. // testListObjectParts - Tests validate listing of object parts when disks go offline.
  1461. func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
  1462. bucketNames := []string{"minio-bucket", "minio-2-bucket"}
  1463. objectNames := []string{"minio-object-1.txt"}
  1464. uploadIDs := []string{}
  1465. globalStorageClass.Update(storageclass.Config{
  1466. RRS: storageclass.StorageClass{
  1467. Parity: 2,
  1468. },
  1469. Standard: storageclass.StorageClass{
  1470. Parity: 4,
  1471. },
  1472. })
  1473. // bucketnames[0].
  1474. // objectNames[0].
  1475. // uploadIds [0].
  1476. // Create bucket before initiating NewMultipartUpload.
  1477. err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{})
  1478. if err != nil {
  1479. // Failed to create newbucket, abort.
  1480. t.Fatalf("%s : %s", instanceType, err.Error())
  1481. }
  1482. opts := ObjectOptions{}
  1483. // Initiate Multipart Upload on the above created bucket.
  1484. res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
  1485. if err != nil {
  1486. // Failed to create NewMultipartUpload, abort.
  1487. t.Fatalf("%s : %s", instanceType, err.Error())
  1488. }
  1489. z := obj.(*erasureServerPools)
  1490. er := z.serverPools[0].sets[0]
  1491. erasureDisks := er.getDisks()
  1492. ridx := rand.Intn(len(erasureDisks))
  1493. z.serverPools[0].erasureDisksMu.Lock()
  1494. er.getDisks = func() []StorageAPI {
  1495. erasureDisks[ridx] = newNaughtyDisk(erasureDisks[ridx], nil, errFaultyDisk)
  1496. return erasureDisks
  1497. }
  1498. z.serverPools[0].erasureDisksMu.Unlock()
  1499. uploadIDs = append(uploadIDs, res.UploadID)
  1500. // Create multipart parts.
  1501. // Need parts to be uploaded before MultipartLists can be called and tested.
  1502. createPartCases := []struct {
  1503. bucketName string
  1504. objName string
  1505. uploadID string
  1506. PartID int
  1507. inputReaderData string
  1508. inputMd5 string
  1509. inputDataSize int64
  1510. expectedMd5 string
  1511. }{
  1512. // Case 1-4.
  1513. // Creating sequence of parts for same uploadID.
  1514. // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
  1515. {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  1516. {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
  1517. {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
  1518. {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
  1519. }
  1520. sha256sum := ""
  1521. // Iterating over creatPartCases to generate multipart chunks.
  1522. for _, testCase := range createPartCases {
  1523. _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts)
  1524. if err != nil {
  1525. t.Fatalf("%s : %s", instanceType, err.Error())
  1526. }
  1527. }
  1528. partInfos := []ListPartsInfo{
  1529. // partinfos - 0.
  1530. {
  1531. Bucket: bucketNames[0],
  1532. Object: objectNames[0],
  1533. MaxParts: 10,
  1534. UploadID: uploadIDs[0],
  1535. Parts: []PartInfo{
  1536. {
  1537. PartNumber: 1,
  1538. Size: 4,
  1539. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1540. },
  1541. {
  1542. PartNumber: 2,
  1543. Size: 4,
  1544. ETag: "1f7690ebdd9b4caf8fab49ca1757bf27",
  1545. },
  1546. {
  1547. PartNumber: 3,
  1548. Size: 4,
  1549. ETag: "09a0877d04abf8759f99adec02baf579",
  1550. },
  1551. {
  1552. PartNumber: 4,
  1553. Size: 4,
  1554. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1555. },
  1556. },
  1557. },
  1558. // partinfos - 1.
  1559. {
  1560. Bucket: bucketNames[0],
  1561. Object: objectNames[0],
  1562. MaxParts: 3,
  1563. NextPartNumberMarker: 3,
  1564. IsTruncated: true,
  1565. UploadID: uploadIDs[0],
  1566. Parts: []PartInfo{
  1567. {
  1568. PartNumber: 1,
  1569. Size: 4,
  1570. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1571. },
  1572. {
  1573. PartNumber: 2,
  1574. Size: 4,
  1575. ETag: "1f7690ebdd9b4caf8fab49ca1757bf27",
  1576. },
  1577. {
  1578. PartNumber: 3,
  1579. Size: 4,
  1580. ETag: "09a0877d04abf8759f99adec02baf579",
  1581. },
  1582. },
  1583. },
  1584. // partinfos - 2.
  1585. {
  1586. Bucket: bucketNames[0],
  1587. Object: objectNames[0],
  1588. MaxParts: 2,
  1589. IsTruncated: false,
  1590. UploadID: uploadIDs[0],
  1591. PartNumberMarker: 3,
  1592. Parts: []PartInfo{
  1593. {
  1594. PartNumber: 4,
  1595. Size: 4,
  1596. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1597. },
  1598. },
  1599. },
  1600. }
  1601. // Collection of non-exhaustive ListObjectParts test cases, valid errors
  1602. // and success responses.
  1603. testCases := []struct {
  1604. bucket string
  1605. object string
  1606. uploadID string
  1607. partNumberMarker int
  1608. maxParts int
  1609. // Expected output of ListPartsInfo.
  1610. expectedResult ListPartsInfo
  1611. expectedErr error
  1612. // Flag indicating whether the test is expected to pass or not.
  1613. shouldPass bool
  1614. }{
  1615. // Test cases with invalid bucket names (Test number 1-4).
  1616. {".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
  1617. {"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
  1618. {"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
  1619. {"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
  1620. // Test cases for listing uploadID with single part.
  1621. // Valid bucket names, but they do not exist (Test number 5-7).
  1622. {"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
  1623. {"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
  1624. {"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
  1625. // Test case for Asserting for invalid objectName (Test number 8).
  1626. {bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
  1627. // Asserting for Invalid UploadID (Test number 9).
  1628. {bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false},
  1629. // Test case for uploadID with multiple parts (Test number 12).
  1630. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true},
  1631. // Test case with maxParts set to less than number of parts (Test number 13).
  1632. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true},
  1633. // Test case with partNumberMarker set (Test number 14)-.
  1634. {bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[2], nil, true},
  1635. }
  1636. for i, testCase := range testCases {
  1637. actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, ObjectOptions{})
  1638. if actualErr != nil && testCase.shouldPass {
  1639. t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
  1640. }
  1641. if actualErr == nil && !testCase.shouldPass {
  1642. t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error())
  1643. }
  1644. // Failed as expected, but does it fail for the expected reason.
  1645. if actualErr != nil && !testCase.shouldPass {
  1646. if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
  1647. t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr, actualErr)
  1648. }
  1649. }
  1650. // Passes as expected, but asserting the results.
  1651. if actualErr == nil && testCase.shouldPass {
  1652. expectedResult := testCase.expectedResult
  1653. // Asserting the MaxParts.
  1654. if actualResult.MaxParts != expectedResult.MaxParts {
  1655. t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts)
  1656. }
  1657. // Asserting Object Name.
  1658. if actualResult.Object != expectedResult.Object {
  1659. t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object)
  1660. }
  1661. // Asserting UploadID.
  1662. if actualResult.UploadID != expectedResult.UploadID {
  1663. t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID)
  1664. }
  1665. // Asserting NextPartNumberMarker.
  1666. if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
  1667. t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
  1668. }
  1669. // Asserting PartNumberMarker.
  1670. if actualResult.PartNumberMarker != expectedResult.PartNumberMarker {
  1671. t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker)
  1672. }
  1673. // Asserting the BucketName.
  1674. if actualResult.Bucket != expectedResult.Bucket {
  1675. t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
  1676. }
  1677. // Asserting IsTruncated.
  1678. if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
  1679. t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
  1680. }
  1681. // Asserting the number of Parts.
  1682. if len(expectedResult.Parts) != len(actualResult.Parts) {
  1683. t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
  1684. } else {
  1685. // Iterating over the partInfos and asserting the fields.
  1686. for j, actualMetaData := range actualResult.Parts {
  1687. // Asserting the PartNumber in the PartInfo.
  1688. if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
  1689. t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
  1690. }
  1691. // Asserting the Size in the PartInfo.
  1692. if actualMetaData.Size != expectedResult.Parts[j].Size {
  1693. t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
  1694. }
  1695. // Asserting the ETag in the PartInfo.
  1696. if actualMetaData.ETag != expectedResult.Parts[j].ETag {
  1697. t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
  1698. }
  1699. }
  1700. }
  1701. }
  1702. }
  1703. }
  1704. // Wrapper for calling TestListObjectParts tests for both Erasure multiple disks and single node setup.
  1705. func TestListObjectParts(t *testing.T) {
  1706. ExecObjectLayerTest(t, testListObjectParts)
  1707. }
  1708. // testListObjectParts - test validate listing of object parts.
  1709. func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) {
  1710. bucketNames := []string{"minio-bucket", "minio-2-bucket"}
  1711. objectNames := []string{"minio-object-1.txt"}
  1712. uploadIDs := []string{}
  1713. opts := ObjectOptions{}
  1714. // bucketnames[0].
  1715. // objectNames[0].
  1716. // uploadIds [0].
  1717. // Create bucket before initiating NewMultipartUpload.
  1718. err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{})
  1719. if err != nil {
  1720. // Failed to create newbucket, abort.
  1721. t.Fatalf("%s : %s", instanceType, err.Error())
  1722. }
  1723. // Initiate Multipart Upload on the above created bucket.
  1724. res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
  1725. if err != nil {
  1726. // Failed to create NewMultipartUpload, abort.
  1727. t.Fatalf("%s : %s", instanceType, err.Error())
  1728. }
  1729. uploadIDs = append(uploadIDs, res.UploadID)
  1730. // Create multipart parts.
  1731. // Need parts to be uploaded before MultipartLists can be called and tested.
  1732. createPartCases := []struct {
  1733. bucketName string
  1734. objName string
  1735. uploadID string
  1736. PartID int
  1737. inputReaderData string
  1738. inputMd5 string
  1739. inputDataSize int64
  1740. expectedMd5 string
  1741. }{
  1742. // Case 1-4.
  1743. // Creating sequence of parts for same uploadID.
  1744. // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
  1745. {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
  1746. {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
  1747. {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
  1748. {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
  1749. }
  1750. sha256sum := ""
  1751. // Iterating over creatPartCases to generate multipart chunks.
  1752. for _, testCase := range createPartCases {
  1753. _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts)
  1754. if err != nil {
  1755. t.Fatalf("%s : %s", instanceType, err.Error())
  1756. }
  1757. }
  1758. partInfos := []ListPartsInfo{
  1759. // partinfos - 0.
  1760. {
  1761. Bucket: bucketNames[0],
  1762. Object: objectNames[0],
  1763. MaxParts: 10,
  1764. UploadID: uploadIDs[0],
  1765. Parts: []PartInfo{
  1766. {
  1767. PartNumber: 1,
  1768. Size: 4,
  1769. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1770. },
  1771. {
  1772. PartNumber: 2,
  1773. Size: 4,
  1774. ETag: "1f7690ebdd9b4caf8fab49ca1757bf27",
  1775. },
  1776. {
  1777. PartNumber: 3,
  1778. Size: 4,
  1779. ETag: "09a0877d04abf8759f99adec02baf579",
  1780. },
  1781. {
  1782. PartNumber: 4,
  1783. Size: 4,
  1784. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1785. },
  1786. },
  1787. },
  1788. // partinfos - 1.
  1789. {
  1790. Bucket: bucketNames[0],
  1791. Object: objectNames[0],
  1792. MaxParts: 3,
  1793. NextPartNumberMarker: 3,
  1794. IsTruncated: true,
  1795. UploadID: uploadIDs[0],
  1796. Parts: []PartInfo{
  1797. {
  1798. PartNumber: 1,
  1799. Size: 4,
  1800. ETag: "e2fc714c4727ee9395f324cd2e7f331f",
  1801. },
  1802. {
  1803. PartNumber: 2,
  1804. Size: 4,
  1805. ETag: "1f7690ebdd9b4caf8fab49ca1757bf27",
  1806. },
  1807. {
  1808. PartNumber: 3,
  1809. Size: 4,
  1810. ETag: "09a0877d04abf8759f99adec02baf579",
  1811. },
  1812. },
  1813. },
  1814. // partinfos - 2.
  1815. {
  1816. Bucket: bucketNames[0],
  1817. Object: objectNames[0],
  1818. MaxParts: 2,
  1819. IsTruncated: false,
  1820. UploadID: uploadIDs[0],
  1821. PartNumberMarker: 3,
  1822. Parts: []PartInfo{
  1823. {
  1824. PartNumber: 4,
  1825. Size: 4,
  1826. ETag: "e132e96a5ddad6da8b07bba6f6131fef",
  1827. },
  1828. },
  1829. },
  1830. }
  1831. // Collection of non-exhaustive ListObjectParts test cases, valid errors
  1832. // and success responses.
  1833. testCases := []struct {
  1834. bucket string
  1835. object string
  1836. uploadID string
  1837. partNumberMarker int
  1838. maxParts int
  1839. // Expected output of ListPartsInfo.
  1840. expectedResult ListPartsInfo
  1841. expectedErr error
  1842. // Flag indicating whether the test is expected to pass or not.
  1843. shouldPass bool
  1844. }{
  1845. // Test cases with invalid bucket names (Test number 1-4).
  1846. {".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
  1847. {"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
  1848. {"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
  1849. {"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
  1850. // Test cases for listing uploadID with single part.
  1851. // Valid bucket names, but they do not exist (Test number 5-7).
  1852. {"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
  1853. {"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
  1854. {"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
  1855. // Test case for Asserting for invalid objectName (Test number 8).
  1856. {bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
  1857. // Asserting for Invalid UploadID (Test number 9).
  1858. {bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false},
  1859. // Test case for uploadID with multiple parts (Test number 12).
  1860. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true},
  1861. // Test case with maxParts set to less than number of parts (Test number 13).
  1862. {bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true},
  1863. // Test case with partNumberMarker set (Test number 14)-.
  1864. {bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[2], nil, true},
  1865. }
  1866. for i, testCase := range testCases {
  1867. actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, opts)
  1868. if actualErr != nil && testCase.shouldPass {
  1869. t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
  1870. }
  1871. if actualErr == nil && !testCase.shouldPass {
  1872. t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error())
  1873. }
  1874. // Failed as expected, but does it fail for the expected reason.
  1875. if actualErr != nil && !testCase.shouldPass {
  1876. if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
  1877. t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr.Error(), actualErr.Error())
  1878. }
  1879. }
  1880. // Passes as expected, but asserting the results.
  1881. if actualErr == nil && testCase.shouldPass {
  1882. expectedResult := testCase.expectedResult
  1883. // Asserting the MaxParts.
  1884. if actualResult.MaxParts != expectedResult.MaxParts {
  1885. t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts)
  1886. }
  1887. // Asserting Object Name.
  1888. if actualResult.Object != expectedResult.Object {
  1889. t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object)
  1890. }
  1891. // Asserting UploadID.
  1892. if actualResult.UploadID != expectedResult.UploadID {
  1893. t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID)
  1894. }
  1895. // Asserting PartNumberMarker.
  1896. if actualResult.PartNumberMarker != expectedResult.PartNumberMarker {
  1897. t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker)
  1898. }
  1899. // Asserting the BucketName.
  1900. if actualResult.Bucket != expectedResult.Bucket {
  1901. t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
  1902. }
  1903. // Asserting IsTruncated.
  1904. if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
  1905. t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
  1906. continue
  1907. }
  1908. // Asserting NextPartNumberMarker.
  1909. if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
  1910. t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
  1911. continue
  1912. }
  1913. // Asserting the number of Parts.
  1914. if len(expectedResult.Parts) != len(actualResult.Parts) {
  1915. t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
  1916. continue
  1917. }
  1918. // Iterating over the partInfos and asserting the fields.
  1919. for j, actualMetaData := range actualResult.Parts {
  1920. // Asserting the PartNumber in the PartInfo.
  1921. if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
  1922. t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
  1923. }
  1924. // Asserting the Size in the PartInfo.
  1925. if actualMetaData.Size != expectedResult.Parts[j].Size {
  1926. t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
  1927. }
  1928. // Asserting the ETag in the PartInfo.
  1929. if actualMetaData.ETag != expectedResult.Parts[j].ETag {
  1930. t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
  1931. }
  1932. }
  1933. }
  1934. }
  1935. }
  1936. // Test for validating complete Multipart upload.
  1937. func TestObjectCompleteMultipartUpload(t *testing.T) {
  1938. ExecExtendedObjectLayerTest(t, testObjectCompleteMultipartUpload)
  1939. }
  1940. // Tests validate CompleteMultipart functionality.
  1941. func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) {
  1942. var err error
  1943. bucketNames := []string{"minio-bucket", "minio-2-bucket"}
  1944. objectNames := []string{"minio-object-1.txt"}
  1945. uploadIDs := []string{}
  1946. // bucketnames[0].
  1947. // objectNames[0].
  1948. // uploadIds [0].
  1949. // Create bucket before initiating NewMultipartUpload.
  1950. err = obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{})
  1951. if err != nil {
  1952. // Failed to create newbucket, abort.
  1953. t.Fatalf("%s : %s", instanceType, err)
  1954. }
  1955. // Initiate Multipart Upload on the above created bucket.
  1956. res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-Id": "id"}})
  1957. if err != nil {
  1958. // Failed to create NewMultipartUpload, abort.
  1959. t.Fatalf("%s : %s", instanceType, err)
  1960. }
  1961. uploadIDs = append(uploadIDs, res.UploadID)
  1962. // Parts with size greater than 5 MiB.
  1963. // Generating a 6MiB byte array.
  1964. validPart := bytes.Repeat([]byte("abcdef"), 1*humanize.MiByte)
  1965. validPartMD5 := getMD5Hash(validPart)
  1966. // Create multipart parts.
  1967. // Need parts to be uploaded before CompleteMultiPartUpload can be called tested.
  1968. parts := []struct {
  1969. bucketName string
  1970. objName string
  1971. uploadID string
  1972. PartID int
  1973. inputReaderData string
  1974. inputMd5 string
  1975. inputDataSize int64
  1976. }{
  1977. // Case 1-4.
  1978. // Creating sequence of parts for same uploadID.
  1979. {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd"))},
  1980. {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh"))},
  1981. {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd"))},
  1982. {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd"))},
  1983. // Part with size larger than 5Mb.
  1984. {bucketNames[0], objectNames[0], uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(validPart))},
  1985. {bucketNames[0], objectNames[0], uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(validPart))},
  1986. {bucketNames[0], objectNames[0], uploadIDs[0], 7, string(validPart), validPartMD5, int64(len(validPart))},
  1987. }
  1988. sha256sum := ""
  1989. var opts ObjectOptions
  1990. // Iterating over creatPartCases to generate multipart chunks.
  1991. for _, part := range parts {
  1992. _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetPutObjReader(t, bytes.NewBufferString(part.inputReaderData), part.inputDataSize, part.inputMd5, sha256sum), opts)
  1993. if err != nil {
  1994. t.Fatalf("%s : %s", instanceType, err)
  1995. }
  1996. }
  1997. // Parts to be sent as input for CompleteMultipartUpload.
  1998. inputParts := []struct {
  1999. parts []CompletePart
  2000. }{
  2001. // inputParts - 0.
  2002. // Case for replicating ETag mismatch.
  2003. {
  2004. []CompletePart{
  2005. {ETag: "abcd", PartNumber: 1},
  2006. },
  2007. },
  2008. // inputParts - 1.
  2009. // should error out with part too small.
  2010. {
  2011. []CompletePart{
  2012. {ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 1},
  2013. {ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", PartNumber: 2},
  2014. },
  2015. },
  2016. // inputParts - 2.
  2017. // Case with invalid Part number.
  2018. {
  2019. []CompletePart{
  2020. {ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 10},
  2021. },
  2022. },
  2023. // inputParts - 3.
  2024. // Case with valid part.
  2025. // Part size greater than 5MB.
  2026. {
  2027. []CompletePart{
  2028. {ETag: fmt.Sprintf("\"\"\"\"\"%s\"\"\"", validPartMD5), PartNumber: 5},
  2029. },
  2030. },
  2031. // inputParts - 4.
  2032. // Used to verify that the other remaining parts are deleted after
  2033. // a successful call to CompleteMultipartUpload.
  2034. {
  2035. []CompletePart{
  2036. {ETag: validPartMD5, PartNumber: 6},
  2037. },
  2038. },
  2039. }
  2040. s3MD5 := getCompleteMultipartMD5(inputParts[3].parts)
  2041. // Test cases with sample input values for CompleteMultipartUpload.
  2042. testCases := []struct {
  2043. bucket string
  2044. object string
  2045. uploadID string
  2046. parts []CompletePart
  2047. // Expected output of CompleteMultipartUpload.
  2048. expectedS3MD5 string
  2049. expectedErr error
  2050. // Flag indicating whether the test is expected to pass or not.
  2051. shouldPass bool
  2052. }{
  2053. // Test cases with invalid bucket names (Test number 1-4).
  2054. {".test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: ".test"}, false},
  2055. {"Test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "Test"}, false},
  2056. {"---", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "---"}, false},
  2057. {"ad", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "ad"}, false},
  2058. // Test cases for listing uploadID with single part.
  2059. // Valid bucket names, but they do not exist (Test number 5-7).
  2060. {"volatile-bucket-1", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-1"}, false},
  2061. {"volatile-bucket-2", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-2"}, false},
  2062. {"volatile-bucket-3", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-3"}, false},
  2063. // Test case for Asserting for invalid objectName (Test number 8).
  2064. {bucketNames[0], "", "", []CompletePart{}, "", ObjectNameInvalid{Bucket: bucketNames[0]}, false},
  2065. // Asserting for Invalid UploadID (Test number 9).
  2066. {bucketNames[0], objectNames[0], "abc", []CompletePart{}, "", InvalidUploadID{UploadID: "abc"}, false},
  2067. // Test case with invalid Part Etag (Test number 10-11).
  2068. {bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abc"}}, "", InvalidPart{}, false},
  2069. {bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcz"}}, "", InvalidPart{}, false},
  2070. // Part number 0 doesn't exist, expecting InvalidPart error (Test number 12).
  2071. {bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcd", PartNumber: 0}}, "", InvalidPart{}, false},
  2072. // // Upload and PartNumber exists, But a deliberate ETag mismatch is introduced (Test number 13).
  2073. {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[0].parts, "", InvalidPart{}, false},
  2074. // Test case with non existent object name (Test number 14).
  2075. {bucketNames[0], "my-object", uploadIDs[0], []CompletePart{{ETag: "abcd", PartNumber: 1}}, "", InvalidUploadID{UploadID: uploadIDs[0]}, false},
  2076. // Testing for Part being too small (Test number 15).
  2077. {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[1].parts, "", PartTooSmall{PartNumber: 1}, false},
  2078. // TestCase with invalid Part Number (Test number 16).
  2079. // Should error with Invalid Part .
  2080. {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[2].parts, "", InvalidPart{}, false},
  2081. // Test case with unsorted parts (Test number 17).
  2082. {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[3].parts, s3MD5, nil, true},
  2083. // The other parts will be flushed after a successful CompletePart (Test number 18).
  2084. // the case above successfully completes CompleteMultipartUpload, the remaining Parts will be flushed.
  2085. // Expecting to fail with Invalid UploadID.
  2086. {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[4].parts, "", InvalidUploadID{UploadID: uploadIDs[0]}, false},
  2087. }
  2088. for _, testCase := range testCases {
  2089. testCase := testCase
  2090. t.(*testing.T).Run("", func(t *testing.T) {
  2091. opts = ObjectOptions{}
  2092. actualResult, actualErr := obj.CompleteMultipartUpload(t.Context(), testCase.bucket, testCase.object, testCase.uploadID, testCase.parts, ObjectOptions{})
  2093. if actualErr != nil && testCase.shouldPass {
  2094. t.Errorf("%s: Expected to pass, but failed with: <ERROR> %s", instanceType, actualErr)
  2095. }
  2096. if actualErr == nil && !testCase.shouldPass {
  2097. t.Errorf("%s: Expected to fail with <ERROR> \"%s\", but passed instead", instanceType, testCase.expectedErr)
  2098. }
  2099. // Failed as expected, but does it fail for the expected reason.
  2100. if actualErr != nil && !testCase.shouldPass {
  2101. if reflect.TypeOf(actualErr) != reflect.TypeOf(testCase.expectedErr) {
  2102. t.Errorf("%s: Expected to fail with error \"%s\", but instead failed with error \"%s\"", instanceType, testCase.expectedErr, actualErr)
  2103. }
  2104. }
  2105. // Passes as expected, but asserting the results.
  2106. if actualErr == nil && testCase.shouldPass {
  2107. // Asserting IsTruncated.
  2108. if actualResult.ETag != testCase.expectedS3MD5 {
  2109. t.Errorf("%s: Expected the result to be \"%v\", but found it to \"%v\"", instanceType, testCase.expectedS3MD5, actualResult)
  2110. }
  2111. }
  2112. })
  2113. }
  2114. }
  2115. // Benchmarks for ObjectLayer.PutObjectPart().
  2116. // The intent is to benchmark PutObjectPart for various sizes ranging from few bytes to 100MB.
  2117. // Also each of these Benchmarks are run both Erasure and FS backends.
  2118. // BenchmarkPutObjectPart5MbFS - Benchmark FS.PutObjectPart() for object size of 5MB.
  2119. func BenchmarkPutObjectPart5MbFS(b *testing.B) {
  2120. benchmarkPutObjectPart(b, "FS", 5*humanize.MiByte)
  2121. }
  2122. // BenchmarkPutObjectPart5MbErasure - Benchmark Erasure.PutObjectPart() for object size of 5MB.
  2123. func BenchmarkPutObjectPart5MbErasure(b *testing.B) {
  2124. benchmarkPutObjectPart(b, "Erasure", 5*humanize.MiByte)
  2125. }
  2126. // BenchmarkPutObjectPart10MbFS - Benchmark FS.PutObjectPart() for object size of 10MB.
  2127. func BenchmarkPutObjectPart10MbFS(b *testing.B) {
  2128. benchmarkPutObjectPart(b, "FS", 10*humanize.MiByte)
  2129. }
  2130. // BenchmarkPutObjectPart10MbErasure - Benchmark Erasure.PutObjectPart() for object size of 10MB.
  2131. func BenchmarkPutObjectPart10MbErasure(b *testing.B) {
  2132. benchmarkPutObjectPart(b, "Erasure", 10*humanize.MiByte)
  2133. }
  2134. // BenchmarkPutObjectPart25MbFS - Benchmark FS.PutObjectPart() for object size of 25MB.
  2135. func BenchmarkPutObjectPart25MbFS(b *testing.B) {
  2136. benchmarkPutObjectPart(b, "FS", 25*humanize.MiByte)
  2137. }
  2138. // BenchmarkPutObjectPart25MbErasure - Benchmark Erasure.PutObjectPart() for object size of 25MB.
  2139. func BenchmarkPutObjectPart25MbErasure(b *testing.B) {
  2140. benchmarkPutObjectPart(b, "Erasure", 25*humanize.MiByte)
  2141. }
  2142. // BenchmarkPutObjectPart50MbFS - Benchmark FS.PutObjectPart() for object size of 50MB.
  2143. func BenchmarkPutObjectPart50MbFS(b *testing.B) {
  2144. benchmarkPutObjectPart(b, "FS", 50*humanize.MiByte)
  2145. }
  2146. // BenchmarkPutObjectPart50MbErasure - Benchmark Erasure.PutObjectPart() for object size of 50MB.
  2147. func BenchmarkPutObjectPart50MbErasure(b *testing.B) {
  2148. benchmarkPutObjectPart(b, "Erasure", 50*humanize.MiByte)
  2149. }