You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
999 lines
41 KiB
999 lines
41 KiB
/*
|
|
* Minimal Object Storage Library, (C) 2015 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Net;
|
|
using System.Linq;
|
|
using RestSharp;
|
|
using System.IO;
|
|
using Minio.Client.Xml;
|
|
using System.Xml.Serialization;
|
|
using System.Xml.Linq;
|
|
using Minio.Client.Errors;
|
|
|
|
namespace Minio.Client
|
|
{
|
|
public class ObjectStorageClient
|
|
{
|
|
private static int PART_SIZE = 5 * 1024 * 1024;
|
|
|
|
private RestClient client;
|
|
private string region;
|
|
|
|
private string SystemUserAgent = "minio-cs/0.0.1 (Windows 8.1; x86_64)";
|
|
private string CustomUserAgent = "";
|
|
|
|
private string FullUserAgent
|
|
{
|
|
get
|
|
{
|
|
return SystemUserAgent + " " + CustomUserAgent;
|
|
}
|
|
}
|
|
|
|
|
|
internal ObjectStorageClient(Uri uri, string accessKey, string secretKey)
|
|
{
|
|
this.client = new RestClient(uri);
|
|
this.region = Regions.GetRegion(uri.Host);
|
|
this.client.UserAgent = this.FullUserAgent;
|
|
if (accessKey != null && secretKey != null)
|
|
{
|
|
this.client.Authenticator = new V4Authenticator(accessKey, secretKey);
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// Creates and returns an object storage client.
|
|
/// </summary>
|
|
/// <param name="uri">Location of the server, supports HTTP and HTTPS.</param>
|
|
/// <returns>Object Storage Client with the uri set as the server location.</returns>
|
|
public static ObjectStorageClient GetClient(Uri uri)
|
|
{
|
|
return GetClient(uri, null, null);
|
|
}
|
|
/// <summary>
|
|
/// Creates and returns an object storage client
|
|
/// </summary>
|
|
/// <param name="uri">Location of the server, supports HTTP and HTTPS</param>
|
|
/// <param name="accessKey">Access Key for authenticated requests</param>
|
|
/// <param name="secretKey">Secret Key for authenticated requests</param>
|
|
/// <returns>Object Storage Client with the uri set as the server location and authentication parameters set.</returns>
|
|
public static ObjectStorageClient GetClient(Uri uri, string accessKey, string secretKey)
|
|
{
|
|
if (uri == null)
|
|
{
|
|
throw new NullReferenceException();
|
|
}
|
|
|
|
if (!(uri.Scheme == "http" || uri.Scheme == "https"))
|
|
{
|
|
throw new UriFormatException("Expecting http or https");
|
|
}
|
|
|
|
if (uri.Query.Length != 0)
|
|
{
|
|
throw new UriFormatException("Expecting no query");
|
|
}
|
|
|
|
if (uri.AbsolutePath.Length == 0 || (uri.AbsolutePath.Length == 1 && uri.AbsolutePath[0] == '/'))
|
|
{
|
|
String path = uri.Scheme + "://" + uri.Host + ":" + uri.Port + "/";
|
|
return new ObjectStorageClient(new Uri(path), accessKey, secretKey);
|
|
}
|
|
throw new UriFormatException("Expecting AbsolutePath to be empty");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates and returns an object storage client
|
|
/// </summary>
|
|
/// <param name="uri">Location of the server, supports HTTP and HTTPS</param>
|
|
/// <returns>Object Storage Client with the uri set as the server location and authentication parameters set.</returns>
|
|
public static ObjectStorageClient GetClient(string url)
|
|
{
|
|
return GetClient(url, null, null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates and returns an object storage client
|
|
/// </summary>
|
|
/// <param name="uri">Location of the server, supports HTTP and HTTPS</param>
|
|
/// <param name="accessKey">Access Key for authenticated requests</param>
|
|
/// <param name="secretKey">Secret Key for authenticated requests</param>
|
|
/// <returns>Object Storage Client with the uri set as the server location and authentication parameters set.</returns>
|
|
public static ObjectStorageClient GetClient(string url, string accessKey, string secretKey)
|
|
{
|
|
Uri uri = new Uri(url);
|
|
return GetClient(uri, accessKey, secretKey);
|
|
}
|
|
|
|
public void SetUserAgent(string product, string version, IEnumerable<string> attributes)
|
|
{
|
|
if (string.IsNullOrEmpty(product))
|
|
{
|
|
throw new ArgumentException("product cannot be null or empty");
|
|
}
|
|
if (string.IsNullOrEmpty(version))
|
|
{
|
|
throw new ArgumentException("version cannot be null or empty");
|
|
}
|
|
string customAgent = product + "/" + version;
|
|
string[] attributesArray = attributes.ToArray();
|
|
if (attributes.Count() > 0)
|
|
{
|
|
customAgent += "(";
|
|
customAgent += string.Join("; ", attributesArray);
|
|
customAgent += ")";
|
|
}
|
|
this.CustomUserAgent = customAgent;
|
|
this.client.UserAgent = this.FullUserAgent;
|
|
this.client.FollowRedirects = false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if the specified bucket exists, otherwise returns false.
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to test existence of</param>
|
|
/// <returns>true if exists and user has access</returns>
|
|
public bool BucketExists(string bucket)
|
|
{
|
|
var request = new RestRequest(bucket, Method.HEAD);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Create a bucket with a given name and canned ACL
|
|
/// </summary>
|
|
/// <param name="bucket">Name of the new bucket</param>
|
|
/// <param name="acl">Canned ACL to set</param>
|
|
public void MakeBucket(string bucket, Acl acl)
|
|
{
|
|
var request = new RestRequest("/" + bucket, Method.PUT);
|
|
|
|
CreateBucketConfiguration config = new CreateBucketConfiguration()
|
|
{
|
|
LocationConstraint = this.region
|
|
};
|
|
|
|
request.AddHeader("x-amz-acl", acl.ToString());
|
|
|
|
request.AddBody(config);
|
|
|
|
var response = client.Execute(request);
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
return;
|
|
}
|
|
|
|
throw ParseError(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Create a private bucket with a give name.
|
|
/// </summary>
|
|
/// <param name="bucket">Name of the new bucket</param>
|
|
public void MakeBucket(string bucket)
|
|
{
|
|
this.MakeBucket(bucket, Acl.Private);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Remove a bucket
|
|
/// </summary>
|
|
/// <param name="bucket">Name of bucket to remove</param>
|
|
public void RemoveBucket(string bucket)
|
|
{
|
|
var request = new RestRequest(bucket, Method.DELETE);
|
|
var response = client.Execute(request);
|
|
|
|
if (!response.StatusCode.Equals(HttpStatusCode.NoContent))
|
|
{
|
|
throw ParseError(response);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get bucket ACL
|
|
/// </summary>
|
|
/// <param name="bucket">NAme of bucket to retrieve canned ACL</param>
|
|
/// <returns>Canned ACL</returns>
|
|
public Acl GetBucketAcl(string bucket)
|
|
{
|
|
var request = new RestRequest(bucket + "?acl", Method.GET);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
var content = StripXmlnsXsi(response.Content);
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
AccessControlPolicy bucketList = (AccessControlPolicy)(new XmlSerializer(typeof(AccessControlPolicy)).Deserialize(stream));
|
|
|
|
bool publicRead = false;
|
|
bool publicWrite = false;
|
|
bool authenticatedRead = false;
|
|
foreach (var x in bucketList.Grants)
|
|
{
|
|
if ("http://acs.amazonaws.com/groups/global/AllUsers".Equals(x.Grantee.URI) && x.Permission.Equals("READ"))
|
|
{
|
|
publicRead = true;
|
|
}
|
|
if ("http://acs.amazonaws.com/groups/global/AllUsers".Equals(x.Grantee.URI) && x.Permission.Equals("WRITE"))
|
|
{
|
|
publicWrite = true;
|
|
}
|
|
if ("http://acs.amazonaws.com/groups/global/AuthenticatedUsers".Equals(x.Grantee.URI) && x.Permission.Equals("READ"))
|
|
{
|
|
authenticatedRead = true;
|
|
}
|
|
}
|
|
if (publicRead && publicWrite && !authenticatedRead)
|
|
{
|
|
return Acl.PublicReadWrite;
|
|
}
|
|
if (publicRead && !publicWrite && !authenticatedRead)
|
|
{
|
|
return Acl.PublicRead;
|
|
}
|
|
if (!publicRead && !publicWrite && authenticatedRead)
|
|
{
|
|
return Acl.AuthenticatedRead;
|
|
}
|
|
return Acl.Private;
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Set a bucket's canned ACL
|
|
/// </summary>
|
|
/// <param name="bucket">Name of bucket to set canned ACL</param>
|
|
/// <param name="acl">Canned ACL to set</param>
|
|
public void SetBucketAcl(string bucket, Acl acl)
|
|
{
|
|
var request = new RestRequest(bucket + "?acl", Method.PUT);
|
|
// TODO add acl header
|
|
request.AddHeader("x-amz-acl", acl.ToString());
|
|
var response = client.Execute(request);
|
|
if (!HttpStatusCode.OK.Equals(response.StatusCode))
|
|
{
|
|
throw ParseError(response);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lists all buckets owned by the user
|
|
/// </summary>
|
|
/// <returns>A list of all buckets owned by the user.</returns>
|
|
public List<Bucket> ListBuckets()
|
|
{
|
|
var request = new RestRequest("/", Method.GET);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
try
|
|
{
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
ListAllMyBucketsResult bucketList = (ListAllMyBucketsResult)(new XmlSerializer(typeof(ListAllMyBucketsResult)).Deserialize(stream));
|
|
return bucketList.Buckets;
|
|
}
|
|
catch (InvalidOperationException ex)
|
|
{
|
|
// unauthed returns html, so we catch xml parse exception and return access denied
|
|
throw new AccessDeniedException();
|
|
}
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get an object. The object will be streamed to the callback given by the user.
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to retrieve object from</param>
|
|
/// <param name="key">Key of object to retrieve</param>
|
|
/// <param name="callback">A stream will be passed to the callback</param>
|
|
public void GetObject(string bucket, string key, Action<Stream> callback)
|
|
{
|
|
RestRequest request = new RestRequest(bucket + "/" + UrlEncode(key), Method.GET);
|
|
request.ResponseWriter = callback;
|
|
var response = client.Execute(request);
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
return;
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get an object starting with the byte specified in offset. The object will be streamed to the callback given by the user
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to retrieve object from</param>
|
|
/// <param name="key">Key of object to retrieve</param>
|
|
/// <param name="offset">Number of bytes to skip</param>
|
|
/// <param name="callback">A stream will be passed to the callback</param>
|
|
public void GetObject(string bucket, string key, ulong offset, Action<Stream> callback)
|
|
{
|
|
var stat = this.StatObject(bucket, key);
|
|
RestRequest request = new RestRequest(bucket + "/" + UrlEncode(key), Method.GET);
|
|
request.AddHeader("Range", "bytes=" + offset + "-" + (stat.Size - 1));
|
|
request.ResponseWriter = callback;
|
|
client.Execute(request);
|
|
// response status code is 0, bug in upstream library, cannot rely on it for errors with PartialContent
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get a byte range of an object given by the offset and length. The object will be streamed to the callback given by the user
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to retrieve object from</param>
|
|
/// <param name="key">Key of object to retrieve</param>
|
|
/// <param name="offset">Number of bytes to skip</param>
|
|
/// <param name="length">Length of requested byte range</param>
|
|
/// <param name="callback">A stream will be passed to the callback</param>
|
|
public void GetObject(string bucket, string key, ulong offset, ulong length, Action<Stream> callback)
|
|
{
|
|
RestRequest request = new RestRequest(bucket + "/" + UrlEncode(key), Method.GET);
|
|
request.AddHeader("Range", "bytes=" + offset + "-" + (offset + length - 1));
|
|
request.ResponseWriter = callback;
|
|
client.Execute(request);
|
|
// response status code is 0, bug in upstream library, cannot rely on it for errors with PartialContent
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests the object's existence and returns metadata about existing objects.
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to test object in</param>
|
|
/// <param name="key">Key of object to stat</param>
|
|
/// <returns>Facts about the object</returns>
|
|
public ObjectStat StatObject(string bucket, string key)
|
|
{
|
|
var request = new RestRequest(bucket + "/" + UrlEncode(key), Method.HEAD);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
long size = 0;
|
|
DateTime lastModified = new DateTime();
|
|
string etag = "";
|
|
foreach (Parameter parameter in response.Headers)
|
|
{
|
|
if (parameter.Name == "Content-Length")
|
|
{
|
|
size = long.Parse(parameter.Value.ToString());
|
|
}
|
|
if (parameter.Name == "Last-Modified")
|
|
{
|
|
DateTime.Parse(parameter.Value.ToString());
|
|
}
|
|
if (parameter.Name == "ETag")
|
|
{
|
|
etag = parameter.Value.ToString().Replace("\"", "");
|
|
}
|
|
}
|
|
|
|
return new ObjectStat(key, size, lastModified, etag);
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates an object
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to create object in</param>
|
|
/// <param name="key">Key of the new object</param>
|
|
/// <param name="size">Total size of bytes to be written, must match with data's length</param>
|
|
/// <param name="contentType">Content type of the new object, null defaults to "application/octet-stream"</param>
|
|
/// <param name="data">Stream of bytes to send</param>
|
|
public void PutObject(string bucket, string key, long size, string contentType, Stream data)
|
|
{
|
|
if (size <= ObjectStorageClient.PART_SIZE)
|
|
{
|
|
var bytes = ReadFull(data, (int)size);
|
|
if (data.ReadByte() > 0)
|
|
{
|
|
throw new DataSizeMismatchException();
|
|
}
|
|
if (bytes.Length != (int)size)
|
|
{
|
|
throw new DataSizeMismatchException()
|
|
{
|
|
Bucket = bucket,
|
|
Key = key,
|
|
UserSpecifiedSize = size,
|
|
ActualReadSize = bytes.Length
|
|
};
|
|
}
|
|
this.DoPutObject(bucket, key, null, 0, contentType, bytes);
|
|
}
|
|
else
|
|
{
|
|
var partSize = CalculatePartSize(size);
|
|
var uploads = this.ListAllIncompleteUploads(bucket, key);
|
|
string uploadId = null;
|
|
Dictionary<int, string> etags = new Dictionary<int, string>();
|
|
if (uploads.Count() > 0)
|
|
{
|
|
uploadId = uploads.Last().UploadId;
|
|
var parts = this.ListParts(bucket, key, uploadId);
|
|
foreach (Part part in parts)
|
|
{
|
|
etags[part.PartNumber] = part.ETag;
|
|
}
|
|
}
|
|
if (uploadId == null)
|
|
{
|
|
uploadId = this.NewMultipartUpload(bucket, key);
|
|
}
|
|
int partNumber = 0;
|
|
long totalWritten = 0;
|
|
while (totalWritten < size)
|
|
{
|
|
partNumber++;
|
|
var currentPartSize = (int)Math.Min((long)partSize, (size - totalWritten));
|
|
byte[] dataToCopy = ReadFull(data, currentPartSize);
|
|
if (dataToCopy == null)
|
|
{
|
|
break;
|
|
}
|
|
System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
|
|
byte[] hash = md5.ComputeHash(dataToCopy);
|
|
string etag = BitConverter.ToString(hash).Replace("-", string.Empty).ToLower();
|
|
if (!etags.ContainsKey(partNumber) || !etags[partNumber].Equals(etag))
|
|
{
|
|
etag = DoPutObject(bucket, key, uploadId, partNumber, contentType, dataToCopy);
|
|
}
|
|
etags[partNumber] = etag;
|
|
totalWritten += dataToCopy.Length;
|
|
}
|
|
|
|
// test if any more data is on the stream
|
|
if (data.ReadByte() != -1)
|
|
{
|
|
throw new DataSizeMismatchException()
|
|
{
|
|
Bucket = bucket,
|
|
Key = key,
|
|
UserSpecifiedSize = size,
|
|
ActualReadSize = totalWritten + 1
|
|
};
|
|
}
|
|
|
|
if (totalWritten != size)
|
|
{
|
|
throw new DataSizeMismatchException()
|
|
{
|
|
Bucket = bucket,
|
|
Key = key,
|
|
UserSpecifiedSize = size,
|
|
ActualReadSize = totalWritten
|
|
};
|
|
}
|
|
|
|
foreach (int curPartNumber in etags.Keys)
|
|
{
|
|
if (curPartNumber > partNumber)
|
|
{
|
|
etags.Remove(curPartNumber);
|
|
}
|
|
}
|
|
|
|
this.CompleteMultipartUpload(bucket, key, uploadId, etags);
|
|
}
|
|
}
|
|
|
|
private byte[] ReadFull(Stream data, int currentPartSize)
|
|
{
|
|
byte[] result = new byte[currentPartSize];
|
|
int totalRead = 0;
|
|
while (totalRead < currentPartSize)
|
|
{
|
|
byte[] curData = new byte[currentPartSize - totalRead];
|
|
int curRead = data.Read(curData, 0, currentPartSize - totalRead);
|
|
if (curRead == 0)
|
|
{
|
|
break;
|
|
}
|
|
for (int i = 0; i < curRead; i++)
|
|
{
|
|
result[totalRead + i] = curData[i];
|
|
}
|
|
totalRead += curRead;
|
|
}
|
|
|
|
if (totalRead == 0) return null;
|
|
|
|
if (totalRead == currentPartSize) return result;
|
|
|
|
byte[] truncatedResult = new byte[totalRead];
|
|
for (int i = 0; i < totalRead; i++)
|
|
{
|
|
truncatedResult[i] = result[i];
|
|
}
|
|
return truncatedResult;
|
|
}
|
|
|
|
private void CompleteMultipartUpload(string bucket, string key, string uploadId, Dictionary<int, string> etags)
|
|
{
|
|
var path = bucket + "/" + UrlEncode(key) + "?uploadId=" + uploadId;
|
|
var request = new RestRequest(path, Method.POST);
|
|
|
|
List<XElement> parts = new List<XElement>();
|
|
|
|
for (int i = 1; i <= etags.Count; i++)
|
|
{
|
|
parts.Add(new XElement("Part",
|
|
new XElement("PartNumber", i),
|
|
new XElement("ETag", etags[i])));
|
|
}
|
|
|
|
var completeMultipartUploadXml = new XElement("CompleteMultipartUpload", parts);
|
|
|
|
var bodyString = completeMultipartUploadXml.ToString();
|
|
|
|
var body = System.Text.Encoding.UTF8.GetBytes(bodyString);
|
|
|
|
request.AddParameter("application/xml", body, RestSharp.ParameterType.RequestBody);
|
|
|
|
var response = client.Execute(request);
|
|
if (response.StatusCode.Equals(HttpStatusCode.OK))
|
|
{
|
|
return;
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
private int CalculatePartSize(long size)
|
|
{
|
|
int minimumPartSize = PART_SIZE; // 5MB
|
|
int partSize = (int)(size / 9999); // using 10000 may cause part size to become too small, and not fit the entire object in
|
|
return Math.Max(minimumPartSize, partSize);
|
|
}
|
|
|
|
private IEnumerable<Part> ListParts(string bucket, string key, string uploadId)
|
|
{
|
|
int nextPartNumberMarker = 0;
|
|
bool isRunning = true;
|
|
while (isRunning)
|
|
{
|
|
var uploads = GetListParts(bucket, key, uploadId);
|
|
foreach (Part part in uploads.Item2)
|
|
{
|
|
yield return part;
|
|
}
|
|
nextPartNumberMarker = uploads.Item1.NextPartNumberMarker;
|
|
isRunning = uploads.Item1.IsTruncated;
|
|
}
|
|
}
|
|
|
|
private Tuple<ListPartsResult, List<Part>> GetListParts(string bucket, string key, string uploadId)
|
|
{
|
|
var path = bucket + "/" + UrlEncode(key) + "?uploadId=" + uploadId;
|
|
var request = new RestRequest(path, Method.GET);
|
|
var response = client.Execute(request);
|
|
if (response.StatusCode.Equals(HttpStatusCode.OK))
|
|
{
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
ListPartsResult listPartsResult = (ListPartsResult)(new XmlSerializer(typeof(ListPartsResult)).Deserialize(stream));
|
|
|
|
XDocument root = XDocument.Parse(response.Content);
|
|
|
|
var uploads = (from c in root.Root.Descendants("{http://s3.amazonaws.com/doc/2006-03-01/}Part")
|
|
select new Part()
|
|
{
|
|
PartNumber = int.Parse(c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}PartNumber").Value),
|
|
ETag = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}ETag").Value.Replace("\"", "")
|
|
});
|
|
|
|
return new Tuple<ListPartsResult, List<Part>>(listPartsResult, new List<Part>());
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
private string NewMultipartUpload(string bucket, string key)
|
|
{
|
|
var path = bucket + "/" + UrlEncode(key) + "?uploads";
|
|
var request = new RestRequest(path, Method.POST);
|
|
var response = client.Execute(request);
|
|
if (response.StatusCode.Equals(HttpStatusCode.OK))
|
|
{
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
InitiateMultipartUploadResult newUpload = (InitiateMultipartUploadResult)(new XmlSerializer(typeof(InitiateMultipartUploadResult)).Deserialize(stream));
|
|
return newUpload.UploadId;
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
private string DoPutObject(string bucket, string key, string uploadId, int partNumber, string contentType, byte[] data)
|
|
{
|
|
var path = bucket + "/" + UrlEncode(key);
|
|
var queries = new List<string>();
|
|
if (uploadId != null)
|
|
{
|
|
path += "?uploadId=" + uploadId + "&partNumber=" + partNumber;
|
|
}
|
|
var request = new RestRequest(path, Method.PUT);
|
|
if (contentType == null)
|
|
{
|
|
contentType = "application/octet-stream";
|
|
}
|
|
|
|
|
|
|
|
request.AddHeader("Content-Type", contentType);
|
|
request.AddParameter(contentType, data, RestSharp.ParameterType.RequestBody);
|
|
var response = client.Execute(request);
|
|
if (response.StatusCode.Equals(HttpStatusCode.OK))
|
|
{
|
|
string etag = null;
|
|
foreach (Parameter parameter in response.Headers)
|
|
{
|
|
if (parameter.Name == "ETag")
|
|
{
|
|
etag = parameter.Value.ToString();
|
|
}
|
|
}
|
|
return etag;
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
private ClientException ParseError(IRestResponse response)
|
|
{
|
|
if (response == null)
|
|
{
|
|
return new ConnectionException();
|
|
}
|
|
if (HttpStatusCode.Redirect.Equals(response.StatusCode) || HttpStatusCode.TemporaryRedirect.Equals(response.StatusCode))
|
|
{
|
|
return new RedirectionException();
|
|
}
|
|
|
|
if (HttpStatusCode.Forbidden.Equals(response.StatusCode) || HttpStatusCode.NotFound.Equals(response.StatusCode))
|
|
{
|
|
ClientException e = null;
|
|
ErrorResponse errorResponse = new ErrorResponse();
|
|
|
|
foreach (Parameter parameter in response.Headers)
|
|
{
|
|
if (parameter.Name.Equals("x-amz-id-2", StringComparison.CurrentCultureIgnoreCase))
|
|
{
|
|
errorResponse.XAmzID2 = parameter.Value.ToString();
|
|
}
|
|
if (parameter.Name.Equals("x-amz-request-id", StringComparison.CurrentCultureIgnoreCase))
|
|
{
|
|
errorResponse.RequestID = parameter.Value.ToString();
|
|
}
|
|
}
|
|
|
|
errorResponse.Resource = response.Request.Resource;
|
|
|
|
if (HttpStatusCode.NotFound.Equals(response.StatusCode))
|
|
{
|
|
int pathLength = response.Request.Resource.Split('/').Count();
|
|
if (pathLength > 1)
|
|
{
|
|
errorResponse.Code = "NoSuchKey";
|
|
e = new ObjectNotFoundException();
|
|
}
|
|
else if (pathLength == 1)
|
|
{
|
|
errorResponse.Code = "NoSuchBucket";
|
|
e = new BucketNotFoundException();
|
|
}
|
|
else
|
|
{
|
|
e = new InternalClientException("404 without body resulted in path with less than two components");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
errorResponse.Code = "Forbidden";
|
|
e = new AccessDeniedException();
|
|
}
|
|
e.Response = errorResponse;
|
|
return e;
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(response.Content))
|
|
{
|
|
throw new InternalClientException("Unsuccessful response from server without XML error: " + response.StatusCode);
|
|
}
|
|
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
ErrorResponse errResponse = (ErrorResponse)(new XmlSerializer(typeof(ErrorResponse)).Deserialize(stream));
|
|
string code = errResponse.Code;
|
|
|
|
ClientException clientException;
|
|
|
|
if ("NoSuchBucket".Equals(code)) clientException = new BucketNotFoundException();
|
|
else if ("NoSuchKey".Equals(code)) clientException = new ObjectNotFoundException();
|
|
else if ("InvalidBucketName".Equals(code)) clientException = new InvalidKeyNameException();
|
|
else if ("InvalidObjectName".Equals(code)) clientException = new InvalidKeyNameException();
|
|
else if ("AccessDenied".Equals(code)) clientException = new AccessDeniedException();
|
|
else if ("BucketAlreadyExists".Equals(code)) clientException = new BucketExistsException();
|
|
else if ("ObjectAlreadyExists".Equals(code)) clientException = new ObjectExistsException();
|
|
else if ("InternalError".Equals(code)) clientException = new InternalServerException();
|
|
else if ("KeyTooLong".Equals(code)) clientException = new InvalidKeyNameException();
|
|
else if ("TooManyBuckets".Equals(code)) clientException = new MaxBucketsReachedException();
|
|
else if ("PermanentRedirect".Equals(code)) clientException = new RedirectionException();
|
|
else if ("MethodNotAllowed".Equals(code)) clientException = new ObjectExistsException();
|
|
else if ("BucketAlreadyOwnedByYou".Equals(code)) clientException = new BucketExistsException();
|
|
else clientException = new InternalClientException(errResponse.ToString());
|
|
|
|
|
|
clientException.Response = errResponse;
|
|
clientException.XmlError = response.Content;
|
|
return clientException;
|
|
}
|
|
|
|
private string StripXmlnsXsi(string input)
|
|
{
|
|
string result = input.Replace("xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\"", "");
|
|
result = result.Replace("xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"Group\"", "");
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// List all objects in a bucket
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to list objects from</param>
|
|
/// <returns>An iterator lazily populated with objects</returns>
|
|
public IEnumerable<Item> ListObjects(string bucket)
|
|
{
|
|
return this.ListObjects(bucket, null, true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// List all objects in a bucket with a given prefix
|
|
/// </summary>
|
|
/// <param name="bucket">BUcket to list objects from</param>
|
|
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
|
|
/// <returns>An iterator lazily populated with objects</returns>
|
|
public IEnumerable<Item> ListObjects(string bucket, string prefix)
|
|
{
|
|
return this.ListObjects(bucket, prefix, true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to list objects from</param>
|
|
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
|
|
/// <param name="recursive">Set to false to emulate a directory</param>
|
|
/// <returns>A iterator lazily populated with objects</returns>
|
|
public IEnumerable<Item> ListObjects(string bucket, string prefix, bool recursive)
|
|
{
|
|
bool isRunning = true;
|
|
|
|
string marker = null;
|
|
|
|
while (isRunning)
|
|
{
|
|
Tuple<ListBucketResult, List<Item>> result = GetObjectList(bucket, prefix, recursive, marker);
|
|
Item lastItem = null;
|
|
foreach (Item item in result.Item2)
|
|
{
|
|
lastItem = item;
|
|
yield return item;
|
|
}
|
|
if (result.Item1.NextMarker != null)
|
|
{
|
|
marker = result.Item1.NextMarker;
|
|
}
|
|
else
|
|
{
|
|
marker = lastItem.Key;
|
|
}
|
|
isRunning = result.Item1.IsTruncated;
|
|
}
|
|
}
|
|
|
|
private Tuple<ListBucketResult, List<Item>> GetObjectList(string bucket, string prefix, bool recursive, string marker)
|
|
{
|
|
var queries = new List<string>();
|
|
if (!recursive)
|
|
{
|
|
queries.Add("delimiter=%2F");
|
|
}
|
|
if (prefix != null)
|
|
{
|
|
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
|
|
}
|
|
if (marker != null)
|
|
{
|
|
queries.Add("marker=" + marker);
|
|
}
|
|
string query = string.Join("&", queries);
|
|
|
|
string path = bucket;
|
|
if (query.Length > 0)
|
|
{
|
|
path += "?" + query;
|
|
}
|
|
var request = new RestRequest(path, Method.GET);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
ListBucketResult listBucketResult = (ListBucketResult)(new XmlSerializer(typeof(ListBucketResult)).Deserialize(stream));
|
|
|
|
XDocument root = XDocument.Parse(response.Content);
|
|
|
|
var items = (from c in root.Root.Descendants("{http://s3.amazonaws.com/doc/2006-03-01/}Contents")
|
|
select new Item()
|
|
{
|
|
Key = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}Key").Value,
|
|
LastModified = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}LastModified").Value,
|
|
ETag = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}ETag").Value,
|
|
Size = UInt64.Parse(c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}Size").Value),
|
|
IsDir = false
|
|
});
|
|
|
|
var prefixes = (from c in root.Root.Descendants("{http://s3.amazonaws.com/doc/2006-03-01/}CommonPrefixes")
|
|
select new Item()
|
|
{
|
|
Key = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}Prefix").Value,
|
|
IsDir = true
|
|
});
|
|
|
|
items = items.Concat(prefixes);
|
|
|
|
return new Tuple<ListBucketResult, List<Item>>(listBucketResult, items.ToList());
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
private Tuple<ListMultipartUploadsResult, List<Upload>> GetMultipartUploadsList(string bucket, string prefix, string keyMarker, string uploadIdMarker)
|
|
{
|
|
var queries = new List<string>();
|
|
queries.Add("uploads");
|
|
if (prefix != null)
|
|
{
|
|
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
|
|
}
|
|
if (keyMarker != null)
|
|
{
|
|
queries.Add("key-marker=" + Uri.EscapeDataString(keyMarker));
|
|
}
|
|
if (uploadIdMarker != null)
|
|
{
|
|
queries.Add("upload-id-marker=" + uploadIdMarker);
|
|
}
|
|
|
|
string query = string.Join("&", queries);
|
|
string path = bucket;
|
|
path += "?" + query;
|
|
|
|
var request = new RestRequest(path, Method.GET);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
|
|
var stream = new MemoryStream(contentBytes);
|
|
ListMultipartUploadsResult listBucketResult = (ListMultipartUploadsResult)(new XmlSerializer(typeof(ListMultipartUploadsResult)).Deserialize(stream));
|
|
|
|
XDocument root = XDocument.Parse(response.Content);
|
|
|
|
var uploads = (from c in root.Root.Descendants("{http://s3.amazonaws.com/doc/2006-03-01/}Upload")
|
|
select new Upload()
|
|
{
|
|
Key = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}Key").Value,
|
|
UploadId = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}UploadId").Value,
|
|
Initiated = c.Element("{http://s3.amazonaws.com/doc/2006-03-01/}Initiated").Value
|
|
});
|
|
|
|
return new Tuple<ListMultipartUploadsResult, List<Upload>>(listBucketResult, uploads.ToList());
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lists all incomplete uploads in a given bucket
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to list all incomplepte uploads from</param>
|
|
/// <returns>A lazily populated list of incomplete uploads</returns>
|
|
public IEnumerable<Upload> ListAllIncompleteUploads(string bucket)
|
|
{
|
|
return this.ListAllIncompleteUploads(bucket, null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lists all incomplete uploads in a given bucket with a given key
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to list incomplete uploads from</param>
|
|
/// <param name="key">Key of object to list incomplete uploads from</param>
|
|
/// <returns></returns>
|
|
public IEnumerable<Upload> ListAllIncompleteUploads(string bucket, string key)
|
|
{
|
|
string nextKeyMarker = null;
|
|
string nextUploadIdMarker = null;
|
|
bool isRunning = true;
|
|
while (isRunning)
|
|
{
|
|
var uploads = GetMultipartUploadsList(bucket, key, nextKeyMarker, nextUploadIdMarker);
|
|
foreach (Upload upload in uploads.Item2)
|
|
{
|
|
if (key != null && !key.Equals(upload.Key))
|
|
{
|
|
continue;
|
|
}
|
|
yield return upload;
|
|
}
|
|
nextKeyMarker = uploads.Item1.NextKeyMarker;
|
|
nextUploadIdMarker = uploads.Item1.NextUploadIdMarker;
|
|
isRunning = uploads.Item1.IsTruncated;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Drop incomplete uploads from a given bucket and key
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to drop incomplete uploads from</param>
|
|
/// <param name="key">Key to drop incomplete uploads from</param>
|
|
public void DropIncompleteUpload(string bucket, string key)
|
|
{
|
|
var uploads = this.ListAllIncompleteUploads(bucket, key);
|
|
foreach (Upload upload in uploads)
|
|
{
|
|
this.DropUpload(bucket, key, upload.UploadId);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Drops all incomplete uploads from a given bucket
|
|
/// </summary>
|
|
/// <param name="bucket">Bucket to drop all incomplete uploads from</param>
|
|
public void DropAllIncompleteUploads(string bucket)
|
|
{
|
|
var uploads = this.ListAllIncompleteUploads(bucket);
|
|
foreach (Upload upload in uploads)
|
|
{
|
|
this.DropUpload(bucket, upload.Key, upload.UploadId);
|
|
}
|
|
}
|
|
|
|
private void DropUpload(string bucket, string key, string uploadId)
|
|
{
|
|
var path = bucket + "/" + UrlEncode(key) + "?uploadId=" + uploadId;
|
|
var request = new RestRequest(path, Method.DELETE);
|
|
var response = client.Execute(request);
|
|
|
|
if (response.StatusCode == HttpStatusCode.NoContent)
|
|
{
|
|
return;
|
|
}
|
|
throw ParseError(response);
|
|
}
|
|
|
|
private string UrlEncode(string input)
|
|
{
|
|
return Uri.EscapeDataString(input).Replace("%2F", "/");
|
|
}
|
|
}
|
|
}
|