Recursive Upload from Disk to Azure Storage


  • 2016 Aug 15 - Updated to work with latest C# libs

using System;

using System.IO;

using System.Threading.Tasks;

using Microsoft.WindowsAzure.Storage;

using Microsoft.WindowsAzure.Storage.Blob;

 

 

namespace UploadToStorage

{

    class Program

    {

        const string StorageAccountConnectionString
=
"DefaultEndpointsProtocol=https;AccountName=[your account];AccountKey=[your key]";

        const string RootContainerName
=
"servetest";

        const string startingFolder
=
@"C:\[your path]";

 

        static System.Collections.Specialized.StringCollection
log =
new System.Collections.Specialized.StringCollection();

        static Int64 folderCount = 0;

        static Int64 fileCount = 0;

        static CloudStorageAccount storageAccount;

        static CloudBlobClient blobClient;

        static CloudBlobContainer rootContainer;

 

        static void Main(string[] args)

        {

            //open the storage account

            storageAccount = CloudStorageAccount.Parse(StorageAccountConnectionString);

 

            //create a blob client

            blobClient = storageAccount.CreateCloudBlobClient();

 

            //get a reference to the container which we’ll use to  create the blob

            rootContainer = blobClient.GetContainerReference(RootContainerName);

 

            //get the DirectoryInfo for the
starting local folder that will be recursed

            DirectoryInfo rootDir = new DirectoryInfo(startingFolder);

 

            //get started

            WalkDirectoryTree(rootDir, rootContainer);

 

            // Write out all the files that could not be processed.

            Console.WriteLine("Files with restricted
access:"
);

 

            foreach (string s in log)

            {

                Console.WriteLine(s);

            }

 

            // Keep the console window open in debug mode.

            Console.WriteLine("Press any key");

            Console.ReadKey();

 

        }

 

        //Co-opted initially from
http://msdn.microsoft.com/en-us/library/bb513869.aspx

        //modified for azure upload and parallel processing

 

        private static void WalkDirectoryTree(System.IO.DirectoryInfo currentPath,
CloudBlobContainer cloudContainer)

        {

            System.IO.FileInfo[]
files =
null;

            System.IO.DirectoryInfo[]
subDirs =
null;

 

            string pathFromRoot
= currentPath.FullName.Substring(startingFolder.Length);

 

            //replace the slashes in order to transition from local filename
path to Azure Storage Blob name

            string cloudFileNamePathPrefix
= pathFromRoot.Replace(
@"\", @"/");

 

            //fixup terminal character if there is a non-empty path

            //If we’re at the root then we don’t need an end delimiter, but
if we are we do

           cloudFileNamePathPrefix = cloudFileNamePathPrefix.Length
== 0 ?
"" : cloudFileNamePathPrefix + "/";

 

            if (cloudFileNamePathPrefix.Length
> 0)

            {

                if (cloudFileNamePathPrefix.Substring(0,
1).Equals(
"/")) { cloudFileNamePathPrefix = cloudFileNamePathPrefix.Substring(1); }

            }

 

            folderCount++;

            Console.WriteLine("Processing folder " + currentPath.FullName);

 

            // First, process all the files directly under this folder

            try

            {

                files = currentPath.GetFiles("*.*");

            }

            // This is thrown if even one of the files requires permissions
greater

            // than the application provides.

            catch (UnauthorizedAccessException e)

            {

                // This code just writes out the message and continues to recurse.

                log.Add(e.Message);

            }

            catch (System.IO.DirectoryNotFoundException e)

            {

                Console.WriteLine(e.Message);

            }

 

            //setting up to as manys as 5 folders
and 10 files per folder working in upload at a time

            //adjust according to bandwidth

            var fileParallelOptions = new ParallelOptions() { MaxDegreeOfParallelism
= 10 };

            var folderParallelOptions = new ParallelOptions() { MaxDegreeOfParallelism
= 5 };

 

            if (files !=
null)

            {

                //build blob name prefix based on delta of root folder path to
current path

                Parallel.ForEach(files, fileParallelOptions, currentFileInfo =>

                {

                    //just keeping a count of files

                    fileCount++;

       

                    //construct new filename for cloud

                    //inlcudes path information from the
root

                    string cloudFilename
= cloudFileNamePathPrefix + currentFileInfo.Name;

                    //string cloudFilename = currentFileInfo.Name;

                    try

                    {

                        var blobRef = rootContainer.GetBlockBlobReference(cloudFilename);

                        //if exists, don’t upload

                        //if uploading to an empty location, this could be refactored
and skip the check

                        if (!blobRef.Exists())

                        {


//upload blob


blobRef.OpenWrite(
null, null, null);


// Console.WriteLine(string.Format("Uploading {
0} as { 1} ", currentFileInfo.Name, cloudFilename));

 

                            //FileMode refers to access on local
file not azure blob


blobRef.UploadFromFile(currentFileInfo.FullName,
null, null, null);//, null);


Console.WriteLine("Upload Complete for " + cloudFilename) ;

                        }

                    }

                    catch (Exception ex)

                    {

                        //TODO:  Log error

                    }

                 }

               );  //end Parallel.ForEach
(Files...

 

                // Now find all the subdirectories under this directory.

                subDirs = currentPath.GetDirectories();

                Parallel.ForEach(subDirs, folderParallelOptions,
currentDirInfo =>

                {

 

                    //check for matching folder in container, create if not present

                    // Resursive call for each
subdirectory.

                    WalkDirectoryTree(currentDirInfo, cloudContainer);

                }

 

                );

 

            } //end if (files != null)

 

            Console.WriteLine("Processing complete for
folder "
+ currentPath.FullName);

            Console.WriteLine("Processed {0} folders and
{1} files "
, folderCount,
fileCount);

 

        }

 

    }

 

}

 

Comments (1)

  1. Bob says:

    This saved me a lot of time!  Thanks!

Skip to main content