Catch访问路径被拒绝异常

这是我的类，它从每个目录返回最新的文件，问题是尽管我把代码放在Try-Catch块中，但我仍然收到路径被拒绝的错误

public class NewestFiles
{
    //public event EventHandler newFileEventHandler;
    // Dictionary: 
    //   Key = The directory name.
    //   Value = The most recently modified file for that directory.
    private Dictionary<string, string> GetNewestFiles(string directory, Dictionary<string, string> dictionary)
    {
        if (dictionary == null)
            dictionary = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
        try
        {
            var files = from file in Directory.GetFiles(directory, "*.*")
                        select new FileInfo(file);
            var latestFile = files.OrderByDescending(file => { return file.LastWriteTimeUtc; }).FirstOrDefault();
            if (latestFile != null)
                dictionary[latestFile.DirectoryName] = latestFile.FullName;
        }
        catch(Exception)
        { }
        foreach (var subDirectory in Directory.GetDirectories(directory))
        {
            try
            {
                GetNewestFiles(subDirectory, dictionary);
            }
            catch { }
        }
        return dictionary;
    }
    public Dictionary<string, string> GetNewestFiles(string directory)
    {
        return GetNewestFiles(directory, null);
    }
}