org.apache.hadoop.hdfs
Class DistributedFileSystem

java.lang.Object
  extended by org.apache.hadoop.conf.Configured
      extended by org.apache.hadoop.fs.FileSystem
          extended by org.apache.hadoop.hdfs.DistributedFileSystem
All Implemented Interfaces:
Closeable, org.apache.hadoop.conf.Configurable

@InterfaceAudience.Private
@InterfaceStability.Evolving
public class DistributedFileSystem
extends org.apache.hadoop.fs.FileSystem

Implementation of the abstract FileSystem for the DFS system. This object is the way end-user code interacts with a Hadoop DistributedFileSystem.


Nested Class Summary
static class DistributedFileSystem.DiskStatus
          Deprecated. Use FsStatus instead
 
Nested classes/interfaces inherited from class org.apache.hadoop.fs.FileSystem
org.apache.hadoop.fs.FileSystem.Statistics
 
Field Summary
 
Fields inherited from class org.apache.hadoop.fs.FileSystem
DEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, statistics
 
Constructor Summary
DistributedFileSystem()
           
DistributedFileSystem(InetSocketAddress namenode, org.apache.hadoop.conf.Configuration conf)
          Deprecated. 
 
Method Summary
 org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f, int bufferSize, org.apache.hadoop.util.Progressable progress)
          This optional operation is not yet supported.
 void cancelDelegationToken(org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> token)
          Cancel an existing delegation token.
protected  void checkPath(org.apache.hadoop.fs.Path path)
          Permit paths which explicitly specify the default port.
 void close()
          
 void concat(org.apache.hadoop.fs.Path trg, org.apache.hadoop.fs.Path[] psrcs)
          THIS IS DFS only operations, it is not part of FileSystem move blocks from srcs to trg and delete srcs afterwards all blocks should be the same size
 org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
           
 org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flag, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
          Same as create(), except fails if parent directory doesn't already exist.
 boolean delete(org.apache.hadoop.fs.Path f, boolean recursive)
           
 UpgradeStatusReport distributedUpgradeProgress(FSConstants.UpgradeAction action)
           
 void finalizeUpgrade()
          Finalize previously upgraded files system state.
 DFSClient getClient()
           
 org.apache.hadoop.fs.ContentSummary getContentSummary(org.apache.hadoop.fs.Path f)
          
 long getCorruptBlocksCount()
          Returns count of blocks with at least one replica marked corrupt.
 DatanodeInfo[] getDataNodeStats()
          Return statistics for each datanode.
 long getDefaultBlockSize()
           
 short getDefaultReplication()
           
 org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> getDelegationToken(org.apache.hadoop.io.Text renewer)
          Get a valid Delegation Token.
 DistributedFileSystem.DiskStatus getDiskStatus()
          Deprecated. Use FileSystem.getStatus() instead
 org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.FileStatus file, long start, long len)
           
 org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.Path p, long start, long len)
           
 org.apache.hadoop.fs.MD5MD5CRC32FileChecksum getFileChecksum(org.apache.hadoop.fs.Path f)
          
 org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path f)
          Returns the stat information about the file.
 org.apache.hadoop.fs.Path getHomeDirectory()
          
 long getMissingBlocksCount()
          Returns count of blocks with no good replicas left.
 long getRawCapacity()
          Deprecated. Use FileSystem.getStatus() instead
 long getRawUsed()
          Deprecated. Use FileSystem.getStatus() instead
 org.apache.hadoop.fs.FsServerDefaults getServerDefaults()
          
 org.apache.hadoop.fs.FsStatus getStatus(org.apache.hadoop.fs.Path p)
          
 long getUnderReplicatedBlocksCount()
          Returns count of blocks with one of more replica missing.
 URI getUri()
           
 org.apache.hadoop.fs.Path getWorkingDirectory()
           
 void initialize(URI uri, org.apache.hadoop.conf.Configuration conf)
           
 org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path p)
          List all the entries of a directory Note that this operation is not atomic for a large directory.
 org.apache.hadoop.fs.Path makeQualified(org.apache.hadoop.fs.Path path)
          Normalize paths that explicitly specify the default port.
 void metaSave(String pathname)
           
 boolean mkdir(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission)
          Create a directory with given name and permission, only when parent directory exists.
 boolean mkdirs(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission)
           
 org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f, int bufferSize)
           
protected  org.apache.hadoop.fs.FSDataOutputStream primitiveCreate(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission absolutePermission, EnumSet<org.apache.hadoop.fs.CreateFlag> flag, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress, int bytesPerChecksum)
           
protected  boolean primitiveMkdir(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission absolutePermission)
           
 void refreshNodes()
          Refreshes the list of hosts and excluded hosts from the configured files.
 boolean rename(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
          
 void rename(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst, org.apache.hadoop.fs.Options.Rename... options)
           This rename operation is guaranteed to be atomic.
 long renewDelegationToken(org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> token)
          Renew an existing delegation token.
 boolean reportChecksumFailure(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.FSDataInputStream in, long inPos, org.apache.hadoop.fs.FSDataInputStream sums, long sumsPos)
          We need to find the blocks that didn't match.
 boolean restoreFailedStorage(String arg)
          enable/disable/check restoreFaileStorage
 void saveNamespace()
          Save namespace image.
 void setOwner(org.apache.hadoop.fs.Path p, String username, String groupname)
          
 void setPermission(org.apache.hadoop.fs.Path p, org.apache.hadoop.fs.permission.FsPermission permission)
          
 void setQuota(org.apache.hadoop.fs.Path src, long namespaceQuota, long diskspaceQuota)
          Set a directory's quotas
 boolean setReplication(org.apache.hadoop.fs.Path src, short replication)
           
 boolean setSafeMode(FSConstants.SafeModeAction action)
          Enter, leave or get safe mode.
 void setTimes(org.apache.hadoop.fs.Path p, long mtime, long atime)
          
 void setVerifyChecksum(boolean verifyChecksum)
           
 void setWorkingDirectory(org.apache.hadoop.fs.Path dir)
           
 String toString()
           
 
Methods inherited from class org.apache.hadoop.fs.FileSystem
append, append, clearStatistics, closeAll, completeLocalOutput, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyToLocalFile, copyToLocalFile, create, create, create, create, create, create, create, create, create, create, createNewFile, delete, deleteOnExit, exists, get, get, get, getAllStatistics, getBlockSize, getDefaultUri, getInitialWorkingDirectory, getLength, getLocal, getName, getNamed, getReplication, getStatistics, getStatistics, getStatus, getUsed, globStatus, globStatus, isDirectory, isFile, listStatus, listStatus, listStatus, mkdirs, mkdirs, moveFromLocalFile, moveFromLocalFile, moveToLocalFile, newInstance, newInstance, newInstance, newInstanceLocal, open, primitiveMkdir, printStatistics, processDeleteOnExit, setDefaultUri, setDefaultUri, startLocalOutput
 
Methods inherited from class org.apache.hadoop.conf.Configured
getConf, setConf
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
 

Constructor Detail

DistributedFileSystem

public DistributedFileSystem()

DistributedFileSystem

@Deprecated
public DistributedFileSystem(InetSocketAddress namenode,
                                        org.apache.hadoop.conf.Configuration conf)
                      throws IOException
Deprecated. 

Throws:
IOException
Method Detail

getUri

public URI getUri()
Specified by:
getUri in class org.apache.hadoop.fs.FileSystem

initialize

public void initialize(URI uri,
                       org.apache.hadoop.conf.Configuration conf)
                throws IOException
Overrides:
initialize in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

checkPath

protected void checkPath(org.apache.hadoop.fs.Path path)
Permit paths which explicitly specify the default port.

Overrides:
checkPath in class org.apache.hadoop.fs.FileSystem

makeQualified

public org.apache.hadoop.fs.Path makeQualified(org.apache.hadoop.fs.Path path)
Normalize paths that explicitly specify the default port.

Overrides:
makeQualified in class org.apache.hadoop.fs.FileSystem

getWorkingDirectory

public org.apache.hadoop.fs.Path getWorkingDirectory()
Specified by:
getWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getDefaultBlockSize

public long getDefaultBlockSize()
Overrides:
getDefaultBlockSize in class org.apache.hadoop.fs.FileSystem

getDefaultReplication

public short getDefaultReplication()
Overrides:
getDefaultReplication in class org.apache.hadoop.fs.FileSystem

setWorkingDirectory

public void setWorkingDirectory(org.apache.hadoop.fs.Path dir)
Specified by:
setWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getHomeDirectory

public org.apache.hadoop.fs.Path getHomeDirectory()

Overrides:
getHomeDirectory in class org.apache.hadoop.fs.FileSystem

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.FileStatus file,
                                                                  long start,
                                                                  long len)
                                                           throws IOException
Overrides:
getFileBlockLocations in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.Path p,
                                                                  long start,
                                                                  long len)
                                                           throws IOException
Overrides:
getFileBlockLocations in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

setVerifyChecksum

public void setVerifyChecksum(boolean verifyChecksum)
Overrides:
setVerifyChecksum in class org.apache.hadoop.fs.FileSystem

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f,
                                                   int bufferSize)
                                            throws IOException
Specified by:
open in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f,
                                                      int bufferSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException
This optional operation is not yet supported.

Specified by:
append in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException
Specified by:
create in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

primitiveCreate

protected org.apache.hadoop.fs.FSDataOutputStream primitiveCreate(org.apache.hadoop.fs.Path f,
                                                                  org.apache.hadoop.fs.permission.FsPermission absolutePermission,
                                                                  EnumSet<org.apache.hadoop.fs.CreateFlag> flag,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress,
                                                                  int bytesPerChecksum)
                                                           throws IOException
Overrides:
primitiveCreate in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  org.apache.hadoop.fs.permission.FsPermission permission,
                                                                  EnumSet<org.apache.hadoop.fs.CreateFlag> flag,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException
Same as create(), except fails if parent directory doesn't already exist.

Throws:
IOException

setReplication

public boolean setReplication(org.apache.hadoop.fs.Path src,
                              short replication)
                       throws IOException
Overrides:
setReplication in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

concat

public void concat(org.apache.hadoop.fs.Path trg,
                   org.apache.hadoop.fs.Path[] psrcs)
            throws IOException
THIS IS DFS only operations, it is not part of FileSystem move blocks from srcs to trg and delete srcs afterwards all blocks should be the same size

Parameters:
trg - existing file to append to
psrcs - list of files (same block size, same replication)
Throws:
IOException

rename

public boolean rename(org.apache.hadoop.fs.Path src,
                      org.apache.hadoop.fs.Path dst)
               throws IOException

Specified by:
rename in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

rename

public void rename(org.apache.hadoop.fs.Path src,
                   org.apache.hadoop.fs.Path dst,
                   org.apache.hadoop.fs.Options.Rename... options)
            throws IOException
This rename operation is guaranteed to be atomic.

Overrides:
rename in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

delete

public boolean delete(org.apache.hadoop.fs.Path f,
                      boolean recursive)
               throws IOException
Specified by:
delete in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

getContentSummary

public org.apache.hadoop.fs.ContentSummary getContentSummary(org.apache.hadoop.fs.Path f)
                                                      throws IOException

Overrides:
getContentSummary in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

setQuota

public void setQuota(org.apache.hadoop.fs.Path src,
                     long namespaceQuota,
                     long diskspaceQuota)
              throws IOException
Set a directory's quotas

Throws:
IOException
See Also:
ClientProtocol.setQuota(String, long, long)

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path p)
                                             throws IOException
List all the entries of a directory Note that this operation is not atomic for a large directory. The entries of a directory may be fetched from NameNode multiple times. It only guarantees that each name occurs once if a directory undergoes changes between the calls.

Specified by:
listStatus in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

mkdir

public boolean mkdir(org.apache.hadoop.fs.Path f,
                     org.apache.hadoop.fs.permission.FsPermission permission)
              throws IOException
Create a directory with given name and permission, only when parent directory exists.

Throws:
IOException

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path f,
                      org.apache.hadoop.fs.permission.FsPermission permission)
               throws IOException
Specified by:
mkdirs in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

primitiveMkdir

protected boolean primitiveMkdir(org.apache.hadoop.fs.Path f,
                                 org.apache.hadoop.fs.permission.FsPermission absolutePermission)
                          throws IOException
Overrides:
primitiveMkdir in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

close

public void close()
           throws IOException

Specified by:
close in interface Closeable
Overrides:
close in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

toString

public String toString()
Overrides:
toString in class Object

getClient

public DFSClient getClient()

getStatus

public org.apache.hadoop.fs.FsStatus getStatus(org.apache.hadoop.fs.Path p)
                                        throws IOException

Overrides:
getStatus in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

getDiskStatus

@Deprecated
public DistributedFileSystem.DiskStatus getDiskStatus()
                                               throws IOException
Deprecated. Use FileSystem.getStatus() instead

Return the disk usage of the filesystem, including total capacity, used space, and remaining space

Throws:
IOException

getRawCapacity

@Deprecated
public long getRawCapacity()
                    throws IOException
Deprecated. Use FileSystem.getStatus() instead

Return the total raw capacity of the filesystem, disregarding replication.

Throws:
IOException

getRawUsed

@Deprecated
public long getRawUsed()
                throws IOException
Deprecated. Use FileSystem.getStatus() instead

Return the total raw used space in the filesystem, disregarding replication.

Throws:
IOException

getMissingBlocksCount

public long getMissingBlocksCount()
                           throws IOException
Returns count of blocks with no good replicas left. Normally should be zero.

Throws:
IOException

getUnderReplicatedBlocksCount

public long getUnderReplicatedBlocksCount()
                                   throws IOException
Returns count of blocks with one of more replica missing.

Throws:
IOException

getCorruptBlocksCount

public long getCorruptBlocksCount()
                           throws IOException
Returns count of blocks with at least one replica marked corrupt.

Throws:
IOException

getDataNodeStats

public DatanodeInfo[] getDataNodeStats()
                                throws IOException
Return statistics for each datanode.

Throws:
IOException

setSafeMode

public boolean setSafeMode(FSConstants.SafeModeAction action)
                    throws IOException
Enter, leave or get safe mode.

Throws:
IOException
See Also:
ClientProtocol.setSafeMode( FSConstants.SafeModeAction)

saveNamespace

public void saveNamespace()
                   throws org.apache.hadoop.security.AccessControlException,
                          IOException
Save namespace image.

Throws:
org.apache.hadoop.security.AccessControlException
IOException
See Also:
ClientProtocol.saveNamespace()

restoreFailedStorage

public boolean restoreFailedStorage(String arg)
                             throws org.apache.hadoop.security.AccessControlException
enable/disable/check restoreFaileStorage

Throws:
org.apache.hadoop.security.AccessControlException
See Also:
ClientProtocol.restoreFailedStorage(String arg)

refreshNodes

public void refreshNodes()
                  throws IOException
Refreshes the list of hosts and excluded hosts from the configured files.

Throws:
IOException

finalizeUpgrade

public void finalizeUpgrade()
                     throws IOException
Finalize previously upgraded files system state.

Throws:
IOException

distributedUpgradeProgress

public UpgradeStatusReport distributedUpgradeProgress(FSConstants.UpgradeAction action)
                                               throws IOException
Throws:
IOException

metaSave

public void metaSave(String pathname)
              throws IOException
Throws:
IOException

getServerDefaults

public org.apache.hadoop.fs.FsServerDefaults getServerDefaults()
                                                        throws IOException

Overrides:
getServerDefaults in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

reportChecksumFailure

public boolean reportChecksumFailure(org.apache.hadoop.fs.Path f,
                                     org.apache.hadoop.fs.FSDataInputStream in,
                                     long inPos,
                                     org.apache.hadoop.fs.FSDataInputStream sums,
                                     long sumsPos)
We need to find the blocks that didn't match. Likely only one is corrupt but we will report both to the namenode. In the future, we can consider figuring out exactly which block is corrupt.


getFileStatus

public org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path f)
                                              throws IOException
Returns the stat information about the file.

Specified by:
getFileStatus in class org.apache.hadoop.fs.FileSystem
Throws:
FileNotFoundException - if the file does not exist.
IOException

getFileChecksum

public org.apache.hadoop.fs.MD5MD5CRC32FileChecksum getFileChecksum(org.apache.hadoop.fs.Path f)
                                                             throws IOException

Overrides:
getFileChecksum in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

setPermission

public void setPermission(org.apache.hadoop.fs.Path p,
                          org.apache.hadoop.fs.permission.FsPermission permission)
                   throws IOException

Overrides:
setPermission in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

setOwner

public void setOwner(org.apache.hadoop.fs.Path p,
                     String username,
                     String groupname)
              throws IOException

Overrides:
setOwner in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

setTimes

public void setTimes(org.apache.hadoop.fs.Path p,
                     long mtime,
                     long atime)
              throws IOException

Overrides:
setTimes in class org.apache.hadoop.fs.FileSystem
Throws:
IOException

getDelegationToken

public org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> getDelegationToken(org.apache.hadoop.io.Text renewer)
                                                                                     throws IOException
Get a valid Delegation Token.

Parameters:
renewer - Name of the designated renewer for the token
Returns:
Token
Throws:
IOException

renewDelegationToken

public long renewDelegationToken(org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> token)
                          throws org.apache.hadoop.security.token.SecretManager.InvalidToken,
                                 IOException
Renew an existing delegation token.

Parameters:
token - delegation token obtained earlier
Returns:
the new expiration time
Throws:
IOException
org.apache.hadoop.security.token.SecretManager.InvalidToken

cancelDelegationToken

public void cancelDelegationToken(org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> token)
                           throws IOException
Cancel an existing delegation token.

Parameters:
token - delegation token
Throws:
IOException


Copyright © 2009 The Apache Software Foundation