package Hdfs.Internal
Get desktop application:
View/edit binary Protocol Buffers messages
* Protocol used from client to the Datanode. See the request and response for details of rpc call.
* Delete the block pool from the datanode.
message
ClientDatanodeProtocol.proto:72* blockPool - block pool to be deleted force - if false, delete the block pool only if it is empty. if true, delete the block pool even if it has blocks.
required string blockPool = 1
required bool force = 2
message
ClientDatanodeProtocol.proto:80* void response
(message has no fields)
* Retrieves the path names of the block file and metadata file stored on the local file system.
message
ClientDatanodeProtocol.proto:90* Gets the file information where block and its metadata is stored block - block for which path information is being requested token - block token This message is deprecated in favor of file descriptor passing.
required block = 1
required token = 2
message
ClientDatanodeProtocol.proto:102* block - block for which file path information is being returned localPath - file path where the block data is stored localMetaPath - file path where the block meta data is stored This message is deprecated in favor of file descriptor passing.
required block = 1
required string localPath = 2
required string localMetaPath = 3
* Retrieve additional HDFS-specific metadata about a set of blocks stored on the local file system.
message
ClientDatanodeProtocol.proto:112* blocks - list of ExtendedBlocks on which we are querying additional info tokens - list of access tokens corresponding to list of ExtendedBlocks
repeated blocks = 1
repeated tokens = 2
message
ClientDatanodeProtocol.proto:123* volumeIds - id of each volume, potentially multiple bytes volumeIndexes - for each block, an index into volumeIds specifying the volume on which it is located. If block is not present on any volume, index is set to MAX_INT.
repeated bytes volumeIds = 1
repeated uint32 volumeIndexes = 2
* Returns the visible length of the replica
message
ClientDatanodeProtocol.proto:44* block - block for which visible length is requested
required block = 1
message
ClientDatanodeProtocol.proto:51* length - visible length of the block
required uint64 length = 1
* Refresh the list of federated namenodes from updated configuration. Adds new namenodes and stops the deleted namenodes.
message
ClientDatanodeProtocol.proto:58* void request
(message has no fields)
message
ClientDatanodeProtocol.proto:64* void response
(message has no fields)
message
ClientNamenodeProtocol.proto:121required b = 1
required string src = 2
required string holder = 3
optional uint64 fileId = 4
message
ClientNamenodeProtocol.proto:128void response
(message has no fields)
message
ClientNamenodeProtocol.proto:131required string src = 1
required string clientName = 2
optional previous = 3
repeated excludeNodes = 4
optional uint64 fileId = 5
default as a bogus id
repeated string favoredNodes = 6
the set of datanodes to use for the block
message
ClientNamenodeProtocol.proto:140required block = 1
message
ClientNamenodeProtocol.proto:425required info = 1
optional uint32 cacheFlags = 2
bits set using CacheFlag
message
ClientNamenodeProtocol.proto:430required int64 id = 1
message
ClientNamenodeProtocol.proto:481required info = 1
message
ClientNamenodeProtocol.proto:485void response
(message has no fields)
message
ClientNamenodeProtocol.proto:628required string snapshotRoot = 1
message
ClientNamenodeProtocol.proto:632(message has no fields)
message
ClientNamenodeProtocol.proto:84required string src = 1
required string clientName = 2
optional uint32 flag = 3
bits set using CreateFlag
message
ClientNamenodeProtocol.proto:90optional block = 1
optional stat = 2
message
Security.proto:57required token = 1
message
Security.proto:61void response
(message has no fields)
message
ClientNamenodeProtocol.proto:158required string src = 1
required string clientName = 2
optional last = 3
optional uint64 fileId = 4
default to GRANDFATHER_INODE_ID
message
ClientNamenodeProtocol.proto:165required bool result = 1
message
ClientNamenodeProtocol.proto:176required string trg = 1
repeated string srcs = 2
message
ClientNamenodeProtocol.proto:181void response
(message has no fields)
message
ClientNamenodeProtocol.proto:70required string src = 1
required masked = 2
required string clientName = 3
required uint32 createFlag = 4
bits set using CreateFlag
required bool createParent = 5
required uint32 replication = 6
Short: Only 16 bits used
required uint64 blockSize = 7
message
ClientNamenodeProtocol.proto:80optional fs = 1
message
ClientNamenodeProtocol.proto:610required string snapshotRoot = 1
optional string snapshotName = 2
message
ClientNamenodeProtocol.proto:615required string snapshotPath = 1
message
ClientNamenodeProtocol.proto:559required string target = 1
required string link = 2
required dirPerm = 3
required bool createParent = 4
message
ClientNamenodeProtocol.proto:566void response
(message has no fields)
message
ClientNamenodeProtocol.proto:229required string src = 1
required bool recursive = 2
message
ClientNamenodeProtocol.proto:234required bool result = 1
message
ClientNamenodeProtocol.proto:642required string snapshotRoot = 1
required string snapshotName = 2
message
ClientNamenodeProtocol.proto:647void response
(message has no fields)
message
ClientNamenodeProtocol.proto:635required string snapshotRoot = 1
message
ClientNamenodeProtocol.proto:639(message has no fields)
message
ClientNamenodeProtocol.proto:362no parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:365void response
(message has no fields)
message
ClientNamenodeProtocol.proto:541required string src = 1
required string client = 2
optional sint64 lastBlockLength = 3
message
ClientNamenodeProtocol.proto:547void response
(message has no fields)
message
ClientNamenodeProtocol.proto:144required string src = 1
required blk = 2
repeated existings = 3
repeated excludes = 4
required uint32 numAdditionalNodes = 5
required string clientName = 6
repeated string existingStorageUuids = 7
message
ClientNamenodeProtocol.proto:154required block = 1
message
ClientNamenodeProtocol.proto:47required string src = 1
file name
required uint64 offset = 2
range start offset
required uint64 length = 3
range length
message
ClientNamenodeProtocol.proto:53optional locations = 1
message
ClientNamenodeProtocol.proto:524required string path = 1
message
ClientNamenodeProtocol.proto:528required summary = 1
message
ClientNamenodeProtocol.proto:603no parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:606optional dataEncryptionKey = 1
message
ClientNamenodeProtocol.proto:304required DatanodeReportTypeProto type = 1
message
ClientNamenodeProtocol.proto:308repeated di = 1
message
Security.proto:41required string renewer = 1
message
Security.proto:45optional token = 1
message
ClientNamenodeProtocol.proto:384required string src = 1
message
ClientNamenodeProtocol.proto:388optional fs = 1
message
ClientNamenodeProtocol.proto:516required string src = 1
message
ClientNamenodeProtocol.proto:520optional fs = 1
message
ClientNamenodeProtocol.proto:286no input paramters
(message has no fields)
message
ClientNamenodeProtocol.proto:289required uint64 capacity = 1
required uint64 used = 2
required uint64 remaining = 3
required uint64 under_replicated = 4
required uint64 corrupt_blocks = 5
required uint64 missing_blocks = 6
message
ClientNamenodeProtocol.proto:194required string src = 1
required string clientName = 2
message
ClientNamenodeProtocol.proto:199void response
(message has no fields)
message
ClientNamenodeProtocol.proto:569required string path = 1
message
ClientNamenodeProtocol.proto:572optional string targetPath = 1
message
ClientNamenodeProtocol.proto:247required string src = 1
required bytes startAfter = 2
required bool needLocation = 3
message
ClientNamenodeProtocol.proto:252optional dirList = 1
message
ClientNamenodeProtocol.proto:312required string filename = 1
message
ClientNamenodeProtocol.proto:316required uint64 bsize = 1
message
ClientNamenodeProtocol.proto:57No parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:60required serverDefaults = 1
message
ClientNamenodeProtocol.proto:262required string snapshotRoot = 1
required string fromSnapshot = 2
required string toSnapshot = 3
message
ClientNamenodeProtocol.proto:267required diffReport = 1
message
ClientNamenodeProtocol.proto:256no input parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:258optional snapshottableDirList = 1
message
ClientNamenodeProtocol.proto:392required string src = 1
message
ClientNamenodeProtocol.proto:396required bool result = 1
message
ClientNamenodeProtocol.proto:449required int64 prevId = 1
required filter = 2
message
ClientNamenodeProtocol.proto:459repeated elements = 1
required bool hasMore = 2
message
ClientNamenodeProtocol.proto:502required string prevPoolName = 1
message
ClientNamenodeProtocol.proto:506repeated entries = 1
required bool hasMore = 2
message
ClientNamenodeProtocol.proto:368required string path = 1
optional string cookie = 2
message
ClientNamenodeProtocol.proto:373required corrupt = 1
message
ClientNamenodeProtocol.proto:377required string filename = 1
message
ClientNamenodeProtocol.proto:381void response
(message has no fields)
message
ClientNamenodeProtocol.proto:238required string src = 1
required masked = 2
required bool createParent = 3
message
ClientNamenodeProtocol.proto:243required bool result = 1
message
ClientNamenodeProtocol.proto:434required info = 1
optional uint32 cacheFlags = 2
bits set using CacheFlag
message
ClientNamenodeProtocol.proto:439(message has no fields)
message
ClientNamenodeProtocol.proto:488required info = 1
message
ClientNamenodeProtocol.proto:492void response
(message has no fields)
message
ClientNamenodeProtocol.proto:278required string src = 1
required string clientName = 2
message
ClientNamenodeProtocol.proto:282required bool result = 1
message
ClientNamenodeProtocol.proto:356no parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:359void response
(message has no fields)
message
ClientNamenodeProtocol.proto:202required string src = 1
required string clientName = 2
message
ClientNamenodeProtocol.proto:207void response
(message has no fields)
message
ClientNamenodeProtocol.proto:442required int64 id = 1
message
ClientNamenodeProtocol.proto:446(message has no fields)
message
ClientNamenodeProtocol.proto:495required string poolName = 1
message
ClientNamenodeProtocol.proto:499void response
(message has no fields)
message
ClientNamenodeProtocol.proto:210required string src = 1
required string dst = 2
message
ClientNamenodeProtocol.proto:215required bool result = 1
message
ClientNamenodeProtocol.proto:220required string src = 1
required string dst = 2
required bool overwriteDest = 3
message
ClientNamenodeProtocol.proto:226void response
(message has no fields)
message
ClientNamenodeProtocol.proto:619required string snapshotRoot = 1
required string snapshotOldName = 2
required string snapshotNewName = 3
message
ClientNamenodeProtocol.proto:625void response
(message has no fields)
message
Security.proto:49required token = 1
message
Security.proto:53required uint64 newExpiryTime = 1
message
ClientNamenodeProtocol.proto:271required string clientName = 1
message
ClientNamenodeProtocol.proto:275void response
(message has no fields)
message
ClientNamenodeProtocol.proto:169repeated blocks = 1
message
ClientNamenodeProtocol.proto:173void response
(message has no fields)
message
ClientNamenodeProtocol.proto:348required string arg = 1
message
ClientNamenodeProtocol.proto:352required bool result = 1
message
ClientNamenodeProtocol.proto:341no parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:344response
required uint64 newSegmentTxId = 1
message
ClientNamenodeProtocol.proto:335no parameters
(message has no fields)
message
ClientNamenodeProtocol.proto:338void response
(message has no fields)
message
ClientNamenodeProtocol.proto:596required int64 bandwidth = 1
message
ClientNamenodeProtocol.proto:600void response
(message has no fields)
message
ClientNamenodeProtocol.proto:112required string src = 1
optional string username = 2
optional string groupname = 3
message
ClientNamenodeProtocol.proto:118void response
(message has no fields)
message
ClientNamenodeProtocol.proto:104required string src = 1
required permission = 2
message
ClientNamenodeProtocol.proto:109void response
(message has no fields)
message
ClientNamenodeProtocol.proto:532required string path = 1
required uint64 namespaceQuota = 2
required uint64 diskspaceQuota = 3
message
ClientNamenodeProtocol.proto:538void response
(message has no fields)
message
ClientNamenodeProtocol.proto:95required string src = 1
required uint32 replication = 2
Short: Only 16 bits used
message
ClientNamenodeProtocol.proto:100required bool result = 1
message
ClientNamenodeProtocol.proto:326required SafeModeActionProto action = 1
optional bool checked = 2
message
ClientNamenodeProtocol.proto:331required bool result = 1
message
ClientNamenodeProtocol.proto:550required string src = 1
required uint64 mtime = 2
required uint64 atime = 3
message
ClientNamenodeProtocol.proto:556void response
(message has no fields)
message
ClientNamenodeProtocol.proto:184required string src = 1
required uint64 newLength = 2
required string clientName = 3
message
ClientNamenodeProtocol.proto:190required bool result = 1
message
ClientNamenodeProtocol.proto:576required block = 1
required string clientName = 2
message
ClientNamenodeProtocol.proto:581required block = 1
message
ClientNamenodeProtocol.proto:585required string clientName = 1
required oldBlock = 2
required newBlock = 3
repeated newNodes = 4
repeated string storageIDs = 5
message
ClientNamenodeProtocol.proto:593void response
(message has no fields)
message
hdfs.proto:201required policy = 1
required bool succeed = 2
optional string errorMsg = 3
message
datatransfer.proto:48Used in: , , , ,
required block = 1
optional token = 2
message
hdfs.proto:448* Block access token information
Used in:
required uint32 keyId = 1
Key identifier
required uint64 expiryDate = 2
Expiry time in milliseconds
optional bytes keyBytes = 3
Key secret
message
datatransfer.proto:189required Status status = 1
optional string firstBadLink = 2
optional checksumResponse = 3
optional readOpChecksumInfo = 4
optional string message = 5
* explanatory text which may be useful to log on the client side
optional uint32 shortCircuitAccessVersion = 6
* If the server chooses to agree to the request of a client for short-circuit access, it will send a response message with the relevant file descriptors attached. In the body of the message, this version number will be set to the specific version number of the block data that the client is about to read.
message
hdfs.proto:396* Block information
Used in:
required uint64 blockId = 1
required uint64 genStamp = 2
optional uint64 numBytes = 3
message
hdfs.proto:405* Block and datanodes where is it located
Used in:
required block = 1
Block
repeated string datanodeUuids = 2
Datanodes with replicas of the block
repeated string storageUuids = 3
Storages with replicas of the block
message
hdfs.proto:414* List of block with locations
repeated blocks = 1
message
ClientNamenodeProtocol.proto:454Used in:
required info = 1
required stats = 2
message
ClientNamenodeProtocol.proto:408Used in:
required int64 millis = 1
required bool isRelative = 2
message
ClientNamenodeProtocol.proto:400Used in: , , ,
optional int64 id = 1
optional string path = 2
optional uint32 replication = 3
optional string pool = 4
optional expiration = 5
message
ClientNamenodeProtocol.proto:413Used in:
required int64 bytesNeeded = 1
required int64 bytesCached = 2
required int64 filesNeeded = 3
required int64 filesCached = 4
required bool hasExpired = 5
FORCE = 1
Ignore pool resource limits
message
ClientNamenodeProtocol.proto:511Used in:
required info = 1
required stats = 2
message
ClientNamenodeProtocol.proto:464Used in: , ,
optional string poolName = 1
optional string ownerName = 2
optional string groupName = 3
optional int32 mode = 4
optional int64 limit = 5
optional int64 maxRelativeExpiry = 6
message
ClientNamenodeProtocol.proto:473Used in:
required int64 bytesNeeded = 1
required int64 bytesCached = 2
required int64 bytesOverlimit = 3
required int64 filesNeeded = 4
required int64 filesCached = 5
message
datatransfer.proto:58Used in: ,
optional bool dropBehind = 1
optional int64 readahead = 2
message
hdfs.proto:385* Command returned from primary to checkpointing namenode. This command has checkpoint signature that identifies checkpoint transaction and is needed for further communication related to checkpointing.
Used in:
required signature = 1
Unique signature to identify checkpoint transation
required bool needToReturnImage = 2
If true, return transfer image to primary upon the completion of checkpoint
message
hdfs.proto:359* Unique signature to identify checkpoint transactions.
Used in:
required string blockPoolId = 1
required uint64 mostRecentCheckpointTxId = 2
required uint64 curSegmentTxId = 3
required storageInfo = 4
message
datatransfer.proto:72Used in: ,
required ChecksumTypeProto type = 1
required uint32 bytesPerChecksum = 2
* Checksum algorithms/types used in HDFS Make sure this enum's integer values match enum values' id properties defined in org.apache.hadoop.util.DataChecksum.Type
Used in: , ,
CHECKSUM_NULL = 0
CHECKSUM_CRC32 = 1
CHECKSUM_CRC32C = 2
message
datatransfer.proto:53Used in: , ,
required baseHeader = 1
required string clientName = 2
message
datatransfer.proto:214* Message sent from the client to the DN after reading the entire read request.
required Status status = 1
message
hdfs.proto:95* Summary of a file or directory
Used in:
required uint64 length = 1
required uint64 fileCount = 2
required uint64 directoryCount = 3
required uint64 quota = 4
required uint64 spaceConsumed = 5
required uint64 spaceQuota = 6
message
hdfs.proto:109* Contains a list of paths corresponding to corrupt files and a cookie used for iterative calls to NameNode.listCorruptFileBlocks.
Used in:
repeated string files = 1
required string cookie = 2
CREATE = 1
Create a file
OVERWRITE = 2
Truncate/overwrite a file. Same as POSIX O_TRUNC
APPEND = 4
Append to a file
message
datatransfer.proto:218required Status status = 1
message
hdfs.proto:157Used in:
required uint32 keyId = 1
required string blockPoolId = 2
required bytes nonce = 3
required bytes encryptionKey = 4
required uint64 expiryDate = 5
optional string encryptionAlgorithm = 6
message
datatransfer.proto:37required DataTransferEncryptorMessageProto.DataTransferEncryptorStatus status = 1
optional bytes payload = 2
optional string message = 3
Used in:
SUCCESS = 0
ERROR_UNKNOWN_KEY = 1
ERROR = 2
message
hdfs.proto:49* Identifies a Datanode
Used in: ,
required string ipAddr = 1
IP address
required string hostName = 2
hostname
required string datanodeUuid = 3
UUID assigned to the Datanode. For
required uint32 xferPort = 4
upgraded clusters this is the same as the original StorageID of the Datanode.
data streaming port
required uint32 infoPort = 5
datanode http port
required uint32 ipcPort = 6
ipc server port
optional uint32 infoSecurePort = 7
datanode https port
message
hdfs.proto:72* The status of a Datanode
Used in: , , , , , , ,
required id = 1
optional uint64 capacity = 2
optional uint64 dfsUsed = 3
optional uint64 remaining = 4
optional uint64 blockPoolUsed = 5
optional uint64 lastUpdate = 6
optional uint32 xceiverCount = 7
optional string location = 8
optional DatanodeInfoProto.AdminState adminState = 10
optional uint64 cacheCapacity = 11
optional uint64 cacheUsed = 12
Used in:
NORMAL = 0
DECOMMISSION_INPROGRESS = 1
DECOMMISSIONED = 2
message
hdfs.proto:65* DatanodeInfo array
repeated datanodes = 1
type of the datanode report
Used in:
ALL = 1
LIVE = 2
DEAD = 3
message
hdfs.proto:286* Directory listing
Used in:
repeated partialListing = 1
required uint32 remainingEntries = 2
message
hdfs.proto:169* ECSchema options entry
Used in:
required string key = 1
required string value = 2
message
hdfs.proto:177* ECSchema for erasurecoding
Used in:
required string codecName = 1
required uint32 dataUnits = 2
required uint32 parityUnits = 3
repeated options = 4
message
hdfs.proto:193Used in: , ,
optional string name = 1
optional schema = 2
optional uint32 cellSize = 3
required uint32 id = 4
Actually a byte - only 8 bits used
optional ErasureCodingPolicyState state = 5
* EC policy state.
Used in:
DISABLED = 1
ENABLED = 2
REMOVED = 3
message
hdfs.proto:457* Current key and set of block keys at the namenode.
required bool isBlockTokenEnabled = 1
required uint64 keyUpdateInterval = 2
required uint64 tokenLifeTime = 3
required currentKey = 4
repeated allKeys = 5
message
hdfs.proto:38* Extended block idenfies a block
Used in: , , , , , , , , , , ,
required string poolId = 1
Block pool id - gloablly unique across clusters
required uint64 blockId = 2
the local id within a pool
required uint64 generationStamp = 3
optional uint64 numBytes = 4
len does not belong in ebid
message
hdfs.proto:117* File or Directory permision - same spec as posix
Used in: , , , , ,
required uint32 perm = 1
Actually a short - only 16bits used
message
hdfs.proto:271* HDFS Server Defaults
Used in:
required uint64 blockSize = 1
required uint32 bytesPerChecksum = 2
required uint32 writePacketSize = 3
required uint32 replication = 4
Actually a short - only 16 bits used
required uint32 fileBufferSize = 5
optional bool encryptDataTransfer = 6
optional uint64 trashInterval = 7
optional ChecksumTypeProto checksumType = 8
message
hdfs.proto:226* Status of a file, directory or symlink Optionally includes a file's block locations if requested by client on the rpc call.
Used in: , , , , ,
required HdfsFileStatusProto.FileType fileType = 1
required bytes path = 2
local name of inode encoded java UTF8
required uint64 length = 3
required permission = 4
required string owner = 5
required string group = 6
required uint64 modification_time = 7
required uint64 access_time = 8
optional bytes symlink = 9
Optional fields for symlink
if symlink, target encoded java UTF8
optional uint32 block_replication = 10
Optional fields for file
only 16bits used
optional uint64 blocksize = 11
optional locations = 12
suppled only if asked by client
optional uint64 fileId = 13
Optional field for fileId
default as an invalid id
optional int32 childrenNum = 14
optional ecPolicy = 17
Optional field for erasure coding
Used in:
IS_DIR = 1
IS_FILE = 2
IS_SYMLINK = 3
message
IpcConnectionContext.proto:44* The connection context is sent as part of the connection establishment. It establishes the context for ALL Rpc calls within the connection.
optional userInfo = 2
UserInfo beyond what is determined as part of security handshake at connection time (kerberos, tokens etc).
optional string protocol = 3
Protocol name for next rpc layer. The client created a proxy with this protocol name
message
hdfs.proto:139* A LocatedBlock gives information about a block and its location.
Used in: , , , , , ,
required b = 1
required uint64 offset = 2
offset of first byte of block in the file
repeated locs = 3
Locations ordered by proximity to client ip
required bool corrupt = 4
true if all replicas of a block are corrupt, else false
required blockToken = 5
repeated bool isCached = 6
if a location in locs is cached
repeated StorageTypeProto storageTypes = 7
repeated string storageIDs = 8
optional bytes blockIndices = 9
striped block related fields
used for striped block to indicate block index for each storage
repeated blockTokens = 10
each internal block has a block token
message
hdfs.proto:210* A set of file blocks and their locations.
Used in: ,
required uint64 fileLength = 1
repeated blocks = 2
required bool underConstruction = 3
optional lastBlock = 4
required bool isLastBlockComplete = 5
optional ecPolicy = 7
Optional field for erasure coding
message
hdfs.proto:369* Command sent from one namenode to another namenode.
required uint32 action = 1
required NamenodeCommandProto.Type type = 2
optional checkpointCmd = 3
Used in:
NamenodeCommand = 0
Base command
CheckPointCommand = 1
Check point command
message
hdfs.proto:344* Information sent by a namenode to identify itself to the primary namenode.
required string rpcAddress = 1
host:port of the namenode RPC address
required string httpAddress = 2
host:port of the namenode http server
required storageInfo = 3
Node information
optional NamenodeRegistrationProto.NamenodeRoleProto role = 4
Namenode role
Used in:
NAMENODE = 1
BACKUP = 2
CHECKPOINT = 3
message
hdfs.proto:437* Namespace information that describes namespace on a namenode
Used in:
required string buildVersion = 1
Software revision version (e.g. an svn or git revision)
required uint32 unused = 2
Retained for backward compatibility
required string blockPoolID = 3
block pool used by the namespace
required storageInfo = 4
Node information
required string softwareVersion = 5
Software version number (e.g. 2.0.0)
message
datatransfer.proto:128required header = 1
message
datatransfer.proto:222Used in:
required uint32 bytesPerCrc = 1
required uint64 crcPerBlock = 2
required bytes md5 = 3
optional ChecksumTypeProto crcType = 4
message
datatransfer.proto:124required header = 1
message
datatransfer.proto:63required header = 1
required uint64 offset = 2
required uint64 len = 3
optional bool sendChecksums = 4
optional cachingStrategy = 5
message
datatransfer.proto:118required header = 1
required string delHint = 2
required source = 3
message
datatransfer.proto:132required header = 1
required uint32 maxVersion = 2
* In order to get short-circuit access to block data, clients must set this to the highest version of the block data that they can understand. Currently 1 is the only version, but more versions may exist in the future if the on-disk format changes.
message
datatransfer.proto:113required header = 1
repeated targets = 2
message
datatransfer.proto:77required header = 1
repeated targets = 2
optional source = 3
required OpWriteBlockProto.BlockConstructionStage stage = 4
required uint32 pipelineSize = 5
required uint64 minBytesRcvd = 6
required uint64 maxBytesRcvd = 7
required uint64 latestGenerationStamp = 8
required requestedChecksum = 9
* The requested checksum mechanism for this block write.
optional cachingStrategy = 10
Used in:
PIPELINE_SETUP_APPEND = 0
PIPELINE_SETUP_APPEND_RECOVERY = 1
pipeline set up for failed PIPELINE_SETUP_APPEND recovery
DATA_STREAMING = 2
data streaming
PIPELINE_SETUP_STREAMING_RECOVERY = 3
pipeline setup for failed data streaming recovery
PIPELINE_CLOSE = 4
close the block and pipeline
PIPELINE_CLOSE_RECOVERY = 5
Recover a failed PIPELINE_CLOSE
PIPELINE_SETUP_CREATE = 6
pipeline set up for block creation
TRANSFER_RBW = 7
transfer RBW for adding datanodes
TRANSFER_FINALIZED = 8
transfer Finalized for adding datanodes
message
datatransfer.proto:143required sfixed64 offsetInBlock = 1
All fields must be fixed-length!
required sfixed64 seqno = 2
required bool lastPacketInBlock = 3
required sfixed32 dataLen = 4
optional bool syncBlock = 5
message
datatransfer.proto:168required sint64 seqno = 1
repeated Status status = 2
optional uint64 downstreamAckTimeNanos = 3
message
datatransfer.proto:178* Sent as part of the BlockOpResponseProto for READ_BLOCK and COPY_BLOCK operations.
Used in:
required checksum = 1
required uint64 chunkOffset = 2
* The offset into the block at which the first packet will start. This is necessary since reads will align backwards to a checksum chunk boundary.
message
hdfs.proto:479* Block that needs to be recovered with at a given location
required uint64 newGenStamp = 1
New genstamp post recovery
required block = 2
Block to be recovered
message
hdfs.proto:430* Enumeration of editlogs available on a remote namenode
repeated logs = 1
message
hdfs.proto:421* Editlog information with available transactions
Used in:
required uint64 startTxId = 1
Starting available edit log transaction
required uint64 endTxId = 2
Ending available edit log transaction
optional bool isInProgress = 3
* State of a block replica at a datanode
FINALIZED = 0
State of a replica when it is not modified
RBW = 1
State of replica that is being written to
RWR = 2
State of replica that is waiting to be recovered
RUR = 3
State of replica that is under recovery
TEMPORARY = 4
State of replica that is created for replication
message
ProtobufRpcEngine.proto:45* This message is the header for the Protobuf Rpc Engine when sending a RPC request from RPC client to the RPC server. The actual request (serialized as protobuf) follows this request. No special header is needed for the Rpc Response for Protobuf Rpc Engine. The normal RPC response header (see RpcHeader.proto) are sufficient.
required string methodName = 1
* Name of the RPC method
required string declaringClassProtocolName = 2
* RPCs for a particular interface (ie protocol) are done using a IPC connection that is setup using rpcProxy. The rpcProxy's has a declared protocol name that is sent form client to server at connection time. Each Rpc call also sends a protocol name (called declaringClassprotocolName). This name is usually the same as the connection protocol name except in some cases. For example metaProtocols such ProtocolInfoProto which get metainfo about the protocol reuse the connection but need to indicate that the actual protocol is different (i.e. the protocol is ProtocolInfoProto) since they reuse the connection; in this case the declaringClassProtocolName field is set to the ProtocolInfoProto
required uint64 clientProtocolVersion = 3
* protocol version of class declaring the called method
* RpcKind determine the rpcEngine and the serialization of the rpc request
Used in:
RPC_BUILTIN = 0
Used for built in calls by tests
RPC_WRITABLE = 1
Use WritableRpcEngine
RPC_PROTOCOL_BUFFER = 2
Use ProtobufRpcEngine
message
RpcHeader.proto:56the header for the RpcRequest
optional RpcKindProto rpcKind = 1
optional RpcRequestHeaderProto.OperationProto rpcOp = 2
required sint32 callId = 3
a sequence number that is sent back in response
required bytes clientId = 4
Globally unique client ID
optional sint32 retryCount = 5
clientId + callId uniquely identifies a request retry count, 1 means this is the first retry
Used in:
RPC_FINAL_PACKET = 0
The final RPC Packet
RPC_CONTINUATION_PACKET = 1
not implemented yet
RPC_CLOSE_CONNECTION = 2
close the rpc connection
message
RpcHeader.proto:93* Rpc Response Header +------------------------------------------------------------------+ | Rpc total response length in bytes (4 bytes int) | | (sum of next two parts) | +------------------------------------------------------------------+ | RpcResponseHeaderProto - serialized delimited ie has len | +------------------------------------------------------------------+ | if request is successful: | | - RpcResponse - The actual rpc response bytes follow | | the response header | | This response is serialized based on RpcKindProto | | if request fails : | | The rpc response header contains the necessary info | +------------------------------------------------------------------+ Note that rpc response header is also used when connection setup fails. Ie the response looks like a rpc response with a fake callId.
* RpcStastus - success or failure The reponseHeader's errDetail, exceptionClassName and errMsg contains further details on the error
required uint32 callId = 1
callId used in Request
required RpcResponseHeaderProto.RpcStatusProto status = 2
optional uint32 serverIpcVersionNum = 3
Sent if success or fail
optional string exceptionClassName = 4
if request fails
optional string errorMsg = 5
if request fails, often contains strack trace
optional RpcResponseHeaderProto.RpcErrorCodeProto errorDetail = 6
in case of error
optional bytes clientId = 7
Globally unique client ID
optional sint32 retryCount = 8
Used in:
ERROR_APPLICATION = 1
Non-fatal Rpc error - connection left open for future rpc calls
RPC Failed - rpc app threw exception
ERROR_NO_SUCH_METHOD = 2
Rpc error - no such method
ERROR_NO_SUCH_PROTOCOL = 3
Rpc error - no such protocol
ERROR_RPC_SERVER = 4
Rpc error on server side
ERROR_SERIALIZING_RESPONSE = 5
error serializign response
ERROR_RPC_VERSION_MISMATCH = 6
Rpc protocol version mismatch
FATAL_UNKNOWN = 10
Fatal Server side Rpc error - connection closed
unknown Fatal error
FATAL_UNSUPPORTED_SERIALIZATION = 11
IPC layer serilization type invalid
FATAL_INVALID_RPC_HEADER = 12
fields of RpcHeader are invalid
FATAL_DESERIALIZING_REQUEST = 13
could not deserilize rpc request
FATAL_VERSION_MISMATCH = 14
Ipc Layer version mismatch
FATAL_UNAUTHORIZED = 15
Auth failed
Used in:
SUCCESS = 0
RPC succeeded
ERROR = 1
RPC or error - connection left open for future calls
FATAL = 2
Fatal error - connection closed
message
RpcHeader.proto:137optional uint32 version = 1
required RpcSaslProto.SaslState state = 2
optional bytes token = 3
repeated auths = 4
message
RpcHeader.proto:147Used in:
required string method = 1
required string mechanism = 2
optional string protocol = 3
optional string serverId = 4
optional bytes challenge = 5
Used in:
SUCCESS = 0
NEGOTIATE = 1
INITIATE = 2
CHALLENGE = 3
RESPONSE = 4
WRAP = 5
Used in:
SAFEMODE_LEAVE = 1
SAFEMODE_ENTER = 2
SAFEMODE_GET = 3
message
hdfs.proto:315* Snapshot diff report entry
Used in:
required bytes fullpath = 1
required string modificationLabel = 2
message
hdfs.proto:323* Snapshot diff report
Used in:
required string snapshotRoot = 1
full path of the directory where snapshots were taken
required string fromSnapshot = 2
required string toSnapshot = 3
repeated diffReportEntries = 4
message
hdfs.proto:501* Information related to a snapshot TODO: add more information
required string snapshotName = 1
required string snapshotRoot = 2
required permission = 3
required string owner = 4
required string group = 5
required string createTime = 6
TODO: do we need access time?
message
hdfs.proto:308* Snapshottable directory listing
Used in:
repeated snapshottableDirListing = 1
message
hdfs.proto:296* Status of a snapshottable directory: besides the normal information for a directory status, also include snapshot quota, number of snapshots, and the full path of the parent directory.
Used in:
required dirStatus = 1
required uint32 snapshot_quota = 2
Fields specific for snapshottable directory
required uint32 snapshot_number = 3
required bytes parent_fullpath = 4
Used in: , , ,
DT_PROTO_SUCCESS = 0
DT_PROTO_ERROR = 1
DT_PROTO_ERROR_CHECKSUM = 2
DT_PROTO_ERROR_INVALID = 3
DT_PROTO_ERROR_EXISTS = 4
DT_PROTO_ERROR_ACCESS_TOKEN = 5
DT_PROTO_CHECKSUM_OK = 6
DT_PROTO_ERROR_UNSUPPORTED = 7
DT_PROTO_OOB_RESTART = 8
Quick restart
DT_PROTO_OOB_RESERVED1 = 9
Reserved
DT_PROTO_OOB_RESERVED2 = 10
Reserved
DT_PROTO_OOB_RESERVED3 = 11
Reserved
DT_PROTO_IN_PROGRESS = 12
message
hdfs.proto:334* Common node information shared by all the nodes in the cluster
Used in: , ,
required uint32 layoutVersion = 1
Layout version of the file system
required uint32 namespceID = 2
File system namespace ID
required string clusterID = 3
ID of the cluster
required uint64 cTime = 4
File system creation time
* Types of recognized storage media.
Used in:
DISK = 1
SSD = 2
message
hdfs.proto:132* A list of storage IDs.
repeated string storageUuids = 1
message
Security.proto:34* Security token identifier
Used in: , , , , , ,
required bytes identifier = 1
required bytes password = 2
required string kind = 3
required string service = 4
message
IpcConnectionContext.proto:35* Spec for UserInformationProto is specified in ProtoUtil#makeIpcConnectionContext
Used in:
optional string effectiveUser = 1
optional string realUser = 2
message
hdfs.proto:487* void request
(message has no fields)
message
hdfs.proto:493* Version response from namenode.
required info = 1