package hadoop.hdfs

required Status status = 1
optional string firstBadLink = 2
optional OpBlockChecksumResponseProto checksumResponse = 3
optional ReadOpChecksumInfoProto readOpChecksumInfo = 4
optional string message = 5
* explanatory text which may be useful to log on the client side
optional uint32 shortCircuitAccessVersion = 6
* If the server chooses to agree to the request of a client for short-circuit access, it will send a response message with the relevant file descriptors attached. In the body of the message, this version number will be set to the specific version number of the block data that the client is about to read.

* Block information

Used in: BlockWithLocationsProto

required uint64 blockId = 1
required uint64 genStamp = 2
optional uint64 numBytes = 3

* Block and datanodes where is it located

Used in: BlocksWithLocationsProto

required BlockProto block = 1
Block
repeated string datanodeUuids = 2
Datanodes with replicas of the block
repeated string storageUuids = 3
Storages with replicas of the block

* List of block with locations

repeated BlockWithLocationsProto blocks = 1

message CacheDirectiveEntryProto

ClientNamenodeProtocol.proto:443

Used in: ListCacheDirectivesResponseProto

required CacheDirectiveInfoProto info = 1
required CacheDirectiveStatsProto stats = 2

Used in: CacheDirectiveInfoProto

required int64 millis = 1
required bool isRelative = 2

Used in: AddCacheDirectiveRequestProto, CacheDirectiveEntryProto, ListCacheDirectivesRequestProto, ModifyCacheDirectiveRequestProto

optional int64 id = 1
optional string path = 2
optional uint32 replication = 3
optional string pool = 4
optional CacheDirectiveInfoExpirationProto expiration = 5

Used in: CacheDirectiveEntryProto

required int64 bytesNeeded = 1
required int64 bytesCached = 2
required int64 filesNeeded = 3
required int64 filesCached = 4
required bool hasExpired = 5

FORCE = 1
Ignore pool resource limits

message CachePoolEntryProto

ClientNamenodeProtocol.proto:500

Used in: ListCachePoolsResponseProto

required CachePoolInfoProto info = 1
required CachePoolStatsProto stats = 2

Used in: AddCachePoolRequestProto, CachePoolEntryProto, ModifyCachePoolRequestProto

optional string poolName = 1
optional string ownerName = 2
optional string groupName = 3
optional int32 mode = 4
optional int64 limit = 5
optional int64 maxRelativeExpiry = 6

Used in: CachePoolEntryProto

required int64 bytesNeeded = 1
required int64 bytesCached = 2
required int64 bytesOverlimit = 3
required int64 filesNeeded = 4
required int64 filesCached = 5

Used in: OpReadBlockProto, OpWriteBlockProto

optional bool dropBehind = 1
optional int64 readahead = 2

* Command returned from primary to checkpointing namenode. This command has checkpoint signature that identifies checkpoint transaction and is needed for further communication related to checkpointing.

Used in: NamenodeCommandProto

required CheckpointSignatureProto signature = 1
Unique signature to identify checkpoint transation
required bool needToReturnImage = 2
If true, return transfer image to primary upon the completion of checkpoint

* Unique signature to identify checkpoint transactions.

Used in: CheckpointCommandProto

required string blockPoolId = 1
required uint64 mostRecentCheckpointTxId = 2
required uint64 curSegmentTxId = 3
required StorageInfoProto storageInfo = 4

Used in: OpWriteBlockProto, ReadOpChecksumInfoProto

required ChecksumTypeProto type = 1
required uint32 bytesPerChecksum = 2

* Checksum algorithms/types used in HDFS Make sure this enum's integer values match enum values' id properties defined in org.apache.hadoop.util.DataChecksum.Type

Used in: ChecksumProto, FsServerDefaultsProto, OpBlockChecksumResponseProto

CHECKSUM_NULL = 0
CHECKSUM_CRC32 = 1
CHECKSUM_CRC32C = 2

Used in: OpReadBlockProto, OpTransferBlockProto, OpWriteBlockProto

required BaseHeaderProto baseHeader = 1
required string clientName = 2

* Message sent from the client to the DN after reading the entire read request.

required Status status = 1

message ContentSummaryProto

hdfs.proto:105

* Summary of a file or directory

Used in: GetContentSummaryResponseProto

required uint64 length = 1
required uint64 fileCount = 2
required uint64 directoryCount = 3
required uint64 quota = 4
required uint64 spaceConsumed = 5
required uint64 spaceQuota = 6

* Contains a list of paths corresponding to corrupt files and a cookie used for iterative calls to NameNode.listCorruptFileBlocks.

Used in: ListCorruptFileBlocksResponseProto

repeated string files = 1
required string cookie = 2

CREATE = 1
Create a file
OVERWRITE = 2
Truncate/overwrite a file. Same as POSIX O_TRUNC
APPEND = 4
Append to a file

required Status status = 1

Used in: GetDataEncryptionKeyResponseProto

required uint32 keyId = 1
required string blockPoolId = 2
required bytes nonce = 3
required bytes encryptionKey = 4
required uint64 expiryDate = 5
optional string encryptionAlgorithm = 6

message DataTransferEncryptorMessageProto

datatransfer.proto:36

required DataTransferEncryptorMessageProto.DataTransferEncryptorStatus status = 1
optional bytes payload = 2
optional string message = 3

enum DataTransferEncryptorMessageProto.DataTransferEncryptorStatus

datatransfer.proto:37

Used in: DataTransferEncryptorMessageProto

SUCCESS = 0
ERROR_UNKNOWN_KEY = 1
ERROR = 2

* Identifies a Datanode

Used in: DatanodeInfoProto, UpdatePipelineRequestProto

required string ipAddr = 1
IP address
required string hostName = 2
hostname
required string datanodeUuid = 3
UUID assigned to the Datanode. For
required uint32 xferPort = 4
upgraded clusters this is the same as the original StorageID of the Datanode.
data streaming port
required uint32 infoPort = 5
datanode http port
required uint32 ipcPort = 6
ipc server port
optional uint32 infoSecurePort = 7
datanode https port

* The status of a Datanode

Used in: AddBlockRequestProto, DatanodeInfosProto, GetAdditionalDatanodeRequestProto, GetDatanodeReportResponseProto, LocatedBlockProto, OpReplaceBlockProto, OpTransferBlockProto, OpWriteBlockProto

required DatanodeIDProto id = 1
optional uint64 capacity = 2
optional uint64 dfsUsed = 3
optional uint64 remaining = 4
optional uint64 blockPoolUsed = 5
optional uint64 lastUpdate = 6
optional uint32 xceiverCount = 7
optional string location = 8
optional DatanodeInfoProto.AdminState adminState = 10
optional uint64 cacheCapacity = 11
optional uint64 cacheUsed = 12

Used in: DatanodeInfoProto

NORMAL = 0
DECOMMISSION_INPROGRESS = 1
DECOMMISSIONED = 2

* DatanodeInfo array

repeated DatanodeInfoProto datanodes = 1

* Datanode local information

Used in: GetDatanodeInfoResponseProto

required string softwareVersion = 1
required string configVersion = 2
required uint64 uptime = 3

type of the datanode report

Used in: GetDatanodeReportRequestProto

ALL = 1
LIVE = 2
DEAD = 3

* Directory listing

Used in: GetListingResponseProto

repeated HdfsFileStatusProto partialListing = 1
required uint32 remainingEntries = 2

* Current key and set of block keys at the namenode.

required bool isBlockTokenEnabled = 1
required uint64 keyUpdateInterval = 2
required uint64 tokenLifeTime = 3
required BlockKeyProto currentKey = 4
repeated BlockKeyProto allKeys = 5

* Extended block idenfies a block

Used in: AbandonBlockRequestProto, AddBlockRequestProto, BaseHeaderProto, CompleteRequestProto, GetAdditionalDatanodeRequestProto, GetBlockLocalPathInfoRequestProto, GetBlockLocalPathInfoResponseProto, GetReplicaVisibleLengthRequestProto, LocatedBlockProto, UpdateBlockForPipelineRequestProto, UpdatePipelineRequestProto

required string poolId = 1
Block pool id - gloablly unique across clusters
required uint64 blockId = 2
the local id within a pool
required uint64 generationStamp = 3
optional uint64 numBytes = 4
len does not belong in ebid

* File or Directory permision - same spec as posix

Used in: CreateRequestProto, CreateSymlinkRequestProto, HdfsFileStatusProto, MkdirsRequestProto, SetPermissionRequestProto, SnapshotInfoProto

required uint32 perm = 1
Actually a short - only 16bits used

* HDFS Server Defaults

Used in: GetServerDefaultsResponseProto

required uint64 blockSize = 1
required uint32 bytesPerChecksum = 2
required uint32 writePacketSize = 3
required uint32 replication = 4
Actually a short - only 16 bits used
required uint32 fileBufferSize = 5
optional bool encryptDataTransfer = 6
optional uint64 trashInterval = 7
optional ChecksumTypeProto checksumType = 8

* Status of a file, directory or symlink Optionally includes a file's block locations if requested by client on the rpc call.

Used in: CreateResponseProto, DirectoryListingProto, GetFileInfoResponseProto, GetFileLinkInfoResponseProto, SnapshottableDirectoryStatusProto

required HdfsFileStatusProto.FileType fileType = 1
required bytes path = 2
local name of inode encoded java UTF8
required uint64 length = 3
required FsPermissionProto permission = 4
required string owner = 5
required string group = 6
required uint64 modification_time = 7
required uint64 access_time = 8
optional bytes symlink = 9
Optional fields for symlink
if symlink, target encoded java UTF8
optional uint32 block_replication = 10
Optional fields for file
only 16bits used
optional uint64 blocksize = 11
optional LocatedBlocksProto locations = 12
suppled only if asked by client
optional uint64 fileId = 13
Optional field for fileId
default as an invalid id
optional int32 childrenNum = 14

Used in: HdfsFileStatusProto

IS_DIR = 1
IS_FILE = 2
IS_SYMLINK = 3

* A LocatedBlock gives information about a block and its location.

Used in: AddBlockResponseProto, AppendResponseProto, GetAdditionalDatanodeResponseProto, LocatedBlocksProto, RecoveringBlockProto, ReportBadBlocksRequestProto, UpdateBlockForPipelineResponseProto

required ExtendedBlockProto b = 1
required uint64 offset = 2
offset of first byte of block in the file
repeated DatanodeInfoProto locs = 3
Locations ordered by proximity to client ip
required bool corrupt = 4
true if all replicas of a block are corrupt, else false
required common.TokenProto blockToken = 5
repeated bool isCached = 6
if a location in locs is cached
repeated StorageTypeProto storageTypes = 7
repeated string storageIDs = 8

* A set of file blocks and their locations.

Used in: GetBlockLocationsResponseProto, HdfsFileStatusProto

required uint64 fileLength = 1
repeated LocatedBlockProto blocks = 2
required bool underConstruction = 3
optional LocatedBlockProto lastBlock = 4
required bool isLastBlockComplete = 5

* Command sent from one namenode to another namenode.

required uint32 action = 1
required NamenodeCommandProto.Type type = 2
optional CheckpointCommandProto checkpointCmd = 3

Used in: NamenodeCommandProto

NamenodeCommand = 0
Base command
CheckPointCommand = 1
Check point command

* Information sent by a namenode to identify itself to the primary namenode.

required string rpcAddress = 1
host:port of the namenode RPC address
required string httpAddress = 2
host:port of the namenode http server
required StorageInfoProto storageInfo = 3
Node information
optional NamenodeRegistrationProto.NamenodeRoleProto role = 4
Namenode role

Used in: NamenodeRegistrationProto

NAMENODE = 1
BACKUP = 2
CHECKPOINT = 3

* Namespace information that describes namespace on a namenode

Used in: VersionResponseProto

required string buildVersion = 1
Software revision version (e.g. an svn or git revision)
required uint32 unused = 2
Retained for backward compatibility
required string blockPoolID = 3
block pool used by the namespace
required StorageInfoProto storageInfo = 4
Node information
required string softwareVersion = 5
Software version number (e.g. 2.0.0)

required BaseHeaderProto header = 1

Used in: BlockOpResponseProto

required uint32 bytesPerCrc = 1
required uint64 crcPerBlock = 2
required bytes md5 = 3
optional ChecksumTypeProto crcType = 4

required BaseHeaderProto header = 1

required ClientOperationHeaderProto header = 1
required uint64 offset = 2
required uint64 len = 3
optional bool sendChecksums = 4
optional CachingStrategyProto cachingStrategy = 5

required BaseHeaderProto header = 1
required string delHint = 2
required DatanodeInfoProto source = 3

required BaseHeaderProto header = 1
required uint32 maxVersion = 2
* In order to get short-circuit access to block data, clients must set this to the highest version of the block data that they can understand. Currently 1 is the only version, but more versions may exist in the future if the on-disk format changes.
optional ShortCircuitShmSlotProto slotId = 3
* The shared memory slot to use, if we are using one.

required ClientOperationHeaderProto header = 1
repeated DatanodeInfoProto targets = 2

required ClientOperationHeaderProto header = 1
repeated DatanodeInfoProto targets = 2
optional DatanodeInfoProto source = 3
required OpWriteBlockProto.BlockConstructionStage stage = 4
required uint32 pipelineSize = 5
required uint64 minBytesRcvd = 6
required uint64 maxBytesRcvd = 7
required uint64 latestGenerationStamp = 8
required ChecksumProto requestedChecksum = 9
* The requested checksum mechanism for this block write.
optional CachingStrategyProto cachingStrategy = 10

Used in: OpWriteBlockProto

PIPELINE_SETUP_APPEND = 0
PIPELINE_SETUP_APPEND_RECOVERY = 1
pipeline set up for failed PIPELINE_SETUP_APPEND recovery
DATA_STREAMING = 2
data streaming
PIPELINE_SETUP_STREAMING_RECOVERY = 3
pipeline setup for failed data streaming recovery
PIPELINE_CLOSE = 4
close the block and pipeline
PIPELINE_CLOSE_RECOVERY = 5
Recover a failed PIPELINE_CLOSE
PIPELINE_SETUP_CREATE = 6
pipeline set up for block creation
TRANSFER_RBW = 7
transfer RBW for adding datanodes
TRANSFER_FINALIZED = 8
transfer Finalized for adding datanodes

required sfixed64 offsetInBlock = 1
All fields must be fixed-length!
required sfixed64 seqno = 2
required bool lastPacketInBlock = 3
required sfixed32 dataLen = 4
optional bool syncBlock = 5

required sint64 seqno = 1
repeated Status status = 2
optional uint64 downstreamAckTimeNanos = 3

* Sent as part of the BlockOpResponseProto for READ_BLOCK and COPY_BLOCK operations.

Used in: BlockOpResponseProto

required ChecksumProto checksum = 1
required uint64 chunkOffset = 2
* The offset into the block at which the first packet will start. This is necessary since reads will align backwards to a checksum chunk boundary.

* Block that needs to be recovered with at a given location

required uint64 newGenStamp = 1
New genstamp post recovery
required LocatedBlockProto block = 2
Block to be recovered

required ShortCircuitShmSlotProto slotId = 1

required Status status = 1
optional string error = 2

* Enumeration of editlogs available on a remote namenode

repeated RemoteEditLogProto logs = 1

* Editlog information with available transactions

Used in: RemoteEditLogManifestProto

required uint64 startTxId = 1
Starting available edit log transaction
required uint64 endTxId = 2
Ending available edit log transaction
optional bool isInProgress = 3

* State of a block replica at a datanode

FINALIZED = 0
State of a replica when it is not modified
RBW = 1
State of replica that is being written to
RWR = 2
State of replica that is waiting to be recovered
RUR = 3
State of replica that is under recovery
TEMPORARY = 4
State of replica that is created for replication

Used in: RollingUpgradeRequestProto

QUERY = 1
START = 2
FINALIZE = 3

Used in: RollingUpgradeResponseProto

required RollingUpgradeStatusProto status = 1
required uint64 startTime = 2
required uint64 finalizeTime = 3
required bool createdRollbackImages = 4

* Rolling upgrade status

Used in: RollingUpgradeInfoProto

required string blockPoolId = 1

Used in: SetSafeModeRequestProto

SAFEMODE_LEAVE = 1
SAFEMODE_ENTER = 2
SAFEMODE_GET = 3

* An ID uniquely identifying a shared memory segment.

Used in: ShortCircuitShmResponseProto, ShortCircuitShmSlotProto

required int64 hi = 1
required int64 lo = 2

required string clientName = 1
The name of the client requesting the shared memory segment. This is purely for logging / debugging purposes.

required Status status = 1
optional string error = 2
optional ShortCircuitShmIdProto id = 3

* An ID uniquely identifying a slot within a shared memory segment.

Used in: OpRequestShortCircuitAccessProto, ReleaseShortCircuitAccessRequestProto

required ShortCircuitShmIdProto shmId = 1
required int32 slotIdx = 2

message SnapshotDiffReportEntryProto

hdfs.proto:275

* Snapshot diff report entry

Used in: SnapshotDiffReportProto

required bytes fullpath = 1
required string modificationLabel = 2

* Snapshot diff report

Used in: GetSnapshotDiffReportResponseProto

required string snapshotRoot = 1
full path of the directory where snapshots were taken
required string fromSnapshot = 2
required string toSnapshot = 3
repeated SnapshotDiffReportEntryProto diffReportEntries = 4

* Information related to a snapshot TODO: add more information

required string snapshotName = 1
required string snapshotRoot = 2
required FsPermissionProto permission = 3
required string owner = 4
required string group = 5
required string createTime = 6
TODO: do we need access time?

* Snapshottable directory listing

Used in: GetSnapshottableDirListingResponseProto

repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1

* Status of a snapshottable directory: besides the normal information for a directory status, also include snapshot quota, number of snapshots, and the full path of the parent directory.

Used in: SnapshottableDirectoryListingProto

required HdfsFileStatusProto dirStatus = 1
required uint32 snapshot_quota = 2
Fields specific for snapshottable directory
required uint32 snapshot_number = 3
required bytes parent_fullpath = 4

Used in: BlockOpResponseProto, ClientReadStatusProto, DNTransferAckProto, PipelineAckProto, ReleaseShortCircuitAccessResponseProto, ShortCircuitShmResponseProto

SUCCESS = 0
ERROR = 1
ERROR_CHECKSUM = 2
ERROR_INVALID = 3
ERROR_EXISTS = 4
ERROR_ACCESS_TOKEN = 5
CHECKSUM_OK = 6
ERROR_UNSUPPORTED = 7
OOB_RESTART = 8
Quick restart
OOB_RESERVED1 = 9
Reserved
OOB_RESERVED2 = 10
Reserved
OOB_RESERVED3 = 11
Reserved

* Common node information shared by all the nodes in the cluster

Used in: CheckpointSignatureProto, NamenodeRegistrationProto, NamespaceInfoProto

required uint32 layoutVersion = 1
Layout version of the file system
required uint32 namespceID = 2
File system namespace ID
required string clusterID = 3
ID of the cluster
required uint64 cTime = 4
File system creation time

* Types of recognized storage media.

Used in: LocatedBlockProto

DISK = 1
SSD = 2

* A list of storage IDs.

repeated string storageUuids = 1

* void request

(message has no fields)

* Version response from namenode.

required NamespaceInfoProto info = 1

package hadoop.hdfs

service ClientDatanodeProtocolService

rpc deleteBlockPool (DeleteBlockPoolRequestProto, DeleteBlockPoolResponseProto)

message DeleteBlockPoolRequestProto

required string blockPool = 1

required bool force = 2

message DeleteBlockPoolResponseProto

rpc getBlockLocalPathInfo (GetBlockLocalPathInfoRequestProto, GetBlockLocalPathInfoResponseProto)

message GetBlockLocalPathInfoRequestProto

required ExtendedBlockProto block = 1

required common.TokenProto token = 2

message GetBlockLocalPathInfoResponseProto

required ExtendedBlockProto block = 1

required string localPath = 2

required string localMetaPath = 3

rpc getDatanodeInfo (GetDatanodeInfoRequestProto, GetDatanodeInfoResponseProto)

message GetDatanodeInfoRequestProto

message GetDatanodeInfoResponseProto

required DatanodeLocalInfoProto localInfo = 1

rpc getHdfsBlockLocations (GetHdfsBlockLocationsRequestProto, GetHdfsBlockLocationsResponseProto)

message GetHdfsBlockLocationsRequestProto

repeated common.TokenProto tokens = 2

required string blockPoolId = 3

repeated sfixed64 blockIds = 4

message GetHdfsBlockLocationsResponseProto

repeated bytes volumeIds = 1

repeated uint32 volumeIndexes = 2

rpc getReplicaVisibleLength (GetReplicaVisibleLengthRequestProto, GetReplicaVisibleLengthResponseProto)

message GetReplicaVisibleLengthRequestProto

required ExtendedBlockProto block = 1

message GetReplicaVisibleLengthResponseProto

required uint64 length = 1

rpc refreshNamenodes (RefreshNamenodesRequestProto, RefreshNamenodesResponseProto)

message RefreshNamenodesRequestProto

message RefreshNamenodesResponseProto

rpc shutdownDatanode (ShutdownDatanodeRequestProto, ShutdownDatanodeResponseProto)

message ShutdownDatanodeRequestProto

required bool forUpgrade = 1

message ShutdownDatanodeResponseProto

service ClientNamenodeProtocol

rpc abandonBlock (AbandonBlockRequestProto, AbandonBlockResponseProto)

message AbandonBlockRequestProto

required ExtendedBlockProto b = 1

required string src = 2

required string holder = 3

message AbandonBlockResponseProto

rpc addBlock (AddBlockRequestProto, AddBlockResponseProto)

message AddBlockRequestProto

required string src = 1

required string clientName = 2

optional ExtendedBlockProto previous = 3

repeated DatanodeInfoProto excludeNodes = 4

optional uint64 fileId = 5

repeated string favoredNodes = 6

message AddBlockResponseProto

required LocatedBlockProto block = 1

rpc addCacheDirective (AddCacheDirectiveRequestProto, AddCacheDirectiveResponseProto)

message AddCacheDirectiveRequestProto

required CacheDirectiveInfoProto info = 1

optional uint32 cacheFlags = 2

message AddCacheDirectiveResponseProto

required int64 id = 1

rpc addCachePool (AddCachePoolRequestProto, AddCachePoolResponseProto)

message AddCachePoolRequestProto

required CachePoolInfoProto info = 1

message AddCachePoolResponseProto

rpc allowSnapshot (AllowSnapshotRequestProto, AllowSnapshotResponseProto)

message AllowSnapshotRequestProto

required string snapshotRoot = 1

message AllowSnapshotResponseProto

rpc append (AppendRequestProto, AppendResponseProto)

message AppendRequestProto

required string src = 1

required string clientName = 2

message AppendResponseProto

optional LocatedBlockProto block = 1

rpc cancelDelegationToken (common.CancelDelegationTokenRequestProto, common.CancelDelegationTokenResponseProto)

rpc complete (CompleteRequestProto, CompleteResponseProto)

message CompleteRequestProto

required string src = 1