package baidu.flume

Get desktop application:
View/edit binary Protocol Buffers messages

• optional int32 delay = 1

Used in: PbScope

• optional uint32 bucket_size = 1

• repeated PbSplit split = 1

Used in: PbExecutor

• optional bytes output = 1

• optional string uri = 1

• optional uint64 offset = 2

Used in: PbJob

• repeated PbDceTask task = 1

• repeated PbDceVertex vertex = 2

• optional string job_name = 3

Used in: PbDceJob

• required uint32 task_index = 1

• required PbExecutor root = 2

• repeated uint32 next_vertex = 3

• optional uint32 concurrency = 4

• optional uint32 partition_offset = 5

  need by merge task

• optional uint32 vertex_index = 6

• optional PbDceTask.MapInput input = 201

• optional PbDceTask.ReduceInput reduce_input = 202

• optional PbDceTask.ReduceOutput reduce_output = 203

Used in: PbDceTask

• required string id = 1

• required PbEntity spliter = 2

• repeated string uri = 3

• optional string input_format = 4

Used in: PbDceTask

• required string id = 1

  external id

• required PbTransferDecoder decoder = 2

• repeated ReduceInput.Channel channel = 3

• optional bool must_keep_empty_group = 4

• optional string session_name = 5

• repeated string broadcast_session_name = 6

Used in: ReduceInput

• required string identity = 1

• required string port = 2

• required uint32 priority = 3

Used in: PbDceTask

• required string id = 1

• repeated ReduceOutput.Channel channel = 2

Used in: ReduceOutput

• required string from = 1

• required PbScope transfer_scope = 2

• required PbTransferEncoder encoder = 3

• required uint32 task_index = 4

• optional uint32 task_concurrency = 5

• optional uint32 task_offset = 6

• optional bool task_use_dynamic_partition = 7

• optional string session_name = 8

• optional bool broadcast = 9

• optional uint32 session_index = 10

Used in: PbDceJob

• repeated uint32 next_vertex_index = 1

  abaci.dag.next.vertex.list.$index = $value_list

• optional uint32 concurrency = 2

  mapred.{map,reduce}.tasks.$index=$value

• repeated uint32 task_index = 3

  task indexs which belong to current vertex

• optional bool use_dynamic_partition = 4

• optional int32 memory_limit = 5

  hadoop.hce.memory.limit.$index = $value

Reflectable classes that contain user defined logic.

Used in: bigflow.python.PbPythonProcessorConfig, PbDceTask.MapInput, PbExecutor.Dispatcher, PbInputScope, PbLoadNode, PbLocalInput, PbLogicalPlan, PbLogicalPlanNode, PbPartialExecutor.Output, PbPhysicalPlan, PbProcessNode, PbProcessorExecutor, PbShuffleNode, PbSinkNode, PbSparkTask.PbHadoopInput, PbStreamProcessorExecutor, PbTmTask.StreamInput, PbWindowScope

• required string name = 1

• optional bytes config = 2

• optional int64 id = 3

  compatible problem, can not be modified

• optional string config_file = 4

  relative path to flume working dir

Used in: PbDceTask, PbLocalTask, PbMonitorTask, PbSparkTask, PbTmTask

• required PbExecutor.Type type = 1

• optional bytes debug_info = 2

• required uint32 scope_level = 3

• repeated string input = 4

• repeated string output = 5

• repeated PbExecutor.Dispatcher dispatcher = 6

• repeated PbExecutor child = 7

• optional string identity = 8

• optional PbExternalExecutor external_executor = 101

• optional PbProcessorExecutor processor_executor = 102

• optional PbLogicalExecutor logical_executor = 103

• optional PbShuffleExecutor shuffle_executor = 104

• optional PbWriteCacheExecutor write_cache_executor = 105

• optional PbPartialExecutor partial_executor = 106

• optional PbCreateEmptyRecordExecutor create_empty_record_executor = 107

• optional PbFilterEmptyRecordExecutor filter_empty_record_executor = 108

• optional PbStreamExternalExecutor stream_external_executor = 201

• optional PbStreamProcessorExecutor stream_processor_executor = 202

• optional PbStreamLogicalExecutor stream_logical_executor = 203

• optional PbStreamShuffleExecutor stream_shuffle_executor = 204

Used in: PbExecutor

• required string identity = 1

• optional PbEntity objector = 2

• optional uint32 priority = 3

• optional int32 usage_count = 4

• optional bool need_dataset = 5

Used in: PbExecutor

• TASK = 0

  drive other executors

• EXTERNAL = 1

  used by different backends to embed engine-specific logic

• PROCESSOR = 2

  used to executor Processor

• LOGICAL = 3

  used to execute a logical plan node (Sink/Load/Union)

• SHUFFLE = 4

  perform shuffle

• WRITE_CACHE = 5

  flush content to persistent storage

• PARTIAL = 6

  process partial inputs at one time

• CREATE_EMPTY_RECORD = 7

  create a record for every empty group

• FILTER_EMPTY_RECORD = 8

  filter records for every empty group

• STREAM_TASK = 9

• STREAM_EXTERNAL = 10

• STREAM_PROCESSOR = 11

• STREAM_LOGICAL = 12

• STREAM_SHUFFLE = 13

Used in: PbExecutor

• required string id = 1

  indicate LocalInput, DceInput or DceShuffle, etc.

Used in: PbInputFormatEntityConfig

• optional string filename_pattern = 1

Used in: PbExecutor

• optional bytes output = 1

• optional int32 step_size = 1

Used in: PbScope

(message has no fields)

• optional bool repeatedly = 1

  1 - 200 for streaming processing

• optional uint64 max_record_num_per_round = 2

• optional uint64 timeout_per_round = 3

  单位为s

• optional PbFileStreamInputFormat file_stream = 200

  200+ for other

• optional string path = 201

• required string uri = 1

• required int64 offset = 2

Used in: PbScope

• repeated string uri = 1

• required PbEntity spliter = 2

Used in: PbPhysicalPlan

• required string id = 1

  uuid

• required PbJob.Type type = 2

• optional bytes debug_info = 3

• optional PbLocalJob local_job = 101

• optional PbDceJob dce_job = 102

• optional PbSparkJob spark_job = 103

• optional PbTmJob tm_job = 104

• optional PbMonitorJob monitor_job = 200

• optional PbJobConfig job_config = 201

Used in: PbJob

• LOCAL = 1

• DAGMR = 2

• SPARK = 3

• TM = 4

• MONITOR = 100

All default values(empty) are meanningful Do not change them unless you know what you're doing.

Used in: PbJob

• optional string tmp_data_path_output = 1

• optional string tmp_data_path_input = 2

• optional string hadoop_client_path = 3

• optional string hadoop_config_path = 4

• optional string job_name = 5

• repeated PbKVConfig kv_config = 6

  such as HadoopJobConf or other kv configs, parameter format in python is "key1=value1 key2=value2"

• optional int32 default_concurrency = 7

• optional int32 default_memory_limit = 8

• optional int32 default_cpu_limit = 9

• optional bool must_keep_empty_group = 10

• optional bool immediately_commit = 11

• optional bool use_hadoop_hash_partitioner = 12

• optional string spark_home_path = 13

  Spark

• optional string prepared_archive_path = 14

• optional int32 round_times = 51

  stream

• optional bool status_as_meta = 101

  status

• optional string status_meta_path = 102

• optional string status_table_path = 103

• optional int32 status_cache_size = 104

• optional bool heap_profile = 201

  switch of heap and cpu profile

• optional bool cpu_profile = 202

• optional bool local_use_leveldb_storage = 301

Used in: PbJobConfig, PbLocalConfig, PbMapReduceConfig, PbSparkConfig, PbTMConfig

• optional bytes key = 1

• optional bytes value = 2

Used in: PbLogicalPlanNode

• repeated string uri = 1

• required PbEntity loader = 2

Used in: bigflow.PbPipeline

• optional string hadoop_client_path = 1

• optional string hadoop_config_path = 2

• repeated PbKVConfig hadoop_conf = 3

• optional int64 round_times = 51

  for stream

• optional bool status_as_meta = 101

  for status

• optional string status_meta_path = 102

• optional string status_table_path = 103

• optional int32 status_cache_size = 104

• optional bool use_leveldb_storage = 125

Used in: PbLocalJob

• required string id = 1

• required PbEntity spliter = 2

• repeated string uri = 3

Used in: PbJob

• required PbLocalTask task = 1

• repeated PbLocalInput input = 2

Used in: PbLocalJob

• required PbExecutor root = 1

Used in: PbExecutor

• required PbLogicalPlanNode node = 1

Used in: bigflow.service.GetStatusRequest, bigflow.service.KillRequest, bigflow.service.LaunchRequest, bigflow.service.SuspendRequest

• repeated PbLogicalPlanNode node = 1

• repeated PbScope scope = 2

• optional PbEntity environment = 3

Used in: PbLogicalExecutor, PbLogicalPlan, PbMonitorTask, PbPartialExecutor, PbShuffleExecutor, PbStreamLogicalExecutor, PbStreamShuffleExecutor

• required string id = 1

  uuid

• required PbLogicalPlanNode.Type type = 2

• optional bytes debug_info = 3

• optional PbEntity objector = 4

• required string scope = 5

  scope uuid

• optional bool cache = 6

  indicate if the node need to be cached

• optional int32 memory_limit = 7

  memory used (MB)

• optional int32 cpu_limit = 8

  cpu used (format, core = format / 100)

• optional bool is_infinite = 9

  indicate if output data is infinite

• optional PbUnionNode union_node = 100

  according to type, only corresponding member makes sense

• optional PbLoadNode load_node = 101

• optional PbSinkNode sink_node = 102

• optional PbProcessNode process_node = 103

• optional PbShuffleNode shuffle_node = 104

Used in: PbLogicalPlanNode

• UNION_NODE = 0

• LOAD_NODE = 1

• SINK_NODE = 2

• PROCESS_NODE = 3

• SHUFFLE_NODE = 4

Used in: bigflow.PbPipeline

• optional string hadoop_client_path = 1

• optional string hadoop_config_path = 2

• optional string tmp_data_path = 3

• optional string job_name = 4

• repeated PbKVConfig hadoop_job_conf = 5

• optional int32 default_concurrency = 6

• optional bool must_keep_empty_group = 7

• optional string prepared_archive_path = 8

• optional bool immediately_commit = 9

• optional bool use_hadoop_hash_partitioner = 10

• optional bool heap_profile = 11

  switch of heap and cpu profile

• optional bool cpu_profile = 12

Used in: PbJob

• optional PbMonitorTask client_stream_task = 1

• optional PbMonitorTask client_prepared_task = 2

• optional PbMonitorTask worker_stream_task = 3

• optional PbMonitorTask worker_prepared_task = 4

Used in: PbMonitorJob

• required PbExecutor root = 1

• optional PbMonitorTask.Input input = 2

• repeated PbMonitorTask.MonitorWriter monitor_writer = 3

• optional PbMonitorTask.MonitorReader monitor_reader = 4

• repeated PbLogicalPlanNode node = 13

• repeated PbScope scope = 14

Used in: PbMonitorTask

• required string id = 1

Used in: PbMonitorTask

• required string id = 1

  external id

• repeated uint32 tag = 2

• repeated string source = 3

Used in: PbMonitorTask

• required string id = 1

  external id

• required MonitorWriter.Type type = 2

• required uint32 tag = 3

Used in: MonitorWriter

• LOCAL = 0

• RPC = 1

• required bool overwrite = 1

• required string path = 2

• optional string commit_path = 3

• optional bool async_mode = 4

• optional string output_format_config = 5

• optional uint32 max_waiting_time = 6

• optional uint32 min_waiting_time = 7

  200+ for other output format

Used in: PbTMConfig

• optional string zk_meta_host = 1

• optional string zk_root_path = 2

• optional string username = 3

• optional string password = 4

• optional string fs_remote_data_path = 7

• optional string fs_local_data_path = 8

• optional string physical_name = 9

• optional string logical_name = 10

• optional string version = 11

• optional int32 rpc_timeout_ms = 51

  for baidu-rpc

  default rpc timeout 30s

• optional string tm_name = 101

  for TM

• optional string tm_token = 102

• optional string tm_archive_path = 103

• optional int32 deleter_chunked_num = 104

• optional int32 deleter_normal_num = 105

• optional int32 deleter_refcnt_num = 106

• optional string deleter_archive_path = 107

Used in: PbExecutor

• repeated PbScope scope = 1

• repeated PbLogicalPlanNode node = 2

• repeated PbPartialExecutor.Output output = 3

• repeated int32 scope_level = 4

  <node->scope_level> one to one mapped

Used in: PbPartialExecutor

• required string identity = 1

• repeated uint32 priority = 2

• required PbEntity objector = 3

• optional bool need_hash = 4

• optional bool need_buffer = 5

• repeated PbJob job = 1

• optional PbEntity environment = 2

Used in: PbLogicalPlanNode

• repeated PbProcessNode.Input input = 1

  node uuid

• required PbEntity processor = 2

• optional int32 least_prepared_inputs = 101

• optional bool is_ignore_group = 102

• optional int32 effective_key_num = 103

• optional bool is_stateful = 104

Used in: PbProcessNode

• required string from = 1

  node uuid

• optional bool is_partial = 101

• optional bool is_prepared = 102

Used in: PbExecutor

• required string identity = 1

• required PbEntity processor = 2

• repeated PbProcessorExecutor.Source source = 3

• optional uint32 partial_key_number = 4

Used in: PbProcessorExecutor

• required string identity = 1

• required SourceType type = 2

• optional bool is_prepared = 3

Used in: Source

• DUMMY = 0

• REQUIRE_STREAM = 1

• REQUIRE_ITERATOR = 2

Used in: PbDceTask.ReduceOutput.Channel, PbLogicalPlan, PbMonitorTask, PbPartialExecutor, PbShuffleExecutor, PbSparkTask.PbShuffleOutput.Channel, PbStreamShuffleExecutor, PbTmTask.TaskOutput.Channel

• required string id = 1

  uuid

• optional PbScope.Type type = 2

  declare optional for compatible reason

• optional string father = 3

  scope uuid

• optional bool is_sorted = 4

• optional uint32 concurrency = 5

  a hint for execution

• optional bool is_infinite = 6

  indicate if output key or output data is infinite

• optional bool is_stream = 7

  indicate if data process is stream

• optional PbInputScope input_scope = 101

• optional PbGroupScope group_scope = 102

• optional PbBucketScope bucket_scope = 103

• optional PbWindowScope window_scope = 104

• optional bool distribute_every = 201

  if this is true, every line is in a different group

Used in: PbScope

• DEFAULT = 0

• INPUT = 1

  only accept single LOAD_NODE under this scope

• GROUP = 2

  accept BROADCAST/KEY SHUFFLE_NODE under this scope

• BUCKET = 3

  accept BROADCAST/SEQUENCE SHUFFLE_NODE under this scope

• WINDOW = 4

  accept WINDOW SHUFFLE_NODE under this scope

Used in: PbExecutor

• required PbScope scope = 1

• required PbShuffleExecutor.Type type = 2

• repeated PbLogicalPlanNode node = 3

• repeated PbShuffleExecutor.MergeSource source = 4

• optional uint32 partition = 5

Used in: PbShuffleExecutor

• required string input = 1

  sources got from outside, with binary encoding format

• required string output = 2

  sources given to child

• optional uint32 priority = 3

Used in: PbShuffleExecutor

• LOCAL = 0

  deal with unordered input streams

• MERGE = 1

  deal with ordered input streams

• COMBINE = 2

  combine ordered and unordered input streams

• LOCAL_DISTRIBUTE = 3

  distribute all input streams to a fixed partition

• BY_RECORD = 4

  distribute every record to a different group

Used in: PbLogicalPlanNode

• required string from = 1

  node uuid

• required PbShuffleNode.Type type = 2

• optional PbEntity key_reader = 3

  for KEY

• optional PbEntity partitioner = 4

  for SEQUENCE

• optional PbEntity time_reader = 5

  for WINDOW

Used in: PbShuffleNode

• BROADCAST = 0

• KEY = 1

• SEQUENCE = 2

• WINDOW = 3

Used in: PbLogicalPlanNode

• required string from = 1

  node uuid

• required PbEntity sinker = 2

Used in: bigflow.PbPipeline

• optional string hadoop_client_path = 1

• optional string hadoop_config_path = 2

• optional string spark_home_path = 3

• optional string tmp_data_path = 4

• optional string job_name = 5

• repeated PbKVConfig kv_config = 6

• optional int32 default_concurrency = 7

• optional bool must_keep_empty_group = 8

• optional string prepared_archive_path = 9

• optional bool immediately_commit = 10

• optional bool use_hadoop_hash_partitioner = 11

• optional bool heap_profile = 201

  switch of heap and cpu profile

• optional bool cpu_profile = 202

Used in: PbJob

• repeated PbSparkRDD rdd = 1

• optional PbSparkJob.PbSparkJobInfo job_info = 2

Used in: PbSparkJob

• repeated string cache_node_id = 1

  cache_node_id -> cache_task_id is a map

  cache node id

• repeated uint32 cache_task_id = 2

  corresponding cache_task id

Used in: PbSparkJob

• optional uint32 rdd_index = 1

• repeated PbSparkTask task = 2

• repeated uint32 parent_rdd_index = 3

• optional uint32 concurrency = 4

• optional int32 memory_limit = 5

Used in: PbSparkRDD

• optional uint32 task_index = 1

• optional PbSparkTask.Type type = 2

• optional uint32 rdd_index = 3

  Not needed maybe

• optional uint32 concurrency = 4

• optional uint32 partition_offset = 5

• optional string cache_node_id = 10

  if it's a CACHE SparkTask, this field is needed to indicate cached node's id.

• optional uint32 cache_task_index = 11

• optional uint32 cache_rdd_index = 12

• optional uint32 cache_effective_key_number = 13

• optional bool do_cpu_profile = 50

  cpu & heap profile

• optional bool do_heap_profile = 51

• optional PbSparkTask.PbHadoopInput hadoop_input = 101

• optional PbSparkTask.PbShuffleInput shuffle_input = 102

• optional PbSparkTask.PbShuffleOutput shuffle_output = 103

• optional PbSparkTask.PbCacheInput cache_input = 104

• optional PbExecutor root = 201

  RDDExecutor(External)

• optional PbSparkTask.PbRuntimeTaskContext runtime_task_ctx = 301

  runtime task context

Used in: PbSparkTask

• optional string id = 1

• optional string cache_node_id = 2

• optional uint32 key_num = 3

Used in: PbSparkTask

• optional string id = 1

• optional PbEntity spliter = 2

• repeated string uri = 3

• optional string input_format = 4

Used in: PbSparkTask

• optional string task_attempt_id = 1

Used in: PbSparkTask

• required string id = 1

  external id

• required PbTransferDecoder decoder = 2

• repeated PbShuffleInput.Channel channel = 3

• optional bool must_keep_empty_group = 4

Used in: PbShuffleInput

• required string identity = 1

• required string port = 2

• required uint32 priority = 3

Used in: PbSparkTask

• required string id = 1

• repeated PbShuffleOutput.Channel channel = 2

Used in: PbShuffleOutput

• required string from = 1

• required PbScope transfer_scope = 2

• required PbTransferEncoder encoder = 3

• required uint32 task_index = 4

• optional uint32 task_concurrency = 5

• optional uint32 task_offset = 6

• optional Channel.TransferType transfer_type = 7

Used in: Channel

• BROADCAST = 0

• KEY = 1

• SEQUENCE = 2

• WINDOW = 3

Move 'type' to PbSparkRDD in the future

Used in: PbSparkTask

• INPUT = 1

• GENERAL = 2

• CACHE = 3

Reflectable classes that contain user defined logic.

Used in: PbCombineSplit

• required PbSplit.Magic magic = 1

  magic number to verify this message

• repeated string location = 2

  list of hostnames where the input split is located

• optional uint64 length = 3

  total number of bytes in the data of the input split

• optional bytes raw_split = 4

  raw_split would be passed to Loader::Load(). If not set, pass the whole PbSplit message to Loader::Load();

Used in: PbSplit

• FLUME = 35863

  'flume' in cellphone's squared keyboard

Used in: PbExecutor

• required string id = 1

  indicate LocalInput, DceInput or DceShuffle, etc.

Used in: PbExecutor

• required PbLogicalPlanNode node = 1

Used in: PbExecutor

• required string identity = 1

  source should add is_batch

• required PbEntity processor = 2

• repeated PbStreamProcessorExecutor.Source source = 3

• optional bool need_status = 4

• optional bool is_ignore_group = 5

Used in: PbStreamProcessorExecutor

• required string identity = 1

• optional bool is_batch = 2

Used in: PbExecutor

• required PbScope scope = 1

• required PbStreamShuffleExecutor.Type type = 2

• repeated PbLogicalPlanNode node = 3

• optional bool need_cache = 4

Used in: PbStreamShuffleExecutor

• LOCAL = 0

  deal with unordered input streams

• LOCAL_DISTRIBUTE = 1

  distribute all input streams to a fixed partition

• DISTRIBUTE_AS_BATCH = 2

  distribute all input streams as batch to a fixed partition

• WINDOW = 3

  distribute all input streams according to time window

Used in: bigflow.PbPipeline

• optional string hadoop_client_path = 1

• optional string hadoop_config_path = 2

• optional string tmp_data_path = 3

• optional string job_name = 4

• optional int32 default_concurrency = 5

• optional int32 default_memory_limit = 6

• optional int32 default_cpu_limit = 7

• optional PbPagodaJobConfig pagoda_job_conf = 8

• repeated PbKVConfig hadoop_job_conf = 9

• repeated PbKVConfig user_job_conf = 10

• optional string abaci_archive_path = 11

• optional string prepared_archive_path = 12

• optional string status_meta_path = 101

  for status

• optional string status_table_path = 102

• optional bool status_as_meta = 103

• optional int32 status_cache_size = 104

• optional PbTextOutputFormatConfig.CompressionType compression_type = 1

• optional bytes record_delimiter = 2

Used in: PbTextOutputFormatConfig

• NONE = 0

• GZIP = 1

Used in: PbJob

• repeated PbTmTask task = 1

• optional string job_name = 2

• optional bool need_load_token = 3

Used in: PbTmJob

• required uint32 task_index = 1

• required PbExecutor root = 2

• repeated uint32 next_tasks = 3

• optional uint32 concurrency = 4

• optional int32 memory_limit = 5

  memory used (MB)

• optional int32 cpu_limit = 6

  cpu used (format, core = format / 100)

• optional string name = 7

  task worker processor name

• optional PbTmTask.StreamInput stream_input = 201

• optional PbTmTask.TaskInput task_input = 202

• optional PbTmTask.TaskOutput task_output = 203

Used in: PbTmTask

• required string id = 1

• required PbEntity spliter = 2

• repeated string uri = 3

Used in: PbTmTask

• required string id = 1

• repeated TaskInput.Channel channel = 2

Used in: TaskInput

• required string identity = 1

• required string port = 2

• required uint32 tag = 3

Used in: PbTmTask

• required string id = 1

• repeated TaskOutput.Channel channel = 2

Used in: TaskOutput

• required string from = 1

• required PbScope transfer_scope = 2

• required uint32 tag = 3

• required uint32 task_index = 4

• required string task_name = 5

• optional uint32 task_concurrency = 6

• optional bool distribute_by_default = 7

Used in: PbDceTask.ReduceInput, PbSparkTask.PbShuffleInput

• required PbTransferEncodingField field = 1

• repeated PbTransferDecoder successor = 2

Used in: PbDceTask.ReduceOutput.Channel, PbSparkTask.PbShuffleOutput.Channel

• required PbTransferEncodingField field = 1

• required uint32 tag = 2

• required PbTransferEncoder.TagEncoding tag_encoding = 3

• optional PbTransferEncoder successor = 4

Used in: PbTransferEncoder

• UNIQUE = 1

• UINT8 = 2

• UINT16 = 3

Used in: PbTransferDecoder, PbTransferEncoder

• required string identity = 1

  executor/dispatcher identity, mainly for debug

• required PbTransferEncodingField.Type type = 2

• optional uint32 length = 3

  set for FIXED_LENGTH_KEY

Used in: PbTransferEncodingField

• RECORD = 1

• GLOBAL_KEY = 2

• LOCAL_PARTITION = 3

• FIXED_LENGTH_KEY = 4

• VARIABLE_LENGTH_KEY = 5

• TAILING_KEY = 6

Used in: PbLogicalPlanNode

• repeated string from = 1

  node uuid

Used in: PbWindowScope

• ACCUMULATING = 0

• DISCARDING = 1

Used in: PbScope

• optional PbEntity window = 1

• optional PbEntity trigger = 2

• optional PbWindowMode mode = 3

Used in: PbExecutor

• required string from = 1

• required string tmp_data_path = 2

  where cache content goes

• optional int32 key_num = 3