package ps

Get desktop application:
View/edit binary Protocol Buffers messages

assignment operators: http://en.cppreference.com/w/cpp/language/operator_assignment

Used in: Task

• ASSIGN = 0

  a = b

• PLUS = 1

  a += b

• MINUS = 2

  a -= b

• TIMES = 3

  a *= b

• DIVIDE = 4

  a -= b

• AND = 5

  a &= b

• OR = 6

  a |= b

• XOR = 7

  a ^= b

Used in: Task

• required Control.Command cmd = 1

• repeated Node node = 2

Used in: Control

• REGISTER_NODE = 2

  a node => the scheduler

• READY_TO_EXIT = 4

  REPORT_PERF = 3;

• READY_TO_RUN = 5

• ADD_NODE = 10

  the scheduler => a node

• REMOVE_NODE = 13

  UPDATE_NODE = 11; REPLACE_NODE = 12;

• EXIT = 14

• required DataConfig.DataFormat format = 1

• optional DataConfig.TextFormat text = 2

• repeated string file = 3

  filenames, supports regular expressions

• optional HDFSConfig hdfs = 5

  files stored in hdfs

• optional bool ignore_feature_group = 6

  ignore the feature group information

• optional int32 max_num_files_per_worker = 7

  the maximal number of files will be assigned to a worker, -1 means no limit

• optional int32 max_num_lines_per_file = 8

  the maximal number of lines will be read from a file, -1 means no limit

• optional bool shuffle = 9

  randomly shuffle the file order

• optional PbRange range = 4

  only valid for the binary format

• optional int32 replica = 10

  duplicate the file several times

Used in: DataConfig

• BIN = 1

• PROTO = 2

• TEXT = 3

see https://github.com/mli/parameter_server/wiki/Data

Used in: DataConfig

• DENSE = 1

• SPARSE = 2

• SPARSE_BINARY = 3

• ADFEA = 4

• LIBSVM = 5

• TERAFEA = 6

• VW = 7

Used in: Task

• OTHER = 0

• INT8 = 1

• INT16 = 2

• INT32 = 3

• INT64 = 4

• UINT8 = 5

• UINT16 = 6

• UINT32 = 7

• UINT64 = 8

• FLOAT = 9

• DOUBLE = 10

• CHAR = 11

Used in: Task

• required Filter.Type type = 1

• optional bool clear_cache = 20

  -- key caching -- if the task is done, then clear the cache (to save memory)

• optional int32 num_bytes = 5

  -- fixing float filter --

• optional float mean = 6

  -- nosie --

• optional float std = 7

• repeated Filter.FixedFloatConfig fixed_point = 4

• optional uint64 signature = 2

• repeated uint64 uncompressed_size = 3

-- runtime parameters used by the system --

Used in: Filter

• optional float min_value = 1

• optional float max_value = 2

Used in: Filter

• KEY_CACHING = 1

  cache the keys at both sender and receiver

• COMPRESSING = 2

  compress data by snappy

• FIXING_FLOAT = 3

  convert a float/double into a fixed-point integer with random rounding

• NOISE = 4

  add noise to data

• DELTA_KEY = 5

  key[i] <- key[i+1] - key[i]

• TRUNCATE_FLOAT = 6

  truncate a float/double into an integer

Used in: DataConfig

• optional string home = 1

  HADOOP_HOME

• optional string ugi = 2

  hadoop.job.ugi, format: user,passwd

• optional string namenode = 4

  fs.default.name

• optional int32 task_id = 1

• optional string hostname = 14

• optional uint32 seconds_since_epoch = 2

  time stamp latest heartbeat report the scheduler has ever received from a specified worker/server

• optional uint32 total_time_milli = 13

• optional uint32 busy_time_milli = 3

• optional uint32 net_in_mb = 4

  recv/sent bytes via zmq

• optional uint32 net_out_mb = 5

• optional uint32 process_cpu_usage = 6

  user+sys (percentage)

• optional uint32 host_cpu_usage = 7

• optional uint32 process_rss_mb = 8

• optional uint32 process_virt_mb = 9

• optional uint32 host_in_use_gb = 10

• optional uint32 host_in_use_percentage = 15

• optional uint32 host_net_in_bw = 11

  host's network in/out bandwidth usage (MB/s)

• optional uint32 host_net_out_bw = 12

Used in: Control

• required Node.Role role = 1

• optional string id = 2

• optional int32 rank = 5

• optional string hostname = 3

  network address

• optional int32 port = 4

• optional PbRange key = 6

Used in: Node

• SERVER = 0

• WORKER = 1

• SCHEDULER = 3

  each running application has a single scheduler

• GROUP = 4

  a virtual node, present a group of node

• UNUSED = 5

  a backup node, could turn into another node

Used in: Task

• optional bool push = 1

  push or pull

• optional bool dyn_val_size = 4

  merge operator optional AsOp op = 2;

• optional TailKeyIFilter tail_filter = 3

• optional bool replica = 10

  it's a replica request

• repeated Timestamp backup = 11

• optional ParamInitConfig.Type type = 1

• optional double constant = 2

• optional double mean = 3

  gaussian random

• optional double std = 4

• optional string file_name = 5

Used in: ParamInitConfig

• ZERO = 1

• CONSTANT = 2

• GAUSSIAN = 3

• FILE = 4

• CLONE = 5

TODO may change it to string

Used in: DataConfig, Node, Task

• required uint64 begin = 1

• required uint64 end = 2

Used in: ParamCall

• optional bool insert_count = 1

• optional int32 freq_threshold = 2

• optional bool query_value = 3

• optional int32 countmin_n = 4

• optional int32 countmin_k = 5

/ entries can be used by user programs:

• optional bytes msg = 17

  the place to store a small amount of data

• optional int32 cmd = 19

  store a user-defined cmd

• optional AsOp op = 21

  a reduce operator

• optional bool control = 1

  true: system control task, typically *ctrl* should be set false: a task for a customer, and *customer_id* should be set

• optional bool request = 2

  true: a request task false: the response task to the request task with the same *time*

• optional int32 customer_id = 3

  the unique id of a customer

• optional int32 time = 5

  the timestamp if this task

• repeated int32 wait_time = 6

  the depended tasks of this one. that is, this task is executed only if all tasks from the same node with time contained in *wait_time* are finished. only valid if *request*=true

• optional PbRange key_range = 7

  the key range of this task

• optional int32 key_channel = 8

  namespace of keys

• optional bool has_key = 9

  true: the message sent with this task will contain a list of keys

• optional DataType key_type = 13

  key type

• repeated DataType value_type = 14

  data type

• repeated Filter filter = 12

  filters applied to the data

• optional Control ctrl = 18

  system control signals

• optional ParamCall param = 20

  for push & pull

Used in: ParamCall

• required string sender = 1

• required int32 time = 2