package PS

Get desktop application:
View/edit binary Protocol Buffers messages

• repeated int64 tp_key = 1

• repeated uint64 tp_count = 2

• repeated int64 fp_key = 3

• repeated uint64 fp_count = 4

assignment operators: http://en.cppreference.com/w/cpp/language/operator_assignment

Used in: ParamCall

• ASSIGN = 0

  a = b

• PLUS = 1

  a += b

• MINUS = 2

  a -= b

• TIMES = 3

  a *= b

• DIVIDE = 4

  a -= b

• AND = 5

  a &= b

• OR = 6

  a |= b

• XOR = 7

  a ^= b

Used in: Task

• required BCDCall.Command cmd = 1

• optional PbRange key = 2

  optional int32 feature_group_id = 3;

• optional double kkt_filter_threshold = 4

• optional bool reset_kkt_filter = 5

• optional int32 iter = 11

• repeated int32 fea_grp = 8

• optional bool hit_cache = 9

• optional DataConfig data = 10

• optional int32 time = 12

Used in: BCDCall

• LOAD_DATA = 1

• PREPROCESS_DATA = 2

• UPDATE_MODEL = 3

• EVALUATE_PROGRESS = 4

• SAVE_MODEL = 5

  save w

• RECOVER = 6

• COMPUTE_VALIDATION_AUC = 7

• REQUEST_WORKLOAD = 8

  SAVE_AS_DENSE = 7; // save X * w in a given key range

Used in: LM.Config

• optional float feature_block_ratio = 1

  Divide a feature group into feature_block_ratio x nnz_feature_per_example blocks

• optional bool random_feature_block_order = 2

  Use a random order to updating feature block. Turn it off to elimiate the randomness (often for debuging), but may affects the convergence rate.

• repeated int32 prior_fea_group = 14

  Updating important feature group at the beginning to get a good initial start point.

• optional int32 num_iter_for_prior_fea_group = 13

• optional int32 max_block_delay = 3

  Bounded-delay consistency

• optional int32 max_pass_of_data = 4

  max number pass of traing data

• optional double epsilon = 5

  convergance critiria. stop if the relative objective <= epsilon

• optional int32 tail_feature_freq = 6

  features which occurs <= *tail_feature_freq* will be filtered before training

• optional int32 countmin_k = 7

  countmin sketch is used to filter the tail features. It has two parameters, k and n.

• optional double countmin_n_ratio = 8

  n = the_first_arrive_key_length * num_workers * countmin_n_ratio

• optional int32 max_num_parallel_groups_in_preprocessing = 9

• optional int32 max_data_buf_size_in_mb = 10

• optional DataConfig local_cache = 11

• optional ParamInitConfig init_w = 12

• repeated FilterConfig comm_filter = 15

• optional int32 save_model_every_n_iter = 16

• optional bool load_local_data = 17

• optional double objective = 1

• optional double relative_obj = 2

• optional uint64 nnz_w = 5

• optional double violation = 6

• optional uint64 nnz_active_set = 7

• optional double total_time = 10

  performance

• repeated double busy_time = 11

Used in: Task

• required Control.Command cmd = 1

• repeated Node node = 2

Used in: Control

• REQUEST_APP = 1

  a node => the scheduler

• REGISTER_NODE = 2

• REPORT_PERF = 3

• READY_TO_EXIT = 4

• ADD_NODE = 10

  the scheduler => a node

• UPDATE_NODE = 11

• REPLACE_NODE = 12

• REMOVE_NODE = 13

• EXIT = 14

Used in: BCDCall, BCDConfig, LM.Config, Workload

• required DataConfig.DataFormat format = 1

• optional DataConfig.TextFormat text = 2

• repeated string file = 3

  filenames, supports regular expressions

• optional HDFSConfig hdfs = 5

  files stored in hdfs

• optional bool ignore_feature_group = 6

  ignore the feature group information

• optional int32 max_num_files_per_worker = 7

  the maximal number of files will be assigned to a worker, -1 means no limit

• optional int32 max_num_lines_per_file = 8

  the maximal number of lines will be read from a file, -1 means no limit

• optional bool shuffle = 9

  randomly shuffle the file order

• optional PbRange range = 4

  only valid for the binary format

• optional int32 replica = 10

  duplicate the file several times

Used in: DataConfig

• BIN = 1

• PROTO = 2

• TEXT = 3

see https://github.com/mli/parameter_server/wiki/Data

Used in: DataConfig

• DENSE = 1

• SPARSE = 2

• SPARSE_BINARY = 3

• ADFEA = 4

• LIBSVM = 5

• TERAFEA = 6

• VW = 7

• CRITEO = 9

Used in: Task

• OTHER = 0

• INT8 = 1

• INT16 = 2

• INT32 = 3

• INT64 = 4

• UINT8 = 5

• UINT16 = 6

• UINT32 = 7

• UINT64 = 8

• FLOAT = 9

• DOUBLE = 10

• CHAR = 11

• repeated Slot slot = 1

Used in: LoadDataResponse

• repeated SlotInfo slot = 1

• optional uint64 num_ex = 2

  total number of instances

Used in: BCDConfig, LM.SGDConfig, Task

• required FilterConfig.Type type = 1

• optional bool clear_cache_if_done = 20

  -- key caching -- if the task is done, then clear the cache (to save memory)

• optional int32 num_bytes = 5

  -- fixing float filter --

• repeated FilterConfig.FixedFloatConfig fixed_point = 4

• optional float mean = 6

  -- nosie --

• optional float std = 7

• optional uint32 signature = 2

  -- runtime parameters used by the system --

• repeated uint64 uncompressed_size = 3

Used in: FilterConfig

• optional float min_value = 1

• optional float max_value = 2

Used in: FilterConfig

• KEY_CACHING = 1

  cache the keys at both sender and receiver

• COMPRESSING = 2

  compress data by snappy

• FIXING_FLOAT = 3

  convert a float/double into a fixed-point integer

• NOISE = 4

  add noise to data

Used in: DataConfig

• optional string home = 1

  HADOOP_HOME

• optional string ugi = 2

  hadoop.job.ugi, format: user,passwd

• optional string namenode = 4

  fs.default.name

• optional int32 task_id = 1

• optional string hostname = 14

• optional uint32 seconds_since_epoch = 2

  time stamp latest heartbeat report the scheduler has ever received from a specified worker/server

• optional uint32 total_time_milli = 13

• optional uint32 busy_time_milli = 3

• optional uint32 net_in_mb = 4

  recv/sent bytes via zmq

• optional uint32 net_out_mb = 5

• optional uint32 process_cpu_usage = 6

  user+sys (percentage)

• optional uint32 host_cpu_usage = 7

• optional uint32 process_rss_mb = 8

• optional uint32 process_virt_mb = 9

• optional uint32 host_in_use_gb = 10

• optional uint32 host_in_use_percentage = 15

• optional uint32 host_net_in_bw = 11

  host's network in/out bandwidth usage (MB/s)

• optional uint32 host_net_out_bw = 12

• optional ExampleInfo example_info = 1

• optional int32 hit_cache = 2

• required MatrixInfo.Type type = 1

• required bool row_major = 2

• optional int32 id = 3

  e.g. feature group id

• required PbRange row = 5

  size

• required PbRange col = 6

• optional uint64 nnz = 7

  number of non-zero entries

• optional uint32 sizeof_index = 8

• required uint32 sizeof_value = 9

Used in: MatrixInfo

• DENSE = 1

• SPARSE = 2

• SPARSE_BINARY = 3

  TODO gpus

Used in: Control

• required Node.Role role = 1

• optional string id = 2

• optional int32 rank = 5

• optional string hostname = 3

  network address

• optional int32 port = 4

• optional PbRange key = 6

Used in: Node

• SERVER = 0

• WORKER = 1

• SCHEDULER = 3

  each running application has a single scheduler

• GROUP = 4

  a virtual node, present a group of node

• UNUSED = 5

  a backup node, could turn into another node

Used in: Task

• optional bool push = 1

  push or pull

• optional AssignOpType op = 2

  merge operator

• optional TailKeyFilter tail_filter = 3

• optional bool replica = 10

  it's a replica request

• repeated Timestamp backup = 11

Used in: BCDConfig

• optional ParamInitConfig.Type type = 1

• optional double constant = 2

• optional double mean = 3

  gaussian random

• optional double std = 4

• optional string file_name = 5

Used in: ParamInitConfig

• ZERO = 1

• CONSTANT = 2

• GAUSSIAN = 3

• FILE = 4

• CLONE = 5

TODO may change it to string

Used in: BCDCall, DataConfig, MatrixInfo, Node, Task

• required uint64 begin = 1

• required uint64 end = 2

Used in: Task

• required SGDCall.Command cmd = 1

• optional Workload load = 2

Used in: SGDCall

• REQUEST_WORKLOAD = 6

• UPDATE_MODEL = 1

• REPORT_PROGRESS = 2

• SAVE_MODEL = 3

• RECOVER = 4

• COMPUTE_VALIDATION_AUC = 5

• repeated double objective = 1

• optional uint64 num_examples_processed = 2

• repeated double accuracy = 3

• repeated double auc = 4

• optional uint64 nnz = 5

• optional double weight_sum = 6

• optional double delta_sum = 7

Used in: Example

• optional int32 id = 1

• repeated uint64 key = 2

• repeated float val = 3

Used in: ExampleInfo

• optional SlotInfo.Format format = 1

• optional int32 id = 2

• optional uint64 min_key = 3

• optional uint64 max_key = 4

• optional uint64 nnz_ele = 5

  total number of non-zero elements

• optional uint64 nnz_ex = 6

  total number of non-zero instance, (non-empty rows)

Used in: SlotInfo

• DENSE = 1

• SPARSE = 2

• SPARSE_BINARY = 3

Used in: ParamCall

• optional bool insert_count = 1

• optional int32 freq_threshold = 2

• optional bool query_value = 3

• optional int32 countmin_n = 4

• optional int32 countmin_k = 5

• optional bool control = 1

  true: system control task, typically *ctrl* should be set false: a task for a customer, and *customer_id* should be set

• optional bool request = 2

  true: a request task false: the response task to the request task with the same *time*

• optional int32 customer_id = 3

  the unique id of a customer

• optional int32 time = 5

  the timestamp if this task

• repeated int32 wait_time = 6

  the depended tasks of this one. that is, this task is executed only if all tasks from the same node with time contained in *wait_time* are finished. only valid if *request*=true

• optional PbRange key_range = 7

  the key range of this task

• optional int32 key_channel = 8

  namespace of keys

• optional bool has_key = 9

  true: the message sent with this task will contain a list of keys

• optional DataType key_type = 13

  data type

• repeated DataType value_type = 14

• repeated FilterConfig filter = 12

  filters applied to the data

• optional bool more = 16

  if true, the tasks will be packed into a single one during sending

• repeated Task task = 15

• optional bytes msg = 17

  the place to store a small amount of data

• optional Control ctrl = 18

  system control signals

• optional ParamCall param = 20

  parameters

• optional SGDCall sgd = 21

• optional BCDCall bcd = 22

Used in: ParamCall

• required string sender = 1

• required int32 time = 2

Used in: SGDCall

• optional int32 id = 1

  the workload id

• optional DataConfig data = 2

  the data associated with this workload

• optional bool shuffle = 3

  randomly shuffle the file order

• optional int32 replica = 4

  duplicate the data several times

• repeated int32 finished = 6

  all finished workload ids

• optional bool all_is_done = 5

  all workload is been done