package android_env

Get desktop application:
View/edit binary Protocol Buffers messages

Used in: SetupStep

• oneof command

  • AdbCall.InstallApk install_apk = 1

  • AdbCall.StartActivity start_activity = 2

  • AdbCall.ForceStop force_stop = 3

  • AdbCall.ClearCache clear_cache = 4

  • AdbCall.GrantPermissions grant_permissions = 5

  • AdbCall.Tap tap = 6

  • AdbCall.PressButton press_button = 7

  • AdbCall.Rotate rotate = 8

  • AdbCall.StartAccessibilityService start_accessibility_service = 9

  • AdbCall.StartScreenPinning start_screen_pinning = 10

  • AdbCall.StartRandomActivity start_random_activity = 11

  • AdbCall.ForceStopRandomActivity force_stop_random_activity = 12

  • AdbCall.ClearCacheRandomActivity clear_cache_random_activity = 13

  • AdbCall.DisableAnimations disable_animations = 14

  • AdbCall.StartIntent start_intent = 15

  • AdbCall.UninstallPackage uninstall_package = 16

  • AdbCall.Command shell_command = 17

  • AdbCall.Command adb_command = 18

Used in: AdbCall

• string package_name = 1

Clear cache for last selected random activity.

Used in: AdbCall

(message has no fields)

Used in: AdbCall

• repeated string command = 1

Disables all Android animations.

Used in: AdbCall

(message has no fields)

Used in: AdbCall

• string package_name = 1

Force stop last selected random activity.

Used in: AdbCall

(message has no fields)

Used in: AdbCall

• string package_name = 1

• repeated string permissions = 2

  Possible values listed at https://developer.android.com/reference/android/Manifest.permission To query an app's required permissions, use the following adb command: > adb shell dumpsys package <package> The output will contain things like android.permission.WRITE_SECURE_SETTINGS

Installs an APK into the simulator.

Used in: AdbCall

• oneof location

  • InstallApk.Filesystem filesystem = 2

A location in the filesystem.

Used in: InstallApk

• string path = 1

Used in: AdbCall

• PressButton.Button button = 1

Used in: PressButton

• HOME = 0

• BACK = 1

Used in: AdbCall

• Rotate.Orientation orientation = 1

Used in: Rotate

• PORTRAIT_0 = 0

  0 degrees counter-clockwise.

• LANDSCAPE_90 = 1

  90 degrees counter-clockwise.

• PORTRAIT_180 = 2

  180 degrees counter-clockwise.

• LANDSCAPE_270 = 3

  270 degrees counter-clockwise.

Used in: AdbCall

• string full_service = 1

  Fully qualified class name of the Accessibility Service.

Used in: AdbCall

• string full_activity = 1

• repeated string extra_args = 2

• string flag = 3

Used in: AdbCall

• string data_uri = 1

• string action = 2

• string package_name = 3

Starts and wait for a randomly chosen activity in the list.

Used in: AdbCall

• repeated string activity_list = 1

• repeated string extra_args = 2

• float timeout_sec = 4

Pins the given activity to the screen. This essentially locks the user into a single app mode (aka "Kiosk mode").

Used in: AdbCall

• string full_activity = 1

Used in: AdbCall

• int32 x = 1

  NOTE: These are absolute coordinates in the range of the screen resolution. They are NOT floats in [0,1].

• int32 y = 2

Used in: AdbCall

• string package_name = 1

An AppScreen identifies a unique configuration that we can observe on the screen of a device.

Used in: Task, WaitForAppScreen

• string activity = 1

  Fully-qualified name of the activity.

• repeated string view_hierarchy_path = 2

  A list of regexes to match at each level of the current view hierarchy. The environment uses this list to determine whether the agent has "exited" this current task. Example: [ "^DecorView@.*\[MainActivity\]$", "^android.widget.LinearLayout\{.*\}$", "^android.widget.FrameLayout\{.*android\:id\/content\}", "^android.widget.RelativeLayout\{.*\}", "^android.widget.FrameLayout\{.*app\:id\/fragment_holder\}", "^android.widget.RelativeLayout\{.*\}", "^com.google.example.games.nostalgicracer.views.RaceView3D\{.*app\:id\/gameplay_screen_3d\}", ],

Used in: Task

• string name = 1

  An identifier for this ArraySpec.

• repeated int32 shape = 2

  The shape of the multi-dimensional values associated with this ArraySpec,

• ArraySpec.DataType dtype = 3

  Data type of elements we expect to see in an array of this spec.

Used in: ArraySpec

• INVALID_DATA_TYPE = 0

• FLOAT = 1

• DOUBLE = 2

• INT8 = 3

• INT16 = 4

• INT32 = 5

• INT64 = 6

• UINT8 = 7

• UINT16 = 8

• UINT32 = 9

• UINT64 = 10

• BOOL = 11

• STRING_U1 = 12

• STRING_U16 = 13

• STRING_U25 = 14

• STRING_U250 = 15

Used in: SuccessCondition

• string package_name = 1

• float timeout_sec = 2

  Maximum time in seconds to wait.

message `EventSlot` {{{ -

Used in: EventSlot.FakeSource, TaskEventSlots

• EventSlot.Type type = 1

• int32 id = 2

• repeated EventSlot.FakeSource events = 3

• repeated int32 prerequisite = 4

  only when all the prerequisites have ever been triggered, this event can be triggered

• repeated string transformation = 5

  Python statement that could be evaluated by `exec` function. Use `x` to indicate the input variable and `y` for the output variable. e.g., use "y = x" to indicate the identity transformation e.g., use expression like "y = 3" to indicate the constant transformation NOTE: for `and`, use `x[i]` to indicate the input from the i-th event NOTE: for `or`, use `x` to represent the input from the exclusive active event

• EventSlot.Repeatability repeatability = 6

• repeated int32 cache_until = 7

  will cache the value and state for a while and not clear them until all the waited events are set simultaneously; -1 for never cleared and 0 for nothing waiting

Used in: EventSlot

• oneof source

  • int32 id = 1

  • EventSlot event = 2

Used in: EventSlot

• UNLIMITED = 0

• LAST = 1

• NONE = 2

Used in: EventSlot

• SINGLE = 0

• AND = 1

• OR = 2

zdy message `EventSource` {{{

Used in: Task

• oneof event

  • EventSource.TextEvent text_recognize = 3

    EventGroup and = 1; EventGroup or = 2;

  • EventSource.TextEvent text_detect = 4

  • EventSource.IconRecogEvent icon_recognize = 5

  • EventSource.IconRecogEvent icon_detect = 6

  • EventSource.IconMatchEvent icon_match = 7

  • EventSource.IconMatchEvent icon_detect_match = 8

  • EventSource.ViewHierarchyEvent view_hierarchy_event = 9

  • EventSource.LogEvent log_event = 10

  • EventSource.ResponseEvent response_event = 11

    bool default_event = 11;

• int32 id = 12

• EventSource.Repeatability repeatability = 14

  repeated int32 prerequisite=13;

  string transformation = 15;

match to the given icon

Used in: EventSource

• string path = 1

• optional Rect rect = 2

recognize the icon of the given class

Used in: EventSource

• string class = 1

• optional Rect rect = 2

Used in: EventSource

• repeated string filters = 1

• string pattern = 2

Used in: ResponseEvent, TextEvent

• REGEX = 0

• DIFFLIB = 1

• FUZZ = 2

• SBERT = 3

floats in range [0, 1]

Used in: IconMatchEvent, IconRecogEvent, TextEvent

• float x0 = 2

• float y0 = 3

• float x1 = 4

• float y1 = 5

Used in: EventSource

• NONE = 0

• LAST = 1

• UNLIMITED = 2

Used in: EventSource

• string pattern = 1

• MatchMode mode = 2

• float threshold = 3

recognize the given text

Used in: EventSource

• string expect = 1

• optional Rect rect = 2

• MatchMode mode = 3

• float threshold = 4

Specified as (EBNF) path = class-regex [ "@" id-regex ] ; If there is "@" in the regex literal, escape it by "\@". repeated string view_hierarchy_path = 1;

Used in: EventSource

• repeated string selector = 3

  Mobile-Env-customized CSS selector several customized syntax: * #"com.wikihow.wikihowapp:id/search_src_text" -> [resource-id="com.wikihow.wikihowapp:id/search_src_text"] * #$"search_src_text" -> [resource-id$=search_src_text] * ."android.widget.EditText" -> [class="android.widget.EditText"] * .$"EditText" -> [class$=EditText] * $"com.wikihow.wikihowapp" -> [package="com.wikihow.wikihowapp"] * @2 -> [index="2"], somewhat the same as :nth-child(3) (index attribute starts from 0) multiple strings will be concatenated by ", ", i.e., selector: ["abc", "def", '."class">@3'] is the same as a group of css selectors selector: 'abc, def, ."class">@3'

• repeated ViewHierarchyEvent.Property properties = 2

Used in: ViewHierarchyEvent

• string property_name = 1

• Sign sign = 2

• oneof property_value

  • string pattern = 3

  • int32 integer = 4

  • float floating = 5

Used in: Property

• EQ = 0

• LE = 1

• LT = 2

• GE = 3

• GT = 4

• NE = 5

A reinforcement learning (RL) observation which agents can use to build their internal state. This comes straight from the Android Emulator.

• int64 timestamp_us = 1

  The moment this RawObservation was generated.

• optional RawObservation.Image screen = 2

  The screenshot representing this raw observation.

A visual image represented as pixels.

Used in: RawObservation

• int32 width = 1

  The width in pixels of this image.

• int32 height = 2

  The height in pixels of this image.

• int32 num_channels = 3

  The number of channels (e.g. 3 in RGB) per pixel.

• bytes data = 4

  The binary data associated with this image.

Used in: Task

• optional SuccessCondition success_condition = 1

• oneof step

  • AdbCall adb_call = 2

  • Sleep sleep = 3

Used in: SetupStep

• float time_sec = 1

Used in: SetupStep

• int32 num_retries = 1

• oneof check

  • WaitForAppScreen wait_for_app_screen = 2

  • CheckInstall check_install = 3

  • WaitForMessage wait_for_message = 4

Description of a reinforcement learning task to be solved by an agent.

• string id = 1

  A globally unique identifier for this task.

• string name = 2

  A human readable name for this task.

• string description = 3

  A description of the task.

• repeated SetupStep setup_steps = 4

• repeated SetupStep reset_steps = 5

• optional AppScreen expected_app_screen = 6

• float max_duration_sec = 7

  AndroidEnv resets the episode after `max_duration_sec` is passed since the last reset(). Note that this is real time as measured by AndroidEnv and is independent of the speed of simulation of Android. If <= 0.0, this logic is disabled.

• int32 max_num_steps = 8

  The maximum number of interactions in a single episode between the environment and an agent. This setting is appropriate for tasks that are not time-dependent or when the performance of the simulation varies dramatically between runs. If <= 0, this logic is disabled.

• repeated EventSource event_sources = 9

  zdy Defines parameters for parsing messages from logcat. LogParsingConfig log_parsing_config = 9;

• optional TaskEventSlots event_slots = 10

• repeated ArraySpec extras_spec = 11

  (Optional): The task may also define extras to help the RL agent. An Extra in AndroidEnv is any information that apps may send to aid the understanding of the task. The type of information sent through this channel is usually something difficult to obtain from raw pixels and may include things such as: - The current board configuration (e.g. of a chess game or a tetris game) - The position of the avatar in a map - Events (e.g. whether a button was pressed or a checkpoint was achieved) Notice that these are entirely optional and may not be available at all. This specification ensures that only extras specified in the Task definition will be passed to the agent, everything else is excluded. The name of an extra must be unique across all extras.

• repeated string command = 12

  zdy

• repeated string vocabulary = 13

Used in: Task

• optional EventSlot score_listener = 1

• optional EventSlot reward_listener = 2

• optional EventSlot episode_end_listener = 3

• optional EventSlot extra_listener = 4

• optional EventSlot json_extra_listener = 5

• optional EventSlot instruction_listener = 6

• optional EventSlot null_listener = 7

Waits for `app_screen` to be the current app screen shown to the user.

Used in: SuccessCondition

• optional AppScreen app_screen = 1

• float timeout_sec = 2

  Maximum time in seconds to wait for the activity to become the current one.

Used in: SuccessCondition

• string message = 1

• float timeout_sec = 2