libcxx/utils/adb_run.py

#!/usr/bin/env python3
#===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===----------------------------------------------------------------------===##

"""adb_run.py is a utility for running a libc++ test program via adb.
"""

import argparse
import hashlib
import os
import re
import shlex
import socket
import subprocess
import sys
from typing import List, Tuple


# Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.
REMOTE_BASE_DIR = "/data/local/tmp/adb_run"

g_job_limit_socket = None
g_verbose = False


def run_adb_sync_command(command: List[str]) -> None:
    """Run an adb command and discard the output, unless the command fails. If
    the command fails, dump the output instead, and exit the script with
    failure.
    """
    if g_verbose:
        sys.stderr.write(f"running: {shlex.join(command)}\n")
    proc = subprocess.run(command, universal_newlines=True,
                          stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT, encoding="utf-8")
    if proc.returncode != 0:
        # adb's stdout (e.g. for adb push) should normally be discarded, but
        # on failure, it should be shown. Print it to stderr because it's
        # unrelated to the test program's stdout output. A common error caught
        # here is "No space left on device".
        sys.stderr.write(f"{proc.stdout}\n"
                         f"error: adb command exited with {proc.returncode}: "
                         f"{shlex.join(command)}\n")
        sys.exit(proc.returncode)


def sync_test_dir(local_dir: str, remote_dir: str) -> None:
    """Sync the libc++ test directory on the host to the remote device."""

    # Optimization: The typical libc++ test directory has only a single
    # *.tmp.exe file in it. In that case, skip the `mkdir` command, which is
    # normally necessary because we don't know if the target directory already
    # exists on the device.
    local_files = os.listdir(local_dir)
    if len(local_files) == 1:
        local_file = os.path.join(local_dir, local_files[0])
        remote_file = os.path.join(remote_dir, local_files[0])
        if not os.path.islink(local_file) and os.path.isfile(local_file):
            run_adb_sync_command(["adb", "push", "--sync", local_file,
                                  remote_file])
            return

    assert os.path.basename(local_dir) == os.path.basename(remote_dir)
    run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])
    run_adb_sync_command(["adb", "push", "--sync", local_dir,
                          os.path.dirname(remote_dir)])


def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:
    components = []
    for arg in env_args:
        k, v = arg.split("=", 1)
        components.append(f"export {k}={shlex.quote(v)}; ")
    for k, v in prepend_path_args:
        components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")
    return "".join(components)


def run_command(args: argparse.Namespace) -> int:
    local_dir = args.execdir
    assert local_dir.startswith("/")
    assert not local_dir.endswith("/")

    # Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using
    # a hash of local_dir so that concurrent adb_run invocations don't create
    # the same intermediate parent directory. At least `adb push` has trouble
    # with concurrent mkdir syscalls on common parent directories. (Somehow
    # mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,
    # b/289311228.)
    local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()
    remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"
    sync_test_dir(local_dir, remote_dir)

    adb_shell_command = (
        # Set the environment early so that PATH can be overridden. Overriding
        # PATH is useful for:
        #  - Replacing older shell utilities with toybox (e.g. on old devices).
        #  - Adding a `bash` command that delegates to `sh` (mksh).
        f"{build_env_arg(args.env, args.prepend_path_env)}"

        # Set a high oom_score_adj so that, if the test program uses too much
        # memory, it is killed before anything else on the device. The default
        # oom_score_adj is -1000, so a test using too much memory typically
        # crashes the device.
        "echo 1000 >/proc/self/oom_score_adj; "

        # If we're running as root, switch to the shell user. The libc++
        # filesystem tests require running without root permissions. Some x86
        # emulator devices (before Android N) do not have a working `adb unroot`
        # and always run as root. Non-debug builds typically lack `su` and only
        # run as the shell user.
        #
        # Some libc++ tests create temporary files in the working directory,
        # which might be owned by root. Before switching to shell, make the
        # cwd writable (and readable+executable) to every user.
        #
        # N.B.:
        #  - Avoid "id -u" because it wasn't supported until Android M.
        #  - The `env` and `which` commands were also added in Android M.
        #  - Starting in Android M, su from root->shell resets PATH, so we need
        #    to modify it again in the new environment.
        #  - Avoid chmod's "a+rwx" syntax because it's not supported until
        #    Android N.
        #  - Defining this function allows specifying the arguments to the test
        #    program (i.e. "$@") only once.
        "run_without_root() {"
        "  chmod 777 .;"
        "  case \"$(id)\" in"
        "    *\"uid=0(root)\"*)"
        "    if command -v env >/dev/null; then"
        "      su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"
        "    else"
        "      su shell \"$@\";"
        "    fi;;"
        "    *) \"$@\";;"
        "  esac;"
        "}; "
    )

    # Older versions of Bionic limit the length of argv[0] to 127 bytes
    # (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this
    # limit. Changing the working directory works around this limit. The limit
    # is increased to 4095 (PATH_MAX-1) in Android M (API 23).
    command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]

    # Prior to the adb feature "shell_v2" (added in Android N), `adb shell`
    # always created a pty:
    #  - This merged stdout and stderr together.
    #  - The pty converts LF to CRLF.
    #  - The exit code of the shell command wasn't propagated.
    # Work around all three limitations, unless "shell_v2" is present.
    proc = subprocess.run(["adb", "features"], check=True,
                          stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                          encoding="utf-8")
    adb_features = set(proc.stdout.strip().split())
    has_shell_v2 = "shell_v2" in adb_features
    if has_shell_v2:
        adb_shell_command += (
            f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"
        )
    else:
        adb_shell_command += (
            f"{{"
            f"  stdout=$("
            f"    cd {remote_dir} && run_without_root {shlex.join(command_line)};"
            f"    echo -n __libcxx_adb_exit__=$?"
            f"  ); "
            f"}} 2>&1; "
            f"echo -n __libcxx_adb_stdout__\"$stdout\""
        )

    adb_command_line = ["adb", "shell", adb_shell_command]
    if g_verbose:
        sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")

    if has_shell_v2:
        proc = subprocess.run(adb_command_line, shell=False, check=False,
                              encoding="utf-8")
        return proc.returncode
    else:
        proc = subprocess.run(adb_command_line, shell=False, check=False,
                              stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                              encoding="utf-8")
        # The old `adb shell` mode used a pty, which converted LF to CRLF.
        # Convert it back.
        output = proc.stdout.replace("\r\n", "\n")

        if proc.returncode:
            sys.stderr.write(f"error: adb failed:\n"
                             f"  command: {shlex.join(adb_command_line)}\n"
                             f"  output: {output}\n")
            return proc.returncode

        match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$",
                     output, re.DOTALL)
        if not match:
            sys.stderr.write(f"error: could not parse adb output:\n"
                             f"  command: {shlex.join(adb_command_line)}\n"
                             f"  output: {output}\n")
            return 1

        sys.stderr.write(match.group(1))
        sys.stdout.write(match.group(2))
        return int(match.group(3))


def connect_to_job_limiter_server(sock_addr: str) -> None:
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

    try:
        sock.connect(sock_addr)
    except (FileNotFoundError, ConnectionRefusedError) as e:
        # Copying-and-pasting an adb_run.py command-line from a lit test failure
        # is likely to fail because the socket no longer exists (or is
        # inactive), so just give a warning.
        sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")
        return

    # The connect call can succeed before the server has called accept, because
    # of the listen backlog, so wait for the server to send a byte.
    sock.recv(1)

    # Keep the socket open until this process ends, then let the OS close the
    # connection automatically.
    global g_job_limit_socket
    g_job_limit_socket = sock


def main() -> int:
    """Main function (pylint wants this docstring)."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--execdir", type=str, required=True)
    parser.add_argument("--env", type=str, required=False, action="append",
                        default=[], metavar="NAME=VALUE")
    parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,
                        action="append", default=[],
                        metavar=("NAME", "PATH"))
    parser.add_argument("--job-limit-socket")
    parser.add_argument("--verbose", "-v", default=False, action="store_true")
    parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
    args = parser.parse_args()

    global g_verbose
    g_verbose = args.verbose
    if args.job_limit_socket is not None:
        connect_to_job_limiter_server(args.job_limit_socket)
    return run_command(args)


if __name__ == '__main__':
    sys.exit(main())