1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5import datetime
6import logging
7import moznetwork
8import select
9import socket
10import time
11import os
12import re
13import posixpath
14import subprocess
15import StringIO
16from devicemanager import DeviceManager, DMError, _pop_last_line
17import errno
18from distutils.version import StrictVersion
19
20
21class DeviceManagerSUT(DeviceManager):
22    """
23    Implementation of DeviceManager interface that speaks to a device over
24    TCP/IP using the "system under test" protocol. A software agent such as
25    Negatus (http://github.com/mozilla/Negatus) or the Mozilla Android SUTAgent
26    app must be present and listening for connections for this to work.
27    """
28
29    _base_prompt = '$>'
30    _base_prompt_re = '\$\>'
31    _prompt_sep = '\x00'
32    _prompt_regex = '.*(' + _base_prompt_re + _prompt_sep + ')'
33    _agentErrorRE = re.compile('^##AGENT-WARNING##\ ?(.*)')
34
35    reboot_timeout = 600
36    reboot_settling_time = 60
37
38    def __init__(self, host, port=20701, retryLimit=5, deviceRoot=None,
39                 logLevel=logging.ERROR, **kwargs):
40        DeviceManager.__init__(self, logLevel=logLevel,
41                               deviceRoot=deviceRoot)
42        self.host = host
43        self.port = port
44        self.retryLimit = retryLimit
45        self._sock = None
46        self._everConnected = False
47
48        # Get version
49        verstring = self._runCmds([{'cmd': 'ver'}])
50        ver_re = re.match('(\S+) Version (\S+)', verstring)
51        self.agentProductName = ver_re.group(1)
52        self.agentVersion = ver_re.group(2)
53
54    def _cmdNeedsResponse(self, cmd):
55        """ Not all commands need a response from the agent:
56            * rebt obviously doesn't get a response
57            * uninstall performs a reboot to ensure starting in a clean state and
58              so also doesn't look for a response
59        """
60        noResponseCmds = [re.compile('^rebt'),
61                          re.compile('^uninst .*$'),
62                          re.compile('^pull .*$')]
63
64        for c in noResponseCmds:
65            if (c.match(cmd)):
66                return False
67
68        # If the command is not in our list, then it gets a response
69        return True
70
71    def _stripPrompt(self, data):
72        """
73        take a data blob and strip instances of the prompt '$>\x00'
74        """
75        promptre = re.compile(self._prompt_regex + '.*')
76        retVal = []
77        lines = data.split('\n')
78        for line in lines:
79            foundPrompt = False
80            try:
81                while (promptre.match(line)):
82                    foundPrompt = True
83                    pieces = line.split(self._prompt_sep)
84                    index = pieces.index('$>')
85                    pieces.pop(index)
86                    line = self._prompt_sep.join(pieces)
87            except(ValueError):
88                pass
89
90            # we don't want to append lines that are blank after stripping the
91            # prompt (those are basically "prompts")
92            if not foundPrompt or line:
93                retVal.append(line)
94
95        return '\n'.join(retVal)
96
97    def _shouldCmdCloseSocket(self, cmd):
98        """
99        Some commands need to close the socket after they are sent:
100          * rebt
101          * uninst
102          * quit
103        """
104        socketClosingCmds = [re.compile('^quit.*'),
105                             re.compile('^rebt.*'),
106                             re.compile('^uninst .*$')]
107
108        for c in socketClosingCmds:
109            if (c.match(cmd)):
110                return True
111        return False
112
113    def _sendCmds(self, cmdlist, outputfile, timeout=None, retryLimit=None):
114        """
115        Wrapper for _doCmds that loops up to retryLimit iterations
116        """
117        # this allows us to move the retry logic outside of the _doCmds() to make it
118        # easier for debugging in the future.
119        # note that since cmdlist is a list of commands, they will all be retried if
120        # one fails.  this is necessary in particular for pushFile(), where we don't want
121        # to accidentally send extra data if a failure occurs during data transmission.
122
123        retryLimit = retryLimit or self.retryLimit
124        retries = 0
125        while retries < retryLimit:
126            try:
127                self._doCmds(cmdlist, outputfile, timeout)
128                return
129            except DMError as err:
130                # re-raise error if it's fatal (i.e. the device got the command but
131                # couldn't execute it). retry otherwise
132                if err.fatal:
133                    raise err
134                self._logger.debug(err)
135                retries += 1
136                # if we lost the connection or failed to establish one, wait a bit
137                if retries < retryLimit and not self._sock:
138                    sleep_time = 5 * retries
139                    self._logger.info('Could not connect; sleeping for %d seconds.' % sleep_time)
140                    time.sleep(sleep_time)
141
142        raise DMError("Remote Device Error: unable to connect to %s after %s attempts" %
143                      (self.host, retryLimit))
144
145    def _runCmds(self, cmdlist, timeout=None, retryLimit=None):
146        """
147        Similar to _sendCmds, but just returns any output as a string instead of
148        writing to a file
149        """
150        retryLimit = retryLimit or self.retryLimit
151        outputfile = StringIO.StringIO()
152        self._sendCmds(cmdlist, outputfile, timeout, retryLimit=retryLimit)
153        outputfile.seek(0)
154        return outputfile.read()
155
156    def _doCmds(self, cmdlist, outputfile, timeout):
157        promptre = re.compile(self._prompt_regex + '$')
158        shouldCloseSocket = False
159
160        if not timeout:
161            # We are asserting that all commands will complete in this time unless
162            # otherwise specified
163            timeout = self.default_timeout
164
165        if not self._sock:
166            try:
167                if self._everConnected:
168                    self._logger.info("reconnecting socket")
169                self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
170            except socket.error as msg:
171                self._sock = None
172                raise DMError("Automation Error: unable to create socket: " + str(msg))
173
174            try:
175                self._sock.settimeout(float(timeout))
176                self._sock.connect((self.host, int(self.port)))
177                self._everConnected = True
178            except socket.error as msg:
179                self._sock = None
180                raise DMError("Remote Device Error: Unable to connect socket: " + str(msg))
181
182            # consume prompt
183            try:
184                self._sock.recv(1024)
185            except socket.error as msg:
186                self._sock.close()
187                self._sock = None
188                raise DMError(
189                    "Remote Device Error: Did not get prompt after connecting: " + str(msg),
190                    fatal=True)
191
192            # future recv() timeouts are handled by select() calls
193            self._sock.settimeout(None)
194
195        for cmd in cmdlist:
196            cmdline = '%s\r\n' % cmd['cmd']
197
198            try:
199                sent = self._sock.send(cmdline)
200                if sent != len(cmdline):
201                    raise DMError("Remote Device Error: our cmd was %s bytes and we "
202                                  "only sent %s" % (len(cmdline), sent))
203                if cmd.get('data'):
204                    totalsent = 0
205                    while totalsent < len(cmd['data']):
206                        sent = self._sock.send(cmd['data'][totalsent:])
207                        self._logger.debug("sent %s bytes of data payload" % sent)
208                        if sent == 0:
209                            raise DMError("Socket connection broken when sending data")
210                        totalsent += sent
211
212                self._logger.debug("sent cmd: %s" % cmd['cmd'])
213            except socket.error as msg:
214                self._sock.close()
215                self._sock = None
216                self._logger.error("Remote Device Error: Error sending data"
217                                   " to socket. cmd=%s; err=%s" % (cmd['cmd'], msg))
218                return False
219
220            # Check if the command should close the socket
221            shouldCloseSocket = self._shouldCmdCloseSocket(cmd['cmd'])
222
223            # Handle responses from commands
224            if self._cmdNeedsResponse(cmd['cmd']):
225                foundPrompt = False
226                data = ""
227                timer = 0
228                select_timeout = 1
229                commandFailed = False
230
231                while not foundPrompt:
232                    socketClosed = False
233                    errStr = ''
234                    temp = ''
235                    self._logger.debug("recv'ing...")
236
237                    # Get our response
238                    try:
239                        # Wait up to a second for socket to become ready for reading...
240                        if select.select([self._sock], [], [], select_timeout)[0]:
241                            temp = self._sock.recv(1024)
242                            self._logger.debug(u"response: %s" % temp.decode('utf8', 'replace'))
243                            timer = 0
244                            if not temp:
245                                socketClosed = True
246                                errStr = 'connection closed'
247                        timer += select_timeout
248                        if timer > timeout:
249                            self._sock.close()
250                            self._sock = None
251                            raise DMError("Automation Error: Timeout in command %s" %
252                                          cmd['cmd'], fatal=True)
253                    except socket.error as err:
254                        socketClosed = True
255                        errStr = str(err)
256                        # This error shows up with we have our tegra rebooted.
257                        if err[0] == errno.ECONNRESET:
258                            errStr += ' - possible reboot'
259
260                    if socketClosed:
261                        self._sock.close()
262                        self._sock = None
263                        raise DMError(
264                            "Automation Error: Error receiving data from socket. "
265                            "cmd=%s; err=%s" % (cmd, errStr))
266
267                    data += temp
268
269                    # If something goes wrong in the agent it will send back a string that
270                    # starts with '##AGENT-WARNING##'
271                    if not commandFailed:
272                        errorMatch = self._agentErrorRE.match(data)
273                        if errorMatch:
274                            # We still need to consume the prompt, so raise an error after
275                            # draining the rest of the buffer.
276                            commandFailed = True
277
278                    for line in data.splitlines():
279                        if promptre.match(line):
280                            foundPrompt = True
281                            data = self._stripPrompt(data)
282                            break
283
284                    # periodically flush data to output file to make sure it doesn't get
285                    # too big/unwieldly
286                    if len(data) > 1024:
287                        outputfile.write(data[0:1024])
288                        data = data[1024:]
289
290                if commandFailed:
291                    raise DMError("Automation Error: Error processing command '%s'; err='%s'" %
292                                  (cmd['cmd'], errorMatch.group(1)), fatal=True)
293
294                # Write any remaining data to outputfile
295                outputfile.write(data)
296
297        if shouldCloseSocket:
298            try:
299                self._sock.close()
300                self._sock = None
301            except:
302                self._sock = None
303                raise DMError("Automation Error: Error closing socket")
304
305    def _setupDeviceRoot(self, deviceRoot):
306        if not deviceRoot:
307            deviceRoot = "%s/tests" % self._runCmds(
308                [{'cmd': 'testroot'}]).strip()
309        self.mkDir(deviceRoot)
310
311        return deviceRoot
312
313    def shell(self, cmd, outputfile, env=None, cwd=None, timeout=None, root=False):
314        cmdline = self._escapedCommandLine(cmd)
315        if env:
316            cmdline = '%s %s' % (self._formatEnvString(env), cmdline)
317
318        # execcwd/execcwdsu currently unsupported in Negatus; see bug 824127.
319        if cwd and self.agentProductName == 'SUTAgentNegatus':
320            raise DMError("Negatus does not support execcwd/execcwdsu")
321
322        haveExecSu = (self.agentProductName == 'SUTAgentNegatus' or
323                      StrictVersion(self.agentVersion) >= StrictVersion('1.13'))
324
325        # Depending on agent version we send one of the following commands here:
326        # * exec (run as normal user)
327        # * execsu (run as privileged user)
328        # * execcwd (run as normal user from specified directory)
329        # * execcwdsu (run as privileged user from specified directory)
330
331        cmd = "exec"
332        if cwd:
333            cmd += "cwd"
334        if root and haveExecSu:
335            cmd += "su"
336
337        if cwd:
338            self._sendCmds([{'cmd': '%s %s %s' % (cmd, cwd, cmdline)}], outputfile, timeout)
339        else:
340            if (not root) or haveExecSu:
341                self._sendCmds([{'cmd': '%s %s' % (cmd, cmdline)}], outputfile, timeout)
342            else:
343                # need to manually inject su -c for backwards compatibility (this may
344                # not work on ICS or above!!)
345                # (FIXME: this backwards compatibility code is really ugly and should
346                # be deprecated at some point in the future)
347                self._sendCmds([{'cmd': '%s su -c "%s"' % (cmd, cmdline)}], outputfile,
348                               timeout)
349
350        # dig through the output to get the return code
351        lastline = _pop_last_line(outputfile)
352        if lastline:
353            m = re.search('return code \[([0-9]+)\]', lastline)
354            if m:
355                return int(m.group(1))
356
357        # woops, we couldn't find an end of line/return value
358        raise DMError(
359            "Automation Error: Error finding end of line/return value when running '%s'" % cmdline)
360
361    def pushFile(self, localname, destname, retryLimit=None, createDir=True):
362        retryLimit = retryLimit or self.retryLimit
363        if createDir:
364            self.mkDirs(destname)
365
366        try:
367            filesize = os.path.getsize(localname)
368            with open(localname, 'rb') as f:
369                remoteHash = self._runCmds([{'cmd': 'push ' + destname + ' ' + str(filesize),
370                                             'data': f.read()}], retryLimit=retryLimit).strip()
371        except OSError:
372            raise DMError("DeviceManager: Error reading file to push")
373
374        self._logger.debug("push returned: %s" % remoteHash)
375
376        localHash = self._getLocalHash(localname)
377
378        if localHash != remoteHash:
379            raise DMError("Automation Error: Push File failed to Validate! (localhash: %s, "
380                          "remotehash: %s)" % (localHash, remoteHash))
381
382    def mkDir(self, name):
383        if not self.dirExists(name):
384            self._runCmds([{'cmd': 'mkdr ' + name}])
385
386    def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
387        retryLimit = retryLimit or self.retryLimit
388        self._logger.info("pushing directory: %s to %s" % (localDir, remoteDir))
389
390        existentDirectories = []
391        for root, dirs, files in os.walk(localDir, followlinks=True):
392            _, subpath = root.split(localDir)
393            subpath = subpath.lstrip('/')
394            remoteRoot = posixpath.join(remoteDir, subpath)
395            for f in files:
396                remoteName = posixpath.join(remoteRoot, f)
397
398                if subpath == "":
399                    remoteRoot = remoteDir
400
401                parent = os.path.dirname(remoteName)
402                if parent not in existentDirectories:
403                    self.mkDirs(remoteName)
404                    existentDirectories.append(parent)
405
406                self.pushFile(os.path.join(root, f), remoteName,
407                              retryLimit=retryLimit, createDir=False)
408
409    def dirExists(self, remotePath):
410        ret = self._runCmds([{'cmd': 'isdir ' + remotePath}]).strip()
411        if not ret:
412            raise DMError('Automation Error: DeviceManager isdir returned null')
413
414        return ret == 'TRUE'
415
416    def fileExists(self, filepath):
417        # Because we always have / style paths we make this a lot easier with some
418        # assumptions
419        filepath = posixpath.normpath(filepath)
420        # / should always exist but we can use this to check for things like
421        # having access to the filesystem
422        if filepath == '/':
423            return self.dirExists(filepath)
424        (containingpath, filename) = posixpath.split(filepath)
425        return filename in self.listFiles(containingpath)
426
427    def listFiles(self, rootdir):
428        rootdir = posixpath.normpath(rootdir)
429        if not self.dirExists(rootdir):
430            return []
431        data = self._runCmds([{'cmd': 'cd ' + rootdir}, {'cmd': 'ls'}])
432
433        files = filter(lambda x: x, data.splitlines())
434        if len(files) == 1 and files[0] == '<empty>':
435            # special case on the agent: empty directories return just the
436            # string "<empty>"
437            return []
438        return files
439
440    def removeFile(self, filename):
441        self._logger.info("removing file: " + filename)
442        if self.fileExists(filename):
443            self._runCmds([{'cmd': 'rm ' + filename}])
444
445    def removeDir(self, remoteDir):
446        if self.dirExists(remoteDir):
447            self._runCmds([{'cmd': 'rmdr ' + remoteDir}])
448
449    def moveTree(self, source, destination):
450        self._runCmds([{'cmd': 'mv %s %s' % (source, destination)}])
451
452    def copyTree(self, source, destination):
453        self._runCmds([{'cmd': 'dd if=%s of=%s' % (source, destination)}])
454
455    def getProcessList(self):
456        data = self._runCmds([{'cmd': 'ps'}])
457
458        processTuples = []
459        for line in data.splitlines():
460            if line:
461                pidproc = line.strip().split()
462                try:
463                    if (len(pidproc) == 2):
464                        processTuples += [[pidproc[0], pidproc[1]]]
465                    elif (len(pidproc) == 3):
466                        # android returns <userID> <procID> <procName>
467                        processTuples += [[int(pidproc[1]), pidproc[2], int(pidproc[0])]]
468                    else:
469                        # unexpected format
470                        raise ValueError
471                except ValueError:
472                    self._logger.error("Unable to parse process list (bug 805969)")
473                    self._logger.error("Line: %s\nFull output of process list:\n%s" % (line, data))
474                    raise DMError("Invalid process line: %s" % line)
475
476        return processTuples
477
478    def fireProcess(self, appname, failIfRunning=False, maxWaitTime=30):
479        """
480        Starts a process
481
482        returns: pid
483
484        DEPRECATED: Use shell() or launchApplication() for new code
485        """
486        if not appname:
487            raise DMError("Automation Error: fireProcess called with no command to run")
488
489        self._logger.info("FIRE PROC: '%s'" % appname)
490
491        if (self.processExist(appname) is None):
492            self._logger.warning("process %s appears to be running already\n" % appname)
493            if (failIfRunning):
494                raise DMError("Automation Error: Process is already running")
495
496        self._runCmds([{'cmd': 'exec ' + appname}])
497
498        # The 'exec' command may wait for the process to start and end, so checking
499        # for the process here may result in process = None.
500        # The normal case is to launch the process and return right away
501        # There is one case with robotium (am instrument) where exec returns at the end
502        pid = None
503        waited = 0
504        while pid is None and waited < maxWaitTime:
505            pid = self.processExist(appname)
506            if pid:
507                break
508            time.sleep(1)
509            waited += 1
510
511        self._logger.debug("got pid: %s for process: %s" % (pid, appname))
512        return pid
513
514    def launchProcess(self, cmd, outputFile="process.txt", cwd='', env='', failIfRunning=False):
515        """
516        Launches a process, redirecting output to standard out
517
518        Returns output filename
519
520        WARNING: Does not work how you expect on Android! The application's
521        own output will be flushed elsewhere.
522
523        DEPRECATED: Use shell() or launchApplication() for new code
524        """
525        if not cmd:
526            self._logger.warning("launchProcess called without command to run")
527            return None
528
529        if cmd[0] == 'am' and hasattr(self, '_getExtraAmStartArgs'):
530            cmd = cmd[:2] + self._getExtraAmStartArgs() + cmd[2:]
531
532        cmdline = subprocess.list2cmdline(cmd)
533        if outputFile == "process.txt" or outputFile is None:
534            outputFile += "%s/process.txt" % self.deviceRoot
535            cmdline += " > " + outputFile
536
537        # Prepend our env to the command
538        cmdline = '%s %s' % (self._formatEnvString(env), cmdline)
539
540        # fireProcess may trigger an exception, but we won't handle it
541        if cmd[0] == "am":
542            # Robocop tests spawn "am instrument". sutAgent's exec ensures that
543            # am has started before returning, so there is no point in having
544            # fireProcess wait for it to start. Also, since "am" does not show
545            # up in the process list while the test is running, waiting for it
546            # in fireProcess is difficult.
547            self.fireProcess(cmdline, failIfRunning, 0)
548        else:
549            self.fireProcess(cmdline, failIfRunning)
550        return outputFile
551
552    def killProcess(self, appname, sig=None):
553        if sig:
554            pid = self.processExist(appname)
555            if pid and pid > 0:
556                try:
557                    self.shellCheckOutput(['kill', '-%d' % sig, str(pid)],
558                                          root=True)
559                except DMError as err:
560                    self._logger.warning("unable to kill -%d %s (pid %s)" %
561                                         (sig, appname, str(pid)))
562                    self._logger.debug(err)
563                    raise err
564            else:
565                self._logger.warning("unable to kill -%d %s -- not running?" %
566                                     (sig, appname))
567        else:
568            retries = 0
569            while retries < self.retryLimit:
570                try:
571                    if self.processExist(appname):
572                        self._runCmds([{'cmd': 'kill ' + appname}])
573                    return
574                except DMError as err:
575                    retries += 1
576                    self._logger.warning("try %d of %d failed to kill %s" %
577                                         (retries, self.retryLimit, appname))
578                    self._logger.debug(err)
579                    if retries >= self.retryLimit:
580                        raise err
581
582    def getTempDir(self):
583        return self._runCmds([{'cmd': 'tmpd'}]).strip()
584
585    def pullFile(self, remoteFile, offset=None, length=None):
586        # The "pull" command is different from other commands in that DeviceManager
587        # has to read a certain number of bytes instead of just reading to the
588        # next prompt.  This is more robust than the "cat" command, which will be
589        # confused if the prompt string exists within the file being catted.
590        # However it means we can't use the response-handling logic in sendCMD().
591
592        def err(error_msg):
593            err_str = 'DeviceManager: pull unsuccessful: %s' % error_msg
594            self._logger.error(err_str)
595            self._sock = None
596            raise DMError(err_str)
597
598        # FIXME: We could possibly move these socket-reading functions up to
599        # the class level if we wanted to refactor sendCMD().  For now they are
600        # only used to pull files.
601
602        def uread(to_recv, error_msg):
603            """ unbuffered read """
604            try:
605                data = ""
606                if select.select([self._sock], [], [], self.default_timeout)[0]:
607                    data = self._sock.recv(to_recv)
608                if not data:
609                    # timed out waiting for response or error response
610                    err(error_msg)
611
612                return data
613            except:
614                err(error_msg)
615
616        def read_until_char(c, buf, error_msg):
617            """ read until 'c' is found; buffer rest """
618            while c not in buf:
619                data = uread(1024, error_msg)
620                buf += data
621            return buf.partition(c)
622
623        def read_exact(total_to_recv, buf, error_msg):
624            """ read exact number of 'total_to_recv' bytes """
625            while len(buf) < total_to_recv:
626                to_recv = min(total_to_recv - len(buf), 1024)
627                data = uread(to_recv, error_msg)
628                buf += data
629            return buf
630
631        prompt = self._base_prompt + self._prompt_sep
632        buf = ''
633
634        # expected return value:
635        # <filename>,<filesize>\n<filedata>
636        # or, if error,
637        # <filename>,-1\n<error message>
638
639        # just send the command first, we read the response inline below
640        if offset is not None and length is not None:
641            cmd = 'pull %s %d %d' % (remoteFile, offset, length)
642        elif offset is not None:
643            cmd = 'pull %s %d' % (remoteFile, offset)
644        else:
645            cmd = 'pull %s' % remoteFile
646
647        self._runCmds([{'cmd': cmd}])
648
649        # read metadata; buffer the rest
650        metadata, sep, buf = read_until_char('\n', buf, 'could not find metadata')
651        if not metadata:
652            return None
653        self._logger.debug('metadata: %s' % metadata)
654
655        filename, sep, filesizestr = metadata.partition(',')
656        if sep == '':
657            err('could not find file size in returned metadata')
658        try:
659            filesize = int(filesizestr)
660        except ValueError:
661            err('invalid file size in returned metadata')
662
663        if filesize == -1:
664            # read error message
665            error_str, sep, buf = read_until_char('\n', buf, 'could not find error message')
666            if not error_str:
667                err("blank error message")
668            # prompt should follow
669            read_exact(len(prompt), buf, 'could not find prompt')
670            # failures are expected, so don't use "Remote Device Error" or we'll RETRY
671            raise DMError("DeviceManager: pulling file '%s' unsuccessful: %s" %
672                          (remoteFile, error_str))
673
674        # read file data
675        total_to_recv = filesize + len(prompt)
676        buf = read_exact(total_to_recv, buf, 'could not get all file data')
677        if buf[-len(prompt):] != prompt:
678            err('no prompt found after file data--DeviceManager may be out of sync with agent')
679            return buf
680        return buf[:-len(prompt)]
681
682    def getFile(self, remoteFile, localFile):
683        data = self.pullFile(remoteFile)
684
685        fhandle = open(localFile, 'wb')
686        fhandle.write(data)
687        fhandle.close()
688        if not self.validateFile(remoteFile, localFile):
689            raise DMError("Automation Error: Failed to validate file when downloading %s" %
690                          remoteFile)
691
692    def getDirectory(self, remoteDir, localDir, checkDir=True):
693        self._logger.info("getting files in '%s'" % remoteDir)
694        if checkDir and not self.dirExists(remoteDir):
695            raise DMError("Automation Error: Error getting directory: %s not a directory" %
696                          remoteDir)
697
698        filelist = self.listFiles(remoteDir)
699        self._logger.debug(filelist)
700        if not os.path.exists(localDir):
701            os.makedirs(localDir)
702
703        for f in filelist:
704            if f == '.' or f == '..':
705                continue
706            remotePath = remoteDir + '/' + f
707            localPath = os.path.join(localDir, f)
708            if self.dirExists(remotePath):
709                self.getDirectory(remotePath, localPath, False)
710            else:
711                self.getFile(remotePath, localPath)
712
713    def validateFile(self, remoteFile, localFile):
714        remoteHash = self._getRemoteHash(remoteFile)
715        localHash = self._getLocalHash(localFile)
716
717        if (remoteHash is None):
718            return False
719
720        if (remoteHash == localHash):
721            return True
722
723        return False
724
725    def _getRemoteHash(self, filename):
726        data = self._runCmds([{'cmd': 'hash ' + filename}]).strip()
727        self._logger.debug("remote hash returned: '%s'" % data)
728        return data
729
730    def unpackFile(self, filePath, destDir=None):
731        """
732        Unzips a bundle to a location on the device
733
734        If destDir is not specified, the bundle is extracted in the same directory
735        """
736        # if no destDir is passed in just set it to filePath's folder
737        if not destDir:
738            destDir = posixpath.dirname(filePath)
739
740        if destDir[-1] != '/':
741            destDir += '/'
742
743        self._runCmds([{'cmd': 'unzp %s %s' % (filePath, destDir)}])
744
745    def _getRebootServerSocket(self, ipAddr):
746        serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
747        serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
748        serverSocket.settimeout(60.0)
749        serverSocket.bind((ipAddr, 0))
750        serverSocket.listen(1)
751        self._logger.debug('Created reboot callback server at %s:%d' %
752                           serverSocket.getsockname())
753        return serverSocket
754
755    def _waitForRebootPing(self, serverSocket):
756        conn = None
757        data = None
758        startTime = datetime.datetime.now()
759        waitTime = datetime.timedelta(seconds=self.reboot_timeout)
760        while not data and datetime.datetime.now() - startTime < waitTime:
761            self._logger.info("Waiting for reboot callback ping from device...")
762            try:
763                if not conn:
764                    conn, _ = serverSocket.accept()
765                # Receiving any data is good enough.
766                data = conn.recv(1024)
767                if data:
768                    self._logger.info("Received reboot callback ping from device!")
769                    conn.sendall('OK')
770                conn.close()
771            except socket.timeout:
772                pass
773            except socket.error as e:
774                if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:
775                    raise
776
777        if not data:
778            raise DMError('Timed out waiting for reboot callback.')
779
780        self._logger.info("Sleeping for %s seconds to wait for device "
781                          "to 'settle'" % self.reboot_settling_time)
782        time.sleep(self.reboot_settling_time)
783
784    def reboot(self, ipAddr=None, port=30000, wait=False):
785        # port ^^^ is here for backwards compatibility only, we now
786        # determine a port automatically and safely
787        wait = (wait or ipAddr)
788
789        cmd = 'rebt'
790
791        self._logger.info("Rebooting device")
792
793        # if we're waiting, create a listening server and pass information on
794        # it to the device before rebooting (we do this instead of just polling
795        # to make sure the device actually rebooted -- yes, there are probably
796        # simpler ways of doing this like polling uptime, but this is what we're
797        # doing for now)
798        if wait:
799            if not ipAddr:
800                ipAddr = moznetwork.get_ip()
801            serverSocket = self._getRebootServerSocket(ipAddr)
802            # The update.info command tells the SUTAgent to send a TCP message
803            # after restarting.
804            destname = '/data/data/com.mozilla.SUTAgentAndroid/files/update.info'
805            data = "%s,%s\rrebooting\r" % serverSocket.getsockname()
806            self._runCmds([{'cmd': 'push %s %s' % (destname, len(data)),
807                            'data': data}])
808            cmd += " %s %s" % serverSocket.getsockname()
809
810        # actually reboot device
811        self._runCmds([{'cmd': cmd}])
812        # if we're waiting, wait for a callback ping from the agent before
813        # continuing (and throw an exception if we don't receive said ping)
814        if wait:
815            self._waitForRebootPing(serverSocket)
816
817    def getInfo(self, directive=None):
818        data = None
819        result = {}
820        collapseSpaces = re.compile('  +')
821
822        directives = ['os', 'id', 'uptime', 'uptimemillis', 'systime', 'screen',
823                      'rotation', 'memory', 'process', 'disk', 'power', 'sutuserinfo',
824                      'temperature']
825        if (directive in directives):
826            directives = [directive]
827
828        for d in directives:
829            data = self._runCmds([{'cmd': 'info ' + d}])
830
831            data = collapseSpaces.sub(' ', data)
832            result[d] = data.split('\n')
833
834        # Get rid of any 0 length members of the arrays
835        for k, v in result.iteritems():
836            result[k] = filter(lambda x: x != '', result[k])
837
838        # Format the process output
839        if 'process' in result:
840            proclist = []
841            for l in result['process']:
842                if l:
843                    proclist.append(l.split('\t'))
844            result['process'] = proclist
845
846        self._logger.debug("results: %s" % result)
847        return result
848
849    def installApp(self, appBundlePath, destPath=None):
850        cmd = 'inst ' + appBundlePath
851        if destPath:
852            cmd += ' ' + destPath
853
854        data = self._runCmds([{'cmd': cmd}])
855
856        if 'installation complete [0]' not in data:
857            raise DMError("Remove Device Error: Error installing app. Error message: %s" % data)
858
859    def uninstallApp(self, appName, installPath=None):
860        cmd = 'uninstall ' + appName
861        if installPath:
862            cmd += ' ' + installPath
863        data = self._runCmds([{'cmd': cmd}])
864
865        status = data.split('\n')[0].strip()
866        self._logger.debug("uninstallApp: '%s'" % status)
867        if status == 'Success':
868            return
869        raise DMError("Remote Device Error: uninstall failed for %s" % appName)
870
871    def uninstallAppAndReboot(self, appName, installPath=None):
872        cmd = 'uninst ' + appName
873        if installPath:
874            cmd += ' ' + installPath
875        data = self._runCmds([{'cmd': cmd}])
876
877        self._logger.debug("uninstallAppAndReboot: %s" % data)
878        return
879
880    def updateApp(self, appBundlePath, processName=None, destPath=None,
881                  ipAddr=None, port=30000, wait=False):
882        # port ^^^ is here for backwards compatibility only, we now
883        # determine a port automatically and safely
884        wait = (wait or ipAddr)
885
886        cmd = 'updt '
887        if processName is None:
888            # Then we pass '' for processName
889            cmd += "'' " + appBundlePath
890        else:
891            cmd += processName + ' ' + appBundlePath
892
893        if destPath:
894            cmd += " " + destPath
895
896        if wait:
897            if not ipAddr:
898                ipAddr = moznetwork.get_ip()
899            serverSocket = self._getRebootServerSocket(ipAddr)
900            cmd += " %s %s" % serverSocket.getsockname()
901
902        self._logger.debug("updateApp using command: " % cmd)
903
904        self._runCmds([{'cmd': cmd}])
905
906        if wait:
907            self._waitForRebootPing(serverSocket)
908
909    def getCurrentTime(self):
910        return int(self._runCmds([{'cmd': 'clok'}]).strip())
911
912    def _formatEnvString(self, env):
913        """
914        Returns a properly formatted env string for the agent.
915
916        Input - env, which is either None, '', or a dict
917        Output - a quoted string of the form: '"envvar1=val1,envvar2=val2..."'
918        If env is None or '' return '' (empty quoted string)
919        """
920        if (env is None or env == ''):
921            return ''
922
923        retVal = '"%s"' % ','.join(map(lambda x: '%s=%s' % (x[0], x[1]), env.iteritems()))
924        if (retVal == '""'):
925            return ''
926
927        return retVal
928
929    def adjustResolution(self, width=1680, height=1050, type='hdmi'):
930        """
931        Adjust the screen resolution on the device, REBOOT REQUIRED
932
933        NOTE: this only works on a tegra ATM
934
935        supported resolutions: 640x480, 800x600, 1024x768, 1152x864, 1200x1024, 1440x900,
936        1680x1050, 1920x1080
937        """
938        if self.getInfo('os')['os'][0].split()[0] != 'harmony-eng':
939            self._logger.warning("unable to adjust screen resolution on non Tegra device")
940            return False
941
942        results = self.getInfo('screen')
943        parts = results['screen'][0].split(':')
944        self._logger.debug("we have a current resolution of %s, %s" %
945                           (parts[1].split()[0], parts[2].split()[0]))
946
947        # verify screen type is valid, and set it to the proper value
948        # (https://bugzilla.mozilla.org/show_bug.cgi?id=632895#c4)
949        screentype = -1
950        if (type == 'hdmi'):
951            screentype = 5
952        elif (type == 'vga' or type == 'crt'):
953            screentype = 3
954        else:
955            return False
956
957        # verify we have numbers
958        if not (isinstance(width, int) and isinstance(height, int)):
959            return False
960
961        if (width < 100 or width > 9999):
962            return False
963
964        if (height < 100 or height > 9999):
965            return False
966
967        self._logger.debug("adjusting screen resolution to %s, %s and rebooting" % (width, height))
968
969        self._runCmds(
970            [{'cmd': "exec setprop persist.tegra.dpy%s.mode.width %s" % (screentype, width)}])
971        self._runCmds(
972            [{'cmd': "exec setprop persist.tegra.dpy%s.mode.height %s" % (screentype, height)}])
973
974    def chmodDir(self, remoteDir, **kwargs):
975        self._runCmds([{'cmd': "chmod " + remoteDir}])
976