xref: /aosp_15_r20/external/toolchain-utils/crosperf/machine_manager.py (revision 760c253c1ed00ce9abd48f8546f08516e57485fe)
1# -*- coding: utf-8 -*-
2# Copyright 2013 The ChromiumOS Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Machine Manager module."""
7
8
9import collections
10import hashlib
11import math
12import os.path
13import re
14import sys
15import threading
16import time
17
18from cros_utils import command_executer
19from cros_utils import logger
20import file_lock_machine
21import image_chromeos
22import test_flag
23
24
25CHECKSUM_FILE = "/usr/local/osimage_checksum_file"
26
27
28class BadChecksum(Exception):
29    """Raised if all machines for a label don't have the same checksum."""
30
31
32class BadChecksumString(Exception):
33    """Raised if all machines for a label don't have the same checksum string."""
34
35
36class MissingLocksDirectory(Exception):
37    """Raised when cannot find/access the machine locks directory."""
38
39
40class CrosCommandError(Exception):
41    """Raised when an error occurs running command on DUT."""
42
43
44class CrosMachine(object):
45    """The machine class."""
46
47    def __init__(self, name, chromeos_root, log_level, cmd_exec=None):
48        self.name = name
49        self.image = None
50        # We relate a dut with a label if we reimage the dut using label or we
51        # detect at the very beginning that the dut is running this label.
52        self.label = None
53        self.checksum = None
54        self.locked = False
55        self.released_time = time.time()
56        self.test_run = None
57        self.chromeos_root = chromeos_root
58        self.log_level = log_level
59        self.cpuinfo = None
60        self.machine_id = None
61        self.checksum_string = None
62        self.meminfo = None
63        self.phys_kbytes = None
64        self.cooldown_wait_time = 0
65        self.ce = cmd_exec or command_executer.GetCommandExecuter(
66            log_level=self.log_level
67        )
68        self.SetUpChecksumInfo()
69
70    def SetUpChecksumInfo(self):
71        if not self.IsReachable():
72            self.machine_checksum = None
73            return
74        self._GetMemoryInfo()
75        self._GetCPUInfo()
76        self._ComputeMachineChecksumString()
77        self._GetMachineID()
78        self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
79        self.machine_id_checksum = self._GetMD5Checksum(self.machine_id)
80
81    def IsReachable(self):
82        command = "ls"
83        ret = self.ce.CrosRunCommand(
84            command, machine=self.name, chromeos_root=self.chromeos_root
85        )
86        if ret:
87            return False
88        return True
89
90    def AddCooldownWaitTime(self, wait_time):
91        self.cooldown_wait_time += wait_time
92
93    def GetCooldownWaitTime(self):
94        return self.cooldown_wait_time
95
96    def _ParseMemoryInfo(self):
97        line = self.meminfo.splitlines()[0]
98        usable_kbytes = int(line.split()[1])
99        # This code is from src/third_party/test/files/client/bin/base_utils.py
100        # usable_kbytes is system's usable DRAM in kbytes,
101        #   as reported by memtotal() from device /proc/meminfo memtotal
102        #   after Linux deducts 1.5% to 9.5% for system table overhead
103        # Undo the unknown actual deduction by rounding up
104        #   to next small multiple of a big power-of-two
105        #   eg  12GB - 5.1% gets rounded back up to 12GB
106        mindeduct = 0.005  # 0.5 percent
107        maxdeduct = 0.095  # 9.5 percent
108        # deduction range 1.5% .. 9.5% supports physical mem sizes
109        #    6GB .. 12GB in steps of .5GB
110        #   12GB .. 24GB in steps of 1 GB
111        #   24GB .. 48GB in steps of 2 GB ...
112        # Finer granularity in physical mem sizes would require
113        #   tighter spread between min and max possible deductions
114
115        # increase mem size by at least min deduction, without rounding
116        min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
117        # increase mem size further by 2**n rounding, by 0..roundKb or more
118        round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
119        # find least binary roundup 2**n that covers worst-cast roundKb
120        mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
121        # have round_kbytes <= mod2n < round_kbytes*2
122        # round min_kbytes up to next multiple of mod2n
123        phys_kbytes = min_kbytes + mod2n - 1
124        phys_kbytes -= phys_kbytes % mod2n  # clear low bits
125        self.phys_kbytes = phys_kbytes
126
127    def _GetMemoryInfo(self):
128        # TODO yunlian: when the machine in rebooting, it will not return
129        # meminfo, the assert does not catch it either
130        command = "cat /proc/meminfo"
131        ret, self.meminfo, _ = self.ce.CrosRunCommandWOutput(
132            command, machine=self.name, chromeos_root=self.chromeos_root
133        )
134        assert ret == 0, "Could not get meminfo from machine: %s" % self.name
135        if ret == 0:
136            self._ParseMemoryInfo()
137
138    def _GetCPUInfo(self):
139        command = "cat /proc/cpuinfo"
140        ret, self.cpuinfo, _ = self.ce.CrosRunCommandWOutput(
141            command, machine=self.name, chromeos_root=self.chromeos_root
142        )
143        assert ret == 0, "Could not get cpuinfo from machine: %s" % self.name
144
145    def _ComputeMachineChecksumString(self):
146        self.checksum_string = ""
147        # Some lines from cpuinfo have to be excluded because they are not
148        # persistent across DUTs.
149        # MHz, BogoMIPS are dynamically changing values.
150        # core id, apicid are identifiers assigned on startup
151        # and may differ on the same type of machine.
152        exclude_lines_list = [
153            "MHz",
154            "BogoMIPS",
155            "bogomips",
156            "core id",
157            "apicid",
158        ]
159        for line in self.cpuinfo.splitlines():
160            if not any(e in line for e in exclude_lines_list):
161                self.checksum_string += line
162        self.checksum_string += " " + str(self.phys_kbytes)
163
164    def _GetMD5Checksum(self, ss):
165        if ss:
166            return hashlib.md5(ss.encode("utf-8")).hexdigest()
167        return ""
168
169    def _GetMachineID(self):
170        command = "dump_vpd_log --full --stdout"
171        _, if_out, _ = self.ce.CrosRunCommandWOutput(
172            command, machine=self.name, chromeos_root=self.chromeos_root
173        )
174        b = if_out.splitlines()
175        a = [l for l in b if "Product" in l]
176        if a:
177            self.machine_id = a[0]
178            return
179        command = "ifconfig"
180        _, if_out, _ = self.ce.CrosRunCommandWOutput(
181            command, machine=self.name, chromeos_root=self.chromeos_root
182        )
183        b = if_out.splitlines()
184        a = [l for l in b if "HWaddr" in l]
185        if a:
186            self.machine_id = "_".join(a)
187            return
188        a = [l for l in b if "ether" in l]
189        if a:
190            self.machine_id = "_".join(a)
191            return
192        assert 0, "Could not get machine_id from machine: %s" % self.name
193
194    def __str__(self):
195        l = []
196        l.append(self.name)
197        l.append(str(self.image))
198        l.append(str(self.checksum))
199        l.append(str(self.locked))
200        l.append(str(self.released_time))
201        return ", ".join(l)
202
203
204class MachineManager(object):
205    """Lock, image and unlock machines locally for benchmark runs.
206
207    This class contains methods and calls to lock, unlock and image
208    machines and distribute machines to each benchmark run.  The assumption is
209    that all of the machines for the experiment have been globally locked
210    in the ExperimentRunner, but the machines still need to be locally
211    locked/unlocked (allocated to benchmark runs) to prevent multiple benchmark
212    runs within the same experiment from trying to use the same machine at the
213    same time.
214    """
215
216    def __init__(
217        self,
218        chromeos_root,
219        acquire_timeout,
220        log_level,
221        locks_dir,
222        cmd_exec=None,
223        lgr=None,
224        keep_stateful: bool = False,
225    ):
226        self._lock = threading.RLock()
227        self._all_machines = []
228        self._machines = []
229        self.image_lock = threading.Lock()
230        self.num_reimages = 0
231        self.chromeos_root = None
232        self.machine_checksum = {}
233        self.machine_checksum_string = {}
234        self.acquire_timeout = acquire_timeout
235        self.log_level = log_level
236        self.locks_dir = locks_dir
237        self.keep_stateful = keep_stateful
238        self.ce = cmd_exec or command_executer.GetCommandExecuter(
239            log_level=self.log_level
240        )
241        self.logger = lgr or logger.GetLogger()
242
243        if self.locks_dir and not os.path.isdir(self.locks_dir):
244            raise MissingLocksDirectory(
245                "Cannot access locks directory: %s" % self.locks_dir
246            )
247
248        self._initialized_machines = []
249        self.chromeos_root = chromeos_root
250
251    def RemoveNonLockedMachines(self, locked_machines):
252        for m in self._all_machines:
253            if m.name not in locked_machines:
254                self._all_machines.remove(m)
255
256        for m in self._machines:
257            if m.name not in locked_machines:
258                self._machines.remove(m)
259
260    def GetChromeVersion(self, machine):
261        """Get the version of Chrome running on the DUT."""
262
263        cmd = "/opt/google/chrome/chrome --version"
264        ret, version, _ = self.ce.CrosRunCommandWOutput(
265            cmd, machine=machine.name, chromeos_root=self.chromeos_root
266        )
267        if ret != 0:
268            raise CrosCommandError(
269                "Couldn't get Chrome version from %s." % machine.name
270            )
271
272        if ret != 0:
273            version = ""
274        return version.rstrip()
275
276    def ImageMachine(self, machine, label):
277        checksum = label.checksum
278
279        if checksum and (machine.checksum == checksum):
280            return
281        chromeos_root = label.chromeos_root
282        if not chromeos_root:
283            chromeos_root = self.chromeos_root
284        image_chromeos_args = [
285            image_chromeos.__file__,
286            "--no_lock",
287            f"--chromeos_root={chromeos_root}",
288            f"--image={label.chromeos_image}",
289            f"--image_args={label.image_args}",
290            f"--remote={machine.name}",
291            f"--logging_level={self.log_level}",
292        ]
293        if label.board:
294            image_chromeos_args.append(f"--board={label.board}")
295        if self.keep_stateful:
296            image_chromeos_args.append("--keep_stateful")
297
298        # Currently can't image two machines at once.
299        # So have to serialized on this lock.
300        save_ce_log_level = self.ce.log_level
301        if self.log_level != "verbose":
302            self.ce.log_level = "average"
303
304        with self.image_lock:
305            if self.log_level != "verbose":
306                self.logger.LogOutput("Pushing image onto machine.")
307                self.logger.LogOutput(
308                    "Running image_chromeos.DoImage with %s"
309                    % " ".join(image_chromeos_args)
310                )
311            retval = 0
312            if not test_flag.GetTestMode():
313                retval = image_chromeos.DoImage(image_chromeos_args)
314            if retval:
315                cmd = "reboot && exit"
316                if self.log_level != "verbose":
317                    self.logger.LogOutput("reboot & exit.")
318                self.ce.CrosRunCommand(
319                    cmd, machine=machine.name, chromeos_root=self.chromeos_root
320                )
321                time.sleep(60)
322                if self.log_level != "verbose":
323                    self.logger.LogOutput("Pushing image onto machine.")
324                    self.logger.LogOutput(
325                        "Running image_chromeos.DoImage with %s"
326                        % " ".join(image_chromeos_args)
327                    )
328                retval = image_chromeos.DoImage(image_chromeos_args)
329            if retval:
330                raise RuntimeError(
331                    "Could not image machine: '%s'." % machine.name
332                )
333
334            self.num_reimages += 1
335            machine.checksum = checksum
336            machine.image = label.chromeos_image
337            machine.label = label
338
339        if not label.chrome_version:
340            label.chrome_version = self.GetChromeVersion(machine)
341
342        self.ce.log_level = save_ce_log_level
343        return retval
344
345    def ComputeCommonCheckSum(self, label):
346        # Since this is used for cache lookups before the machines have been
347        # compared/verified, check here to make sure they all have the same
348        # checksum (otherwise the cache lookup may not be valid).
349        base = None
350        for machine in self.GetMachines(label):
351            # Make sure the machine's checksums are calculated.
352            if not machine.machine_checksum:
353                machine.SetUpChecksumInfo()
354            # Use the first machine as the basis for comparison.
355            if not base:
356                base = machine
357            # Make sure this machine's checksum matches our 'common' checksum.
358            if base.machine_checksum != machine.machine_checksum:
359                # Found a difference. Fatal error.
360                # Extract non-matching part and report it.
361                for mismatch_index in range(len(base.checksum_string)):
362                    if (
363                        mismatch_index >= len(machine.checksum_string)
364                        or base.checksum_string[mismatch_index]
365                        != machine.checksum_string[mismatch_index]
366                    ):
367                        break
368                # We want to show some context after the mismatch.
369                end_ind = mismatch_index + 8
370                # Print a mismatching string.
371                raise BadChecksum(
372                    "Machine checksums do not match!\n"
373                    "Diff:\n"
374                    f"{base.name}: {base.checksum_string[:end_ind]}\n"
375                    f"{machine.name}: {machine.checksum_string[:end_ind]}\n"
376                    "\nCheck for matching /proc/cpuinfo and /proc/meminfo on DUTs.\n"
377                )
378        self.machine_checksum[label.name] = base.machine_checksum
379
380    def ComputeCommonCheckSumString(self, label):
381        # The assumption is that this function is only called AFTER
382        # ComputeCommonCheckSum, so there is no need to verify the machines
383        # are the same here.  If this is ever changed, this function should be
384        # modified to verify that all the machines for a given label are the
385        # same.
386        for machine in self.GetMachines(label):
387            if machine.checksum_string:
388                self.machine_checksum_string[
389                    label.name
390                ] = machine.checksum_string
391                break
392
393    def _TryToLockMachine(self, cros_machine):
394        with self._lock:
395            assert cros_machine, "Machine can't be None"
396            for m in self._machines:
397                if m.name == cros_machine.name:
398                    return
399            locked = True
400            if self.locks_dir:
401                locked = file_lock_machine.Machine(
402                    cros_machine.name, self.locks_dir
403                ).Lock(True, sys.argv[0])
404            if locked:
405                self._machines.append(cros_machine)
406                command = "cat %s" % CHECKSUM_FILE
407                ret, out, _ = self.ce.CrosRunCommandWOutput(
408                    command,
409                    chromeos_root=self.chromeos_root,
410                    machine=cros_machine.name,
411                )
412                if ret == 0:
413                    cros_machine.checksum = out.strip()
414            elif self.locks_dir:
415                self.logger.LogOutput("Couldn't lock: %s" % cros_machine.name)
416
417    # This is called from single threaded mode.
418    def AddMachine(self, machine_name):
419        with self._lock:
420            for m in self._all_machines:
421                assert m.name != machine_name, (
422                    "Tried to double-add %s" % machine_name
423                )
424
425            if self.log_level != "verbose":
426                self.logger.LogOutput(
427                    "Setting up remote access to %s" % machine_name
428                )
429                self.logger.LogOutput(
430                    "Checking machine characteristics for %s" % machine_name
431                )
432            cm = CrosMachine(machine_name, self.chromeos_root, self.log_level)
433            if cm.machine_checksum:
434                self._all_machines.append(cm)
435
436    def RemoveMachine(self, machine_name):
437        with self._lock:
438            self._machines = [
439                m for m in self._machines if m.name != machine_name
440            ]
441            if self.locks_dir:
442                res = file_lock_machine.Machine(
443                    machine_name, self.locks_dir
444                ).Unlock(True)
445                if not res:
446                    self.logger.LogError(
447                        "Could not unlock machine: '%s'." % machine_name
448                    )
449
450    def ForceSameImageToAllMachines(self, label):
451        machines = self.GetMachines(label)
452        for m in machines:
453            self.ImageMachine(m, label)
454            m.SetUpChecksumInfo()
455
456    def AcquireMachine(self, label):
457        image_checksum = label.checksum
458        machines = self.GetMachines(label)
459        check_interval_time = 120
460        with self._lock:
461            # Lazily external lock machines
462            while self.acquire_timeout >= 0:
463                for m in machines:
464                    new_machine = m not in self._all_machines
465                    self._TryToLockMachine(m)
466                    if new_machine:
467                        m.released_time = time.time()
468                if self.GetAvailableMachines(label):
469                    break
470                sleep_time = max(
471                    1, min(self.acquire_timeout, check_interval_time)
472                )
473                time.sleep(sleep_time)
474                self.acquire_timeout -= sleep_time
475
476            if self.acquire_timeout < 0:
477                self.logger.LogFatal(
478                    "Could not acquire any of the "
479                    "following machines: '%s'"
480                    % ", ".join(machine.name for machine in machines)
481                )
482
483            ###      for m in self._machines:
484            ###        if (m.locked and time.time() - m.released_time < 10 and
485            ###            m.checksum == image_checksum):
486            ###          return None
487            unlocked_machines = [
488                machine
489                for machine in self.GetAvailableMachines(label)
490                if not machine.locked
491            ]
492            for m in unlocked_machines:
493                if image_checksum and m.checksum == image_checksum:
494                    m.locked = True
495                    m.test_run = threading.current_thread()
496                    return m
497            for m in unlocked_machines:
498                if not m.checksum:
499                    m.locked = True
500                    m.test_run = threading.current_thread()
501                    return m
502            # This logic ensures that threads waiting on a machine will get a machine
503            # with a checksum equal to their image over other threads. This saves time
504            # when crosperf initially assigns the machines to threads by minimizing
505            # the number of re-images.
506            # TODO(asharif): If we centralize the thread-scheduler, we wont need this
507            # code and can implement minimal reimaging code more cleanly.
508            for m in unlocked_machines:
509                if time.time() - m.released_time > 15:
510                    # The release time gap is too large, so it is probably in the start
511                    # stage, we need to reset the released_time.
512                    m.released_time = time.time()
513                elif time.time() - m.released_time > 8:
514                    m.locked = True
515                    m.test_run = threading.current_thread()
516                    return m
517        return None
518
519    def GetAvailableMachines(self, label=None):
520        if not label:
521            return self._machines
522        return [m for m in self._machines if m.name in label.remote]
523
524    def GetMachines(self, label=None):
525        if not label:
526            return self._all_machines
527        return [m for m in self._all_machines if m.name in label.remote]
528
529    def ReleaseMachine(self, machine):
530        with self._lock:
531            for m in self._machines:
532                if machine.name == m.name:
533                    assert m.locked, "Tried to double-release %s" % m.name
534                    m.released_time = time.time()
535                    m.locked = False
536                    m.status = "Available"
537                    break
538
539    def Cleanup(self):
540        with self._lock:
541            # Unlock all machines (via file lock)
542            for m in self._machines:
543                res = file_lock_machine.Machine(m.name, self.locks_dir).Unlock(
544                    True
545                )
546
547                if not res:
548                    self.logger.LogError(
549                        "Could not unlock machine: '%s'." % m.name
550                    )
551
552    def __str__(self):
553        with self._lock:
554            l = ["MachineManager Status:"] + [str(m) for m in self._machines]
555            return "\n".join(l)
556
557    def AsString(self):
558        with self._lock:
559            stringify_fmt = "%-30s %-10s %-4s %-25s %-32s"
560            header = stringify_fmt % (
561                "Machine",
562                "Thread",
563                "Lock",
564                "Status",
565                "Checksum",
566            )
567            table = [header]
568            for m in self._machines:
569                if m.test_run:
570                    test_name = m.test_run.name
571                    test_status = m.test_run.timeline.GetLastEvent()
572                else:
573                    test_name = ""
574                    test_status = ""
575
576                try:
577                    machine_string = stringify_fmt % (
578                        m.name,
579                        test_name,
580                        m.locked,
581                        test_status,
582                        m.checksum,
583                    )
584                except ValueError:
585                    machine_string = ""
586                table.append(machine_string)
587            return "Machine Status:\n%s" % "\n".join(table)
588
589    def GetAllCPUInfo(self, labels):
590        """Get cpuinfo for labels, merge them if their cpuinfo are the same."""
591        dic = collections.defaultdict(list)
592        for label in labels:
593            for machine in self._all_machines:
594                if machine.name in label.remote:
595                    dic[machine.cpuinfo].append(label.name)
596                    break
597        output_segs = []
598        for key, v in dic.items():
599            output = " ".join(v)
600            output += "\n-------------------\n"
601            output += key
602            output += "\n\n\n"
603            output_segs.append(output)
604        return "".join(output_segs)
605
606    def GetAllMachines(self):
607        return self._all_machines
608
609
610class MockCrosMachine(CrosMachine):
611    """Mock cros machine class."""
612
613    # pylint: disable=super-init-not-called
614
615    MEMINFO_STRING = """MemTotal:        3990332 kB
616MemFree:         2608396 kB
617Buffers:          147168 kB
618Cached:           811560 kB
619SwapCached:            0 kB
620Active:           503480 kB
621Inactive:         628572 kB
622Active(anon):     174532 kB
623Inactive(anon):    88576 kB
624Active(file):     328948 kB
625Inactive(file):   539996 kB
626Unevictable:           0 kB
627Mlocked:               0 kB
628SwapTotal:       5845212 kB
629SwapFree:        5845212 kB
630Dirty:              9384 kB
631Writeback:             0 kB
632AnonPages:        173408 kB
633Mapped:           146268 kB
634Shmem:             89676 kB
635Slab:             188260 kB
636SReclaimable:     169208 kB
637SUnreclaim:        19052 kB
638KernelStack:        2032 kB
639PageTables:         7120 kB
640NFS_Unstable:          0 kB
641Bounce:                0 kB
642WritebackTmp:          0 kB
643CommitLimit:     7840376 kB
644Committed_AS:    1082032 kB
645VmallocTotal:   34359738367 kB
646VmallocUsed:      364980 kB
647VmallocChunk:   34359369407 kB
648DirectMap4k:       45824 kB
649DirectMap2M:     4096000 kB
650"""
651
652    CPUINFO_STRING = """processor: 0
653vendor_id: GenuineIntel
654cpu family: 6
655model: 42
656model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
657stepping: 7
658microcode: 0x25
659cpu MHz: 1300.000
660cache size: 2048 KB
661physical id: 0
662siblings: 2
663core id: 0
664cpu cores: 2
665apicid: 0
666initial apicid: 0
667fpu: yes
668fpu_exception: yes
669cpuid level: 13
670wp: yes
671flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
672bogomips: 2594.17
673clflush size: 64
674cache_alignment: 64
675address sizes: 36 bits physical, 48 bits virtual
676power management:
677
678processor: 1
679vendor_id: GenuineIntel
680cpu family: 6
681model: 42
682model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
683stepping: 7
684microcode: 0x25
685cpu MHz: 1300.000
686cache size: 2048 KB
687physical id: 0
688siblings: 2
689core id: 1
690cpu cores: 2
691apicid: 2
692initial apicid: 2
693fpu: yes
694fpu_exception: yes
695cpuid level: 13
696wp: yes
697flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
698bogomips: 2594.17
699clflush size: 64
700cache_alignment: 64
701address sizes: 36 bits physical, 48 bits virtual
702power management:
703"""
704
705    def __init__(self, name, chromeos_root, log_level):
706        self.name = name
707        self.image = None
708        self.checksum = None
709        self.locked = False
710        self.released_time = time.time()
711        self.test_run = None
712        self.chromeos_root = chromeos_root
713        self.checksum_string = re.sub(r"\d", "", name)
714        # In test, we assume "lumpy1", "lumpy2" are the same machine.
715        self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
716        self.log_level = log_level
717        self.label = None
718        self.cooldown_wait_time = 0
719        self.ce = command_executer.GetCommandExecuter(log_level=self.log_level)
720        self._GetCPUInfo()
721
722    def IsReachable(self):
723        return True
724
725    def _GetMemoryInfo(self):
726        self.meminfo = self.MEMINFO_STRING
727        self._ParseMemoryInfo()
728
729    def _GetCPUInfo(self):
730        self.cpuinfo = self.CPUINFO_STRING
731
732
733class MockMachineManager(MachineManager):
734    """Mock machine manager class."""
735
736    def __init__(
737        self,
738        chromeos_root,
739        acquire_timeout,
740        log_level,
741        locks_dir,
742        keep_stateful: bool = False,
743    ):
744        super(MockMachineManager, self).__init__(
745            chromeos_root,
746            acquire_timeout,
747            log_level,
748            locks_dir,
749            keep_stateful=keep_stateful,
750        )
751
752    def _TryToLockMachine(self, cros_machine):
753        self._machines.append(cros_machine)
754        cros_machine.checksum = ""
755
756    def AddMachine(self, machine_name):
757        with self._lock:
758            for m in self._all_machines:
759                assert m.name != machine_name, (
760                    "Tried to double-add %s" % machine_name
761                )
762            cm = MockCrosMachine(
763                machine_name, self.chromeos_root, self.log_level
764            )
765            assert cm.machine_checksum, (
766                "Could not find checksum for machine %s" % machine_name
767            )
768            # In Original MachineManager, the test is 'if cm.machine_checksum:' - if a
769            # machine is unreachable, then its machine_checksum is None. Here we
770            # cannot do this, because machine_checksum is always faked, so we directly
771            # test cm.IsReachable, which is properly mocked.
772            if cm.IsReachable():
773                self._all_machines.append(cm)
774
775    def GetChromeVersion(self, machine):
776        return "Mock Chrome Version R50"
777
778    def AcquireMachine(self, label):
779        for machine in self._all_machines:
780            if not machine.locked:
781                machine.locked = True
782                return machine
783        return None
784
785    def ImageMachine(self, machine, label):
786        if machine or label:
787            return 0
788        return 1
789
790    def ReleaseMachine(self, machine):
791        machine.locked = False
792
793    def GetMachines(self, label=None):
794        return self._all_machines
795
796    def GetAvailableMachines(self, label=None):
797        return self._all_machines
798
799    def ForceSameImageToAllMachines(self, label=None):
800        return 0
801
802    def ComputeCommonCheckSum(self, label=None):
803        common_checksum = 12345
804        for machine in self.GetMachines(label):
805            machine.machine_checksum = common_checksum
806        self.machine_checksum[label.name] = common_checksum
807
808    def GetAllMachines(self):
809        return self._all_machines
810