xref: /XiangShan/scripts/constantHelper.py (revision 7cf78eb25d45a53e97cb143ddd0185eddc10672c) 
1import os
2import random
3from subprocess import Popen, PIPE
4import psutil
5import json
6import sys
7import math
8
9# usage: python3 constantHelper.py JSON_FILE_PATH
10#
11# an example json config file is as follow:
12# visit https://bosc.yuque.com/yny0gi/gr7hyo/oy3dagqi9v97p696 for detail
13# {
14#     "constants": [
15#         {
16#             "name": "block_cycles_cache_0",
17#             "width": 7,
18#             "guide": 20,
19#             "init": 11
20#         },
21#         {
22#             "name": "block_cycles_cache_1",
23#             "width": 7,
24#             "init": 18
25#         },
26#         {
27#             "name": "block_cycles_cache_2",
28#             "width": 7,
29#             "init": 127
30#         },
31#         {
32#             "name": "block_cycles_cache_3",
33#             "width": 7,
34#             "init": 17
35#         }
36#     ],
37#     "opt_target": [
38#         {"successfully_forward_channel_D": {"policy" :"max", "baseline" :0} },
39#         {"successfully_forward_mshr": {"policy" :"max", "baseline" :0} },
40#         {"dcache.missQueue.entries_0: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :250396} },
41#         {"dcache.missQueue.entries_1: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :5634} },
42#         {"dcache.missQueue.entries_2: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :4599} },
43#         {"dcache.missQueue.entries_3: load_miss_penalty_to_use,": {"policy" :"min", "baseline" :4146} }
44#     ],
45
46#     "population_num": 50,
47#     "iteration_num": 50,
48#     "crossover_rate": 50,
49#     "mutation_rate": 50,
50
51#     "emu_threads": 16,
52#     "concurrent_emu": 4,
53#     "max_instr": 1000000,
54#     "seed": 3888,
55#     "work_load": "~/nexus-am/apps/maprobe/build/maprobe-riscv64-xs.bin"
56# }
57
58
59# parameters according to noop
60NOOP_HOME = os.getenv("NOOP_HOME")
61DIFF_PATH = os.path.join(NOOP_HOME, "ready-to-run", "riscv64-nemu-interpreter-so")
62BUILD_PATH = os.path.join(NOOP_HOME, "build")
63EMU_PATH = os.path.join(BUILD_PATH, "emu")
64
65CONFIG_FILE_PREFIX = ".constant_result_"
66PERF_FILE_POSTFIX = "tmp"
67MAXVAL = (1 << 63) - 1
68
69class Constant:
70    def __init__(self, obj: dict) -> None:
71        self.name = obj['name']
72        self.width = obj['width']
73        self.guide = (1 << self.width - 1) - 1 if 'guide' not in obj.keys() else obj['guide']
74        self.init = random.randint(0, self.guide) if 'init' not in obj.keys() else obj['init']
75    def maxrange(self) -> int:
76        return (1 << self.width) - 1
77
78
79class Config:
80    def __init__(self, constants, opt_target, population_num, iteration_num, crossover_rate, mutation_rate, emu_threads, concurrent_emu, max_instr, seed, work_load) -> None:
81        self.constants = constants
82        self.opt_target = opt_target
83        self.population_num = int(population_num)
84        self.iteration_num = int(iteration_num)
85        self.crossover_rate = int(crossover_rate)
86        self.mutation_rate = int(mutation_rate)
87        self.emu_threads = int(emu_threads)
88        self.concurrent_emu = int(concurrent_emu)
89        self.max_instr = int(max_instr)
90        self.seed = int(seed)
91        self.work_load = work_load
92    def get_ith_constant(self, i) -> Constant:
93        return self.constants[i]
94    def get_constain_num(self) -> int:
95        return len(self.constants)
96
97
98def loadConfig(json_path) -> Config:
99    obj = json.load(open(json_path, "r"))
100    constants = [Constant(obj['constants'][i]) for i in range(len(obj['constants']))]
101    config = Config(constants, obj['opt_target'], obj['population_num'], obj['iteration_num'], obj['crossover_rate'], obj['mutation_rate'], obj['emu_threads'], obj['concurrent_emu'], obj['max_instr'], obj['seed'], obj['work_load'])
102    return config
103
104class RunContext:
105    def __init__(self, config: Config) -> None:
106        self.config = config
107    def checkCoreFree(self) -> None:
108        percent_per_core = psutil.cpu_percent(interval=1 ,percpu=True)
109        acc = 0
110        for i in range(self.config.concurrent_emu * self.config.emu_threads):
111            acc += percent_per_core[i]
112        if acc < (0.1 * (100 * self.config.concurrent_emu * self.config.emu_threads)):
113            return True
114        else:
115            print("no free {} core, core usage:".format(self.config.concurrent_emu * self.config.emu_threads))
116            print(percent_per_core)
117            return False
118    def getStdIn(self, population: list, id: int) -> str:
119        res = 'echo \"'
120        res += str(len(population[id]))
121        res += '\\n'
122        for item in population[id]:
123            res += item[0] + ' ' + str(item[1]) + '\\n'
124        res += '\"'
125        return res
126
127    def genRunCMD(self, population, id) -> str:
128        coreStart = (id % self.config.concurrent_emu) * self.config.emu_threads
129        coreEnd = ((id % self.config.concurrent_emu) + 1) * self.config.emu_threads - 1
130        stdinStr = self.getStdIn(population, id)
131        return "{} | numactl -m 1 -C {}-{} {} --i {} --diff {} -I {} -s {} 2>{}.{}".format(stdinStr, coreStart, coreEnd, EMU_PATH, self.config.work_load, DIFF_PATH, self.config.max_instr, self.config.seed, os.path.join(BUILD_PATH, CONFIG_FILE_PREFIX + str(id)), PERF_FILE_POSTFIX)
132
133class Solution:
134    def __init__(self, config: Config) -> None:
135        self.config = config
136        self.context = RunContext(config)
137    def genFirstPopulation(self) -> list:
138        res = []
139        used = []
140        config = self.config
141        for i in range(config.population_num):
142            candidate = [[config.get_ith_constant(i).name, random.randint(0, config.get_ith_constant(i).maxrange()) % config.get_ith_constant(i).guide] for i in range(config.get_constain_num())]
143            while(candidate in used):
144                candidate = [[config.get_ith_constant(i).name, random.randint(0, config.get_ith_constant(i).maxrange()) % config.get_ith_constant(i).guide] for i in range(config.get_constain_num())]
145            used.append(candidate)
146            res.append(candidate)
147        assert(len(res) == config.population_num)
148        return res
149    def profilling_fitness(self) -> list:
150        fitness = []
151        lines = []
152        for idx in range(self.config.population_num):
153            perfFilePath = os.path.join(BUILD_PATH, CONFIG_FILE_PREFIX + str(idx) + '.' + PERF_FILE_POSTFIX)
154            with open(perfFilePath, "r") as fp:
155                lines = fp.readlines()
156                res = 0
157                for line in lines:
158                    for opt in config.opt_target:
159                        if list(opt.keys())[0] in line:
160                            # max and min policy
161                            if list(opt.values())[0]['policy'] == 'max':
162                                res += int(list(filter(lambda x: x != '', line.split(' ')))[-1]) - int(list(opt.values())[0]['baseline'])
163                            elif list(opt.values())[0]['policy'] == 'min':
164                                res += int(list(opt.values())[0]['baseline']) - int(list(filter(lambda x: x != '', line.split(' ')))[-1])
165                fitness.append(res)
166        assert(len(fitness) == self.config.population_num)
167        return fitness
168    def run_one_round(self, population: list) -> None:
169        procs = []
170        i = 0
171        while i < len(population):
172            if i % self.config.concurrent_emu == 0:
173                for proc in procs:
174                    proc.wait()
175                procs.clear()
176            # print(self.context.genRunCMD(population, i))
177            print(population[i])
178            procs.append(Popen(args=self.context.genRunCMD(population, i), shell=True, encoding='utf-8', stdin=PIPE, stdout=PIPE, stderr=PIPE))
179            i += 1
180        for proc in procs:
181            proc.wait()
182    def mutation(self, item: list) -> list:
183        res = []
184        for val in item:
185            width = 0
186            guide = 0
187            for constant in self.config.constants:
188                if(constant.name == val[0]):
189                    width = constant.width
190                    guide = constant.guide
191            mask = 1 << random.randint(0, width - 1)
192            if random.randint(0, 100) > self.config.mutation_rate:
193                res.append(val)
194            else:
195                val[1] = (((val[1] & mask) ^ mask) | val[1]) % guide
196                res.append(val)
197        assert(len(item) == len(res))
198        return res
199    def crossover(self, poplulation: list) -> list:
200        res = []
201        if len(poplulation) < 2:
202            return poplulation
203        for individual in poplulation:
204            indivi = []
205            for (index, constant) in enumerate(individual):
206                const = constant
207                if random.randint(0, 100) < self.config.crossover_rate:
208                    crossover_target_id = 0
209                    while crossover_target_id == index:
210                        crossover_target_id = random.randint(0, len(poplulation) - 1)
211                    maskMax = 0
212                    guide = 0
213                    for config_const in self.config.constants:
214                        if config_const.name == constant[0]:
215                            maskMax = config_const.width
216                            guide = config_const.guide
217                    maskMax = int(math.log2(guide)) + 1 if (int(math.log2(guide)) + 1 < maskMax) else maskMax
218                    maskLen = random.randint(1, maskMax)
219                    mask = (1 << maskLen) - 1
220                    shiftLen = random.randint(0, maskMax - maskLen)
221                    mask = mask << shiftLen
222                    const_now = const[1]
223                    target_now = poplulation[crossover_target_id][index][1]
224                    const_now = ((const_now & ~(mask)) | (target_now & mask)) % guide
225                    const = [constant[0], const_now]
226                indivi.append(const)
227            res.append(indivi)
228        assert(len(poplulation) == len(res))
229        return res
230    def genNextPop(self, curPop, fitness) -> list:
231        nextgen = []
232        tmp = sorted(zip(curPop, fitness), key=lambda x : x[1], reverse=True)
233        print()
234        print("opt constant in this round is ", list(tmp)[0][0], " fitness is ", int(list(tmp)[0][1]))
235        cross = []
236        for i in range(len(tmp)):
237            if i < (len(tmp) // 2):
238                # select
239                nextgen.append(tmp[i][0])
240            else:
241                cross.append(tmp[i][0])
242        # crossover
243        cross = self.crossover(cross)
244        nextgen = nextgen + cross
245        # mutation
246        for i in range(len(tmp)):
247            nextgen[i] = self.mutation(nextgen[i])
248        assert(len(curPop) == len(nextgen))
249        return nextgen
250
251    class HashList:
252        def __init__(self, obj: list) -> None:
253            # obj: [['test1', 38], ['test2', 15]]
254            self.obj = obj
255        def __hash__(self) -> str:
256            res = ''
257            for const in self.obj:
258                res += ' '.join(map(lambda x : str(x), const))
259            return hash(res)
260        def __eq__(self, __o: object) -> bool:
261            for (idx, const) in enumerate(self.obj):
262                if const != __o.obj[idx]:
263                    return False
264            return True
265
266    def gene_cal(self) -> None:
267        globalMap = dict()
268        if(self.config.population_num % 2 != 0):
269            print("gene algrithom must ensure that population_num is an even value")
270            return
271        parentPoplation = self.genFirstPopulation()
272        init_indiv = []
273        for constant in self.config.constants:
274            const = []
275            const.append(constant.name)
276            const.append(constant.init)
277            init_indiv.append(const)
278        parentPoplation.pop()
279        parentPoplation.append(init_indiv)
280        for i in range(self.config.iteration_num):
281            if i != 0:
282                print()
283            print("iteration ", i, " begins")
284            print()
285            while True:
286                if self.context.checkCoreFree():
287                    self.run_one_round(parentPoplation)
288                    fitness = self.profilling_fitness()
289                    for (pop, fit) in zip(parentPoplation, fitness):
290                        globalMap[self.HashList(pop)] = fit
291                    parentPoplation = self.genNextPop(parentPoplation, fitness)
292                    break
293        globalMap = zip(globalMap.keys(), globalMap.values())
294        globalMap = sorted(globalMap, key=lambda x : x[1], reverse=True)
295        print("opt constant for gene algrithom is ", list(globalMap)[0][0].obj, " fitness", int(list(globalMap)[0][1]))
296
297config = loadConfig(sys.argv[1])
298Solution(config).gene_cal()
299