You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
494 lines
16 KiB
494 lines
16 KiB
from __future__ import print_function # python >= 2.6
|
|
from os import makedirs, path, listdir, rename, remove
|
|
from subprocess import Popen
|
|
from filecmp import cmp
|
|
from glob import iglob
|
|
from shutil import rmtree
|
|
from json import loads as json_loads, dump as json_dump
|
|
from sys import stderr, argv, path as sys_path
|
|
|
|
"""
|
|
gen_config_combinations_uniq_output.py
|
|
|
|
Creates from a given set of options all possible option settings
|
|
combinations, formats files with those and displays how much non equal
|
|
formatted outputs have been created.
|
|
|
|
Expects arg1 to be a filepath to a json config file
|
|
(see config example below)
|
|
|
|
:author: Daniel Chumak
|
|
:license: GPL v2+
|
|
"""
|
|
|
|
|
|
# config = {
|
|
# "option_settings": {
|
|
# "AT_BOOL": ["False", "True"],
|
|
# "AT_IARF": ["ignore", "add", "remove", "force"],
|
|
# "AT_POS": ["ignore", "join", "lead", "lead_break", "lead_force",
|
|
# "trail", "trail_break", "trail_force"],
|
|
# "AT_LINE": ["auto", "lf", "crlf", "cr"],
|
|
# "AT_NUM": [-2, -1, 0, 1, 2, 3],
|
|
# "AT_UNUM": [0, 1, 2, 3]
|
|
# },
|
|
# "options": [{
|
|
# "name": "nl_func_type_name",
|
|
# "type": "AT_IARF"
|
|
# }, {
|
|
# "name": "nl_template_class",
|
|
# "type": "AT_IARF"
|
|
# }],
|
|
# "out_dir": "./Out",
|
|
# "in_files": ["./t.cpp", "./t2.cpp"],
|
|
# "unc_bin": "../build/uncrustify",
|
|
# "cleanup_lvl": 2,
|
|
# "force_cleanup": false,
|
|
# "json_output": false
|
|
# }
|
|
#
|
|
|
|
|
|
def len_index_combinations(max_indices):
|
|
"""generator function that yields a list starting from
|
|
n_0 = 0, ... n_m-1 = 0, n_m = 0
|
|
...
|
|
n_0 = 0, ... n_m-1 = 0, n_m = n_m_max
|
|
...
|
|
n_0 = 0, ... n_m-1 = 1, n_m = 0
|
|
n_0 = 0, ... n_m-1 = 1, n_m = 1
|
|
...
|
|
n_0 = 0, ... n_m-1 = n_m-1_max, n_m = n_m_max
|
|
...
|
|
n_0 = n_0_max, ... n_m-1 = n_m-1_max, n_m = n_m_max
|
|
|
|
|
|
:param max_indices: list of max values every position is going to reach
|
|
|
|
:yield: list of values at the current step
|
|
"""
|
|
|
|
fields = len(max_indices)
|
|
accu = [0] * fields
|
|
|
|
# increment last position n, on max val move pos by one (n-1) and increment
|
|
# if (n-1) is max move again (n-2) and increment, ...
|
|
pos = fields
|
|
while pos >= 0:
|
|
yield (accu)
|
|
|
|
pos = fields - 1
|
|
accu[pos] += 1
|
|
|
|
# on reaching max reset value, move pos and increment at pos
|
|
while pos >= 0 and accu[pos] >= max_indices[pos]:
|
|
accu[pos] = 0
|
|
pos -= 1
|
|
|
|
if pos >= 0:
|
|
accu[pos] += 1
|
|
|
|
|
|
def write_config_files(config):
|
|
"""Writes a configuration file for each possible combination of 'option'
|
|
settings
|
|
|
|
:param config: configuration object, expects that it was processed by
|
|
check_config
|
|
"""
|
|
|
|
options_len = len(config["options"])
|
|
|
|
# populate len_options with amount of settings for the types of each option
|
|
len_options = [0] * options_len
|
|
for i in range(options_len):
|
|
option_setting = config["options"][i]["type"]
|
|
len_options[i] = len(config["option_settings"][option_setting])
|
|
|
|
# write configuration files, one per possible combination
|
|
for combination in len_index_combinations(len_options):
|
|
len_indices = len(combination)
|
|
|
|
# generate output filepath
|
|
file_path = config['out_dir'] + "/"
|
|
for i in range(len_indices):
|
|
option_name = config["options"][i]["name"]
|
|
file_path += ("%s__" % option_name)
|
|
for i in range(len_indices):
|
|
option_type = config["options"][i]["type"]
|
|
option_setting = combination[i]
|
|
file_path += ("%d__" % option_setting)
|
|
file_path += "unc.cfg"
|
|
|
|
# write configuration file
|
|
with open(file_path, 'w') as f:
|
|
for i in range(len_indices):
|
|
option_name = config["options"][i]["name"]
|
|
option_type = config["options"][i]["type"]
|
|
option_setting = config["option_settings"][option_type][
|
|
combination[i]]
|
|
|
|
f.write("%s = %s\n" % (option_name, option_setting))
|
|
|
|
|
|
def gen_equal_output_map(config):
|
|
"""Formats 'in_files' with configs inside the 'out_dir' with Uncrustify and
|
|
groups formatted files with equal content together.
|
|
Expects config filename format generated by write_config_files
|
|
|
|
:param config: configuration object, expects that it was processed by
|
|
check_config
|
|
:return: dict of files with equal content
|
|
key -- group index
|
|
value -- filepath list
|
|
"""
|
|
|
|
# maps that will hold configurations that produce the same formatted files
|
|
equal_output_map = {}
|
|
# map len counter
|
|
map_val_idx = 0
|
|
|
|
# iterate through all generated config file names
|
|
|
|
for cfg_path in sorted(iglob('%s/*.cfg' % config["out_dir"])):
|
|
for in_file_idx in range(len(config["in_files"])):
|
|
# extract substring form config gile name (removes __unc.cfg)
|
|
splits_file = cfg_path.split("__unc")
|
|
if len(splits_file) < 1:
|
|
raise Exception('split with "__unc" | Wrong split len: %d'
|
|
% len(splits_file))
|
|
|
|
out_path = ("%s__%d" % (splits_file[0], in_file_idx))
|
|
|
|
# gen formatted files with uncrustify binary
|
|
proc = Popen([config["unc_bin"],
|
|
"-c", cfg_path,
|
|
"-f", config["in_files"][in_file_idx],
|
|
"-o", out_path,
|
|
])
|
|
proc.wait()
|
|
if proc.returncode != 0:
|
|
continue
|
|
|
|
# populate 'equal_output_map' map
|
|
if len(equal_output_map) == 0:
|
|
equal_output_map[0] = [out_path]
|
|
map_val_idx += 1
|
|
else:
|
|
found_flag = False
|
|
for i in range(map_val_idx):
|
|
# compare first file of group i with the generated file
|
|
if cmp(equal_output_map[i][0], out_path):
|
|
equal_output_map[i].append(out_path)
|
|
found_flag = True
|
|
break
|
|
# create new group if files do not match
|
|
if not found_flag:
|
|
equal_output_map[map_val_idx] = [out_path]
|
|
map_val_idx += 1
|
|
|
|
return equal_output_map
|
|
|
|
|
|
def gen_output_dict(config, equal_output_map):
|
|
"""Makes an output dict with the generated results.
|
|
|
|
:param config: configuration object, expects that it was processed by
|
|
check_config
|
|
|
|
:param equal_output_map: dict of files with equal content,
|
|
expects format generated by gen_equal_output_map
|
|
:return: output dict, format:
|
|
copies objects option_settings, options and in_files (renamed as
|
|
files) from the config object. Additionally has the object groups
|
|
that holds gourp - file - settings combination data
|
|
format:
|
|
groups = [ [fileIdx0[
|
|
[settingIdx0, settingIdx1, ...],
|
|
[settingIdx0, settingIdx1, ...] ] ]
|
|
[fileIdx1[
|
|
[settingIdx0, settingIdx1, ...],
|
|
[settingIdx0, settingIdx1, ...] ] ]
|
|
]
|
|
"""
|
|
|
|
output_dict = {"option_settings": config["option_settings"],
|
|
"options": config["options"],
|
|
"files": config["in_files"],
|
|
"groups": []}
|
|
|
|
options_len = len(output_dict["options"])
|
|
files_len = len(output_dict["files"])
|
|
|
|
for key in equal_output_map:
|
|
group_dict = []
|
|
for file_arr_idx in range(files_len):
|
|
group_dict.append([])
|
|
|
|
for list_value in equal_output_map[key]:
|
|
split = list_value.rsplit("/", 1)
|
|
split = split[len(split) - 1].split("__")
|
|
split_len = len(split)
|
|
|
|
# n option names + n option values + file idx
|
|
if split_len < options_len * 2 + 1:
|
|
print(" wrong split len on %s\n" % list_value, file=stderr)
|
|
continue
|
|
|
|
file_idx = int(split[split_len - 1])
|
|
file_combinations = [int(i) for i in split[options_len:split_len-1]]
|
|
|
|
group_dict[file_idx].append(file_combinations)
|
|
|
|
output_dict["groups"].append(group_dict)
|
|
|
|
return output_dict
|
|
|
|
|
|
def write_output_dict_pretty(out_dict, out_path):
|
|
"""pretty prints the output dict into a file
|
|
|
|
:param out_dict: dict that will be printed, expects format generated by
|
|
gen_output_dict
|
|
|
|
:param out_path: output filepath
|
|
"""
|
|
|
|
group_id = 0
|
|
options_len = len(out_dict["options"])
|
|
|
|
with open(out_path, 'w') as f:
|
|
|
|
f.write("Files:\n")
|
|
for in_file_idx in range(len(out_dict["files"])):
|
|
f.write(" %d: %s\n" % (in_file_idx,
|
|
out_dict["files"][in_file_idx]))
|
|
|
|
f.write("\nOptions:\n")
|
|
for option_idx in range(options_len):
|
|
f.write(" %d: %s\n" % (option_idx,
|
|
out_dict["options"][option_idx]["name"]))
|
|
f.write("\n\n")
|
|
|
|
for group in out_dict["groups"]:
|
|
f.write("Group: %d\n" % group_id)
|
|
group_id += 1
|
|
|
|
for file_idx in range(len(group)):
|
|
file = group[file_idx]
|
|
|
|
for combinations in file:
|
|
combination_strings = []
|
|
for combination_idx in range(len(combinations)):
|
|
|
|
combination_id = combinations[combination_idx]
|
|
combination_string = out_dict["option_settings"][
|
|
out_dict["options"][combination_idx]["type"]][
|
|
combination_id]
|
|
combination_strings.append(str(combination_string))
|
|
f.write(" (%s: %s)\n" % (file_idx,
|
|
" - ".join(combination_strings)))
|
|
f.write("\n")
|
|
|
|
|
|
def load_config(file_path):
|
|
"""reads a file and parses it as json
|
|
|
|
:param file_path: path to the json file
|
|
|
|
:return: json object
|
|
"""
|
|
|
|
with open(file_path, 'r') as f:
|
|
string = f.read()
|
|
json = json_loads(string)
|
|
|
|
return json
|
|
|
|
|
|
def make_abs_path(basis_abs_path, rel_path):
|
|
return path.normpath(path.join(basis_abs_path, rel_path))
|
|
|
|
|
|
def check_config(config, cfg_path=""):
|
|
"""checks if the provided config has all needed options, sets default
|
|
settings for optional options and transform relative paths into absolute
|
|
paths.
|
|
|
|
:param config: config dict that will be checked
|
|
|
|
:param cfg_path: if not empty transforms relative to absolute paths,
|
|
paths will be based upon the cfg_path.
|
|
"""
|
|
|
|
extend_relative_paths = True if len(cfg_path) > 0 else False
|
|
cfg_path = path.abspath(path.dirname(cfg_path))
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
if "option_settings" not in config:
|
|
raise Exception("config file: 'option_settings' missing")
|
|
|
|
if len(config["option_settings"]) == 0:
|
|
raise Exception("config file: 'option_settings' values missing")
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
if "options" not in config:
|
|
raise Exception("config file: 'options' missing")
|
|
|
|
if len(config["options"]) < 2:
|
|
raise Exception("config file: 'options' min. two options needed")
|
|
|
|
for option_obj in config["options"]:
|
|
if "name" not in option_obj:
|
|
raise Exception("config file: 'options[{}]' name missing")
|
|
if "type" not in option_obj:
|
|
raise Exception("config file: 'options[{}]' type missing")
|
|
if option_obj["type"] not in config["option_settings"]:
|
|
raise Exception("config file: 'options[{type='%s'}]' not in option_"
|
|
"settings" % option_obj["type"])
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
if "out_dir" not in config:
|
|
raise Exception("config file: 'out_dir' missing")
|
|
|
|
if len(config['out_dir']) == 0:
|
|
raise Exception("config file: 'out_dir' value missing")
|
|
|
|
if extend_relative_paths and not path.isabs(config['out_dir']):
|
|
config['out_dir'] = make_abs_path(cfg_path, config['out_dir'])
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
if "in_files" not in config:
|
|
raise Exception("config file: 'in_files' missing")
|
|
|
|
if len(config['in_files']) == 0:
|
|
raise Exception("config file: 'in_files' values missing")
|
|
|
|
for file_idx in range(len(config['in_files'])):
|
|
if extend_relative_paths and not path.isabs(
|
|
config['in_files'][file_idx]):
|
|
config['in_files'][file_idx] = make_abs_path(cfg_path,
|
|
config['in_files'][
|
|
file_idx])
|
|
|
|
if not path.isfile(config['in_files'][file_idx]):
|
|
raise Exception("config file: '%s' is not a file"
|
|
% config['in_files'][file_idx])
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
if "unc_bin" not in config:
|
|
raise Exception("config file: 'in_files' missing")
|
|
|
|
if extend_relative_paths and not path.isabs(config['unc_bin']):
|
|
config['unc_bin'] = make_abs_path(cfg_path, config['unc_bin'])
|
|
|
|
if not path.isfile(config['unc_bin']):
|
|
raise Exception("config file: '%s' is not a file" % config['unc_bin'])
|
|
|
|
# Optional -----------------------------------------------------------------
|
|
|
|
if "cleanup_lvl" not in config:
|
|
config["cleanup_lvl"] = 1
|
|
|
|
if "force_cleanup" not in config:
|
|
config["force_cleanup"] = False
|
|
|
|
if "json_output" not in config:
|
|
config["json_output"] = False
|
|
|
|
|
|
def cleanup(level, eq_map, clean_target_dir, keep_files=()):
|
|
"""cleans up output_dir
|
|
|
|
:param level: 0 - do nothing,
|
|
1 - keep `keep_files` and 1 file for each group,
|
|
2 - remove everything
|
|
|
|
:param equal_output_map: dict of files with equal content,
|
|
expects format generated by gen_equal_output_map
|
|
|
|
:param clean_target_dir: directory which content will be cleaned
|
|
|
|
:param keep_files: list of files should not be removed
|
|
"""
|
|
|
|
if level == 0:
|
|
return
|
|
|
|
if level == 2:
|
|
rmtree(clean_target_dir)
|
|
|
|
if level == 1:
|
|
rm_files = [clean_target_dir + "/" + f for f in
|
|
listdir(clean_target_dir)]
|
|
|
|
for f in keep_files:
|
|
rm_files.remove(f)
|
|
|
|
for idx in eq_map:
|
|
old_path = eq_map[idx][0]
|
|
new_path = ("%s/g_%d" % (path.dirname(path.abspath(old_path)), idx))
|
|
rename(old_path, new_path)
|
|
|
|
try:
|
|
rm_files.remove(old_path)
|
|
except ValueError:
|
|
pass # ignore that it is missing
|
|
|
|
try:
|
|
rm_files.remove(new_path)
|
|
except ValueError:
|
|
pass # ignore that it is missing
|
|
|
|
for f in rm_files:
|
|
remove(f)
|
|
|
|
|
|
def main(args):
|
|
config = load_config(args[0])
|
|
check_config(config, args[0])
|
|
|
|
# gen output directory
|
|
if path.isfile(config["out_dir"]):
|
|
raise Exception("%s is a file" % config["out_dir"])
|
|
|
|
if not path.isdir(config["out_dir"]):
|
|
makedirs(config["out_dir"])
|
|
elif not config["force_cleanup"] and config["cleanup_lvl"] > 0:
|
|
raise Exception("cleanup_lvl > 0 on an existing directory: %s"
|
|
% config["out_dir"])
|
|
|
|
write_config_files(config)
|
|
eq_map = gen_equal_output_map(config)
|
|
output_dict = gen_output_dict(config, eq_map)
|
|
|
|
# write output as txt file
|
|
output_dict_path = path.join(config["out_dir"], "out.txt")
|
|
write_output_dict_pretty(output_dict, output_dict_path)
|
|
|
|
# read ouput txt file to print it
|
|
with open(output_dict_path, 'r') as f:
|
|
print()
|
|
print(f.read())
|
|
|
|
keep_files = [output_dict_path]
|
|
|
|
# write output as json file
|
|
if config["json_output"]:
|
|
output_dict_json_path = path.join(config["out_dir"], "out.json")
|
|
with open(output_dict_json_path, 'w') as f:
|
|
json_dump(output_dict, f)
|
|
keep_files.append(output_dict_json_path)
|
|
|
|
# clean output directory
|
|
cleanup(config["cleanup_lvl"], eq_map, config["out_dir"], keep_files)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(argv[1:])
|