From 6c22d12640b1e31e28685bf6259f25d408462566 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Mon, 3 Oct 2022 15:47:49 +0200 Subject: [PATCH 01/28] Makefile and parseClang.py --- srcs/Makefile | 2 +- srcs/extractertool/parserClang.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/srcs/Makefile b/srcs/Makefile index 260bede..8426612 100644 --- a/srcs/Makefile +++ b/srcs/Makefile @@ -36,7 +36,7 @@ deps: $(GOGET) github.com/akamensky/argparse $(GOGET) github.com/awalterschulze/gographviz $(GOGET) github.com/sergi/go-diff/... - $(GOGET) github.com/AlecAivazis/survey + $(GOGET) gopkg.in/AlecAivazis/survey.v1 # Cross compilation build-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v diff --git a/srcs/extractertool/parserClang.py b/srcs/extractertool/parserClang.py index 138b1df..56c1f9d 100755 --- a/srcs/extractertool/parserClang.py +++ b/srcs/extractertool/parserClang.py @@ -5,7 +5,7 @@ # pip3 install clang # # cd /usr/lib/x86_64-linux-gnu/ -# sudo ln -s libclang-X.Y.so.1 libclang.so (X.Y the version number) +# sudo ln -s libclang-X.Y.so.1 libclang-14.so (X.Y the version number) # # (*) Run: # From d641975f97c2eddb163e37ad8bb101662ef2a720 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Fri, 14 Oct 2022 18:48:49 +0200 Subject: [PATCH 02/28] handleCreationApp and createUnikraftApp corrected --- srcs/buildtool/unikraft_files_process.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 96e8b4e..28aa3ef 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -109,7 +109,10 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) { if !created { u.PrintWarning(appFolder + " already exists.") - handleCreationApp(&appFolder) + appFolder = handleCreationApp(appFolder) + if _, err := u.CreateFolder(appFolder); err != nil { + return nil, err + } } return &appFolder, nil @@ -117,7 +120,7 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) { // -----------------------------Create App folder------------------------------- -func handleCreationApp(appFolder *string) { +func handleCreationApp(appFolder string) string { fmt.Println("Make your choice:\n1: Copy and overwrite files\n2: " + "Enter manually the name of the folder\n3: exit program") var input int @@ -128,13 +131,13 @@ func handleCreationApp(appFolder *string) { } else { switch input { case 1: - return + return appFolder case 2: fmt.Print("Enter text: ") reader := bufio.NewReader(os.Stdin) text, _ := reader.ReadString('\n') - appFolder = &text - return + appFolder = strings.Split(text, "\n")[0] + u.SEP + return appFolder case 3: os.Exit(1) default: @@ -142,6 +145,8 @@ func handleCreationApp(appFolder *string) { } } } + + return appFolder } // -------------------------MOVE FILES TO APP FOLDER---------------------------- From 7f26ca5d3da72f3e80d124e58101e4eaf19d2de4 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Tue, 25 Oct 2022 18:39:36 +0200 Subject: [PATCH 03/28] matchLibs and searchInternalDependencies fixed --- srcs/buildtool/microlibs_process.go | 52 +++++++++++++-------- srcs/buildtool/run_buildtool.go | 70 ++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/srcs/buildtool/microlibs_process.go b/srcs/buildtool/microlibs_process.go index b03cecb..86b3b11 100644 --- a/srcs/buildtool/microlibs_process.go +++ b/srcs/buildtool/microlibs_process.go @@ -41,7 +41,6 @@ type MicroLibsFunction struct { // -----------------------------Match micro-libs-------------------------------- // processSymbols adds symbols within the 'exportsyms.uk' file into a map. -// func processSymbols(microLib, output string, mapSymbols map[string][]string) { lines := strings.Split(output, "\n") @@ -134,6 +133,32 @@ func fetchSymbolsExternalLibs(folder string, return externalLibs, nil } +// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static and +// dynamic analyses together into a map structure. +// +// It returns the map containing all the symbols and system calls. +func putJsonSymbolsTogether(data *u.Data) map[string]string { + dataMap := make(map[string]string) + + for k, v := range data.StaticData.Symbols { + dataMap[k] = v + } + + for k := range data.StaticData.SystemCalls { + dataMap[k] = "" + } + + for k, v := range data.DynamicData.Symbols { + dataMap[k] = v + } + + for k := range data.DynamicData.SystemCalls { + dataMap[k] = "" + } + + return dataMap +} + // matchSymbols performs the matching between Unikraft's micro-libs and // libraries used by a given application based on the list of symbols that both // contain. @@ -144,13 +169,6 @@ func matchSymbols(matchedLibs []string, data map[string]string, for key := range data { if values, ok := microLibs[key]; ok { for _, value := range values { - - // todo remove - if strings.Compare(NOLIBC, value) == 0 { - value = NEWLIB - } - // remove above - if !u.Contains(matchedLibs, value) { matchedLibs = append(matchedLibs, value) } @@ -172,11 +190,6 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, matchedLibs := make([]string, 0) - //todo remove - matchedLibs = append(matchedLibs, POSIXLIBDL) - matchedLibs = append(matchedLibs, POSIXSYSINFO) - matchedLibs = append(matchedLibs, UKMMAP) - folder := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal") if err := fetchSymbolsInternalLibs(folder, mapSymbols); err != nil { return nil, nil, err @@ -189,11 +202,13 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, return nil, nil, err } - // Perform the matching symbols on static data - matchedLibs = matchSymbols(matchedLibs, data.StaticData.Symbols, mapSymbols) - - // Perform the matching symbols on dynamic data - matchedLibs = matchSymbols(matchedLibs, data.DynamicData.Symbols, mapSymbols) + dataMap := putJsonSymbolsTogether(data) + //matchedLibs = append(matchedLibs, POSIXPROCESS) + //matchedLibs = append(matchedLibs, POSIXUSER) + //matchedLibs = append(matchedLibs, POSIXSYSINFO) + //matchedLibs = append(matchedLibs, POSIXLIBDL) + // Perform the symbol matching + matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols) return matchedLibs, externalLibs, nil } @@ -223,7 +238,6 @@ func cloneGitRepo(url, unikraftPathLibs, lib string) error { // cloneLibsFolders clones all the needed micro-libs that are needed by a // given application -// func cloneLibsFolders(workspacePath string, matchedLibs []string, externalLibs map[string]string) { diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 4b38f6a..3d967d1 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -7,13 +7,15 @@ package buildtool import ( - "github.com/AlecAivazis/survey/v2" + "fmt" "io/ioutil" "os" "path/filepath" "regexp" "strings" u "tools/srcs/common" + + "github.com/AlecAivazis/survey/v2" ) // STATES @@ -51,7 +53,8 @@ func generateConfigUk(filename, programName string, matchedLibs []string) error // execution of the 'make' command. // // It returns an integer that defines the result of 'make': -// +// +// func checkMakeOutput(appFolder string, stderr *string) int { if stderr == nil { @@ -107,7 +110,6 @@ func parseMakeOutput(output string) string { // RunBuildTool runs the automatic build tool to build a unikernel of a // given application. -// func RunBuildTool(homeDir string, data *u.Data) { // Init and parse local arguments @@ -216,7 +218,8 @@ func RunBuildTool(homeDir string, data *u.Data) { if err != nil { u.PrintErr(err) } - + fmt.Println("\nPREFINAL\n") + fmt.Println(matchedLibs) // Clone the external git repositories cloneLibsFolders(workspacePath, matchedLibs, externalLibs) @@ -229,7 +232,8 @@ func RunBuildTool(homeDir string, data *u.Data) { for _, lib := range matchedLibs { u.PrintOk("Match lib: " + lib) } - + fmt.Println("\nFINAL\n") + fmt.Println(matchedLibs) // Clone the external git repositories (if changed) cloneLibsFolders(workspacePath, matchedLibs, externalLibs) @@ -274,11 +278,6 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, config = strings.TrimPrefix(config, "LIB") } - // Replace underscore by dash - if strings.Contains(config, "_") { - config = strings.ReplaceAll(config, "_", "-") - } - // Check if matchedLibs already contains the lib config = strings.ToLower(config) if !u.Contains(*matchedLibs, config) { @@ -291,6 +290,57 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, return nil } +/* // This version considers also internal lib dependencies +func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, + externalLibs map[string]string) error { + + for _, lib := range *matchedLibs { + if strings.Contains(lib, "_") { + lib = strings.ReplaceAll(lib, "_", "-") + } + + // Get and read Config.UK from lib + var configUk string + + if _, ok := externalLibs[lib]; ok { + configUk = unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk" + } else { + configUk = unikraftPath + u.UNIKRAFTFOLDER + "lib" + u.SEP + lib + u.SEP + "Config.uk" + } + + lines, err := u.ReadLinesFile(configUk) + if err != nil { + return err + } + + // Process Config.UK file + mapConfig := make(map[string][]string) + u.ProcessConfigUK(lines, true, mapConfig, nil) + + for config := range mapConfig { + + // Remove LIB prefix + if strings.Contains(config, "LIB") { + config = strings.TrimPrefix(config, "LIB") + } + + // Replace underscore by dash + //if strings.Contains(config, "_") { + // config = strings.ReplaceAll(config, "_", "-") + //} + + // Check if matchedLibs already contains the lib + config = strings.ToLower(config) + if !u.Contains(*matchedLibs, config) { + *matchedLibs = append(*matchedLibs, config) + } + } + } + + return nil +} +*/ + func generateMake(programName, appFolder, workspacePath, makefile string, matchedLibs, sourceFiles []string, externalLibs map[string]string) error { // Generate Makefile From 6d88b02718b88a66b4d0a841cdf1b1315506ae04 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sat, 5 Nov 2022 16:53:12 +0100 Subject: [PATCH 04/28] dependtool first improvement --- srcs/buildtool/microlibs_process.go | 5 +- srcs/dependtool/parserClang.py | 243 ++++++++++++++++++++++++++++ srcs/dependtool/static_analyser.go | 110 ++++++++++++- 3 files changed, 351 insertions(+), 7 deletions(-) create mode 100755 srcs/dependtool/parserClang.py diff --git a/srcs/buildtool/microlibs_process.go b/srcs/buildtool/microlibs_process.go index 86b3b11..ff8eed9 100644 --- a/srcs/buildtool/microlibs_process.go +++ b/srcs/buildtool/microlibs_process.go @@ -203,10 +203,7 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, } dataMap := putJsonSymbolsTogether(data) - //matchedLibs = append(matchedLibs, POSIXPROCESS) - //matchedLibs = append(matchedLibs, POSIXUSER) - //matchedLibs = append(matchedLibs, POSIXSYSINFO) - //matchedLibs = append(matchedLibs, POSIXLIBDL) + // Perform the symbol matching matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols) diff --git a/srcs/dependtool/parserClang.py b/srcs/dependtool/parserClang.py new file mode 100755 index 0000000..502ed2c --- /dev/null +++ b/srcs/dependtool/parserClang.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +#--------------------------------------------------------------------- +# (*) Installation: +# +# pip3 install clang +# +# cd /usr/lib/x86_64-linux-gnu/ +# sudo ln -s libclang-X.Y.so.1 libclang-14.so (X.Y the version number) +# +# (*) Run: +# +# python3 parserClang.py [includepathsfile] +# +# where filepath can be a repository/folder or a file (c/cpp/h/hpp) +# +# +# Gaulthier Gain +# License: BSD +#--------------------------------------------------------------------- + +import getopt +import os +import sys +import json +import clang.cindex +import clang +import platform +from clang.cindex import CursorKind +from collections import Counter + +verbose = False # Change it to verbose mode + +global_funcs = Counter() +global_calls = Counter() + +silent_flag = False + +# Check if a path is a directory or a file +def check_input_path(path, includePaths): + if os.path.isdir(path): + iterate_root_folder(path, includePaths) + elif os.path.isfile(path): + check_type_file(path, includePaths) + else: + print("Unable to analyse this file") + exit(1) + +def get_include_paths(rootdir, includepathsFile): + paths = [] + with open(includepathsFile, 'r') as file: + for includePath in file.readlines(): + path = '-isystem ' + rootdir + includePath.replace('\n', '') + paths.append(path) + + return ' '.join(paths) + +# Check type/exenstion of a given file +def check_type_file(filepath, includePaths): + cplusplusOptions = '-x c++ --std=c++11' + cOptions = '' + + if includePaths is not None: + cplusplusOptions = cplusplusOptions + ' ' + includePaths + cOptions = cOptions + ' ' + includePaths + if silent_flag is False: + print("Gathering symbols of " + filepath) + if filepath.endswith(".cpp") or filepath.endswith(".hpp"): + parse_file(filepath, cplusplusOptions) + elif filepath.endswith(".c") or filepath.endswith(".h"): + parse_file(filepath, cOptions) + +# Iterate through a root folder +def iterate_root_folder(rootdir, includePaths): + for subdir, dirs, files in os.walk(rootdir): + for file in files: + filepath = subdir + os.sep + file + check_type_file(filepath, includePaths) + +# Print info about symbols (verbose mode) +def display_info_function(funcs, calls): + for f in funcs: + print(fully_qualified(f), f.location) + for c in calls: + if is_function_call(f, c): + print('-', c.location) + print() + +# Parse a given file to generate a AST +def parse_file(filepath, arguments): + + idx = clang.cindex.Index.create() + args = arguments.split() + tu = idx.parse(filepath, args=args) + funcs, calls = find_funcs_and_calls(tu) + if verbose: + display_info_function(funcs, calls) + print(list(tu.diagnostics)) + + +# Retrieve a fully qualified function name (with namespaces) +def fully_qualified(c): + if c is None: + return '' + elif c.kind == CursorKind.TRANSLATION_UNIT: + return '' + else: + res = fully_qualified(c.semantic_parent) + if res != '': + return res + '::' + c.spelling + return c.spelling + +# Determine where a call-expression cursor refers to a particular +# function declaration +def is_function_call(funcdecl, c): + defn = c.get_definition() + return (defn is not None) and (defn == funcdecl) + +# Filter name to take only the function name (remove "(args)") +def filter_func_name(displayname): + if "(" in displayname: + funcName = displayname.split('(')[0] + else: + funcName = displayname + return funcName + +# Retrieve lists of function declarations and call expressions in a +#translation unit +def find_funcs_and_calls(tu): + filename = tu.cursor.spelling + calls = [] + funcs = [] + for c in tu.cursor.walk_preorder(): + if c.kind == CursorKind.CALL_EXPR: + calls.append(c) + # filter name to take only the name if necessary + funcName = filter_func_name(c.displayname) + global_calls[funcName] += 1 + elif c.kind == CursorKind.FUNCTION_DECL: + funcs.append(c) + # filter name to take only the name if necessary + funcName = filter_func_name(c.displayname) + global_funcs[funcName] += 1 + return funcs, calls + +# Write data to json file +def write_to_json(output_filename, data): + with open(output_filename + '.json', 'w') as fp: + json.dump(data, fp, indent=4, sort_keys=True) + +# Open data to json file +def read_from_json(filename): + with open(output_filename + '.json', 'r') as fp: + data = json.load(fp) + return data + +# Read the list of syscalls (text file) +def read_syscalls_list(filename): + syscalls = set() + with open(filename) as f: + for line in f: + syscalls.add(line.strip()) + return syscalls + +# Check which syscall is called +def compare_syscalls(syscalls): + if silent_flag is False: + print("Gathered syscalls from function calls:") + + return [key for key in global_calls.keys() if key not in syscalls] + + + +# Main function +def main(): + optlist, args = getopt.getopt(sys.argv[1:], "o:qvt") + input_file_names = None + includepathsFile = None + output_file_name = None + textFormat = False + for opt in optlist: + if opt[0] == "-i": + includepathFile = opt[1] + if opt[0] == "-o": + output_file_name = opt[1] + if opt[0] == "-q": + global silent_flag + silent_flag = True + if opt[0] == "-v": + global verbose + verbose = True + if opt[0] == "-t": + textFormat = True + + + + input_file_names = args + if len(input_file_names) == 0: + if silent_flag is False: + print("No input files supplied") + exit(1) + if includepathsFile is not None: + includePaths = get_include_paths(input_file_name, includepathsFile) + for input_file_name in input_file_names: + check_input_path(input_file_name, includePaths) + else: + for input_file_name in input_file_names: + check_input_path(input_file_name, None) + + if silent_flag is False: + print("---------------------------------------------------------") + + if textFormat: + i = 0 + for key,value in global_funcs.items(): + if i < len(global_funcs.items())-1: + print(key, end=',') + else: + print(key) + i = i + 1 + else: + # Dump function declarations and calls to json + output_dikt = { + 'functions':'', + 'calls':'' + } + output_dikt['functions'] = [{'name':key, 'value':value} for key,value in + global_funcs.items()] + output_dikt['calls'] = [{'name':key, 'value':value} for key,value in global_calls.items()] + if includepathsFile is not None: + # Read syscalls from txt file + all_syscalls = read_syscalls_list('syscall_list.txt') + called_syscalls = compare_syscalls(all_syscalls) + output_dikt['syscalls'] = called_syscalls + + output_file = sys.stdout + json.dump(output_dikt, output_file) + + +if __name__== "__main__": + if platform.system() == "Darwin": + clang.cindex.Config.set_library_file + ("/Applications/Xcode.app/Contents/Frameworks/libclang.dylib") + main() diff --git a/srcs/dependtool/static_analyser.go b/srcs/dependtool/static_analyser.go index 6790e59..bc90a41 100644 --- a/srcs/dependtool/static_analyser.go +++ b/srcs/dependtool/static_analyser.go @@ -11,6 +11,9 @@ import ( "debug/elf" "fmt" "os" + "os/exec" + "path/filepath" + "strings" u "tools/srcs/common" ) @@ -191,12 +194,106 @@ func executeDependAptCache(programName string, data *u.StaticData, return nil } +// findSourcesFiles puts together all C/C++ source files found in a given application folder. +// +// It returns a slice containing the found source file names and an error if any. Otherwise, it +// returns nil. +func findSourcesFiles(workspace string) ([]string, error) { + + var filenames []string + + err := filepath.Walk(workspace, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + ext := filepath.Ext(info.Name()) + if ext == ".c" || ext == ".cpp" { + filenames = append(filenames, path) + } + return nil + }) + if err != nil { + return nil, err + } + return filenames, nil +} + +// TODO REPLACE +// ExecuteCommand a single command without displaying the output. +// +// It returns a string which represents stdout and an error if any, otherwise +// it returns nil. +func ExecuteCommand(command string, arguments []string) (string, error) { + out, err := exec.Command(command, arguments...).CombinedOutput() + if err != nil { + return "", err + } + return string(out), nil +} + +// addSourceFileSymbols adds all the symbols present in 'output' to the static data field in +// 'data'. +func addSourceFileSymbols(output string, data *u.Data) { + outputTab := strings.Split(output, ",") + + // Get the list of system calls + systemCalls := initSystemCalls() + + for _, s := range outputTab { + if _, isSyscall := systemCalls[s]; isSyscall { + data.StaticData.SystemCalls[s] = systemCalls[s] + } else { + data.StaticData.Symbols[s] = "" + } + } +} + +// extractPrototype executes the parserClang.py script on each source file to extracts all possible +// symbols of each of these files. +// +// It returns an error if any, otherwise it returns nil. +func extractPrototype(sourcesFiltered []string, data *u.Data) error { + + for _, f := range sourcesFiltered { + script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool", + "parserClang.py") + output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f}) + if err != nil { + u.PrintWarning("Incomplete analysis with file " + f) + continue + } + addSourceFileSymbols(output, data) + } + return nil +} + +// gatherSourceFileSymbols gathers symbols of source files from a given application folder. +// +// It returns an error if any, otherwise it returns nil. +func gatherSourceFileSymbols(data *u.Data, programPath string) error { + + tmp := strings.Split(programPath, "/") + folderPath := strings.Join(tmp[:len(tmp)-1], "/") + + files, err := findSourcesFiles(folderPath) + if err != nil { + u.PrintErr(err) + } + + if err := extractPrototype(files, data); err != nil { + u.PrintErr(err) + } + return nil +} + // -------------------------------------Run------------------------------------- // staticAnalyser runs the static analysis to get shared libraries, // system calls and library calls of a given application. -// -func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments, data *u.Data, programPath string) { +func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments, data *u.Data, + programPath string) { programName := *args.StringArg[programArg] fullDeps := *args.BoolArg[fullDepsArg] @@ -240,9 +337,16 @@ func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments } } + // Detect symbols from source files + u.PrintHeader2("(*) Gathering symbols from source files") + if err := gatherSourceFileSymbols(data, programPath); err != nil { + u.PrintWarning(err) + } + // Detect symbols from shared libraries if fullStaticAnalysis && isLinux { - u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary file") + u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary" + + "file") for key, path := range staticData.SharedLibs { if len(path) > 0 { fmt.Printf("\t-> Analysing %s - %s\n", key, path[0]) From b9bcfb971054444f6806022f69fd7465f822c777 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sat, 5 Nov 2022 20:25:13 +0100 Subject: [PATCH 05/28] dependtool first improvement --- srcs/dependtool/static_analyser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/srcs/dependtool/static_analyser.go b/srcs/dependtool/static_analyser.go index bc90a41..8955cd4 100644 --- a/srcs/dependtool/static_analyser.go +++ b/srcs/dependtool/static_analyser.go @@ -196,7 +196,7 @@ func executeDependAptCache(programName string, data *u.StaticData, // findSourcesFiles puts together all C/C++ source files found in a given application folder. // -// It returns a slice containing the found source file names and an error if any. Otherwise, it +// It returns a slice containing the found source file names and an error if any, otherwise it // returns nil. func findSourcesFiles(workspace string) ([]string, error) { From 4fed79c2d8fb129acdfa6ea477d6cfd8001ceb03 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sat, 12 Nov 2022 17:53:40 +0100 Subject: [PATCH 06/28] progamPath bug fix --- srcs/dependtool/static_analyser.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/srcs/dependtool/static_analyser.go b/srcs/dependtool/static_analyser.go index 8955cd4..d465e7c 100644 --- a/srcs/dependtool/static_analyser.go +++ b/srcs/dependtool/static_analyser.go @@ -275,7 +275,13 @@ func extractPrototype(sourcesFiltered []string, data *u.Data) error { func gatherSourceFileSymbols(data *u.Data, programPath string) error { tmp := strings.Split(programPath, "/") - folderPath := strings.Join(tmp[:len(tmp)-1], "/") + i := 2 + for ; i < len(tmp); i++ { + if tmp[len(tmp)-i] == "apps" { + break + } + } + folderPath := strings.Join(tmp[:len(tmp)-i+2], "/") files, err := findSourcesFiles(folderPath) if err != nil { From bb0c63f43d2ad83f0b7d106e371bca0b0a1bde72 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 16 Nov 2022 20:20:20 +0100 Subject: [PATCH 07/28] searchInternalDependencies fix --- srcs/buildtool/run_buildtool.go | 57 ++++++++------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 3d967d1..d00b747 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -253,51 +253,25 @@ func RunBuildTool(homeDir string, data *u.Data) { runMake(programName, appFolder) } -func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, - externalLibs map[string]string) error { - - for _, lib := range *matchedLibs { - // Consider only external libs - if _, ok := externalLibs[lib]; ok { - - // Get and read Config.UK from external lib - configUk := unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk" - lines, err := u.ReadLinesFile(configUk) - if err != nil { - return err - } - - // Process Config.UK file - mapConfig := make(map[string][]string) - u.ProcessConfigUK(lines, false, mapConfig, nil) - - for config := range mapConfig { - - // Remove LIB prefix - if strings.Contains(config, "LIB") { - config = strings.TrimPrefix(config, "LIB") - } - - // Check if matchedLibs already contains the lib - config = strings.ToLower(config) - if !u.Contains(*matchedLibs, config) { - *matchedLibs = append(*matchedLibs, config) - } - } - } +// retFolderCompat modifies its string argument in order to replace its underscore by a dash when +// necessary for the searchInternalDependencies function to find the corresponding folder in the +// 'unikraft' folder. +// +// It returns its string argument whose underscore has been replaced by a dash if necessary, +// otherwise it returns its argument unchanged. +func retFolderForCompat(lib string) string { + if strings.Contains(lib, "posix_") { + folder := strings.ReplaceAll(lib, "posix_", "posix-") + return folder } - return nil + return lib } -/* // This version considers also internal lib dependencies func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, externalLibs map[string]string) error { for _, lib := range *matchedLibs { - if strings.Contains(lib, "_") { - lib = strings.ReplaceAll(lib, "_", "-") - } // Get and read Config.UK from lib var configUk string @@ -305,7 +279,8 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, if _, ok := externalLibs[lib]; ok { configUk = unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk" } else { - configUk = unikraftPath + u.UNIKRAFTFOLDER + "lib" + u.SEP + lib + u.SEP + "Config.uk" + configUk = unikraftPath + u.UNIKRAFTFOLDER + "lib" + u.SEP + retFolderForCompat(lib) + + u.SEP + "Config.uk" } lines, err := u.ReadLinesFile(configUk) @@ -324,11 +299,6 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, config = strings.TrimPrefix(config, "LIB") } - // Replace underscore by dash - //if strings.Contains(config, "_") { - // config = strings.ReplaceAll(config, "_", "-") - //} - // Check if matchedLibs already contains the lib config = strings.ToLower(config) if !u.Contains(*matchedLibs, config) { @@ -339,7 +309,6 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, return nil } -*/ func generateMake(programName, appFolder, workspacePath, makefile string, matchedLibs, sourceFiles []string, externalLibs map[string]string) error { From e7c459d86d0124602ad9c5ae56ccb41696ea56f8 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Thu, 24 Nov 2022 19:18:47 +0100 Subject: [PATCH 08/28] extractertool tar gz files bug fix --- srcs/dependtool/parserClang.py | 3 ++- srcs/extractertool/parserClang.py | 3 ++- srcs/extractertool/run_extractertool.go | 24 +++++++++++++++++++----- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/srcs/dependtool/parserClang.py b/srcs/dependtool/parserClang.py index 502ed2c..1ae84dd 100755 --- a/srcs/dependtool/parserClang.py +++ b/srcs/dependtool/parserClang.py @@ -64,7 +64,8 @@ def check_type_file(filepath, includePaths): cOptions = cOptions + ' ' + includePaths if silent_flag is False: print("Gathering symbols of " + filepath) - if filepath.endswith(".cpp") or filepath.endswith(".hpp"): + if filepath.endswith(".cpp") or filepath.endswith(".hpp") or filepath.endswith(".cc") or \ + filepath.endswith(".hcc"): parse_file(filepath, cplusplusOptions) elif filepath.endswith(".c") or filepath.endswith(".h"): parse_file(filepath, cOptions) diff --git a/srcs/extractertool/parserClang.py b/srcs/extractertool/parserClang.py index 56c1f9d..3d3816b 100755 --- a/srcs/extractertool/parserClang.py +++ b/srcs/extractertool/parserClang.py @@ -64,7 +64,8 @@ def check_type_file(filepath, includePaths): cOptions = cOptions + ' ' + includePaths if silent_flag is False: print("Gathering symbols of " + filepath) - if filepath.endswith(".cpp") or filepath.endswith(".hpp"): + if filepath.endswith(".cpp") or filepath.endswith(".hpp") or filepath.endswith(".cc") or \ + filepath.endswith(".hcc"): parse_file(filepath, cplusplusOptions) elif filepath.endswith(".c") or filepath.endswith(".h"): parse_file(filepath, cOptions) diff --git a/srcs/extractertool/run_extractertool.go b/srcs/extractertool/run_extractertool.go index 56bc79e..2acaf4c 100644 --- a/srcs/extractertool/run_extractertool.go +++ b/srcs/extractertool/run_extractertool.go @@ -103,7 +103,7 @@ func detectURL(mapVariables map[string]*Variables) *string { return nil } -//TODO REPLACE +// TODO REPLACE func CreateFolder(path string) (bool, error) { if _, err := os.Stat(path); os.IsNotExist(err) { if err = os.Mkdir(path, 0755); err != nil { @@ -157,7 +157,7 @@ func findSourcesFiles(workspace string) ([]string, error) { return filenames, nil } -//TODO REPLACE +// TODO REPLACE // ExecuteCommand a single command without displaying the output. // // It returns a string which represents stdout and an error if any, otherwise @@ -261,7 +261,14 @@ func RunExtracterTool(homeDir string) { return } - fileExtension := filepath.Ext(*url) + var fileExtension string + urlSplit := strings.Split(*url, "/") + if urlSplit[len(urlSplit)-1] == "download" { + fileExtension = filepath.Ext(urlSplit[len(urlSplit)-2]) + } else { + fileExtension = filepath.Ext(*url) + } + folderName := lib + "_sources_folder" created, err := CreateFolder(folderName) if err != nil { @@ -269,7 +276,15 @@ func RunExtracterTool(homeDir string) { } var files []string - archiveName := lib + "_sources" + fileExtension + var archiveName string + + if fileExtension == ".gz" { + archiveName = lib + "_sources.tar" + fileExtension + + } else { + archiveName = lib + "_sources" + fileExtension + } + if created { u.PrintInfo(*url + " is found. Download the lib sources...") err := DownloadFile(archiveName, *url) @@ -327,7 +342,6 @@ func RunExtracterTool(homeDir string) { } u.PrintOk(strconv.Itoa(len(mapSymbols)) + " symbols from " + lib + " have been extracted.") - filename := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib) if err := u.RecordDataJson(filename, mf); err != nil { u.PrintErr(err) From 816d1958adfb8d34a1b15ef5ce907e52d6027870 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Thu, 8 Dec 2022 15:21:25 +0100 Subject: [PATCH 09/28] extractertool first improvement --- srcs/buildtool/microlibs_process.go | 17 ++- srcs/buildtool/run_buildtool.go | 3 +- srcs/common/git.go | 2 +- srcs/extractertool/run_extractertool.go | 163 +++++++++++++----------- 4 files changed, 106 insertions(+), 79 deletions(-) diff --git a/srcs/buildtool/microlibs_process.go b/srcs/buildtool/microlibs_process.go index ff8eed9..8f2413f 100644 --- a/srcs/buildtool/microlibs_process.go +++ b/srcs/buildtool/microlibs_process.go @@ -159,6 +159,19 @@ func putJsonSymbolsTogether(data *u.Data) map[string]string { return dataMap } +// retNameCompat modifies its string argument in order to replace its underscore by a dash when +// necessary. +// +// It returns its string argument whose underscore has been replaced by a dash if necessary, +// otherwise it returns its argument unchanged. +func retNameForCompat(value string) string { + if strings.Contains(value, "posix-") { + return strings.ReplaceAll(value, "posix-", "posix_") + } + + return value +} + // matchSymbols performs the matching between Unikraft's micro-libs and // libraries used by a given application based on the list of symbols that both // contain. @@ -169,8 +182,8 @@ func matchSymbols(matchedLibs []string, data map[string]string, for key := range data { if values, ok := microLibs[key]; ok { for _, value := range values { - if !u.Contains(matchedLibs, value) { - matchedLibs = append(matchedLibs, value) + if !u.Contains(matchedLibs, retNameForCompat(value)) { + matchedLibs = append(matchedLibs, retNameForCompat(value)) } } } diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index d00b747..035bd95 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -261,8 +261,7 @@ func RunBuildTool(homeDir string, data *u.Data) { // otherwise it returns its argument unchanged. func retFolderForCompat(lib string) string { if strings.Contains(lib, "posix_") { - folder := strings.ReplaceAll(lib, "posix_", "posix-") - return folder + return strings.ReplaceAll(lib, "posix_", "posix-") } return lib diff --git a/srcs/common/git.go b/srcs/common/git.go index 880c798..5fa8dda 100644 --- a/srcs/common/git.go +++ b/srcs/common/git.go @@ -6,7 +6,7 @@ package common -const branch = "RELEASE-0.7.0" +const branch = "RELEASE-0.10.0" // GitCloneRepository clones a git repository at the the given url. // diff --git a/srcs/extractertool/run_extractertool.go b/srcs/extractertool/run_extractertool.go index 2acaf4c..a1712ba 100644 --- a/srcs/extractertool/run_extractertool.go +++ b/srcs/extractertool/run_extractertool.go @@ -32,7 +32,7 @@ type Variables struct { } func getMakefileSources(content string, mapSources map[string]string) { - var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp`) + var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp|\/.*\.hpp|\/.*\.cc|\/.*\.hcc`) for _, match := range re.FindAllString(content, -1) { vars := strings.Split(match, "/") mapSources[vars[len(vars)-1]] = match @@ -51,7 +51,7 @@ func findVariables(content string, mapVariables map[string]*Variables) { value: "", } - regexVar := regexp.MustCompile("(?m)" + v.name + "=.*$") + regexVar := regexp.MustCompile("(?m)" + v.name + "[ \t]*=.*$") for _, matchVar := range regexVar.FindAllString(content, -1) { v.value = matchVar break @@ -93,7 +93,8 @@ func resolveVariables(mapVariables map[string]*Variables) { func detectURL(mapVariables map[string]*Variables) *string { for key, value := range mapVariables { if strings.Contains(key, URL) && strings.Contains(value.value, HTTP) { - vars := strings.Split(value.value, "=") + spaceDel := strings.Join(strings.Split(value.value, " "), "") + vars := strings.Split(spaceDel, "=") if len(vars) > 1 { return &vars[1] } @@ -146,7 +147,8 @@ func findSourcesFiles(workspace string) ([]string, error) { } ext := filepath.Ext(info.Name()) - if ext == ".c" || ext == ".cpp" { + if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" || + ext == ".hcc" { filenames = append(filenames, path) } return nil @@ -179,7 +181,7 @@ func saveSymbols(output string, mapSymbols map[string]string, libName string) { for _, s := range symbols { if len(s) > 0 { if _, ok := mapSymbols[s]; !ok { - if s == "main" || strings.Contains(s, "test") { + if s == "main" || strings.Contains(s, "test") || strings.Contains(s, "TEST") { u.PrintWarning("Ignore function: " + s) } else { mapSymbols[s] = libName @@ -252,78 +254,81 @@ func RunExtracterTool(homeDir string) { mapSources := make(map[string]string) getMakefileSources(content, mapSources) - findVariables(content, mapVariables) resolveVariables(mapVariables) url := detectURL(mapVariables) - if url == nil { - u.PrintErr(errors.New("url of the lib not found")) - return - } - - var fileExtension string - urlSplit := strings.Split(*url, "/") - if urlSplit[len(urlSplit)-1] == "download" { - fileExtension = filepath.Ext(urlSplit[len(urlSplit)-2]) - } else { - fileExtension = filepath.Ext(*url) - } - folderName := lib + "_sources_folder" - created, err := CreateFolder(folderName) + var archiveName string + var sourcesFiltered []string + + if url != nil { + var fileExtension string + urlSplit := strings.Split(*url, "/") + if urlSplit[len(urlSplit)-1] == "download" { + fileExtension = filepath.Ext(urlSplit[len(urlSplit)-2]) + } else { + fileExtension = filepath.Ext(*url) + } + + created, err := CreateFolder(folderName) + if err != nil { + u.PrintErr(err) + } + + var files []string + + if fileExtension == ".gz" { + archiveName = lib + "_sources.tar" + fileExtension + + } else { + archiveName = lib + "_sources" + fileExtension + } + + if created { + u.PrintInfo(*url + " is found. Download the lib sources...") + err := DownloadFile(archiveName, *url) + if err != nil { + u.PrintErr(err) + } + u.PrintOk(*url + " successfully downloaded.") + + u.PrintInfo("Extracting " + archiveName + "...") + if fileExtension == ".zip" { + files, err = Unzip(archiveName, folderName) + if err != nil { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) + u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") + } + + } else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == ".tgz" { + files, err = unTarGz(archiveName, folderName) + if err != nil { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) + u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") + } + + } else { + u.PrintErr(errors.New("unknown extension for archive")) + } + } + + u.PrintInfo("Inspecting folder " + folderName + " for sources...") + folderFiles, err := findSourcesFiles(folderName) + if err != nil { + u.PrintErr(err) + } + + sourcesFiltered = filterSourcesFiles(files, mapSources) + sourcesFiltered = append(sourcesFiltered, folderFiles...) + } + + libpathFiles, err := findSourcesFiles(libpath) if err != nil { u.PrintErr(err) } - - var files []string - var archiveName string - - if fileExtension == ".gz" { - archiveName = lib + "_sources.tar" + fileExtension - - } else { - archiveName = lib + "_sources" + fileExtension - } - - if created { - u.PrintInfo(*url + " is found. Download the lib sources...") - err := DownloadFile(archiveName, *url) - if err != nil { - u.PrintErr(err) - } - u.PrintOk(*url + " successfully downloaded.") - - u.PrintInfo("Extracting " + archiveName + "...") - if fileExtension == ".zip" { - files, err = Unzip(archiveName, folderName) - if err != nil { - _ = os.Remove(archiveName) - _ = os.RemoveAll(folderName) - u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") - } - - } else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == ".tgz" { - files, err = unTarGz(archiveName, folderName) - if err != nil { - _ = os.Remove(archiveName) - _ = os.RemoveAll(folderName) - u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") - } - - } else { - u.PrintErr(errors.New("unknown extension for archive")) - } - } - - if len(files) == 0 { - u.PrintInfo("Inspecting folder " + folderName + " for sources...") - files, err = findSourcesFiles(folderName) - if err != nil { - u.PrintErr(err) - } - } - - sourcesFiltered := filterSourcesFiles(files, mapSources) + sourcesFiltered = append(sourcesFiltered, libpathFiles...) u.PrintInfo("Find " + strconv.Itoa(len(sourcesFiltered)) + " files to analyse") @@ -342,14 +347,24 @@ func RunExtracterTool(homeDir string) { } u.PrintOk(strconv.Itoa(len(mapSymbols)) + " symbols from " + lib + " have been extracted.") - filename := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib) + + var filename string + if url != nil { + filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib) + + } else { + filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal", lib) + } + if err := u.RecordDataJson(filename, mf); err != nil { u.PrintErr(err) } else { u.PrintOk("Symbols file have been written to " + filename + ".json") } - u.PrintInfo("Remove folders " + archiveName + " and " + folderName) - _ = os.Remove(archiveName) - _ = os.RemoveAll(folderName) + if url != nil { + u.PrintInfo("Remove folders " + archiveName + " and " + folderName) + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) + } } From 53aeeba760a2976a05a460da0114f6acbca8590f Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Mon, 2 Jan 2023 15:58:29 +0100 Subject: [PATCH 10/28] added config files argument --- srcs/buildtool/args.go | 10 +++++++--- srcs/buildtool/run_buildtool.go | 25 ++++++++++++++++++++++++ srcs/buildtool/unikraft_files_process.go | 2 +- srcs/common/arguments.go | 15 ++++++++++---- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/srcs/buildtool/args.go b/srcs/buildtool/args.go index 52c665c..80e77df 100644 --- a/srcs/buildtool/args.go +++ b/srcs/buildtool/args.go @@ -7,9 +7,10 @@ package buildtool import ( - "github.com/akamensky/argparse" "os" u "tools/srcs/common" + + "github.com/akamensky/argparse" ) const ( @@ -18,6 +19,7 @@ const ( sourcesArg = "sources" objsArg = "objects" makefileArg = "makefile" + configArg = "config" ) // ParseArguments parses arguments of the application. @@ -34,11 +36,13 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { &argparse.Options{Required: true, Help: "App Sources " + "Folder"}) args.InitArgParse(p, args, u.BOOL, "o", objsArg, - &argparse.Options{Required: false, Default: false, Help: "Add objects from external build system " + - "Folder"}) + &argparse.Options{Required: false, Default: false, Help: "Add objects from external" + + "build system Folder"}) args.InitArgParse(p, args, u.STRING, "m", makefileArg, &argparse.Options{Required: false, Help: "Add additional properties " + "for Makefile"}) + args.InitArgParse(p, args, u.STRINGLIST, "c", configArg, + &argparse.Options{Required: false, Help: "Add configuration files"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 035bd95..8bfc63f 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -123,6 +123,8 @@ func RunBuildTool(homeDir string, data *u.Data) { u.PrintErr(err) } + fmt.Println(args) + fmt.Println(*args.StringListArg[configArg]) // Get program Name programName := *args.StringArg[programArg] @@ -213,11 +215,33 @@ func RunBuildTool(homeDir string, data *u.Data) { panic(err) } + // Move config files to Unikraft folder + configArg := *args.StringListArg[configArg] + for _, configFilePath := range configArg { + pathSplit := strings.Split(configFilePath, "/") + file := pathSplit[len(pathSplit)-1] + fileSplit := strings.Split(file, ".") + ext := fileSplit[len(fileSplit)-1] + + if ext == "h" || ext == "hpp" || ext == "hcc" { + if err = u.CopyFileContents(configFilePath, *includeFolder+file); err != nil { + u.PrintErr(err) + } + } else if ext == "c" || ext == "cpp" || ext == "cc" { + if err = u.CopyFileContents(configFilePath, appFolder+file); err != nil { + u.PrintErr(err) + } + } else { + u.PrintWarning("Unsupported extension for file: " + file) + } + } + // Match micro-libs matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data) if err != nil { u.PrintErr(err) } + fmt.Println("\nPREFINAL\n") fmt.Println(matchedLibs) // Clone the external git repositories @@ -232,6 +256,7 @@ func RunBuildTool(homeDir string, data *u.Data) { for _, lib := range matchedLibs { u.PrintOk("Match lib: " + lib) } + fmt.Println("\nFINAL\n") fmt.Println(matchedLibs) // Clone the external git repositories (if changed) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 28aa3ef..034260b 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -195,7 +195,7 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil { return err } - } else if extension == ".h" { + } else if extension == ".h" || extension == ".hpp" || extension == ".hcc" { // Add source files to includesFiles list includesFiles = append(includesFiles, info.Name()) diff --git a/srcs/common/arguments.go b/srcs/common/arguments.go index 38e171e..5a9fd84 100644 --- a/srcs/common/arguments.go +++ b/srcs/common/arguments.go @@ -8,9 +8,10 @@ package common import ( "errors" - "github.com/akamensky/argparse" "os" "strings" + + "github.com/akamensky/argparse" ) // Exported constants to determine arguments type. @@ -18,6 +19,7 @@ const ( INT = iota BOOL STRING + STRINGLIST ) // Exported constants to determine which tool is used. @@ -38,9 +40,10 @@ const ( // Exported constants to represent different types of arguments. type Arguments struct { - IntArg map[string]*int - BoolArg map[string]*bool - StringArg map[string]*string + IntArg map[string]*int + BoolArg map[string]*bool + StringArg map[string]*string + StringListArg map[string]*[]string } // InitArguments allows to initialize the parser in order to parse given @@ -52,6 +55,7 @@ func (args *Arguments) InitArguments(name, description string) (*argparse.Parser args.IntArg = make(map[string]*int) args.BoolArg = make(map[string]*bool) args.StringArg = make(map[string]*string) + args.StringListArg = make(map[string]*[]string) p := argparse.NewParser(name, description) @@ -130,5 +134,8 @@ func (*Arguments) InitArgParse(p *argparse.Parser, args *Arguments, typeVar int, case STRING: args.StringArg[long] = new(string) args.StringArg[long] = p.String(short, long, options) + case STRINGLIST: + args.StringListArg[long] = new([]string) + args.StringListArg[long] = p.StringList(short, long, options) } } From 820e00c0bae11be1c06b8dc4d995b9029228b4a2 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Mon, 2 Jan 2023 16:02:41 +0100 Subject: [PATCH 11/28] added config files argument --- srcs/dependtool/static_analyser.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/srcs/dependtool/static_analyser.go b/srcs/dependtool/static_analyser.go index d465e7c..c54ae7f 100644 --- a/srcs/dependtool/static_analyser.go +++ b/srcs/dependtool/static_analyser.go @@ -209,7 +209,8 @@ func findSourcesFiles(workspace string) ([]string, error) { } ext := filepath.Ext(info.Name()) - if ext == ".c" || ext == ".cpp" { + if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" || + ext == ".hcc" { filenames = append(filenames, path) } return nil From 6b5ed45943af022fa8103379c7fa9101ee5708b5 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Mon, 2 Jan 2023 20:23:17 +0100 Subject: [PATCH 12/28] added config files argument --- srcs/dependtool/run_deptool.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/srcs/dependtool/run_deptool.go b/srcs/dependtool/run_deptool.go index 2c87c6c..4041dd5 100644 --- a/srcs/dependtool/run_deptool.go +++ b/srcs/dependtool/run_deptool.go @@ -4,10 +4,11 @@ import ( "debug/elf" "errors" "fmt" - "github.com/fatih/color" "runtime" "strings" u "tools/srcs/common" + + "github.com/fatih/color" ) // RunAnalyserTool allows to run the dependency analyser tool. @@ -222,7 +223,7 @@ func saveGraph(programName, outFolder string, data *u.Data) { programName+"_dependencies", data.StaticData.Dependencies, nil) } - if len(data.StaticData.SharedLibs) > 0 { + if len(data.DynamicData.SharedLibs) > 0 { u.GenerateGraph(programName, outFolder+"dynamic"+u.SEP+ programName+"_shared_libs", data.DynamicData.SharedLibs, nil) } From b5ec3c33293b60fc1decf9e1da0ba45b5eac0e25 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sun, 29 Jan 2023 18:30:39 +0100 Subject: [PATCH 13/28] added interdependence graph generation argument --- srcs/buildtool/run_buildtool.go | 53 ++++----- srcs/dependtool/args.go | 7 +- srcs/dependtool/run_deptool.go | 176 ++++++++++++++++++++++++++++- srcs/dependtool/static_analyser.go | 37 +++--- 4 files changed, 229 insertions(+), 44 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 8bfc63f..8607148 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -7,7 +7,6 @@ package buildtool import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -106,6 +105,32 @@ func parseMakeOutput(output string) string { return sb.String() } +// addConfigFiles adds user-provided configuration files to the app unikernel folder. +func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder, + appFolder string) { + + for _, configFilePath := range configFiles { + configFile := filepath.Base(configFilePath) + fileExt := filepath.Ext(configFile) + + // Copy config file + if fileExt == ".h" || fileExt == ".hpp" || fileExt == ".hcc" { + if err := u.CopyFileContents(configFilePath, includeFolder+configFile); err != nil { + u.PrintErr(err) + } + } else if fileExt == ".c" || fileExt == ".cpp" || fileExt == ".cc" { + if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil { + u.PrintErr(err) + } + + // Add Makefile.uk entry + *selectedFiles = append(*selectedFiles, configFile) + } else { + u.PrintWarning("Unsupported extension for file: " + configFile) + } + } +} + // -------------------------------------Run------------------------------------- // RunBuildTool runs the automatic build tool to build a unikernel of a @@ -123,8 +148,6 @@ func RunBuildTool(homeDir string, data *u.Data) { u.PrintErr(err) } - fmt.Println(args) - fmt.Println(*args.StringListArg[configArg]) // Get program Name programName := *args.StringArg[programArg] @@ -216,25 +239,7 @@ func RunBuildTool(homeDir string, data *u.Data) { } // Move config files to Unikraft folder - configArg := *args.StringListArg[configArg] - for _, configFilePath := range configArg { - pathSplit := strings.Split(configFilePath, "/") - file := pathSplit[len(pathSplit)-1] - fileSplit := strings.Split(file, ".") - ext := fileSplit[len(fileSplit)-1] - - if ext == "h" || ext == "hpp" || ext == "hcc" { - if err = u.CopyFileContents(configFilePath, *includeFolder+file); err != nil { - u.PrintErr(err) - } - } else if ext == "c" || ext == "cpp" || ext == "cc" { - if err = u.CopyFileContents(configFilePath, appFolder+file); err != nil { - u.PrintErr(err) - } - } else { - u.PrintWarning("Unsupported extension for file: " + file) - } - } + addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder) // Match micro-libs matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data) @@ -242,8 +247,6 @@ func RunBuildTool(homeDir string, data *u.Data) { u.PrintErr(err) } - fmt.Println("\nPREFINAL\n") - fmt.Println(matchedLibs) // Clone the external git repositories cloneLibsFolders(workspacePath, matchedLibs, externalLibs) @@ -257,8 +260,6 @@ func RunBuildTool(homeDir string, data *u.Data) { u.PrintOk("Match lib: " + lib) } - fmt.Println("\nFINAL\n") - fmt.Println(matchedLibs) // Clone the external git repositories (if changed) cloneLibsFolders(workspacePath, matchedLibs, externalLibs) diff --git a/srcs/dependtool/args.go b/srcs/dependtool/args.go index 8cba529..f209847 100644 --- a/srcs/dependtool/args.go +++ b/srcs/dependtool/args.go @@ -7,9 +7,10 @@ package dependtool import ( - "github.com/akamensky/argparse" "os" u "tools/srcs/common" + + "github.com/akamensky/argparse" ) const ( @@ -22,6 +23,7 @@ const ( fullDepsArg = "fullDeps" fullStaticAnalysis = "fullStaticAnalysis" typeAnalysis = "typeAnalysis" + interdependArg = "interdepend" ) // parseLocalArguments parses arguments of the application. @@ -53,6 +55,9 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { args.InitArgParse(p, args, u.INT, "", typeAnalysis, &argparse.Options{Required: false, Default: 0, Help: "Kind of analysis (0: both; 1: static; 2: dynamic)"}) + args.InitArgParse(p, args, u.BOOL, "i", interdependArg, + &argparse.Options{Required: false, Default: false, + Help: "Create the source files interdependence graph"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/dependtool/run_deptool.go b/srcs/dependtool/run_deptool.go index 4041dd5..85ee901 100644 --- a/srcs/dependtool/run_deptool.go +++ b/srcs/dependtool/run_deptool.go @@ -4,6 +4,8 @@ import ( "debug/elf" "errors" "fmt" + "os" + "path/filepath" "runtime" "strings" u "tools/srcs/common" @@ -11,6 +13,110 @@ import ( "github.com/fatih/color" ) +// sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file. +// +// It returns a slice containing the found header names. +func sourceFileIncludesAnalysis(sourceFile string) []string { + + var fileIncludes []string + + fileLines, err := u.ReadLinesFile(sourceFile) + if err != nil { + u.PrintErr(err) + } + + for _, line := range fileLines { + if strings.Contains(line, "#include") { + line = strings.ReplaceAll(line, " ", "") + line := strings.Split(line, "#include")[1] + if strings.HasPrefix(line, "\"") { + fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], "\"")+1]) + } else if strings.HasPrefix(line, "<") { + fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], ">")+1]) + } + } + } + + return fileIncludes +} + +// gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using +// the gcc preprocessor. +// +// It returns a slice containing the found header names. +func gccSourceFileIncludesAnalysis(sourceFile, tmpFolder string) []string { + + var fileIncludes []string + + outputStr, _ := ExecuteCommand("gcc", []string{"-E", sourceFile, "-I", tmpFolder}) + outputSlice := strings.Split(outputStr, "\n") + + for _, line := range outputSlice { + + // Only interested in headers not coming from the standard library + if strings.Contains(line, "\""+tmpFolder) { + line = strings.Split(line, "\""+tmpFolder)[1] + includeDirective := line[0:strings.Index(line[0:], "\"")] + if !u.Contains(fileIncludes, includeDirective) { + fileIncludes = append(fileIncludes, includeDirective) + } + } + } + + return fileIncludes +} + +// runInterdependAnalyser collects all the included headers names (i.e., dependencies) from each +// C/C++ source file of a program and builds an interdependence graph (dot file) between all these +// source files. +func runInterdependAnalyser(programPath, programName, outFolder string) { + + // Find all program source files + sourceFiles, err := findSourcesFiles(getProgramFolder(programPath)) + if err != nil { + u.PrintErr(err) + } + + // Create a temporary folder and copy all source files into it for use with the gcc + // preprocessor + tmpFolder := "tmp/" + _, err = u.CreateFolder(tmpFolder) + if err != nil { + u.PrintErr(err) + } + var tmpFiles []string + for _, sourceFilePath := range sourceFiles { + if err := u.CopyFileContents(sourceFilePath, + tmpFolder+filepath.Base(sourceFilePath)); err != nil { + u.PrintErr(err) + } + tmpFiles = append(tmpFiles, tmpFolder+filepath.Base(sourceFilePath)) + } + + // Analyse source files include directives and collect header names. Source files are first + // analysed "by hand" to get all their include directives and then by gcc to make sure to avoid + // directives that are commented or subjected to a macro. + interdependMap := make(map[string][]string) + for _, tmpFile := range tmpFiles { + interdependMap[filepath.Base(tmpFile)] = make([]string, 0) + analysis := sourceFileIncludesAnalysis(tmpFile) + gccAnalysis := gccSourceFileIncludesAnalysis(tmpFile, tmpFolder) + for _, includeDirective := range gccAnalysis { + if u.Contains(analysis, includeDirective) { + interdependMap[filepath.Base(tmpFile)] = + append(interdependMap[filepath.Base(tmpFile)], includeDirective) + } + } + } + + // Create dot file + u.GenerateGraph(programName, outFolder+programName, interdependMap, nil) + + // Remove tmp folder + u.PrintInfo("Remove folder " + tmpFolder) + _ = os.RemoveAll(tmpFolder) +} + // RunAnalyserTool allows to run the dependency analyser tool. func RunAnalyserTool(homeDir string, data *u.Data) { @@ -95,6 +201,11 @@ func RunAnalyserTool(homeDir string, data *u.Data) { if *args.BoolArg[fullDepsArg] { saveGraph(programName, outFolder, data) } + + // Create source files interdependence graph if interdependence option is set + if *args.BoolArg[interdependArg] { + runInterdependAnalyser(programPath, programName, outFolder) + } } // displayProgramDetails display various information such path, background, ... @@ -157,8 +268,8 @@ func checkElf(programPath *string) (*elf.File, bool) { } // runStaticAnalyser runs the static analyser -func runStaticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args *u.Arguments, programName, programPath, - outFolder string, data *u.Data) { +func runStaticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args *u.Arguments, programName, + programPath, outFolder string, data *u.Data) { staticAnalyser(elfFile, isDynamic, isLinux, *args, data, programPath) @@ -228,3 +339,64 @@ func saveGraph(programName, outFolder string, data *u.Data) { programName+"_shared_libs", data.DynamicData.SharedLibs, nil) } } + +/* +// /!\ MISSING "/" !!! +stringFile := "#include\n/* #include ta mère *\nint main() {\n\t// Salut bitch !\n\treturn 0;\n}" + +for { + comStartIndex := strings.Index(stringFile, "/*") + if comStartIndex != -1 { + comEndIndex := strings.Index(stringFile, "*") + stringFile = strings.Join([]string{stringFile[:comStartIndex], + stringFile[comEndIndex+2:]}, "") + } else { + break + } +} +//what to do with "\t" in lines ? +var finalFile []string +sliceFile := strings.Split(stringFile, "\n") +for i := 0; i < len(sliceFile); i++ { + if !strings.HasPrefix(sliceFile[i], "//") { + finalFile = append(finalFile, sliceFile[i]) + } +} +} + +// Remove dependencies whose files are not in program directory (e.g., stdio, stdlib, ...) + for internalFile, dependencies := range interdependMap { + var internalDep []string + for _, dependency := range dependencies { + if _, ok := interdependMap[dependency]; ok { + a++ + internalDep = append(internalDep, dependency) + } + } + interdependMap[internalFile] = internalDep + } + +// Detect and print removable program source files (i.e., files that no other file depends + // on) + var removableFiles []string + for internalFile := range interdependMap { + depends := false + for _, dependencies := range interdependMap { + for _, dependency := range dependencies { + if internalFile == dependency { + depends = true + break + } + } + if depends { + break + } + } + + if !depends { + removableFiles = append(removableFiles, internalFile) + } + } + fmt.Println("Removable program source files of ", programName, ":") + fmt.Println(removableFiles) +*/ diff --git a/srcs/dependtool/static_analyser.go b/srcs/dependtool/static_analyser.go index c54ae7f..c09f630 100644 --- a/srcs/dependtool/static_analyser.go +++ b/srcs/dependtool/static_analyser.go @@ -194,6 +194,25 @@ func executeDependAptCache(programName string, data *u.StaticData, return nil } +// getProgramFolder gets the folder path in which the given program is located, according to the +// Unikraft standard (e.g., /home/.../apps/programFolder/.../program). +// +// It returns the folder containing the program files according to the standard described above. +func getProgramFolder(programPath string) string { + + tmp := strings.Split(programPath, "/") + i := 2 + + for ; i < len(tmp); i++ { + if tmp[len(tmp)-i] == "apps" { + break + } + } + + folderPath := strings.Join(tmp[:len(tmp)-i+2], "/") + return folderPath +} + // findSourcesFiles puts together all C/C++ source files found in a given application folder. // // It returns a slice containing the found source file names and an error if any, otherwise it @@ -228,10 +247,7 @@ func findSourcesFiles(workspace string) ([]string, error) { // it returns nil. func ExecuteCommand(command string, arguments []string) (string, error) { out, err := exec.Command(command, arguments...).CombinedOutput() - if err != nil { - return "", err - } - return string(out), nil + return string(out), err } // addSourceFileSymbols adds all the symbols present in 'output' to the static data field in @@ -275,21 +291,12 @@ func extractPrototype(sourcesFiltered []string, data *u.Data) error { // It returns an error if any, otherwise it returns nil. func gatherSourceFileSymbols(data *u.Data, programPath string) error { - tmp := strings.Split(programPath, "/") - i := 2 - for ; i < len(tmp); i++ { - if tmp[len(tmp)-i] == "apps" { - break - } - } - folderPath := strings.Join(tmp[:len(tmp)-i+2], "/") - - files, err := findSourcesFiles(folderPath) + sourceFiles, err := findSourcesFiles(getProgramFolder(programPath)) if err != nil { u.PrintErr(err) } - if err := extractPrototype(files, data); err != nil { + if err := extractPrototype(sourceFiles, data); err != nil { u.PrintErr(err) } return nil From a4ee7e752696a848db61f4d375715bdf78d44113 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 15 Mar 2023 11:29:53 +0100 Subject: [PATCH 14/28] code reorganisation --- srcs/buildtool/args.go | 17 +- srcs/buildtool/microlibs_process.go | 13 +- srcs/common/data.go | 9 +- srcs/dependtool/args.go | 7 +- srcs/dependtool/interdependence_graph.go | 263 +++++++++++++++++++++++ srcs/dependtool/run_deptool.go | 176 ++++++--------- srcs/dependtool/sources_analyser.go | 119 ++++++++++ 7 files changed, 475 insertions(+), 129 deletions(-) create mode 100644 srcs/dependtool/interdependence_graph.go create mode 100644 srcs/dependtool/sources_analyser.go diff --git a/srcs/buildtool/args.go b/srcs/buildtool/args.go index 80e77df..0791ae1 100644 --- a/srcs/buildtool/args.go +++ b/srcs/buildtool/args.go @@ -14,12 +14,13 @@ import ( ) const ( - programArg = "program" - workspaceArg = "workspace" - sourcesArg = "sources" - objsArg = "objects" - makefileArg = "makefile" - configArg = "config" + programArg = "program" + workspaceArg = "workspace" + sourcesArg = "sources" + objsArg = "objects" + makefileArg = "makefile" + configArg = "config" + interdependArg = "interdepend" ) // ParseArguments parses arguments of the application. @@ -43,6 +44,10 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { "for Makefile"}) args.InitArgParse(p, args, u.STRINGLIST, "c", configArg, &argparse.Options{Required: false, Help: "Add configuration files"}) + args.InitArgParse(p, args, u.BOOL, "i", interdependArg, + &argparse.Options{Required: false, Default: false, + Help: "Use the source files filtered by the deptool interdependence graph to build " + + "the app"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/buildtool/microlibs_process.go b/srcs/buildtool/microlibs_process.go index 8f2413f..8a5ced8 100644 --- a/srcs/buildtool/microlibs_process.go +++ b/srcs/buildtool/microlibs_process.go @@ -133,8 +133,8 @@ func fetchSymbolsExternalLibs(folder string, return externalLibs, nil } -// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static and -// dynamic analyses together into a map structure. +// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static, +// dynamic and source files analyses together into a map structure. // // It returns the map containing all the symbols and system calls. func putJsonSymbolsTogether(data *u.Data) map[string]string { @@ -156,6 +156,14 @@ func putJsonSymbolsTogether(data *u.Data) map[string]string { dataMap[k] = "" } + for k, v := range data.SourcesData.Symbols { + dataMap[k] = v + } + + for k := range data.SourcesData.SystemCalls { + dataMap[k] = "" + } + return dataMap } @@ -219,6 +227,7 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, // Perform the symbol matching matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols) + //matchedLibs = append(matchedLibs, LWIP) return matchedLibs, externalLibs, nil } diff --git a/srcs/common/data.go b/srcs/common/data.go index df1262b..72d6920 100644 --- a/srcs/common/data.go +++ b/srcs/common/data.go @@ -6,10 +6,11 @@ package common -// Exported struct that represents static and dynamic data. +// Exported struct that represents static, dynamic and sources data. type Data struct { StaticData StaticData `json:"static_data"` DynamicData DynamicData `json:"dynamic_data"` + SourcesData SourcesData `json:"sources_data"` } // Exported struct that represents data for static dependency analysis. @@ -26,3 +27,9 @@ type DynamicData struct { SystemCalls map[string]int `json:"system_calls"` Symbols map[string]string `json:"symbols"` } + +// Exported struct that represents data for sources dependency analysis. +type SourcesData struct { + SystemCalls map[string]int `json:"system_calls"` + Symbols map[string]string `json:"symbols"` +} diff --git a/srcs/dependtool/args.go b/srcs/dependtool/args.go index f209847..41bd083 100644 --- a/srcs/dependtool/args.go +++ b/srcs/dependtool/args.go @@ -23,7 +23,6 @@ const ( fullDepsArg = "fullDeps" fullStaticAnalysis = "fullStaticAnalysis" typeAnalysis = "typeAnalysis" - interdependArg = "interdepend" ) // parseLocalArguments parses arguments of the application. @@ -54,10 +53,8 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { Help: "Full static analysis (analyse shared libraries too)"}) args.InitArgParse(p, args, u.INT, "", typeAnalysis, &argparse.Options{Required: false, Default: 0, - Help: "Kind of analysis (0: both; 1: static; 2: dynamic)"}) - args.InitArgParse(p, args, u.BOOL, "i", interdependArg, - &argparse.Options{Required: false, Default: false, - Help: "Create the source files interdependence graph"}) + Help: "Kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: " + + "sources; 5: stripped-down app and json for buildtool)"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/dependtool/interdependence_graph.go b/srcs/dependtool/interdependence_graph.go new file mode 100644 index 0000000..2516351 --- /dev/null +++ b/srcs/dependtool/interdependence_graph.go @@ -0,0 +1,263 @@ +package dependtool + +import ( + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + u "tools/srcs/common" +) + +// ---------------------------------Gather Data--------------------------------- + +// getProgramFolder gets the folder path in which the given program is located, according to the +// Unikraft standard (e.g., /home/.../apps/programFolder/.../program). +// +// It returns the folder containing the program files according to the standard described above. +func getProgramFolder(programPath string) string { + + tmp := strings.Split(programPath, u.SEP) + i := 2 + + for ; i < len(tmp); i++ { + if tmp[len(tmp)-i] == "apps" { + break + } + } + + folderPath := strings.Join(tmp[:len(tmp)-i+2], u.SEP) + return folderPath +} + +// sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file. +// +// It returns a slice containing the found header names. +func sourceFileIncludesAnalysis(sourceFile string) []string { + + var fileIncludes []string + + fileLines, err := u.ReadLinesFile(sourceFile) + if err != nil { + u.PrintErr(err) + } + + for _, line := range fileLines { + if strings.Contains(line, "#include") { + line = strings.ReplaceAll(line, " ", "") + line = strings.Split(line, "#include")[1] + if strings.HasPrefix(line, "\"") { + fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], "\"")+1]) + } else if strings.HasPrefix(line, "<") { + fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], ">")+1]) + } + } + } + + return fileIncludes +} + +// gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using +// the gcc preprocessor. +// +// It returns a slice containing the found header names. +func gccSourceFileIncludesAnalysis(sourceFile, outAppFolder string) ([]string, error) { + + var fileIncludes []string + + // gcc command + outputStr, err := ExecuteCommand("gcc", []string{"-E", sourceFile, "-I", outAppFolder}) + + // if gcc command returns an error, prune file: it contains non-standard libs + if err != nil { + return make([]string, 0), err + } + outputSlice := strings.Split(outputStr, "\n") + + for _, line := range outputSlice { + + // Only interested in headers not coming from the standard library + if strings.Contains(line, "\""+outAppFolder) { + line = strings.Split(line, "\""+outAppFolder)[1] + includeDirective := line[0:strings.Index(line[0:], "\"")] + if !u.Contains(fileIncludes, includeDirective) { + fileIncludes = append(fileIncludes, includeDirective) + } + } + } + + return fileIncludes, nil +} + +// pruneRemovableFiles prunes interdependence graph elements if the latter are unused header files. +func pruneRemovableFiles(interdependMap *map[string][]string) { + + for internalFile := range *interdependMap { + + // No removal of C/C++ source files + if filepath.Ext(internalFile) != ".c" && filepath.Ext(internalFile) != ".cpp" && + filepath.Ext(internalFile) != ".cc" { + + // Lookup for files depending on the current header file + depends := false + for _, dependencies := range *interdependMap { + for _, dependency := range dependencies { + if internalFile == dependency { + depends = true + break + } + } + if depends { + break + } + } + + // Prune header file if unused + if !depends { + delete(*interdependMap, internalFile) + } + } + } +} + +// pruneElemFiles prunes interdependence graph elements if the latter contain the substring in +// argument. +func pruneElemFiles(interdependMap *map[string][]string, pruneElem string) { + + // Lookup for key elements containing the substring and prune them + for internalFile := range *interdependMap { + if strings.Contains(internalFile, pruneElem) { + delete(*interdependMap, internalFile) + + // Lookup for key elements that depend on the key found above and prune them + for file, dependencies := range *interdependMap { + for _, dependency := range dependencies { + if dependency == internalFile { + pruneElemFiles(interdependMap, file) + } + } + } + } + } +} + +// requestUnikraftExtLibs collects all the GitHub repositories of Unikraft through the GitHub API +// and returns the whole list of Unikraft external libraries. +func requestUnikraftExtLibs() []string { + + var extLibsList, appsList []string + + // Only 2 Web pages of repos as for february 2023 (125 repos - 100 repos per page) + nbPages := 2 + + for i := 1; i <= nbPages; i++ { + + // HTTP Get request + resp, err := http.Get("https://api.github.com/orgs/unikraft/repos?page=" + + strconv.Itoa(i) + "&per_page=100") + if err != nil { + u.PrintErr(err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + u.PrintErr(err) + } + + // Collect libs + fileLines := strings.Split(string(body), "\"name\":\"lib-") + for i := 1; i < len(fileLines); i++ { + extLibsList = append(extLibsList, fileLines[i][0:strings.Index(fileLines[i][0:], + "\"")]) + } + + // Collect apps + fileLines = strings.Split(string(body), "\"name\":\"app-") + for i := 1; i < len(fileLines); i++ { + appsList = append(appsList, fileLines[i][0:strings.Index(fileLines[i][0:], + "\"")]) + } + } + + // Avoid libs that are also apps (e.g. nginx, redis) + for i, lib := range extLibsList { + if u.Contains(appsList, lib) { + extLibsList = append(extLibsList[:i], extLibsList[i+1:]...) + } + } + return extLibsList +} + +// -------------------------------------Run------------------------------------- + +// runInterdependAnalyser collects all the included headers names (i.e., dependencies) from each +// C/C++ source file of a program and builds an interdependence graph (dot file) between all these +// source files. +func interdependAnalyser(programPath, programName, outFolder string) string { + + // Find all program source files + sourceFiles, err := findSourcesFiles(getProgramFolder(programPath)) + if err != nil { + u.PrintErr(err) + } + + // Create a folder and copy all source files into it for use with the gcc preprocessor + //tmp := strings.Split(getProgramFolder(programPath), u.SEP) + //outAppFolder := strings.Join(tmp[:len(tmp)-1], u.SEP) + u.SEP + programName + + //"_deptool_output" + u.SEP + outAppFolder := outFolder + programName + u.SEP + _, err = u.CreateFolder(outAppFolder) + if err != nil { + u.PrintErr(err) + } + var outAppFiles []string + for _, sourceFilePath := range sourceFiles { + if err := u.CopyFileContents(sourceFilePath, + outAppFolder+filepath.Base(sourceFilePath)); err != nil { + u.PrintErr(err) + } + outAppFiles = append(outAppFiles, outAppFolder+filepath.Base(sourceFilePath)) + } + + // Analyse source files include directives and collect header names. Source files are first + // analysed by gcc to make sure to avoid directives that are commented or subjected to a macro + // and then "by hand" to sort the include directives of the gcc analysis (i.e., to avoid + // include directives that are not present in the source file currently analysed). + interdependMap := make(map[string][]string) + for _, outAppFile := range outAppFiles { + analysis := sourceFileIncludesAnalysis(outAppFile) + gccAnalysis, err := gccSourceFileIncludesAnalysis(outAppFile, outAppFolder) + if err != nil { + continue + } + + interdependMap[filepath.Base(outAppFile)] = make([]string, 0) + for _, includeDirective := range gccAnalysis { + if u.Contains(analysis, includeDirective) { + interdependMap[filepath.Base(outAppFile)] = + append(interdependMap[filepath.Base(outAppFile)], includeDirective) + } + } + } + + // Prune interdependence graph + extLibsList := requestUnikraftExtLibs() + extLibsList = append(extLibsList, "win32", "test", "TEST") + for _, extLib := range extLibsList { + pruneElemFiles(&interdependMap, extLib) + } + pruneRemovableFiles(&interdependMap) + + // Remove pruned files from the out app folder + for _, outAppFile := range outAppFiles { + if _, ok := interdependMap[filepath.Base(outAppFile)]; !ok { + os.Remove(outAppFile) + } + } + + // Create dot file + u.GenerateGraph(programName, outFolder+programName, interdependMap, nil) + + return outAppFolder +} diff --git a/srcs/dependtool/run_deptool.go b/srcs/dependtool/run_deptool.go index 85ee901..79eb01d 100644 --- a/srcs/dependtool/run_deptool.go +++ b/srcs/dependtool/run_deptool.go @@ -4,8 +4,6 @@ import ( "debug/elf" "errors" "fmt" - "os" - "path/filepath" "runtime" "strings" u "tools/srcs/common" @@ -13,110 +11,6 @@ import ( "github.com/fatih/color" ) -// sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file. -// -// It returns a slice containing the found header names. -func sourceFileIncludesAnalysis(sourceFile string) []string { - - var fileIncludes []string - - fileLines, err := u.ReadLinesFile(sourceFile) - if err != nil { - u.PrintErr(err) - } - - for _, line := range fileLines { - if strings.Contains(line, "#include") { - line = strings.ReplaceAll(line, " ", "") - line := strings.Split(line, "#include")[1] - if strings.HasPrefix(line, "\"") { - fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], "\"")+1]) - } else if strings.HasPrefix(line, "<") { - fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], ">")+1]) - } - } - } - - return fileIncludes -} - -// gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using -// the gcc preprocessor. -// -// It returns a slice containing the found header names. -func gccSourceFileIncludesAnalysis(sourceFile, tmpFolder string) []string { - - var fileIncludes []string - - outputStr, _ := ExecuteCommand("gcc", []string{"-E", sourceFile, "-I", tmpFolder}) - outputSlice := strings.Split(outputStr, "\n") - - for _, line := range outputSlice { - - // Only interested in headers not coming from the standard library - if strings.Contains(line, "\""+tmpFolder) { - line = strings.Split(line, "\""+tmpFolder)[1] - includeDirective := line[0:strings.Index(line[0:], "\"")] - if !u.Contains(fileIncludes, includeDirective) { - fileIncludes = append(fileIncludes, includeDirective) - } - } - } - - return fileIncludes -} - -// runInterdependAnalyser collects all the included headers names (i.e., dependencies) from each -// C/C++ source file of a program and builds an interdependence graph (dot file) between all these -// source files. -func runInterdependAnalyser(programPath, programName, outFolder string) { - - // Find all program source files - sourceFiles, err := findSourcesFiles(getProgramFolder(programPath)) - if err != nil { - u.PrintErr(err) - } - - // Create a temporary folder and copy all source files into it for use with the gcc - // preprocessor - tmpFolder := "tmp/" - _, err = u.CreateFolder(tmpFolder) - if err != nil { - u.PrintErr(err) - } - var tmpFiles []string - for _, sourceFilePath := range sourceFiles { - if err := u.CopyFileContents(sourceFilePath, - tmpFolder+filepath.Base(sourceFilePath)); err != nil { - u.PrintErr(err) - } - tmpFiles = append(tmpFiles, tmpFolder+filepath.Base(sourceFilePath)) - } - - // Analyse source files include directives and collect header names. Source files are first - // analysed "by hand" to get all their include directives and then by gcc to make sure to avoid - // directives that are commented or subjected to a macro. - interdependMap := make(map[string][]string) - for _, tmpFile := range tmpFiles { - interdependMap[filepath.Base(tmpFile)] = make([]string, 0) - analysis := sourceFileIncludesAnalysis(tmpFile) - gccAnalysis := gccSourceFileIncludesAnalysis(tmpFile, tmpFolder) - for _, includeDirective := range gccAnalysis { - if u.Contains(analysis, includeDirective) { - interdependMap[filepath.Base(tmpFile)] = - append(interdependMap[filepath.Base(tmpFile)], includeDirective) - } - } - } - - // Create dot file - u.GenerateGraph(programName, outFolder+programName, interdependMap, nil) - - // Remove tmp folder - u.PrintInfo("Remove folder " + tmpFolder) - _ = os.RemoveAll(tmpFolder) -} - // RunAnalyserTool allows to run the dependency analyser tool. func RunAnalyserTool(homeDir string, data *u.Data) { @@ -136,10 +30,11 @@ func RunAnalyserTool(homeDir string, data *u.Data) { u.PrintErr(err) } - // Get the kind of analysis (0: both; 1: static; 2: dynamic) + // Get the kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: sources; 5: + // stripped-down app and json for buildtool) typeAnalysis := *args.IntArg[typeAnalysis] - if typeAnalysis < 0 || typeAnalysis > 2 { - u.PrintErr(errors.New("analysis argument must be between [0,2]")) + if typeAnalysis < 0 || typeAnalysis > 5 { + u.PrintErr(errors.New("analysis argument must be between [0,5]")) } // Get program path @@ -175,7 +70,8 @@ func RunAnalyserTool(homeDir string, data *u.Data) { // Run static analyser if typeAnalysis == 0 || typeAnalysis == 1 { u.PrintHeader1("(1.1) RUN STATIC ANALYSIS") - runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder, data) + runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder, + data) } // Run dynamic analyser @@ -189,6 +85,24 @@ func RunAnalyserTool(homeDir string, data *u.Data) { } } + // Run interdependence analyser + if typeAnalysis == 0 || typeAnalysis == 3 { + u.PrintHeader1("(1.3) RUN INTERDEPENDENCE ANALYSIS") + _ = runInterdependAnalyser(programPath, programName, outFolder) + } + + // Run sources analyser + if typeAnalysis == 0 || typeAnalysis == 4 { + u.PrintHeader1("(1.4) RUN SOURCES ANALYSIS") + runSourcesAnalyser(getProgramFolder(programPath), data) + } + + // Prepare stripped-down app for buildtool + if typeAnalysis == 5 { + u.PrintHeader1("(1.5) PREPARE STRIPPED-DOWN APP AND JSON FOR BUILDTOOL") + runSourcesAnalyser(runInterdependAnalyser(programPath, programName, outFolder), data) + } + // Save Data to JSON if err = u.RecordDataJson(outFolder+programName, data); err != nil { u.PrintErr(err) @@ -201,11 +115,6 @@ func RunAnalyserTool(homeDir string, data *u.Data) { if *args.BoolArg[fullDepsArg] { saveGraph(programName, outFolder, data) } - - // Create source files interdependence graph if interdependence option is set - if *args.BoolArg[interdependArg] { - runInterdependAnalyser(programPath, programName, outFolder) - } } // displayProgramDetails display various information such path, background, ... @@ -321,6 +230,12 @@ func runDynamicAnalyser(args *u.Arguments, programName, programPath, } } +// runDynamicAnalyser runs the sources analyser. +func runSourcesAnalyser(programPath string, data *u.Data) { + + sourcesAnalyser(data, programPath) +} + // saveGraph saves dependency graphs of a given app into the output folder. func saveGraph(programName, outFolder string, data *u.Data) { @@ -340,6 +255,12 @@ func saveGraph(programName, outFolder string, data *u.Data) { } } +// runInterdependAnalyser runs the interdependence analyser. +func runInterdependAnalyser(programPath, programName, outFolder string) string { + + return interdependAnalyser(programPath, programName, outFolder) +} + /* // /!\ MISSING "/" !!! stringFile := "#include\n/* #include ta mère *\nint main() {\n\t// Salut bitch !\n\treturn 0;\n}" @@ -364,6 +285,7 @@ for i := 0; i < len(sliceFile); i++ { } } + // Remove dependencies whose files are not in program directory (e.g., stdio, stdlib, ...) for internalFile, dependencies := range interdependMap { var internalDep []string @@ -376,6 +298,7 @@ for i := 0; i < len(sliceFile); i++ { interdependMap[internalFile] = internalDep } + // Detect and print removable program source files (i.e., files that no other file depends // on) var removableFiles []string @@ -399,4 +322,27 @@ for i := 0; i < len(sliceFile); i++ { } fmt.Println("Removable program source files of ", programName, ":") fmt.Println(removableFiles) + + +func isADependency(internalFile string, interdependMap* map[string][]string) bool { + for _, dependencies := range *interdependMap { + for _, dependency := range dependencies { + if internalFile == dependency { + return true + } + } + } + return false +} + + +mymap := make(map[string][]string) + mymap["file_win32"] = make([]string, 0) + mymap["file_1"] = []string{"file_win32"} + mymap["file_2"] = []string{"file_1"} + mymap["file_3"] = make([]string, 0) + mymap["file_4"] = []string{"file_3"} + mymap["file_openssl"] = []string{"file_3"} + mymap["file_5"] = []string{"file_openssl"} + mymap["file_6"] = []string{"file_5"} */ diff --git a/srcs/dependtool/sources_analyser.go b/srcs/dependtool/sources_analyser.go new file mode 100644 index 0000000..a842eca --- /dev/null +++ b/srcs/dependtool/sources_analyser.go @@ -0,0 +1,119 @@ +package dependtool + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + u "tools/srcs/common" +) + +// ---------------------------------Gather Data--------------------------------- + +// findSourcesFiles puts together all C/C++ source files found in a given application folder. +// +// It returns a slice containing the found source file names and an error if any, otherwise it +// returns nil. +func findSourcesFiles(workspace string) ([]string, error) { + + var filenames []string + + err := filepath.Walk(workspace, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + ext := filepath.Ext(info.Name()) + if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" || + ext == ".hcc" { + filenames = append(filenames, path) + } + return nil + }) + if err != nil { + return nil, err + } + return filenames, nil +} + +// TODO REPLACE +// ExecuteCommand a single command without displaying the output. +// +// It returns a string which represents stdout and an error if any, otherwise +// it returns nil. +func ExecuteCommand(command string, arguments []string) (string, error) { + out, err := exec.Command(command, arguments...).CombinedOutput() + return string(out), err +} + +// addSourceFileSymbols adds all the symbols present in 'output' to the static data field in +// 'data'. +func addSourceFileSymbols(output string, data *u.SourcesData) { + outputTab := strings.Split(output, ",") + + // Get the list of system calls + systemCalls := initSystemCalls() + + for _, s := range outputTab { + if _, isSyscall := systemCalls[s]; isSyscall { + data.SystemCalls[s] = systemCalls[s] + } else { + data.Symbols[s] = "" + } + } +} + +// extractPrototype executes the parserClang.py script on each source file to extracts all possible +// symbols of each of these files. +// +// It returns an error if any, otherwise it returns nil. +func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error { + + for _, f := range sourcesFiltered { + script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool", + "parserClang.py") + output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f}) + if err != nil { + u.PrintWarning("Incomplete analysis with file " + f) + continue + } + addSourceFileSymbols(output, data) + } + return nil +} + +// gatherSourceFileSymbols gathers symbols of source files from a given application folder. +// +// It returns an error if any, otherwise it returns nil. +func gatherSourceFileSymbols(data *u.SourcesData, programPath string) error { + + sourceFiles, err := findSourcesFiles(programPath) + if err != nil { + u.PrintErr(err) + } + + if err := extractPrototype(sourceFiles, data); err != nil { + u.PrintErr(err) + } + return nil +} + +// -------------------------------------Run------------------------------------- + +// staticAnalyser runs the static analysis to get system calls and library calls of a given +// application. +func sourcesAnalyser(data *u.Data, programPath string) { + + sourcesData := &data.SourcesData + + // Init symbols members + sourcesData.Symbols = make(map[string]string) + sourcesData.SystemCalls = make(map[string]int) + + // Detect symbols from source files + u.PrintHeader2("(*) Gathering symbols from source files") + if err := gatherSourceFileSymbols(sourcesData, programPath); err != nil { + u.PrintWarning(err) + } +} From 5338fc37edfb3e5f7d5682209751f7cecc31037c Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 15 Mar 2023 11:32:58 +0100 Subject: [PATCH 15/28] code reorganisation --- srcs/dependtool/static_analyser.go | 117 ----------------------------- 1 file changed, 117 deletions(-) diff --git a/srcs/dependtool/static_analyser.go b/srcs/dependtool/static_analyser.go index c09f630..0445c98 100644 --- a/srcs/dependtool/static_analyser.go +++ b/srcs/dependtool/static_analyser.go @@ -11,9 +11,6 @@ import ( "debug/elf" "fmt" "os" - "os/exec" - "path/filepath" - "strings" u "tools/srcs/common" ) @@ -194,114 +191,6 @@ func executeDependAptCache(programName string, data *u.StaticData, return nil } -// getProgramFolder gets the folder path in which the given program is located, according to the -// Unikraft standard (e.g., /home/.../apps/programFolder/.../program). -// -// It returns the folder containing the program files according to the standard described above. -func getProgramFolder(programPath string) string { - - tmp := strings.Split(programPath, "/") - i := 2 - - for ; i < len(tmp); i++ { - if tmp[len(tmp)-i] == "apps" { - break - } - } - - folderPath := strings.Join(tmp[:len(tmp)-i+2], "/") - return folderPath -} - -// findSourcesFiles puts together all C/C++ source files found in a given application folder. -// -// It returns a slice containing the found source file names and an error if any, otherwise it -// returns nil. -func findSourcesFiles(workspace string) ([]string, error) { - - var filenames []string - - err := filepath.Walk(workspace, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - ext := filepath.Ext(info.Name()) - if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" || - ext == ".hcc" { - filenames = append(filenames, path) - } - return nil - }) - if err != nil { - return nil, err - } - return filenames, nil -} - -// TODO REPLACE -// ExecuteCommand a single command without displaying the output. -// -// It returns a string which represents stdout and an error if any, otherwise -// it returns nil. -func ExecuteCommand(command string, arguments []string) (string, error) { - out, err := exec.Command(command, arguments...).CombinedOutput() - return string(out), err -} - -// addSourceFileSymbols adds all the symbols present in 'output' to the static data field in -// 'data'. -func addSourceFileSymbols(output string, data *u.Data) { - outputTab := strings.Split(output, ",") - - // Get the list of system calls - systemCalls := initSystemCalls() - - for _, s := range outputTab { - if _, isSyscall := systemCalls[s]; isSyscall { - data.StaticData.SystemCalls[s] = systemCalls[s] - } else { - data.StaticData.Symbols[s] = "" - } - } -} - -// extractPrototype executes the parserClang.py script on each source file to extracts all possible -// symbols of each of these files. -// -// It returns an error if any, otherwise it returns nil. -func extractPrototype(sourcesFiltered []string, data *u.Data) error { - - for _, f := range sourcesFiltered { - script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool", - "parserClang.py") - output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f}) - if err != nil { - u.PrintWarning("Incomplete analysis with file " + f) - continue - } - addSourceFileSymbols(output, data) - } - return nil -} - -// gatherSourceFileSymbols gathers symbols of source files from a given application folder. -// -// It returns an error if any, otherwise it returns nil. -func gatherSourceFileSymbols(data *u.Data, programPath string) error { - - sourceFiles, err := findSourcesFiles(getProgramFolder(programPath)) - if err != nil { - u.PrintErr(err) - } - - if err := extractPrototype(sourceFiles, data); err != nil { - u.PrintErr(err) - } - return nil -} - // -------------------------------------Run------------------------------------- // staticAnalyser runs the static analysis to get shared libraries, @@ -351,12 +240,6 @@ func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments } } - // Detect symbols from source files - u.PrintHeader2("(*) Gathering symbols from source files") - if err := gatherSourceFileSymbols(data, programPath); err != nil { - u.PrintWarning(err) - } - // Detect symbols from shared libraries if fullStaticAnalysis && isLinux { u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary" + From eb2a2c9d3a4eab70b0569ea51fa956ff8967caeb Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 15 Mar 2023 13:02:57 +0100 Subject: [PATCH 16/28] code reorganisation --- srcs/buildtool/microlibs_process.go | 1 - 1 file changed, 1 deletion(-) diff --git a/srcs/buildtool/microlibs_process.go b/srcs/buildtool/microlibs_process.go index 8a5ced8..43a01c4 100644 --- a/srcs/buildtool/microlibs_process.go +++ b/srcs/buildtool/microlibs_process.go @@ -227,7 +227,6 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, // Perform the symbol matching matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols) - //matchedLibs = append(matchedLibs, LWIP) return matchedLibs, externalLibs, nil } From 428780009e39d830cafdd2910f3778c4055e7190 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Thu, 23 Mar 2023 21:39:05 +0100 Subject: [PATCH 17/28] add extractertool tar xz files extraction added --- srcs/buildtool/args.go | 17 +++---- srcs/dependtool/interdependence_graph.go | 11 +++++ srcs/dependtool/sources_analyser.go | 13 +---- srcs/extractertool/run_extractertool.go | 61 ++++++++++++++---------- 4 files changed, 53 insertions(+), 49 deletions(-) diff --git a/srcs/buildtool/args.go b/srcs/buildtool/args.go index 0791ae1..80e77df 100644 --- a/srcs/buildtool/args.go +++ b/srcs/buildtool/args.go @@ -14,13 +14,12 @@ import ( ) const ( - programArg = "program" - workspaceArg = "workspace" - sourcesArg = "sources" - objsArg = "objects" - makefileArg = "makefile" - configArg = "config" - interdependArg = "interdepend" + programArg = "program" + workspaceArg = "workspace" + sourcesArg = "sources" + objsArg = "objects" + makefileArg = "makefile" + configArg = "config" ) // ParseArguments parses arguments of the application. @@ -44,10 +43,6 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { "for Makefile"}) args.InitArgParse(p, args, u.STRINGLIST, "c", configArg, &argparse.Options{Required: false, Help: "Add configuration files"}) - args.InitArgParse(p, args, u.BOOL, "i", interdependArg, - &argparse.Options{Required: false, Default: false, - Help: "Use the source files filtered by the deptool interdependence graph to build " + - "the app"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/dependtool/interdependence_graph.go b/srcs/dependtool/interdependence_graph.go index 2516351..e9fed66 100644 --- a/srcs/dependtool/interdependence_graph.go +++ b/srcs/dependtool/interdependence_graph.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "net/http" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -58,6 +59,16 @@ func sourceFileIncludesAnalysis(sourceFile string) []string { return fileIncludes } +// TODO REPLACE +// ExecuteCommand a single command without displaying the output. +// +// It returns a string which represents stdout and an error if any, otherwise +// it returns nil. +func ExecuteCommand(command string, arguments []string) (string, error) { + out, err := exec.Command(command, arguments...).CombinedOutput() + return string(out), err +} + // gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using // the gcc preprocessor. // diff --git a/srcs/dependtool/sources_analyser.go b/srcs/dependtool/sources_analyser.go index a842eca..68b7c43 100644 --- a/srcs/dependtool/sources_analyser.go +++ b/srcs/dependtool/sources_analyser.go @@ -2,7 +2,6 @@ package dependtool import ( "os" - "os/exec" "path/filepath" "strings" u "tools/srcs/common" @@ -37,16 +36,6 @@ func findSourcesFiles(workspace string) ([]string, error) { return filenames, nil } -// TODO REPLACE -// ExecuteCommand a single command without displaying the output. -// -// It returns a string which represents stdout and an error if any, otherwise -// it returns nil. -func ExecuteCommand(command string, arguments []string) (string, error) { - out, err := exec.Command(command, arguments...).CombinedOutput() - return string(out), err -} - // addSourceFileSymbols adds all the symbols present in 'output' to the static data field in // 'data'. func addSourceFileSymbols(output string, data *u.SourcesData) { @@ -73,7 +62,7 @@ func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error { for _, f := range sourcesFiltered { script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool", "parserClang.py") - output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f}) + output, err := u.ExecuteCommand("python3", []string{script, "-q", "-t", f}) if err != nil { u.PrintWarning("Incomplete analysis with file " + f) continue diff --git a/srcs/extractertool/run_extractertool.go b/srcs/extractertool/run_extractertool.go index a1712ba..6dfd731 100644 --- a/srcs/extractertool/run_extractertool.go +++ b/srcs/extractertool/run_extractertool.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "net/http" "os" - "os/exec" "path/filepath" "regexp" "strconv" @@ -159,19 +158,6 @@ func findSourcesFiles(workspace string) ([]string, error) { return filenames, nil } -// TODO REPLACE -// ExecuteCommand a single command without displaying the output. -// -// It returns a string which represents stdout and an error if any, otherwise -// it returns nil. -func ExecuteCommand(command string, arguments []string) (string, error) { - out, err := exec.Command(command, arguments...).CombinedOutput() - if err != nil { - return "", err - } - return string(out), nil -} - func saveSymbols(output string, mapSymbols map[string]string, libName string) { if strings.Contains(output, "\n") { output = strings.TrimSuffix(output, "\n") @@ -191,11 +177,13 @@ func saveSymbols(output string, mapSymbols map[string]string, libName string) { } } -func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string, libName string) error { +func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string, + libName string) error { for _, f := range sourcesFiltered { - script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool", "parserClang.py") - output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f}) + script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool", + "parserClang.py") + output, err := u.ExecuteCommand("python3", []string{script, "-q", "-t", f}) if err != nil { u.PrintWarning("Incomplete analysis with file " + f) continue @@ -224,7 +212,8 @@ func RunExtracterTool(homeDir string) { // Init and parse local arguments args := new(u.Arguments) p, err := args.InitArguments("--extracter", - "The extracter tool allows to extract all the symbols (functions) of an external/internal library") + "The extracter tool allows to extract all the symbols (functions) of an "+ + "external/internal library") if err != nil { u.PrintErr(err) } @@ -277,7 +266,7 @@ func RunExtracterTool(homeDir string) { var files []string - if fileExtension == ".gz" { + if fileExtension == ".gz" || fileExtension == ".xz" { archiveName = lib + "_sources.tar" + fileExtension } else { @@ -288,6 +277,7 @@ func RunExtracterTool(homeDir string) { u.PrintInfo(*url + " is found. Download the lib sources...") err := DownloadFile(archiveName, *url) if err != nil { + _ = os.RemoveAll(folderName) u.PrintErr(err) } u.PrintOk(*url + " successfully downloaded.") @@ -301,7 +291,8 @@ func RunExtracterTool(homeDir string) { u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") } - } else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == ".tgz" { + } else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == + ".tgz" { files, err = unTarGz(archiveName, folderName) if err != nil { _ = os.Remove(archiveName) @@ -309,7 +300,18 @@ func RunExtracterTool(homeDir string) { u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") } + } else if fileExtension == ".tar" || fileExtension == ".xz" || fileExtension == + ".txz" { + _, err := u.ExecuteCommand("tar", []string{"-xf", archiveName, "-C", folderName}) + if err != nil { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) + u.PrintErr(err.Error() + ". Corrupted archive. Please try again.") + } + } else { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) u.PrintErr(errors.New("unknown extension for archive")) } } @@ -317,6 +319,8 @@ func RunExtracterTool(homeDir string) { u.PrintInfo("Inspecting folder " + folderName + " for sources...") folderFiles, err := findSourcesFiles(folderName) if err != nil { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) u.PrintErr(err) } @@ -326,6 +330,10 @@ func RunExtracterTool(homeDir string) { libpathFiles, err := findSourcesFiles(libpath) if err != nil { + if url != nil { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) + } u.PrintErr(err) } sourcesFiltered = append(sourcesFiltered, libpathFiles...) @@ -335,6 +343,10 @@ func RunExtracterTool(homeDir string) { mapSymbols := make(map[string]string) u.PrintInfo("Extracting symbols from all sources of " + lib + ". This may take some times...") if err := extractPrototype(sourcesFiltered, mapSymbols, lib); err != nil { + if url != nil { + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) + } u.PrintErr(err) } @@ -350,6 +362,9 @@ func RunExtracterTool(homeDir string) { var filename string if url != nil { + u.PrintInfo("Remove folders " + archiveName + " and " + folderName) + _ = os.Remove(archiveName) + _ = os.RemoveAll(folderName) filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib) } else { @@ -361,10 +376,4 @@ func RunExtracterTool(homeDir string) { } else { u.PrintOk("Symbols file have been written to " + filename + ".json") } - - if url != nil { - u.PrintInfo("Remove folders " + archiveName + " and " + folderName) - _ = os.Remove(archiveName) - _ = os.RemoveAll(folderName) - } } From f460bc7f1a4bda0706982639a2064f223fa1f3ac Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sat, 1 Apr 2023 19:09:14 +0200 Subject: [PATCH 18/28] include directive paths problem fix --- srcs/buildtool/run_buildtool.go | 5 ++- srcs/buildtool/unikraft_files_process.go | 49 ++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 8607148..55e615e 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -14,7 +14,7 @@ import ( "strings" u "tools/srcs/common" - "github.com/AlecAivazis/survey/v2" + "gopkg.in/AlecAivazis/survey.v1" ) // STATES @@ -234,7 +234,7 @@ func RunBuildTool(homeDir string, data *u.Data) { } var selectedFiles []string - if err := survey.AskOne(prompt, &selectedFiles); err != nil { + if err := survey.AskOne(prompt, &selectedFiles, nil); err != nil { panic(err) } @@ -277,6 +277,7 @@ func RunBuildTool(homeDir string, data *u.Data) { // Run make runMake(programName, appFolder) + } // retFolderCompat modifies its string argument in order to replace its underscore by a dash when diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 034260b..611db21 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -12,6 +12,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" u "tools/srcs/common" @@ -175,6 +176,49 @@ func filterSourcesFiles(sourceFiles []string) []string { return filterSrcFiles } +// conformIncDirAndCopyFile conforms all the include directives from a C/C++ source file so that no +// directive contains a path to a header file but the header file name only (i.e., the last element +// of the path). It also copies the content of the source file in the same way as CopyFileContents. +// +// It returns an error if any, otherwise it returns nil. +func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { + + fileLines, err := u.ReadLinesFile(sourcePath) + if err != nil { + return err + } + + // Find include directives using regexp + var re = regexp.MustCompile(`(.*)(#include)(.*)(<|")(.*)(>|")(.*)`) + + for index := range fileLines { + for _, match := range re.FindAllStringSubmatch(fileLines[index], -1) { + + // Only interested in include directives containing a path to a header file + if !strings.Contains(match[0], "/") { + continue + } + + // Replace the path by its last element + for i := 1; i < len(match); i++ { + if match[i] == "<" || match[i] == "\"" { + match[i+1] = filepath.Base(match[i+1]) + fileLines[index] = strings.Join(match[1:], "") + "\n" + break + } + } + } + } + + // Write the modified content to a file in the unikernel folder + err = u.WriteToFile(destPath, []byte(strings.Join(fileLines, ""))) + if err != nil { + return err + } + + return nil +} + func processSourceFiles(sourcesPath, appFolder, includeFolder string, sourceFiles, includesFiles []string) ([]string, error) { @@ -182,7 +226,6 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, err error) error { if !info.IsDir() { - extension := filepath.Ext(info.Name()) if _, ok := srcLanguages[extension]; ok { // Add source files to sourceFiles list @@ -192,7 +235,7 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, srcLanguages[extension] += 1 // Copy source files to the appFolder - if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil { + if err = conformIncDirAndCopyFile(path, appFolder+info.Name()); err != nil { return err } } else if extension == ".h" || extension == ".hpp" || extension == ".hcc" { @@ -200,7 +243,7 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, includesFiles = append(includesFiles, info.Name()) // Copy header files to the INCLUDEFOLDER - if err = u.CopyFileContents(path, includeFolder+info.Name()); err != nil { + if err = conformIncDirAndCopyFile(path, includeFolder+info.Name()); err != nil { return err } } else { From 6768e1d930467597eb3b5dbaeb53e46de15f49a8 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 5 Apr 2023 19:26:29 +0200 Subject: [PATCH 19/28] include directive paths problem fix 2 --- srcs/buildtool/unikraft_files_process.go | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 611db21..84f4fc1 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -156,11 +156,11 @@ var srcLanguages = map[string]int{ ".c": 0, ".cpp": 0, ".cc": 0, - ".S": 0, - ".s": 0, - ".asm": 0, - ".py": 0, - ".go": 0, + //".S": 0, + //".s": 0, + //".asm": 0, + //".py": 0, + //".go": 0, } func filterSourcesFiles(sourceFiles []string) []string { @@ -176,9 +176,10 @@ func filterSourcesFiles(sourceFiles []string) []string { return filterSrcFiles } -// conformIncDirAndCopyFile conforms all the include directives from a C/C++ source file so that no -// directive contains a path to a header file but the header file name only (i.e., the last element -// of the path). It also copies the content of the source file in the same way as CopyFileContents. +// conformIncDirAndCopyFile conforms all the user-defined include directives from a C/C++ source +// file so that none of these directives contains a path to a header file but the header file name +// only (i.e., the last element of the path). It also copies the content of the source file in the +// same way as CopyFileContents. // // It returns an error if any, otherwise it returns nil. func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { @@ -188,20 +189,20 @@ func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { return err } - // Find include directives using regexp - var re = regexp.MustCompile(`(.*)(#include)(.*)(<|")(.*)(>|")(.*)`) + // Find user-defined include directives using regexp + var re = regexp.MustCompile(`(.*)(#include)(.*)(")(.*)(")(.*)`) for index := range fileLines { for _, match := range re.FindAllStringSubmatch(fileLines[index], -1) { - // Only interested in include directives containing a path to a header file + // Only interested in user-defined directives containing a path to a header file if !strings.Contains(match[0], "/") { continue } // Replace the path by its last element for i := 1; i < len(match); i++ { - if match[i] == "<" || match[i] == "\"" { + if match[i] == "\"" { match[i+1] = filepath.Base(match[i+1]) fileLines[index] = strings.Join(match[1:], "") + "\n" break From 302af1e1a246581eb8130b0d9cc90b8875a4a8b9 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 19 Apr 2023 09:57:13 +0200 Subject: [PATCH 20/28] include directive paths problem fix 3 --- srcs/buildtool/unikraft_files_process.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 84f4fc1..ffb5756 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -195,15 +195,10 @@ func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { for index := range fileLines { for _, match := range re.FindAllStringSubmatch(fileLines[index], -1) { - // Only interested in user-defined directives containing a path to a header file - if !strings.Contains(match[0], "/") { - continue - } - // Replace the path by its last element for i := 1; i < len(match); i++ { if match[i] == "\"" { - match[i+1] = filepath.Base(match[i+1]) + match[i+1] = "include/" + filepath.Base(match[i+1]) fileLines[index] = strings.Join(match[1:], "") + "\n" break } @@ -244,7 +239,7 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, includesFiles = append(includesFiles, info.Name()) // Copy header files to the INCLUDEFOLDER - if err = conformIncDirAndCopyFile(path, includeFolder+info.Name()); err != nil { + if err = u.CopyFileContents(path, includeFolder+info.Name()); err != nil { return err } } else { From 4ef09a6b2b96a37c137e72054c3a1fe14b38bb22 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sun, 23 Apr 2023 18:52:33 +0200 Subject: [PATCH 21/28] include directive paths problem fix 4 --- srcs/buildtool/run_buildtool.go | 6 ++- srcs/buildtool/unikraft_files_process.go | 51 +++++++++++++++++++----- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 55e615e..eb9f49a 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -222,7 +222,7 @@ func RunBuildTool(homeDir string, data *u.Data) { } // Filter source files to limit build errors (e.g., remove test files, - //multiple main file, ...) + // multiple main file, ...) filterSourceFiles := filterSourcesFiles(sourceFiles) // Prompt file selection @@ -238,6 +238,10 @@ func RunBuildTool(homeDir string, data *u.Data) { panic(err) } + if err := conformIncludeDirectives(appFolder); err != nil { + u.PrintErr(err) + } + // Move config files to Unikraft folder addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index ffb5756..8bccaf7 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -176,15 +176,41 @@ func filterSourcesFiles(sourceFiles []string) []string { return filterSrcFiles } -// conformIncDirAndCopyFile conforms all the user-defined include directives from a C/C++ source -// file so that none of these directives contains a path to a header file but the header file name -// only (i.e., the last element of the path). It also copies the content of the source file in the -// same way as CopyFileContents. +func conformIncludeDirectives(sourcesPath string) error { + err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo, + err error) error { + if !info.IsDir() { + extension := filepath.Ext(info.Name()) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + if err = conformFile(path, true); err != nil { + return err + } + } else if extension == ".c" || extension == ".cpp" || extension == ".cc" { + if err = conformFile(path, false); err != nil { + return err + } + } else { + u.PrintWarning("Unsupported extension for file: " + info.Name()) + } + } + return nil + }) + + if err != nil { + return err + } + return nil +} + +// conformFile conforms all the user-defined include directives from a C/C++ source file so that +// none of these directives contains a path to a header file but the header file name only (i.e., +// the last element of the path). It also copies the content of the source file in the same way as +// CopyFileContents. // // It returns an error if any, otherwise it returns nil. -func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { +func conformFile(path string, isHeader bool) (err error) { - fileLines, err := u.ReadLinesFile(sourcePath) + fileLines, err := u.ReadLinesFile(path) if err != nil { return err } @@ -198,7 +224,11 @@ func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { // Replace the path by its last element for i := 1; i < len(match); i++ { if match[i] == "\"" { - match[i+1] = "include/" + filepath.Base(match[i+1]) + if isHeader { + match[i+1] = filepath.Base(match[i+1]) + } else { + match[i+1] = "include" + u.SEP + filepath.Base(match[i+1]) + } fileLines[index] = strings.Join(match[1:], "") + "\n" break } @@ -207,7 +237,7 @@ func conformIncDirAndCopyFile(sourcePath, destPath string) (err error) { } // Write the modified content to a file in the unikernel folder - err = u.WriteToFile(destPath, []byte(strings.Join(fileLines, ""))) + err = u.WriteToFile(path, []byte(strings.Join(fileLines, ""))) if err != nil { return err } @@ -231,7 +261,10 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, srcLanguages[extension] += 1 // Copy source files to the appFolder - if err = conformIncDirAndCopyFile(path, appFolder+info.Name()); err != nil { + //if err = conformIncDirAndCopyFile(path, appFolder+info.Name()); err != nil { + // return err + //} + if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil { return err } } else if extension == ".h" || extension == ".hpp" || extension == ".hcc" { From 5664ad23ca084f78fd1b3d5b3428f013b572880f Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sun, 23 Apr 2023 19:38:53 +0200 Subject: [PATCH 22/28] include directive paths problem fix 5 --- srcs/buildtool/run_buildtool.go | 3 ++- srcs/buildtool/unikraft_files_process.go | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index eb9f49a..3fe54e3 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -238,11 +238,12 @@ func RunBuildTool(homeDir string, data *u.Data) { panic(err) } + // Conform file include directives to the new unikernel folder organisation if err := conformIncludeDirectives(appFolder); err != nil { u.PrintErr(err) } - // Move config files to Unikraft folder + // Move config files to the unikernel folder addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder) // Match micro-libs diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 8bccaf7..ccf028c 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -125,7 +125,7 @@ func handleCreationApp(appFolder string) string { fmt.Println("Make your choice:\n1: Copy and overwrite files\n2: " + "Enter manually the name of the folder\n3: exit program") var input int - for true { + for { fmt.Print("Please enter your choice (0 to exit): ") if _, err := fmt.Scanf("%d", &input); err != nil { u.PrintWarning("Choice must be numeric! Try again") @@ -146,8 +146,6 @@ func handleCreationApp(appFolder string) string { } } } - - return appFolder } // -------------------------MOVE FILES TO APP FOLDER---------------------------- @@ -176,9 +174,16 @@ func filterSourcesFiles(sourceFiles []string) []string { return filterSrcFiles } +// conformIncludeDirectives conforms all the user-defined include directives of all C/C++ source +// files to the unikernel directory format so that all these directives are paths to header files +// located in the include folder of the unikernel directory. +// +// It returns an error if any, otherwise it returns nil. func conformIncludeDirectives(sourcesPath string) error { + err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { extension := filepath.Ext(info.Name()) if extension == ".h" || extension == ".hpp" || extension == ".hcc" { @@ -202,10 +207,9 @@ func conformIncludeDirectives(sourcesPath string) error { return nil } -// conformFile conforms all the user-defined include directives from a C/C++ source file so that -// none of these directives contains a path to a header file but the header file name only (i.e., -// the last element of the path). It also copies the content of the source file in the same way as -// CopyFileContents. +// conformFile conforms all the user-defined include directives of a C/C++ source file to the +// unikernel directory format so that all these directives are paths to header files located in the +// include folder. // // It returns an error if any, otherwise it returns nil. func conformFile(path string, isHeader bool) (err error) { @@ -261,9 +265,6 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, srcLanguages[extension] += 1 // Copy source files to the appFolder - //if err = conformIncDirAndCopyFile(path, appFolder+info.Name()); err != nil { - // return err - //} if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil { return err } From bac05d60b8e9059d650973717f28b357d1c6779c Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Tue, 2 May 2023 13:09:56 +0200 Subject: [PATCH 23/28] added patch files argument --- srcs/buildtool/args.go | 3 + srcs/buildtool/run_buildtool.go | 17 ++- srcs/buildtool/unikraft_files_process.go | 161 +++++++++++++++++++++-- srcs/common/git.go | 2 +- 4 files changed, 167 insertions(+), 16 deletions(-) diff --git a/srcs/buildtool/args.go b/srcs/buildtool/args.go index 80e77df..8b3e13e 100644 --- a/srcs/buildtool/args.go +++ b/srcs/buildtool/args.go @@ -20,6 +20,7 @@ const ( objsArg = "objects" makefileArg = "makefile" configArg = "config" + patchArg = "patch" ) // ParseArguments parses arguments of the application. @@ -43,6 +44,8 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { "for Makefile"}) args.InitArgParse(p, args, u.STRINGLIST, "c", configArg, &argparse.Options{Required: false, Help: "Add configuration files"}) + args.InitArgParse(p, args, u.STRING, "", patchArg, + &argparse.Options{Required: false, Help: "Add patch files"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 3fe54e3..d1d2d39 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -238,6 +238,19 @@ func RunBuildTool(homeDir string, data *u.Data) { panic(err) } + // Copy, conform and apply user-provided patches to the new unikernel folder + if len(*args.StringArg[patchArg]) > 0 { + // Create patch folder + patchFolder, err := createPatchFolder(appFolder) + if err != nil { + u.PrintErr(err) + } + if err := addAndApplyPatchFiles(*args.StringArg[patchArg], *patchFolder, appFolder); err != + nil { + u.PrintErr(err) + } + } + // Conform file include directives to the new unikernel folder organisation if err := conformIncludeDirectives(appFolder); err != nil { u.PrintErr(err) @@ -248,6 +261,7 @@ func RunBuildTool(homeDir string, data *u.Data) { // Match micro-libs matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data) + if err != nil { u.PrintErr(err) } @@ -257,6 +271,7 @@ func RunBuildTool(homeDir string, data *u.Data) { // Match internal dependencies between micro-libs if err := searchInternalDependencies(workspacePath, &matchedLibs, + externalLibs); err != nil { u.PrintErr(err) } @@ -270,6 +285,7 @@ func RunBuildTool(homeDir string, data *u.Data) { // Generate Makefiles if err := generateMake(programName, appFolder, workspacePath, *args.StringArg[makefileArg], + matchedLibs, selectedFiles, externalLibs); err != nil { u.PrintErr(err) } @@ -282,7 +298,6 @@ func RunBuildTool(homeDir string, data *u.Data) { // Run make runMake(programName, appFolder) - } // retFolderCompat modifies its string argument in order to replace its underscore by a dash when diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index ccf028c..1bb31de 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -18,6 +18,18 @@ import ( u "tools/srcs/common" ) +// ---------------------------Create Patches Folder----------------------------- + +func createPatchFolder(appFolder string) (*string, error) { + + patchesFolder := appFolder + "patches" + u.SEP + if _, err := u.CreateFolder(patchesFolder); err != nil { + return nil, err + } + + return &patchesFolder, nil +} + // ---------------------------Create Include Folder----------------------------- func createIncludeFolder(appFolder string) (*string, error) { @@ -154,11 +166,11 @@ var srcLanguages = map[string]int{ ".c": 0, ".cpp": 0, ".cc": 0, - //".S": 0, - //".s": 0, - //".asm": 0, - //".py": 0, - //".go": 0, + ".S": 0, + ".s": 0, + ".asm": 0, + ".py": 0, + ".go": 0, } func filterSourcesFiles(sourceFiles []string) []string { @@ -174,14 +186,134 @@ func filterSourcesFiles(sourceFiles []string) []string { return filterSrcFiles } +// addAndApplyPatchFiles copies all the user-provided patch files to the unikernel directory, +// conforms them to the unikernel directory format so that all paths in the patch files are paths +// to source files located in the unikernel folder and applies the patches. +// +// It returns an error if any, otherwise it returns nil. +func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) error { + + // Copy and conform patch files + err := filepath.Walk(patchPath, func(filePath string, info os.FileInfo, + err error) error { + + if !info.IsDir() { + extension := filepath.Ext(info.Name()) + if extension == ".patch" { + if err = conformPatchFile(filePath, patchFolder+info.Name()); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + + // Initialise git repo to be able to apply patches + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "init") + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "add", ".") + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "commit", "-m", "first commit") + + // Apply patches + err = filepath.Walk(patchPath, func(filePath string, info os.FileInfo, + err error) error { + + if !info.IsDir() { + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "am", patchFolder+info.Name()) + } + return nil + }) + if err != nil { + return err + } + + return nil +} + +// conformPatchPath conforms a path in a user-provided patch file to the unikernel directory format +// so that this path describes a source file located in the unikernel folder. +func conformPatchPath(match *[]string, index int) { + extension := filepath.Ext((*match)[index]) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + (*match)[index] = "include" + u.SEP + filepath.Base((*match)[index]) + } else if extension == ".c" || extension == ".cpp" || extension == ".cc" { + (*match)[index] = filepath.Base((*match)[index]) + } else { + u.PrintWarning("Unsupported extension for file: " + + filepath.Base((*match)[index])) + } +} + +// conformPatchFile copies all the user-provided patch files to the unikernel directory and +// conforms them to the unikernel directory format so that all paths in the patch files are paths +// to source files located in the unikernel folder. +// +// It returns an error if any, otherwise it returns nil. +func conformPatchFile(patchPath, newPatchPath string) error { + + patchLines, err := u.ReadLinesFile(patchPath) + if err != nil { + return err + } + + // Find paths in patch file using regexp + var re1 = regexp.MustCompile(`( )(.*)( \| )(.*)`) + var re2 = regexp.MustCompile(`(diff --git )(a/)?(.*)( )(b/)?(.*)`) + var re3 = regexp.MustCompile(`(--- )(a/)?(.*)`) + var re4 = regexp.MustCompile(`(\+\+\+ )(b/)?(.*)`) + + for lineIndex := range patchLines { + + // All paths to files to be modified by the patch are listed under "---" + if patchLines[lineIndex] == "---\n" { + lineIndex++ + for ; !strings.Contains(patchLines[lineIndex], "changed"); lineIndex++ { + for _, match := range re1.FindAllStringSubmatch(patchLines[lineIndex], -1) { + conformPatchPath(&match, 2) + patchLines[lineIndex] = strings.Join(match[1:], "") + "\n" + } + } + } + + // All diff lines contain paths to files to be modified by the patch + if len(patchLines[lineIndex]) > 10 && patchLines[lineIndex][:10] == "diff --git" { + for _, match := range re2.FindAllStringSubmatch(patchLines[lineIndex], -1) { + conformPatchPath(&match, 3) + conformPatchPath(&match, 6) + patchLines[lineIndex] = strings.Join(match[1:], "") + "\n" + } + + // Same observation for the two lines following the index line + for _, match := range re3.FindAllStringSubmatch(patchLines[lineIndex+2], -1) { + conformPatchPath(&match, 3) + patchLines[lineIndex+2] = strings.Join(match[1:], "") + "\n" + } + for _, match := range re4.FindAllStringSubmatch(patchLines[lineIndex+3], -1) { + conformPatchPath(&match, 3) + patchLines[lineIndex+3] = strings.Join(match[1:], "") + "\n" + } + } + } + + // Write the modified content to a file in the unikernel folder + err = u.WriteToFile(newPatchPath, []byte(strings.Join(patchLines, ""))) + if err != nil { + return err + } + + return nil +} + // conformIncludeDirectives conforms all the user-defined include directives of all C/C++ source // files to the unikernel directory format so that all these directives are paths to header files // located in the include folder of the unikernel directory. // // It returns an error if any, otherwise it returns nil. -func conformIncludeDirectives(sourcesPath string) error { +func conformIncludeDirectives(sourcePath string) error { - err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo, + err := filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { if !info.IsDir() { @@ -204,6 +336,7 @@ func conformIncludeDirectives(sourcesPath string) error { if err != nil { return err } + return nil } @@ -212,9 +345,9 @@ func conformIncludeDirectives(sourcesPath string) error { // include folder. // // It returns an error if any, otherwise it returns nil. -func conformFile(path string, isHeader bool) (err error) { +func conformFile(filePath string, isHeader bool) (err error) { - fileLines, err := u.ReadLinesFile(path) + fileLines, err := u.ReadLinesFile(filePath) if err != nil { return err } @@ -222,10 +355,10 @@ func conformFile(path string, isHeader bool) (err error) { // Find user-defined include directives using regexp var re = regexp.MustCompile(`(.*)(#include)(.*)(")(.*)(")(.*)`) - for index := range fileLines { - for _, match := range re.FindAllStringSubmatch(fileLines[index], -1) { + for lineIndex := range fileLines { + for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) { - // Replace the path by its last element + // Replace the path by (include/ +) its last element for i := 1; i < len(match); i++ { if match[i] == "\"" { if isHeader { @@ -233,7 +366,7 @@ func conformFile(path string, isHeader bool) (err error) { } else { match[i+1] = "include" + u.SEP + filepath.Base(match[i+1]) } - fileLines[index] = strings.Join(match[1:], "") + "\n" + fileLines[lineIndex] = strings.Join(match[1:], "") + "\n" break } } @@ -241,7 +374,7 @@ func conformFile(path string, isHeader bool) (err error) { } // Write the modified content to a file in the unikernel folder - err = u.WriteToFile(path, []byte(strings.Join(fileLines, ""))) + err = u.WriteToFile(filePath, []byte(strings.Join(fileLines, ""))) if err != nil { return err } diff --git a/srcs/common/git.go b/srcs/common/git.go index 5fa8dda..0ff16f5 100644 --- a/srcs/common/git.go +++ b/srcs/common/git.go @@ -6,7 +6,7 @@ package common -const branch = "RELEASE-0.10.0" +const branch = "RELEASE-0.9.0" // GitCloneRepository clones a git repository at the the given url. // From de071b8c0cbed96f5319986a4350f8610cfa8108 Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Wed, 17 May 2023 21:48:04 +0200 Subject: [PATCH 24/28] small corrections --- srcs/buildtool/run_buildtool.go | 8 +- srcs/buildtool/unikraft_files_process.go | 7 -- srcs/dependtool/interdependence_graph.go | 47 ++++------ srcs/dependtool/run_deptool.go | 107 +++++------------------ 4 files changed, 38 insertions(+), 131 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index d1d2d39..d27f292 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -126,7 +126,9 @@ func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder // Add Makefile.uk entry *selectedFiles = append(*selectedFiles, configFile) } else { - u.PrintWarning("Unsupported extension for file: " + configFile) + if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil { + u.PrintErr(err) + } } } } @@ -261,7 +263,6 @@ func RunBuildTool(homeDir string, data *u.Data) { // Match micro-libs matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data) - if err != nil { u.PrintErr(err) } @@ -271,7 +272,6 @@ func RunBuildTool(homeDir string, data *u.Data) { // Match internal dependencies between micro-libs if err := searchInternalDependencies(workspacePath, &matchedLibs, - externalLibs); err != nil { u.PrintErr(err) } @@ -285,7 +285,6 @@ func RunBuildTool(homeDir string, data *u.Data) { // Generate Makefiles if err := generateMake(programName, appFolder, workspacePath, *args.StringArg[makefileArg], - matchedLibs, selectedFiles, externalLibs); err != nil { u.PrintErr(err) } @@ -337,7 +336,6 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, // Process Config.UK file mapConfig := make(map[string][]string) u.ProcessConfigUK(lines, true, mapConfig, nil) - for config := range mapConfig { // Remove LIB prefix diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 1bb31de..35be7f1 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -196,7 +196,6 @@ func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) erro // Copy and conform patch files err := filepath.Walk(patchPath, func(filePath string, info os.FileInfo, err error) error { - if !info.IsDir() { extension := filepath.Ext(info.Name()) if extension == ".patch" { @@ -219,7 +218,6 @@ func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) erro // Apply patches err = filepath.Walk(patchPath, func(filePath string, info os.FileInfo, err error) error { - if !info.IsDir() { _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "am", patchFolder+info.Name()) } @@ -315,7 +313,6 @@ func conformIncludeDirectives(sourcePath string) error { err := filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { extension := filepath.Ext(info.Name()) if extension == ".h" || extension == ".hpp" || extension == ".hcc" { @@ -332,7 +329,6 @@ func conformIncludeDirectives(sourcePath string) error { } return nil }) - if err != nil { return err } @@ -387,7 +383,6 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { extension := filepath.Ext(info.Name()) if _, ok := srcLanguages[extension]; ok { @@ -413,10 +408,8 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string, u.PrintWarning("Unsupported extension for file: " + info.Name()) } } - return nil }) - if err != nil { return nil, err } diff --git a/srcs/dependtool/interdependence_graph.go b/srcs/dependtool/interdependence_graph.go index e9fed66..943388b 100644 --- a/srcs/dependtool/interdependence_graph.go +++ b/srcs/dependtool/interdependence_graph.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" u "tools/srcs/common" @@ -13,25 +14,6 @@ import ( // ---------------------------------Gather Data--------------------------------- -// getProgramFolder gets the folder path in which the given program is located, according to the -// Unikraft standard (e.g., /home/.../apps/programFolder/.../program). -// -// It returns the folder containing the program files according to the standard described above. -func getProgramFolder(programPath string) string { - - tmp := strings.Split(programPath, u.SEP) - i := 2 - - for ; i < len(tmp); i++ { - if tmp[len(tmp)-i] == "apps" { - break - } - } - - folderPath := strings.Join(tmp[:len(tmp)-i+2], u.SEP) - return folderPath -} - // sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file. // // It returns a slice containing the found header names. @@ -44,14 +26,18 @@ func sourceFileIncludesAnalysis(sourceFile string) []string { u.PrintErr(err) } - for _, line := range fileLines { - if strings.Contains(line, "#include") { - line = strings.ReplaceAll(line, " ", "") - line = strings.Split(line, "#include")[1] - if strings.HasPrefix(line, "\"") { - fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], "\"")+1]) - } else if strings.HasPrefix(line, "<") { - fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], ">")+1]) + // Find user-defined include directives using regexp + var re = regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`) + + for lineIndex := range fileLines { + for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) { + + // Find the last element of the path and append it to the list of found header names + for i := 1; i < len(match); i++ { + if match[i] == "\"" || match[i] == "<" { + fileIncludes = append(fileIncludes, filepath.Base(match[i+1])) + break + } } } } @@ -91,7 +77,7 @@ func gccSourceFileIncludesAnalysis(sourceFile, outAppFolder string) ([]string, e // Only interested in headers not coming from the standard library if strings.Contains(line, "\""+outAppFolder) { line = strings.Split(line, "\""+outAppFolder)[1] - includeDirective := line[0:strings.Index(line[0:], "\"")] + includeDirective := filepath.Base(line[0:strings.Index(line[0:], "\"")]) if !u.Contains(fileIncludes, includeDirective) { fileIncludes = append(fileIncludes, includeDirective) } @@ -208,15 +194,12 @@ func requestUnikraftExtLibs() []string { func interdependAnalyser(programPath, programName, outFolder string) string { // Find all program source files - sourceFiles, err := findSourcesFiles(getProgramFolder(programPath)) + sourceFiles, err := findSourcesFiles(programPath) if err != nil { u.PrintErr(err) } // Create a folder and copy all source files into it for use with the gcc preprocessor - //tmp := strings.Split(getProgramFolder(programPath), u.SEP) - //outAppFolder := strings.Join(tmp[:len(tmp)-1], u.SEP) + u.SEP + programName + - //"_deptool_output" + u.SEP outAppFolder := outFolder + programName + u.SEP _, err = u.CreateFolder(outAppFolder) if err != nil { diff --git a/srcs/dependtool/run_deptool.go b/srcs/dependtool/run_deptool.go index 79eb01d..2683cd6 100644 --- a/srcs/dependtool/run_deptool.go +++ b/srcs/dependtool/run_deptool.go @@ -11,6 +11,25 @@ import ( "github.com/fatih/color" ) +// getProgramFolder gets the folder path in which the given program is located, according to the +// Unikraft standard (e.g., /home/.../apps/programFolder/.../program). +// +// It returns the folder containing the program files according to the standard described above. +func getProgramFolder(programPath string) string { + + tmp := strings.Split(programPath, u.SEP) + i := 2 + + for ; i < len(tmp); i++ { + if tmp[len(tmp)-i] == "apps" { + break + } + } + + folderPath := strings.Join(tmp[:len(tmp)-i+2], u.SEP) + return folderPath +} + // RunAnalyserTool allows to run the dependency analyser tool. func RunAnalyserTool(homeDir string, data *u.Data) { @@ -88,7 +107,7 @@ func RunAnalyserTool(homeDir string, data *u.Data) { // Run interdependence analyser if typeAnalysis == 0 || typeAnalysis == 3 { u.PrintHeader1("(1.3) RUN INTERDEPENDENCE ANALYSIS") - _ = runInterdependAnalyser(programPath, programName, outFolder) + _ = runInterdependAnalyser(getProgramFolder(programPath), programName, outFolder) } // Run sources analyser @@ -260,89 +279,3 @@ func runInterdependAnalyser(programPath, programName, outFolder string) string { return interdependAnalyser(programPath, programName, outFolder) } - -/* -// /!\ MISSING "/" !!! -stringFile := "#include\n/* #include ta mère *\nint main() {\n\t// Salut bitch !\n\treturn 0;\n}" - -for { - comStartIndex := strings.Index(stringFile, "/*") - if comStartIndex != -1 { - comEndIndex := strings.Index(stringFile, "*") - stringFile = strings.Join([]string{stringFile[:comStartIndex], - stringFile[comEndIndex+2:]}, "") - } else { - break - } -} -//what to do with "\t" in lines ? -var finalFile []string -sliceFile := strings.Split(stringFile, "\n") -for i := 0; i < len(sliceFile); i++ { - if !strings.HasPrefix(sliceFile[i], "//") { - finalFile = append(finalFile, sliceFile[i]) - } -} -} - - -// Remove dependencies whose files are not in program directory (e.g., stdio, stdlib, ...) - for internalFile, dependencies := range interdependMap { - var internalDep []string - for _, dependency := range dependencies { - if _, ok := interdependMap[dependency]; ok { - a++ - internalDep = append(internalDep, dependency) - } - } - interdependMap[internalFile] = internalDep - } - - -// Detect and print removable program source files (i.e., files that no other file depends - // on) - var removableFiles []string - for internalFile := range interdependMap { - depends := false - for _, dependencies := range interdependMap { - for _, dependency := range dependencies { - if internalFile == dependency { - depends = true - break - } - } - if depends { - break - } - } - - if !depends { - removableFiles = append(removableFiles, internalFile) - } - } - fmt.Println("Removable program source files of ", programName, ":") - fmt.Println(removableFiles) - - -func isADependency(internalFile string, interdependMap* map[string][]string) bool { - for _, dependencies := range *interdependMap { - for _, dependency := range dependencies { - if internalFile == dependency { - return true - } - } - } - return false -} - - -mymap := make(map[string][]string) - mymap["file_win32"] = make([]string, 0) - mymap["file_1"] = []string{"file_win32"} - mymap["file_2"] = []string{"file_1"} - mymap["file_3"] = make([]string, 0) - mymap["file_4"] = []string{"file_3"} - mymap["file_openssl"] = []string{"file_3"} - mymap["file_5"] = []string{"file_openssl"} - mymap["file_6"] = []string{"file_5"} -*/ From c8b7d3ddd14eb8717dbbc81bf76998020c03e13c Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Fri, 19 May 2023 11:27:41 +0200 Subject: [PATCH 25/28] small corrections 2 --- srcs/buildtool/unikraft_files_process.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 35be7f1..d5e434a 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -236,11 +236,8 @@ func conformPatchPath(match *[]string, index int) { extension := filepath.Ext((*match)[index]) if extension == ".h" || extension == ".hpp" || extension == ".hcc" { (*match)[index] = "include" + u.SEP + filepath.Base((*match)[index]) - } else if extension == ".c" || extension == ".cpp" || extension == ".cc" { - (*match)[index] = filepath.Base((*match)[index]) } else { - u.PrintWarning("Unsupported extension for file: " + - filepath.Base((*match)[index])) + (*match)[index] = filepath.Base((*match)[index]) } } From c5107793e367a0e8351ff472e0eafeaf03a614bb Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Thu, 25 May 2023 23:15:00 +0200 Subject: [PATCH 26/28] small corrections 3 --- srcs/buildtool/run_buildtool.go | 6 +++--- srcs/buildtool/unikraft_files_process.go | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index d27f292..cf6a02e 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -105,7 +105,8 @@ func parseMakeOutput(output string) string { return sb.String() } -// addConfigFiles adds user-provided configuration files to the app unikernel folder. +// addConfigFiles adds user-provided configuration files in the order in which they come to the +// unikernel folder. func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder, appFolder string) { @@ -113,7 +114,6 @@ func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder configFile := filepath.Base(configFilePath) fileExt := filepath.Ext(configFile) - // Copy config file if fileExt == ".h" || fileExt == ".hpp" || fileExt == ".hcc" { if err := u.CopyFileContents(configFilePath, includeFolder+configFile); err != nil { u.PrintErr(err) @@ -258,7 +258,7 @@ func RunBuildTool(homeDir string, data *u.Data) { u.PrintErr(err) } - // Move config files to the unikernel folder + // Add config files to the unikernel folder addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder) // Match micro-libs diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index d5e434a..fc5023a 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -166,11 +166,11 @@ var srcLanguages = map[string]int{ ".c": 0, ".cpp": 0, ".cc": 0, - ".S": 0, + /*".S": 0, ".s": 0, ".asm": 0, ".py": 0, - ".go": 0, + ".go": 0,*/ } func filterSourcesFiles(sourceFiles []string) []string { @@ -320,8 +320,6 @@ func conformIncludeDirectives(sourcePath string) error { if err = conformFile(path, false); err != nil { return err } - } else { - u.PrintWarning("Unsupported extension for file: " + info.Name()) } } return nil From 09192bdccd66d2ebc76e729548019195c276285b Mon Sep 17 00:00:00 2001 From: Rob1103 Date: Sat, 27 May 2023 16:10:46 +0200 Subject: [PATCH 27/28] small corrections 4 --- srcs/buildtool/unikraft_files_process.go | 113 ++++++++++++++++++----- srcs/dependtool/interdependence_graph.go | 2 +- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index fc5023a..640d808 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -230,17 +230,6 @@ func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) erro return nil } -// conformPatchPath conforms a path in a user-provided patch file to the unikernel directory format -// so that this path describes a source file located in the unikernel folder. -func conformPatchPath(match *[]string, index int) { - extension := filepath.Ext((*match)[index]) - if extension == ".h" || extension == ".hpp" || extension == ".hcc" { - (*match)[index] = "include" + u.SEP + filepath.Base((*match)[index]) - } else { - (*match)[index] = filepath.Base((*match)[index]) - } -} - // conformPatchFile copies all the user-provided patch files to the unikernel directory and // conforms them to the unikernel directory format so that all paths in the patch files are paths // to source files located in the unikernel folder. @@ -254,10 +243,10 @@ func conformPatchFile(patchPath, newPatchPath string) error { } // Find paths in patch file using regexp - var re1 = regexp.MustCompile(`( )(.*)( \| )(.*)`) - var re2 = regexp.MustCompile(`(diff --git )(a/)?(.*)( )(b/)?(.*)`) - var re3 = regexp.MustCompile(`(--- )(a/)?(.*)`) - var re4 = regexp.MustCompile(`(\+\+\+ )(b/)?(.*)`) + re1 := regexp.MustCompile(`( )(.*)( \| )(.*)`) + re2 := regexp.MustCompile(`(diff --git )(a/)?(.*)( )(b/)?(.*)`) + re3 := regexp.MustCompile(`(--- )(a/)?(.*)`) + re4 := regexp.MustCompile(`(\+\+\+ )(b/)?(.*)`) for lineIndex := range patchLines { @@ -301,8 +290,19 @@ func conformPatchFile(patchPath, newPatchPath string) error { return nil } +// conformPatchPath conforms a path in a user-provided patch file to the unikernel directory format +// so that this path describes a source file located in the unikernel folder. +func conformPatchPath(match *[]string, index int) { + extension := filepath.Ext((*match)[index]) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + (*match)[index] = "include" + u.SEP + filepath.Base((*match)[index]) + } else { + (*match)[index] = filepath.Base((*match)[index]) + } +} + // conformIncludeDirectives conforms all the user-defined include directives of all C/C++ source -// files to the unikernel directory format so that all these directives are paths to header files +// files to the unikernel directory format so that all these directives are paths to source files // located in the include folder of the unikernel directory. // // It returns an error if any, otherwise it returns nil. @@ -332,7 +332,7 @@ func conformIncludeDirectives(sourcePath string) error { } // conformFile conforms all the user-defined include directives of a C/C++ source file to the -// unikernel directory format so that all these directives are paths to header files located in the +// unikernel directory format so that all these directives are paths to source files located in the // include folder. // // It returns an error if any, otherwise it returns nil. @@ -343,24 +343,50 @@ func conformFile(filePath string, isHeader bool) (err error) { return err } - // Find user-defined include directives using regexp - var re = regexp.MustCompile(`(.*)(#include)(.*)(")(.*)(")(.*)`) + // Find include directives using regexp + re := regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`) for lineIndex := range fileLines { for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) { - // Replace the path by (include/ +) its last element + // Only interested in included files not coming from the standard library + if incDirContainsStdFold(fileLines[lineIndex]) { + break + } + for i := 1; i < len(match); i++ { - if match[i] == "\"" { - if isHeader { - match[i+1] = filepath.Base(match[i+1]) + if match[i] == "\"" || match[i] == "<" { + + // Determine the included source file extension to know what the path to it + // must be in the current file + var extIsHeader bool + extension := filepath.Ext(match[i+1]) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + extIsHeader = true + } else if extension == ".c" || extension == ".cpp" || extension == ".cc" { + extIsHeader = false } else { - match[i+1] = "include" + u.SEP + filepath.Base(match[i+1]) + + // C++ header + break } + + // Modify the include path + match[i] = "\"" + if isHeader && !extIsHeader { + match[i+1] = "../" + filepath.Base(match[i+1]) + } else if !isHeader && extIsHeader { + match[i+1] = "include" + u.SEP + filepath.Base(match[i+1]) + } else { + match[i+1] = filepath.Base(match[i+1]) + } + match[i+2] = "\"" fileLines[lineIndex] = strings.Join(match[1:], "") + "\n" break } } + + break } } @@ -373,6 +399,45 @@ func conformFile(filePath string, isHeader bool) (err error) { return nil } +// incDirContainsStdFold determines if an include directive is a path to a standard header. +// +// It returns true if it is the case, false otherwise. +func incDirContainsStdFold(fileLine string) bool { + + // Standard header list + stdHeaders := []string{ + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", " Date: Mon, 12 Jun 2023 15:36:08 +0200 Subject: [PATCH 28/28] big corrections 1 --- srcs/dependtool/interdependence_graph.go | 253 ++++++++++++++++------- srcs/dependtool/sources_analyser.go | 9 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/srcs/dependtool/interdependence_graph.go b/srcs/dependtool/interdependence_graph.go index e2de823..705804f 100644 --- a/srcs/dependtool/interdependence_graph.go +++ b/srcs/dependtool/interdependence_graph.go @@ -1,6 +1,7 @@ package dependtool import ( + "fmt" "io/ioutil" "net/http" "os" @@ -14,35 +15,109 @@ import ( // ---------------------------------Gather Data--------------------------------- -// sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file. +// findSourceFilesAndFolders puts together all the application C/C++ source files found on one hand +// and all the application (sub-)folder paths on the other hand. // -// It returns a slice containing the found header names. +// It returns two slices: one containing the found source file paths and one containing the found +// (sub-)folder paths, and an error if any, otherwise it returns nil. +func findSourceFilesAndFolders(workspace string) ([]string, []string, error) { + + var filenames []string + var foldernames []string + + err := filepath.Walk(workspace, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore hidden elements + if string(info.Name()[0]) == "." { + return nil + } + + // Found (sub-)folder + if info.IsDir() { + foldernames = append(foldernames, "-I") + foldernames = append(foldernames, path) + return nil + } + + // Found source file + ext := filepath.Ext(info.Name()) + if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" || + ext == ".hcc" { + filenames = append(filenames, path) + } + return nil + }) + if err != nil { + return nil, nil, err + } + return filenames, foldernames, nil +} + +// checkIncDir checks that an inclusion directive path does not contain the two dots ".." which +// refer to the parent directory and that can cause trouble when pruning the map and generating the +// graph. If the path does contain these dots, it is simplified. +// +// It returns the simplified directive path or the path itself if it contains no dots. +func checkIncDir(directive string) string { + + // No problem + if !strings.Contains(directive, "../") { + return directive + } + + // Must be simplified + splitDir := strings.Split(directive, u.SEP) + var i int + for i = 0; i < len(splitDir); i++ { + if splitDir[i] == ".." { + + // Dots at the beginning of the path + if i == 0 { + splitDir = splitDir[1:] + i-- + + // Dots somewhere else in the path + } else { + splitDir = append(splitDir[:i-1], splitDir[i+1:]...) + i -= 2 + } + } + } + return strings.Join(splitDir, u.SEP) +} + +// sourceFileIncludesAnalysis collects all the inclusion directives from a C/C++ source file. +// +// It returns a slice containing all the relative paths contained in the inclusion directives. func sourceFileIncludesAnalysis(sourceFile string) []string { - var fileIncludes []string + var fileIncDir []string fileLines, err := u.ReadLinesFile(sourceFile) if err != nil { u.PrintErr(err) } - // Find user-defined include directives using regexp + // Find inclusion directives using regexp var re = regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`) for lineIndex := range fileLines { for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) { - // Find the last element of the path and append it to the list of found header names + // Append the relative path to the list of relative paths for i := 1; i < len(match); i++ { if match[i] == "\"" || match[i] == "<" { - fileIncludes = append(fileIncludes, filepath.Base(match[i+1])) + fileIncDir = append(fileIncDir, checkIncDir(match[i+1])) break } } } } - - return fileIncludes + return fileIncDir } // TODO REPLACE @@ -50,57 +125,64 @@ func sourceFileIncludesAnalysis(sourceFile string) []string { // // It returns a string which represents stdout and an error if any, otherwise // it returns nil. -func ExecuteCommand(command string, arguments []string) (string, error) { +func executeCommand(command string, arguments []string) (string, error) { + out, err := exec.Command(command, arguments...).CombinedOutput() return string(out), err } -// gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using -// the gcc preprocessor. +// gppSourceFileIncludesAnalysis collects all the inclusion directives from a C/C++ source file +// using the gpp preprocessor. // -// It returns a slice containing the found header names. -func gccSourceFileIncludesAnalysis(sourceFile, outAppFolder string) ([]string, error) { +// It returns a slice containing all the absolute paths contained in the inclusion directives. +func gppSourceFileIncludesAnalysis(sourceFile, programPath string, + sourceFolders []string) ([]string, error) { - var fileIncludes []string + var fileIncDir []string - // gcc command - outputStr, err := ExecuteCommand("gcc", []string{"-E", sourceFile, "-I", outAppFolder}) + // g++ command + outputStr, err := executeCommand("g++", append([]string{"-E", sourceFile}, sourceFolders...)) - // if gcc command returns an error, prune file: it contains non-standard libs + // If command g++ returns an error, prune file: it contains non-standard libraries if err != nil { - return make([]string, 0), err + return nil, err } - outputSlice := strings.Split(outputStr, "\n") + // Find inclusion directive paths + outputSlice := strings.Split(outputStr, "\n") for _, line := range outputSlice { - // Only interested in headers not coming from the standard library - if strings.Contains(line, "\""+outAppFolder) { - line = strings.Split(line, "\""+outAppFolder)[1] - includeDirective := filepath.Base(line[0:strings.Index(line[0:], "\"")]) - if !u.Contains(fileIncludes, includeDirective) { - fileIncludes = append(fileIncludes, includeDirective) + // If warnings or errors are present, ignore their paths + if strings.Contains(line, ":") { + continue + } + + // Only interested in file paths not coming from the standard library + if strings.Contains(line, programPath) { + includeDirective := + checkIncDir(line[strings.Index(line, "\"")+1 : strings.LastIndex(line, "\"")]) + if !u.Contains(fileIncDir, includeDirective) { + fileIncDir = append(fileIncDir, includeDirective) } } } - - return fileIncludes, nil + return fileIncDir, nil } // pruneRemovableFiles prunes interdependence graph elements if the latter are unused header files. func pruneRemovableFiles(interdependMap *map[string][]string) { - for internalFile := range *interdependMap { + for file := range *interdependMap { // No removal of C/C++ source files - if filepath.Ext(internalFile) != ".c" && filepath.Ext(internalFile) != ".cpp" && - filepath.Ext(internalFile) != ".cc" { + if filepath.Ext(file) != ".c" && filepath.Ext(file) != ".cpp" && + filepath.Ext(file) != ".cc" { // Lookup for files depending on the current header file depends := false for _, dependencies := range *interdependMap { for _, dependency := range dependencies { - if internalFile == dependency { + if file == dependency { depends = true break } @@ -112,7 +194,7 @@ func pruneRemovableFiles(interdependMap *map[string][]string) { // Prune header file if unused if !depends { - delete(*interdependMap, internalFile) + delete(*interdependMap, file) } } } @@ -123,14 +205,14 @@ func pruneRemovableFiles(interdependMap *map[string][]string) { func pruneElemFiles(interdependMap *map[string][]string, pruneElem string) { // Lookup for key elements containing the substring and prune them - for internalFile := range *interdependMap { - if strings.Contains(internalFile, pruneElem) { - delete(*interdependMap, internalFile) + for key := range *interdependMap { + if strings.Contains(key, pruneElem) { + delete(*interdependMap, key) // Lookup for key elements that depend on the key found above and prune them for file, dependencies := range *interdependMap { for _, dependency := range dependencies { - if dependency == internalFile { + if dependency == key { pruneElemFiles(interdependMap, file) } } @@ -141,11 +223,13 @@ func pruneElemFiles(interdependMap *map[string][]string, pruneElem string) { // requestUnikraftExtLibs collects all the GitHub repositories of Unikraft through the GitHub API // and returns the whole list of Unikraft external libraries. +// +// It returns a slice containing all the libraries names. func requestUnikraftExtLibs() []string { var extLibsList, appsList []string - // Only 2 Web pages of repos as for february 2023 (125 repos - 100 repos per page) + // Only 2 Web pages of repositories as for february 2023 (125 repos - 100 repos per page) nbPages := 2 for i := 1; i <= nbPages; i++ { @@ -162,14 +246,14 @@ func requestUnikraftExtLibs() []string { u.PrintErr(err) } - // Collect libs + // Collect libraries fileLines := strings.Split(string(body), "\"name\":\"lib-") for i := 1; i < len(fileLines); i++ { extLibsList = append(extLibsList, fileLines[i][0:strings.Index(fileLines[i][0:], "\"")]) } - // Collect apps + // Collect applications fileLines = strings.Split(string(body), "\"name\":\"app-") for i := 1; i < len(fileLines); i++ { appsList = append(appsList, fileLines[i][0:strings.Index(fileLines[i][0:], @@ -188,54 +272,46 @@ func requestUnikraftExtLibs() []string { // -------------------------------------Run------------------------------------- -// runInterdependAnalyser collects all the included headers names (i.e., dependencies) from each -// C/C++ source file of a program and builds an interdependence graph (dot file) between all these -// source files. +// runInterdependAnalyser collects all the inclusion directives (i.e., dependencies) from each +// C/C++ application source file and builds an interdependence graph (dot file and png image) +// between the source files that it deemed compilable with Unikraft. +// +// It returns the path to the folder containing the source files that have been kept for generating +// the graph. func interdependAnalyser(programPath, programName, outFolder string) string { - // Find all program source files - sourceFiles, err := findSourcesFiles(programPath) + // Find all application source files and (sub-)folders + sourceFiles, sourceFolders, err := findSourceFilesAndFolders(programPath) if err != nil { u.PrintErr(err) } + u.PrintOk(fmt.Sprint(len(sourceFiles)) + " source files found") + u.PrintOk(fmt.Sprint(len(sourceFolders)) + " source subfolders found") - // Create a folder and copy all source files into it for later use with build tool - outAppFolder := outFolder + programName + u.SEP - _, err = u.CreateFolder(outAppFolder) - if err != nil { - u.PrintErr(err) - } - var outAppFiles []string - for _, sourceFilePath := range sourceFiles { - if err := u.CopyFileContents(sourceFilePath, - outAppFolder+filepath.Base(sourceFilePath)); err != nil { - u.PrintErr(err) - } - outAppFiles = append(outAppFiles, outAppFolder+filepath.Base(sourceFilePath)) - } - - // Analyse source files include directives and collect header names. Source files are first - // analysed by gcc to make sure to avoid directives that are commented or subjected to a macro - // and then "by hand" to sort the include directives of the gcc analysis (i.e., to avoid - // include directives that are not present in the source file currently analysed). + // Collect source file inclusion directives. Source files are first analysed by g++ to make + // sure to avoid directives that are commented or subjected to a macro and then "by hand" to + // sort the include directives of the g++ analysis (i.e. to avoid inclusion directives that are + // not present in the source file currently analysed). interdependMap := make(map[string][]string) - for _, outAppFile := range outAppFiles { - analysis := sourceFileIncludesAnalysis(outAppFile) - gccAnalysis, err := gccSourceFileIncludesAnalysis(outAppFile, outAppFolder) + for _, sourceFile := range sourceFiles { + analysis := sourceFileIncludesAnalysis(sourceFile) + gppAnalysis, err := gppSourceFileIncludesAnalysis(sourceFile, programPath, sourceFolders) if err != nil { continue } - interdependMap[filepath.Base(outAppFile)] = make([]string, 0) - for _, includeDirective := range gccAnalysis { - if u.Contains(analysis, includeDirective) { - interdependMap[filepath.Base(outAppFile)] = - append(interdependMap[filepath.Base(outAppFile)], includeDirective) + // Build the interdependence map with the paths contained in the inclusion directives + interdependMap[sourceFile] = make([]string, 0) + for _, directive := range analysis { + for _, gppDirective := range gppAnalysis { + if strings.Contains(gppDirective, directive) { + interdependMap[sourceFile] = append(interdependMap[sourceFile], gppDirective) + } } } } - // Prune interdependence graph + // Prune the interdependence graph extLibsList := requestUnikraftExtLibs() extLibsList = append(extLibsList, "win32", "test", "TEST") for _, extLib := range extLibsList { @@ -243,15 +319,36 @@ func interdependAnalyser(programPath, programName, outFolder string) string { } pruneRemovableFiles(&interdependMap) - // Remove pruned files from the out app folder - for _, outAppFile := range outAppFiles { - if _, ok := interdependMap[filepath.Base(outAppFile)]; !ok { - os.Remove(outAppFile) + // Create a folder and copy all the kept source files into it for later use with build tool + outAppFolder := outFolder + programName + u.SEP + _, err = u.CreateFolder(outAppFolder) + if err != nil { + u.PrintErr(err) + } + for _, sourceFile := range sourceFiles { + if _, ok := interdependMap[sourceFile]; ok { + if err := u.CopyFileContents(sourceFile, + outAppFolder+filepath.Base(sourceFile)); err != nil { + u.PrintErr(err) + } + } + } + u.PrintOk(fmt.Sprint(len(interdependMap)) + " source files kept and copied to " + outAppFolder) + + // Change the absolute paths in the interdependence map into relative paths for more + // readability in the png image + graphMap := make(map[string][]string) + for appFilePath := range interdependMap { + appFile := strings.Split(appFilePath, programPath)[1][1:] + graphMap[appFile] = make([]string, 0) + for _, fileDepPath := range interdependMap[appFilePath] { + graphMap[appFile] = append(graphMap[appFile], strings.Split(fileDepPath, + programPath)[1][1:]) } } - // Create dot file - u.GenerateGraph(programName, outFolder+programName, interdependMap, nil) + // Create dot and png files + u.GenerateGraph(programName, outFolder+programName, graphMap, nil) return outAppFolder } diff --git a/srcs/dependtool/sources_analyser.go b/srcs/dependtool/sources_analyser.go index 68b7c43..d96e467 100644 --- a/srcs/dependtool/sources_analyser.go +++ b/srcs/dependtool/sources_analyser.go @@ -1,6 +1,7 @@ package dependtool import ( + "fmt" "os" "path/filepath" "strings" @@ -13,7 +14,7 @@ import ( // // It returns a slice containing the found source file names and an error if any, otherwise it // returns nil. -func findSourcesFiles(workspace string) ([]string, error) { +func findSourceFiles(workspace string) ([]string, error) { var filenames []string @@ -39,6 +40,7 @@ func findSourcesFiles(workspace string) ([]string, error) { // addSourceFileSymbols adds all the symbols present in 'output' to the static data field in // 'data'. func addSourceFileSymbols(output string, data *u.SourcesData) { + outputTab := strings.Split(output, ",") // Get the list of system calls @@ -69,6 +71,7 @@ func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error { } addSourceFileSymbols(output, data) } + return nil } @@ -77,14 +80,16 @@ func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error { // It returns an error if any, otherwise it returns nil. func gatherSourceFileSymbols(data *u.SourcesData, programPath string) error { - sourceFiles, err := findSourcesFiles(programPath) + sourceFiles, err := findSourceFiles(programPath) if err != nil { u.PrintErr(err) } + u.PrintOk(fmt.Sprint(len(sourceFiles)) + " source files found") if err := extractPrototype(sourceFiles, data); err != nil { u.PrintErr(err) } + return nil }