diff --git a/srcs/Makefile b/srcs/Makefile index 260bede..8426612 100644 --- a/srcs/Makefile +++ b/srcs/Makefile @@ -36,7 +36,7 @@ deps: $(GOGET) github.com/akamensky/argparse $(GOGET) github.com/awalterschulze/gographviz $(GOGET) github.com/sergi/go-diff/... - $(GOGET) github.com/AlecAivazis/survey + $(GOGET) gopkg.in/AlecAivazis/survey.v1 # Cross compilation build-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v diff --git a/srcs/buildtool/args.go b/srcs/buildtool/args.go index 52c665c..8b3e13e 100644 --- a/srcs/buildtool/args.go +++ b/srcs/buildtool/args.go @@ -7,9 +7,10 @@ package buildtool import ( - "github.com/akamensky/argparse" "os" u "tools/srcs/common" + + "github.com/akamensky/argparse" ) const ( @@ -18,6 +19,8 @@ const ( sourcesArg = "sources" objsArg = "objects" makefileArg = "makefile" + configArg = "config" + patchArg = "patch" ) // ParseArguments parses arguments of the application. @@ -34,11 +37,15 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { &argparse.Options{Required: true, Help: "App Sources " + "Folder"}) args.InitArgParse(p, args, u.BOOL, "o", objsArg, - &argparse.Options{Required: false, Default: false, Help: "Add objects from external build system " + - "Folder"}) + &argparse.Options{Required: false, Default: false, Help: "Add objects from external" + + "build system Folder"}) args.InitArgParse(p, args, u.STRING, "m", makefileArg, &argparse.Options{Required: false, Help: "Add additional properties " + "for Makefile"}) + args.InitArgParse(p, args, u.STRINGLIST, "c", configArg, + &argparse.Options{Required: false, Help: "Add configuration files"}) + args.InitArgParse(p, args, u.STRING, "", patchArg, + &argparse.Options{Required: false, Help: "Add patch files"}) return u.ParserWrapper(p, os.Args) } diff --git a/srcs/buildtool/microlibs_process.go b/srcs/buildtool/microlibs_process.go index b03cecb..43a01c4 100644 --- a/srcs/buildtool/microlibs_process.go +++ b/srcs/buildtool/microlibs_process.go @@ -41,7 +41,6 @@ type MicroLibsFunction struct { // -----------------------------Match micro-libs-------------------------------- // processSymbols adds symbols within the 'exportsyms.uk' file into a map. -// func processSymbols(microLib, output string, mapSymbols map[string][]string) { lines := strings.Split(output, "\n") @@ -134,6 +133,53 @@ func fetchSymbolsExternalLibs(folder string, return externalLibs, nil } +// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static, +// dynamic and source files analyses together into a map structure. +// +// It returns the map containing all the symbols and system calls. +func putJsonSymbolsTogether(data *u.Data) map[string]string { + dataMap := make(map[string]string) + + for k, v := range data.StaticData.Symbols { + dataMap[k] = v + } + + for k := range data.StaticData.SystemCalls { + dataMap[k] = "" + } + + for k, v := range data.DynamicData.Symbols { + dataMap[k] = v + } + + for k := range data.DynamicData.SystemCalls { + dataMap[k] = "" + } + + for k, v := range data.SourcesData.Symbols { + dataMap[k] = v + } + + for k := range data.SourcesData.SystemCalls { + dataMap[k] = "" + } + + return dataMap +} + +// retNameCompat modifies its string argument in order to replace its underscore by a dash when +// necessary. +// +// It returns its string argument whose underscore has been replaced by a dash if necessary, +// otherwise it returns its argument unchanged. +func retNameForCompat(value string) string { + if strings.Contains(value, "posix-") { + return strings.ReplaceAll(value, "posix-", "posix_") + } + + return value +} + // matchSymbols performs the matching between Unikraft's micro-libs and // libraries used by a given application based on the list of symbols that both // contain. @@ -144,15 +190,8 @@ func matchSymbols(matchedLibs []string, data map[string]string, for key := range data { if values, ok := microLibs[key]; ok { for _, value := range values { - - // todo remove - if strings.Compare(NOLIBC, value) == 0 { - value = NEWLIB - } - // remove above - - if !u.Contains(matchedLibs, value) { - matchedLibs = append(matchedLibs, value) + if !u.Contains(matchedLibs, retNameForCompat(value)) { + matchedLibs = append(matchedLibs, retNameForCompat(value)) } } } @@ -172,11 +211,6 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, matchedLibs := make([]string, 0) - //todo remove - matchedLibs = append(matchedLibs, POSIXLIBDL) - matchedLibs = append(matchedLibs, POSIXSYSINFO) - matchedLibs = append(matchedLibs, UKMMAP) - folder := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal") if err := fetchSymbolsInternalLibs(folder, mapSymbols); err != nil { return nil, nil, err @@ -189,11 +223,10 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, return nil, nil, err } - // Perform the matching symbols on static data - matchedLibs = matchSymbols(matchedLibs, data.StaticData.Symbols, mapSymbols) + dataMap := putJsonSymbolsTogether(data) - // Perform the matching symbols on dynamic data - matchedLibs = matchSymbols(matchedLibs, data.DynamicData.Symbols, mapSymbols) + // Perform the symbol matching + matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols) return matchedLibs, externalLibs, nil } @@ -223,7 +256,6 @@ func cloneGitRepo(url, unikraftPathLibs, lib string) error { // cloneLibsFolders clones all the needed micro-libs that are needed by a // given application -// func cloneLibsFolders(workspacePath string, matchedLibs []string, externalLibs map[string]string) { diff --git a/srcs/buildtool/run_buildtool.go b/srcs/buildtool/run_buildtool.go index 4b38f6a..cf6a02e 100644 --- a/srcs/buildtool/run_buildtool.go +++ b/srcs/buildtool/run_buildtool.go @@ -7,13 +7,14 @@ package buildtool import ( - "github.com/AlecAivazis/survey/v2" "io/ioutil" "os" "path/filepath" "regexp" "strings" u "tools/srcs/common" + + "gopkg.in/AlecAivazis/survey.v1" ) // STATES @@ -51,7 +52,8 @@ func generateConfigUk(filename, programName string, matchedLibs []string) error // execution of the 'make' command. // // It returns an integer that defines the result of 'make': -// +// +// func checkMakeOutput(appFolder string, stderr *string) int { if stderr == nil { @@ -103,11 +105,38 @@ func parseMakeOutput(output string) string { return sb.String() } +// addConfigFiles adds user-provided configuration files in the order in which they come to the +// unikernel folder. +func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder, + appFolder string) { + + for _, configFilePath := range configFiles { + configFile := filepath.Base(configFilePath) + fileExt := filepath.Ext(configFile) + + if fileExt == ".h" || fileExt == ".hpp" || fileExt == ".hcc" { + if err := u.CopyFileContents(configFilePath, includeFolder+configFile); err != nil { + u.PrintErr(err) + } + } else if fileExt == ".c" || fileExt == ".cpp" || fileExt == ".cc" { + if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil { + u.PrintErr(err) + } + + // Add Makefile.uk entry + *selectedFiles = append(*selectedFiles, configFile) + } else { + if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil { + u.PrintErr(err) + } + } + } +} + // -------------------------------------Run------------------------------------- // RunBuildTool runs the automatic build tool to build a unikernel of a // given application. -// func RunBuildTool(homeDir string, data *u.Data) { // Init and parse local arguments @@ -195,7 +224,7 @@ func RunBuildTool(homeDir string, data *u.Data) { } // Filter source files to limit build errors (e.g., remove test files, - //multiple main file, ...) + // multiple main file, ...) filterSourceFiles := filterSourcesFiles(sourceFiles) // Prompt file selection @@ -207,10 +236,31 @@ func RunBuildTool(homeDir string, data *u.Data) { } var selectedFiles []string - if err := survey.AskOne(prompt, &selectedFiles); err != nil { + if err := survey.AskOne(prompt, &selectedFiles, nil); err != nil { panic(err) } + // Copy, conform and apply user-provided patches to the new unikernel folder + if len(*args.StringArg[patchArg]) > 0 { + // Create patch folder + patchFolder, err := createPatchFolder(appFolder) + if err != nil { + u.PrintErr(err) + } + if err := addAndApplyPatchFiles(*args.StringArg[patchArg], *patchFolder, appFolder); err != + nil { + u.PrintErr(err) + } + } + + // Conform file include directives to the new unikernel folder organisation + if err := conformIncludeDirectives(appFolder); err != nil { + u.PrintErr(err) + } + + // Add config files to the unikernel folder + addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder) + // Match micro-libs matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data) if err != nil { @@ -249,41 +299,54 @@ func RunBuildTool(homeDir string, data *u.Data) { runMake(programName, appFolder) } +// retFolderCompat modifies its string argument in order to replace its underscore by a dash when +// necessary for the searchInternalDependencies function to find the corresponding folder in the +// 'unikraft' folder. +// +// It returns its string argument whose underscore has been replaced by a dash if necessary, +// otherwise it returns its argument unchanged. +func retFolderForCompat(lib string) string { + if strings.Contains(lib, "posix_") { + return strings.ReplaceAll(lib, "posix_", "posix-") + } + + return lib +} + func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, externalLibs map[string]string) error { for _, lib := range *matchedLibs { - // Consider only external libs - if _, ok := externalLibs[lib]; ok { - // Get and read Config.UK from external lib - configUk := unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk" - lines, err := u.ReadLinesFile(configUk) - if err != nil { - return err + // Get and read Config.UK from lib + var configUk string + + if _, ok := externalLibs[lib]; ok { + configUk = unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk" + } else { + configUk = unikraftPath + u.UNIKRAFTFOLDER + "lib" + u.SEP + retFolderForCompat(lib) + + u.SEP + "Config.uk" + } + + lines, err := u.ReadLinesFile(configUk) + if err != nil { + return err + } + + // Process Config.UK file + mapConfig := make(map[string][]string) + u.ProcessConfigUK(lines, true, mapConfig, nil) + for config := range mapConfig { + + // Remove LIB prefix + if strings.Contains(config, "LIB") { + config = strings.TrimPrefix(config, "LIB") } - // Process Config.UK file - mapConfig := make(map[string][]string) - u.ProcessConfigUK(lines, false, mapConfig, nil) - - for config := range mapConfig { - - // Remove LIB prefix - if strings.Contains(config, "LIB") { - config = strings.TrimPrefix(config, "LIB") - } - - // Replace underscore by dash - if strings.Contains(config, "_") { - config = strings.ReplaceAll(config, "_", "-") - } - - // Check if matchedLibs already contains the lib - config = strings.ToLower(config) - if !u.Contains(*matchedLibs, config) { - *matchedLibs = append(*matchedLibs, config) - } + // Check if matchedLibs already contains the lib + config = strings.ToLower(config) + if !u.Contains(*matchedLibs, config) { + *matchedLibs = append(*matchedLibs, config) } } } diff --git a/srcs/buildtool/unikraft_files_process.go b/srcs/buildtool/unikraft_files_process.go index 96e8b4e..640d808 100644 --- a/srcs/buildtool/unikraft_files_process.go +++ b/srcs/buildtool/unikraft_files_process.go @@ -12,11 +12,24 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" u "tools/srcs/common" ) +// ---------------------------Create Patches Folder----------------------------- + +func createPatchFolder(appFolder string) (*string, error) { + + patchesFolder := appFolder + "patches" + u.SEP + if _, err := u.CreateFolder(patchesFolder); err != nil { + return nil, err + } + + return &patchesFolder, nil +} + // ---------------------------Create Include Folder----------------------------- func createIncludeFolder(appFolder string) (*string, error) { @@ -109,7 +122,10 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) { if !created { u.PrintWarning(appFolder + " already exists.") - handleCreationApp(&appFolder) + appFolder = handleCreationApp(appFolder) + if _, err := u.CreateFolder(appFolder); err != nil { + return nil, err + } } return &appFolder, nil @@ -117,24 +133,24 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) { // -----------------------------Create App folder------------------------------- -func handleCreationApp(appFolder *string) { +func handleCreationApp(appFolder string) string { fmt.Println("Make your choice:\n1: Copy and overwrite files\n2: " + "Enter manually the name of the folder\n3: exit program") var input int - for true { + for { fmt.Print("Please enter your choice (0 to exit): ") if _, err := fmt.Scanf("%d", &input); err != nil { u.PrintWarning("Choice must be numeric! Try again") } else { switch input { case 1: - return + return appFolder case 2: fmt.Print("Enter text: ") reader := bufio.NewReader(os.Stdin) text, _ := reader.ReadString('\n') - appFolder = &text - return + appFolder = strings.Split(text, "\n")[0] + u.SEP + return appFolder case 3: os.Exit(1) default: @@ -150,11 +166,11 @@ var srcLanguages = map[string]int{ ".c": 0, ".cpp": 0, ".cc": 0, - ".S": 0, + /*".S": 0, ".s": 0, ".asm": 0, ".py": 0, - ".go": 0, + ".go": 0,*/ } func filterSourcesFiles(sourceFiles []string) []string { @@ -170,14 +186,264 @@ func filterSourcesFiles(sourceFiles []string) []string { return filterSrcFiles } +// addAndApplyPatchFiles copies all the user-provided patch files to the unikernel directory, +// conforms them to the unikernel directory format so that all paths in the patch files are paths +// to source files located in the unikernel folder and applies the patches. +// +// It returns an error if any, otherwise it returns nil. +func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) error { + + // Copy and conform patch files + err := filepath.Walk(patchPath, func(filePath string, info os.FileInfo, + err error) error { + if !info.IsDir() { + extension := filepath.Ext(info.Name()) + if extension == ".patch" { + if err = conformPatchFile(filePath, patchFolder+info.Name()); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + + // Initialise git repo to be able to apply patches + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "init") + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "add", ".") + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "commit", "-m", "first commit") + + // Apply patches + err = filepath.Walk(patchPath, func(filePath string, info os.FileInfo, + err error) error { + if !info.IsDir() { + _, _, _ = u.ExecuteRunCmd("git", appFolder, true, "am", patchFolder+info.Name()) + } + return nil + }) + if err != nil { + return err + } + + return nil +} + +// conformPatchFile copies all the user-provided patch files to the unikernel directory and +// conforms them to the unikernel directory format so that all paths in the patch files are paths +// to source files located in the unikernel folder. +// +// It returns an error if any, otherwise it returns nil. +func conformPatchFile(patchPath, newPatchPath string) error { + + patchLines, err := u.ReadLinesFile(patchPath) + if err != nil { + return err + } + + // Find paths in patch file using regexp + re1 := regexp.MustCompile(`( )(.*)( \| )(.*)`) + re2 := regexp.MustCompile(`(diff --git )(a/)?(.*)( )(b/)?(.*)`) + re3 := regexp.MustCompile(`(--- )(a/)?(.*)`) + re4 := regexp.MustCompile(`(\+\+\+ )(b/)?(.*)`) + + for lineIndex := range patchLines { + + // All paths to files to be modified by the patch are listed under "---" + if patchLines[lineIndex] == "---\n" { + lineIndex++ + for ; !strings.Contains(patchLines[lineIndex], "changed"); lineIndex++ { + for _, match := range re1.FindAllStringSubmatch(patchLines[lineIndex], -1) { + conformPatchPath(&match, 2) + patchLines[lineIndex] = strings.Join(match[1:], "") + "\n" + } + } + } + + // All diff lines contain paths to files to be modified by the patch + if len(patchLines[lineIndex]) > 10 && patchLines[lineIndex][:10] == "diff --git" { + for _, match := range re2.FindAllStringSubmatch(patchLines[lineIndex], -1) { + conformPatchPath(&match, 3) + conformPatchPath(&match, 6) + patchLines[lineIndex] = strings.Join(match[1:], "") + "\n" + } + + // Same observation for the two lines following the index line + for _, match := range re3.FindAllStringSubmatch(patchLines[lineIndex+2], -1) { + conformPatchPath(&match, 3) + patchLines[lineIndex+2] = strings.Join(match[1:], "") + "\n" + } + for _, match := range re4.FindAllStringSubmatch(patchLines[lineIndex+3], -1) { + conformPatchPath(&match, 3) + patchLines[lineIndex+3] = strings.Join(match[1:], "") + "\n" + } + } + } + + // Write the modified content to a file in the unikernel folder + err = u.WriteToFile(newPatchPath, []byte(strings.Join(patchLines, ""))) + if err != nil { + return err + } + + return nil +} + +// conformPatchPath conforms a path in a user-provided patch file to the unikernel directory format +// so that this path describes a source file located in the unikernel folder. +func conformPatchPath(match *[]string, index int) { + extension := filepath.Ext((*match)[index]) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + (*match)[index] = "include" + u.SEP + filepath.Base((*match)[index]) + } else { + (*match)[index] = filepath.Base((*match)[index]) + } +} + +// conformIncludeDirectives conforms all the user-defined include directives of all C/C++ source +// files to the unikernel directory format so that all these directives are paths to source files +// located in the include folder of the unikernel directory. +// +// It returns an error if any, otherwise it returns nil. +func conformIncludeDirectives(sourcePath string) error { + + err := filepath.Walk(sourcePath, func(path string, info os.FileInfo, + err error) error { + if !info.IsDir() { + extension := filepath.Ext(info.Name()) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + if err = conformFile(path, true); err != nil { + return err + } + } else if extension == ".c" || extension == ".cpp" || extension == ".cc" { + if err = conformFile(path, false); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + + return nil +} + +// conformFile conforms all the user-defined include directives of a C/C++ source file to the +// unikernel directory format so that all these directives are paths to source files located in the +// include folder. +// +// It returns an error if any, otherwise it returns nil. +func conformFile(filePath string, isHeader bool) (err error) { + + fileLines, err := u.ReadLinesFile(filePath) + if err != nil { + return err + } + + // Find include directives using regexp + re := regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`) + + for lineIndex := range fileLines { + for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) { + + // Only interested in included files not coming from the standard library + if incDirContainsStdFold(fileLines[lineIndex]) { + break + } + + for i := 1; i < len(match); i++ { + if match[i] == "\"" || match[i] == "<" { + + // Determine the included source file extension to know what the path to it + // must be in the current file + var extIsHeader bool + extension := filepath.Ext(match[i+1]) + if extension == ".h" || extension == ".hpp" || extension == ".hcc" { + extIsHeader = true + } else if extension == ".c" || extension == ".cpp" || extension == ".cc" { + extIsHeader = false + } else { + + // C++ header + break + } + + // Modify the include path + match[i] = "\"" + if isHeader && !extIsHeader { + match[i+1] = "../" + filepath.Base(match[i+1]) + } else if !isHeader && extIsHeader { + match[i+1] = "include" + u.SEP + filepath.Base(match[i+1]) + } else { + match[i+1] = filepath.Base(match[i+1]) + } + match[i+2] = "\"" + fileLines[lineIndex] = strings.Join(match[1:], "") + "\n" + break + } + } + + break + } + } + + // Write the modified content to a file in the unikernel folder + err = u.WriteToFile(filePath, []byte(strings.Join(fileLines, ""))) + if err != nil { + return err + } + + return nil +} + +// incDirContainsStdFold determines if an include directive is a path to a standard header. +// +// It returns true if it is the case, false otherwise. +func incDirContainsStdFold(fileLine string) bool { + + // Standard header list + stdHeaders := []string{ + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "", "", "", + "", "