Merge pull request #1 from gaulthiergain/perski/dev

Perski/dev

srcs/Makefile

@@ -36,7 +36,7 @@ deps:
 	$(GOGET) github.com/akamensky/argparse
 	$(GOGET) github.com/awalterschulze/gographviz
 	$(GOGET) github.com/sergi/go-diff/...
-	$(GOGET) github.com/AlecAivazis/survey
+	$(GOGET) gopkg.in/AlecAivazis/survey.v1
 # Cross compilation
 build-linux:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v

srcs/buildtool/args.go

@@ -7,9 +7,10 @@
 package buildtool
 
 import (
-	"github.com/akamensky/argparse"
 	"os"
 	u "tools/srcs/common"
+
+	"github.com/akamensky/argparse"
 )
 
 const (
@@ -18,6 +19,8 @@ const (
 	sourcesArg   = "sources"
 	objsArg      = "objects"
 	makefileArg  = "makefile"
+	configArg    = "config"
+	patchArg     = "patch"
 )
 
 // ParseArguments parses arguments of the application.
@@ -34,11 +37,15 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error {
 		&argparse.Options{Required: true, Help: "App Sources " +
 			"Folder"})
 	args.InitArgParse(p, args, u.BOOL, "o", objsArg,
-		&argparse.Options{Required: false, Default: false, Help: "Add objects from external build system " +
+		&argparse.Options{Required: false, Default: false, Help: "Add objects from external" +
-			"Folder"})
+			"build system Folder"})
 	args.InitArgParse(p, args, u.STRING, "m", makefileArg,
 		&argparse.Options{Required: false, Help: "Add additional properties " +
 			"for Makefile"})
+	args.InitArgParse(p, args, u.STRINGLIST, "c", configArg,
+		&argparse.Options{Required: false, Help: "Add configuration files"})
+	args.InitArgParse(p, args, u.STRING, "", patchArg,
+		&argparse.Options{Required: false, Help: "Add patch files"})
 
 	return u.ParserWrapper(p, os.Args)
 }

srcs/buildtool/microlibs_process.go

@@ -41,7 +41,6 @@ type MicroLibsFunction struct {
 // -----------------------------Match micro-libs--------------------------------
 
 // processSymbols adds symbols within the 'exportsyms.uk' file into a map.
-//
 func processSymbols(microLib, output string, mapSymbols map[string][]string) {
 
 	lines := strings.Split(output, "\n")
@@ -134,6 +133,53 @@ func fetchSymbolsExternalLibs(folder string,
 	return externalLibs, nil
 }
 
+// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static,
+// dynamic and source files analyses together into a map structure.
+//
+// It returns the map containing all the symbols and system calls.
+func putJsonSymbolsTogether(data *u.Data) map[string]string {
+	dataMap := make(map[string]string)
+
+	for k, v := range data.StaticData.Symbols {
+		dataMap[k] = v
+	}
+
+	for k := range data.StaticData.SystemCalls {
+		dataMap[k] = ""
+	}
+
+	for k, v := range data.DynamicData.Symbols {
+		dataMap[k] = v
+	}
+
+	for k := range data.DynamicData.SystemCalls {
+		dataMap[k] = ""
+	}
+
+	for k, v := range data.SourcesData.Symbols {
+		dataMap[k] = v
+	}
+
+	for k := range data.SourcesData.SystemCalls {
+		dataMap[k] = ""
+	}
+
+	return dataMap
+}
+
+// retNameCompat modifies its string argument in order to replace its underscore by a dash when
+// necessary.
+//
+// It returns its string argument whose underscore has been replaced by a dash if necessary,
+// otherwise it returns its argument unchanged.
+func retNameForCompat(value string) string {
+	if strings.Contains(value, "posix-") {
+		return strings.ReplaceAll(value, "posix-", "posix_")
+	}
+
+	return value
+}
+
 // matchSymbols performs the matching between Unikraft's micro-libs and
 // libraries used by a given application based on the list of symbols that both
 // contain.
@@ -144,15 +190,8 @@ func matchSymbols(matchedLibs []string, data map[string]string,
 	for key := range data {
 		if values, ok := microLibs[key]; ok {
 			for _, value := range values {
-
+				if !u.Contains(matchedLibs, retNameForCompat(value)) {
-				// todo remove
+					matchedLibs = append(matchedLibs, retNameForCompat(value))
-				if strings.Compare(NOLIBC, value) == 0 {
-					value = NEWLIB
-				}
-				// remove above
-
-				if !u.Contains(matchedLibs, value) {
-					matchedLibs = append(matchedLibs, value)
 				}
 			}
 		}
@@ -172,11 +211,6 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string,
 
 	matchedLibs := make([]string, 0)
 
-	//todo remove
-	matchedLibs = append(matchedLibs, POSIXLIBDL)
-	matchedLibs = append(matchedLibs, POSIXSYSINFO)
-	matchedLibs = append(matchedLibs, UKMMAP)
-
 	folder := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal")
 	if err := fetchSymbolsInternalLibs(folder, mapSymbols); err != nil {
 		return nil, nil, err
@@ -189,11 +223,10 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string,
 		return nil, nil, err
 	}
 
-	// Perform the matching symbols on static data
+	dataMap := putJsonSymbolsTogether(data)
-	matchedLibs = matchSymbols(matchedLibs, data.StaticData.Symbols, mapSymbols)
 
-	// Perform the matching symbols on dynamic data
+	// Perform the symbol matching
-	matchedLibs = matchSymbols(matchedLibs, data.DynamicData.Symbols, mapSymbols)
+	matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols)
 
 	return matchedLibs, externalLibs, nil
 }
@@ -223,7 +256,6 @@ func cloneGitRepo(url, unikraftPathLibs, lib string) error {
 
 // cloneLibsFolders clones all the needed micro-libs that are needed by a
 // given application
-//
 func cloneLibsFolders(workspacePath string, matchedLibs []string,
 	externalLibs map[string]string) {
 

srcs/buildtool/run_buildtool.go

@@ -7,13 +7,14 @@
 package buildtool
 
 import (
-	"github.com/AlecAivazis/survey/v2"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	u "tools/srcs/common"
+
+	"gopkg.in/AlecAivazis/survey.v1"
 )
 
 // STATES
@@ -51,7 +52,8 @@ func generateConfigUk(filename, programName string, matchedLibs []string) error
 // execution of the 'make' command.
 //
 // It returns an integer that defines the result of 'make':
-// 	<SUCCESS, LINKING_ERROR, COMPILER_ERROR>
+//
+//	<SUCCESS, LINKING_ERROR, COMPILER_ERROR>
 func checkMakeOutput(appFolder string, stderr *string) int {
 
 	if stderr == nil {
@@ -103,11 +105,38 @@ func parseMakeOutput(output string) string {
 	return sb.String()
 }
 
+// addConfigFiles adds user-provided configuration files in the order in which they come to the
+// unikernel folder.
+func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder,
+	appFolder string) {
+
+	for _, configFilePath := range configFiles {
+		configFile := filepath.Base(configFilePath)
+		fileExt := filepath.Ext(configFile)
+
+		if fileExt == ".h" || fileExt == ".hpp" || fileExt == ".hcc" {
+			if err := u.CopyFileContents(configFilePath, includeFolder+configFile); err != nil {
+				u.PrintErr(err)
+			}
+		} else if fileExt == ".c" || fileExt == ".cpp" || fileExt == ".cc" {
+			if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil {
+				u.PrintErr(err)
+			}
+
+			// Add Makefile.uk entry
+			*selectedFiles = append(*selectedFiles, configFile)
+		} else {
+			if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil {
+				u.PrintErr(err)
+			}
+		}
+	}
+}
+
 // -------------------------------------Run-------------------------------------
 
 // RunBuildTool runs the automatic build tool to build a unikernel of a
 // given application.
-//
 func RunBuildTool(homeDir string, data *u.Data) {
 
 	// Init and parse local arguments
@@ -195,7 +224,7 @@ func RunBuildTool(homeDir string, data *u.Data) {
 	}
 
 	// Filter source files to limit build errors (e.g., remove test files,
-	//multiple main file, ...)
+	// multiple main file, ...)
 	filterSourceFiles := filterSourcesFiles(sourceFiles)
 
 	// Prompt file selection
@@ -207,10 +236,31 @@ func RunBuildTool(homeDir string, data *u.Data) {
 	}
 
 	var selectedFiles []string
-	if err := survey.AskOne(prompt, &selectedFiles); err != nil {
+	if err := survey.AskOne(prompt, &selectedFiles, nil); err != nil {
 		panic(err)
 	}
 
+	// Copy, conform and apply user-provided patches to the new unikernel folder
+	if len(*args.StringArg[patchArg]) > 0 {
+		// Create patch folder
+		patchFolder, err := createPatchFolder(appFolder)
+		if err != nil {
+			u.PrintErr(err)
+		}
+		if err := addAndApplyPatchFiles(*args.StringArg[patchArg], *patchFolder, appFolder); err !=
+			nil {
+			u.PrintErr(err)
+		}
+	}
+
+	// Conform file include directives to the new unikernel folder organisation
+	if err := conformIncludeDirectives(appFolder); err != nil {
+		u.PrintErr(err)
+	}
+
+	// Add config files to the unikernel folder
+	addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder)
+
 	// Match micro-libs
 	matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data)
 	if err != nil {
@@ -249,41 +299,54 @@ func RunBuildTool(homeDir string, data *u.Data) {
 	runMake(programName, appFolder)
 }
 
+// retFolderCompat modifies its string argument in order to replace its underscore by a dash when
+// necessary for the searchInternalDependencies function to find the corresponding folder in the
+// 'unikraft' folder.
+//
+// It returns its string argument whose underscore has been replaced by a dash if necessary,
+// otherwise it returns its argument unchanged.
+func retFolderForCompat(lib string) string {
+	if strings.Contains(lib, "posix_") {
+		return strings.ReplaceAll(lib, "posix_", "posix-")
+	}
+
+	return lib
+}
+
 func searchInternalDependencies(unikraftPath string, matchedLibs *[]string,
 	externalLibs map[string]string) error {
 
 	for _, lib := range *matchedLibs {
-		// Consider only external libs
-		if _, ok := externalLibs[lib]; ok {
 
-			// Get and read Config.UK from external lib
+		// Get and read Config.UK from lib
-			configUk := unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk"
+		var configUk string
-			lines, err := u.ReadLinesFile(configUk)
+
-			if err != nil {
+		if _, ok := externalLibs[lib]; ok {
-				return err
+			configUk = unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk"
+		} else {
+			configUk = unikraftPath + u.UNIKRAFTFOLDER + "lib" + u.SEP + retFolderForCompat(lib) +
+				u.SEP + "Config.uk"
+		}
+
+		lines, err := u.ReadLinesFile(configUk)
+		if err != nil {
+			return err
+		}
+
+		// Process Config.UK file
+		mapConfig := make(map[string][]string)
+		u.ProcessConfigUK(lines, true, mapConfig, nil)
+		for config := range mapConfig {
+
+			// Remove LIB prefix
+			if strings.Contains(config, "LIB") {
+				config = strings.TrimPrefix(config, "LIB")
 			}
 
-			// Process Config.UK file
+			// Check if matchedLibs already contains the lib
-			mapConfig := make(map[string][]string)
+			config = strings.ToLower(config)
-			u.ProcessConfigUK(lines, false, mapConfig, nil)
+			if !u.Contains(*matchedLibs, config) {
-
+				*matchedLibs = append(*matchedLibs, config)
-			for config := range mapConfig {
-
-				// Remove LIB prefix
-				if strings.Contains(config, "LIB") {
-					config = strings.TrimPrefix(config, "LIB")
-				}
-
-				// Replace underscore by dash
-				if strings.Contains(config, "_") {
-					config = strings.ReplaceAll(config, "_", "-")
-				}
-
-				// Check if matchedLibs already contains the lib
-				config = strings.ToLower(config)
-				if !u.Contains(*matchedLibs, config) {
-					*matchedLibs = append(*matchedLibs, config)
-				}
 			}
 		}
 	}

srcs/buildtool/unikraft_files_process.go

@@ -12,11 +12,24 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	u "tools/srcs/common"
 )
 
+// ---------------------------Create Patches Folder-----------------------------
+
+func createPatchFolder(appFolder string) (*string, error) {
+
+	patchesFolder := appFolder + "patches" + u.SEP
+	if _, err := u.CreateFolder(patchesFolder); err != nil {
+		return nil, err
+	}
+
+	return &patchesFolder, nil
+}
+
 // ---------------------------Create Include Folder-----------------------------
 
 func createIncludeFolder(appFolder string) (*string, error) {
@@ -109,7 +122,10 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) {
 
 	if !created {
 		u.PrintWarning(appFolder + " already exists.")
-		handleCreationApp(&appFolder)
+		appFolder = handleCreationApp(appFolder)
+		if _, err := u.CreateFolder(appFolder); err != nil {
+			return nil, err
+		}
 	}
 
 	return &appFolder, nil
@@ -117,24 +133,24 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) {
 
 // -----------------------------Create App folder-------------------------------
 
-func handleCreationApp(appFolder *string) {
+func handleCreationApp(appFolder string) string {
 	fmt.Println("Make your choice:\n1: Copy and overwrite files\n2: " +
 		"Enter manually the name of the folder\n3: exit program")
 	var input int
-	for true {
+	for {
 		fmt.Print("Please enter your choice (0 to exit): ")
 		if _, err := fmt.Scanf("%d", &input); err != nil {
 			u.PrintWarning("Choice must be numeric! Try again")
 		} else {
 			switch input {
 			case 1:
-				return
+				return appFolder
 			case 2:
 				fmt.Print("Enter text: ")
 				reader := bufio.NewReader(os.Stdin)
 				text, _ := reader.ReadString('\n')
-				appFolder = &text
+				appFolder = strings.Split(text, "\n")[0] + u.SEP
-				return
+				return appFolder
 			case 3:
 				os.Exit(1)
 			default:
@@ -150,11 +166,11 @@ var srcLanguages = map[string]int{
 	".c":   0,
 	".cpp": 0,
 	".cc":  0,
-	".S":   0,
+	/*".S":   0,
 	".s":   0,
 	".asm": 0,
 	".py":  0,
-	".go":  0,
+	".go":  0,*/
 }
 
 func filterSourcesFiles(sourceFiles []string) []string {
@@ -170,14 +186,264 @@ func filterSourcesFiles(sourceFiles []string) []string {
 	return filterSrcFiles
 }
 
+// addAndApplyPatchFiles copies all the user-provided patch files to the unikernel directory,
+// conforms them to the unikernel directory format so that all paths in the patch files are paths
+// to source files located in the unikernel folder and applies the patches.
+//
+// It returns an error if any, otherwise it returns nil.
+func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) error {
+
+	// Copy and conform patch files
+	err := filepath.Walk(patchPath, func(filePath string, info os.FileInfo,
+		err error) error {
+		if !info.IsDir() {
+			extension := filepath.Ext(info.Name())
+			if extension == ".patch" {
+				if err = conformPatchFile(filePath, patchFolder+info.Name()); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Initialise git repo to be able to apply patches
+	_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "init")
+	_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "add", ".")
+	_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "commit", "-m", "first commit")
+
+	// Apply patches
+	err = filepath.Walk(patchPath, func(filePath string, info os.FileInfo,
+		err error) error {
+		if !info.IsDir() {
+			_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "am", patchFolder+info.Name())
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// conformPatchFile copies all the user-provided patch files to the unikernel directory and
+// conforms them to the unikernel directory format so that all paths in the patch files are paths
+// to source files located in the unikernel folder.
+//
+// It returns an error if any, otherwise it returns nil.
+func conformPatchFile(patchPath, newPatchPath string) error {
+
+	patchLines, err := u.ReadLinesFile(patchPath)
+	if err != nil {
+		return err
+	}
+
+	// Find paths in patch file using regexp
+	re1 := regexp.MustCompile(`( )(.*)( \| )(.*)`)
+	re2 := regexp.MustCompile(`(diff --git )(a/)?(.*)( )(b/)?(.*)`)
+	re3 := regexp.MustCompile(`(--- )(a/)?(.*)`)
+	re4 := regexp.MustCompile(`(\+\+\+ )(b/)?(.*)`)
+
+	for lineIndex := range patchLines {
+
+		// All paths to files to be modified by the patch are listed under "---"
+		if patchLines[lineIndex] == "---\n" {
+			lineIndex++
+			for ; !strings.Contains(patchLines[lineIndex], "changed"); lineIndex++ {
+				for _, match := range re1.FindAllStringSubmatch(patchLines[lineIndex], -1) {
+					conformPatchPath(&match, 2)
+					patchLines[lineIndex] = strings.Join(match[1:], "") + "\n"
+				}
+			}
+		}
+
+		// All diff lines contain paths to files to be modified by the patch
+		if len(patchLines[lineIndex]) > 10 && patchLines[lineIndex][:10] == "diff --git" {
+			for _, match := range re2.FindAllStringSubmatch(patchLines[lineIndex], -1) {
+				conformPatchPath(&match, 3)
+				conformPatchPath(&match, 6)
+				patchLines[lineIndex] = strings.Join(match[1:], "") + "\n"
+			}
+
+			// Same observation for the two lines following the index line
+			for _, match := range re3.FindAllStringSubmatch(patchLines[lineIndex+2], -1) {
+				conformPatchPath(&match, 3)
+				patchLines[lineIndex+2] = strings.Join(match[1:], "") + "\n"
+			}
+			for _, match := range re4.FindAllStringSubmatch(patchLines[lineIndex+3], -1) {
+				conformPatchPath(&match, 3)
+				patchLines[lineIndex+3] = strings.Join(match[1:], "") + "\n"
+			}
+		}
+	}
+
+	// Write the modified content to a file in the unikernel folder
+	err = u.WriteToFile(newPatchPath, []byte(strings.Join(patchLines, "")))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// conformPatchPath conforms a path in a user-provided patch file to the unikernel directory format
+// so that this path describes a source file located in the unikernel folder.
+func conformPatchPath(match *[]string, index int) {
+	extension := filepath.Ext((*match)[index])
+	if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
+		(*match)[index] = "include" + u.SEP + filepath.Base((*match)[index])
+	} else {
+		(*match)[index] = filepath.Base((*match)[index])
+	}
+}
+
+// conformIncludeDirectives conforms all the user-defined include directives of all C/C++ source
+// files to the unikernel directory format so that all these directives are paths to source files
+// located in the include folder of the unikernel directory.
+//
+// It returns an error if any, otherwise it returns nil.
+func conformIncludeDirectives(sourcePath string) error {
+
+	err := filepath.Walk(sourcePath, func(path string, info os.FileInfo,
+		err error) error {
+		if !info.IsDir() {
+			extension := filepath.Ext(info.Name())
+			if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
+				if err = conformFile(path, true); err != nil {
+					return err
+				}
+			} else if extension == ".c" || extension == ".cpp" || extension == ".cc" {
+				if err = conformFile(path, false); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// conformFile conforms all the user-defined include directives of a C/C++ source file to the
+// unikernel directory format so that all these directives are paths to source files located in the
+// include folder.
+//
+// It returns an error if any, otherwise it returns nil.
+func conformFile(filePath string, isHeader bool) (err error) {
+
+	fileLines, err := u.ReadLinesFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	// Find include directives using regexp
+	re := regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`)
+
+	for lineIndex := range fileLines {
+		for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) {
+
+			// Only interested in included files not coming from the standard library
+			if incDirContainsStdFold(fileLines[lineIndex]) {
+				break
+			}
+
+			for i := 1; i < len(match); i++ {
+				if match[i] == "\"" || match[i] == "<" {
+
+					// Determine the included source file extension to know what the path to it
+					// must be in the current file
+					var extIsHeader bool
+					extension := filepath.Ext(match[i+1])
+					if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
+						extIsHeader = true
+					} else if extension == ".c" || extension == ".cpp" || extension == ".cc" {
+						extIsHeader = false
+					} else {
+
+						// C++ header
+						break
+					}
+
+					// Modify the include path
+					match[i] = "\""
+					if isHeader && !extIsHeader {
+						match[i+1] = "../" + filepath.Base(match[i+1])
+					} else if !isHeader && extIsHeader {
+						match[i+1] = "include" + u.SEP + filepath.Base(match[i+1])
+					} else {
+						match[i+1] = filepath.Base(match[i+1])
+					}
+					match[i+2] = "\""
+					fileLines[lineIndex] = strings.Join(match[1:], "") + "\n"
+					break
+				}
+			}
+
+			break
+		}
+	}
+
+	// Write the modified content to a file in the unikernel folder
+	err = u.WriteToFile(filePath, []byte(strings.Join(fileLines, "")))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// incDirContainsStdFold determines if an include directive is a path to a standard header.
+//
+// It returns true if it is the case, false otherwise.
+func incDirContainsStdFold(fileLine string) bool {
+
+	// Standard header list
+	stdHeaders := []string{
+		"<aio.h>", "<libgen.h>", "<spawn.h>", "<sys/time.h>",
+		"<arpa/inet.h>", "<limits.h>", "<stdarg.h>", "<sys/times.h>",
+		"<assert.h>", "<locale.h>", "<stdbool.h>", "<sys/types.h>",
+		"<complex.h>", "<math.h>", "<stddef.h>", "<sys/uio.h>",
+		"<cpio.h>", "<monetary.h>", "<stdint.h>", "<sys/un.h>",
+		"<ctype.h>", "<mqueue.h>", "<stdio.h>", "<sys/utsname.h>",
+		"<dirent.h>", "<ndbm.h>", "<stdlib.h>", "<sys/wait.h>",
+		"<dlfcn.h>", "<net/if.h>", "<string.h>", "<syslog.h>",
+		"<errno.h>", "<netdb.h>", "<strings.h>", "<tar.h>",
+		"<fcntl.h>", "<netinet/in.h>", "<stropts.h>", "<termios.h>",
+		"<fenv.h>", "<netinet/tcp.h>", "<sys/ipc.h>", "<tgmath.h>",
+		"<float.h>", "<nl_types.h>", "<sys/mman.h>", "<time.h>",
+		"<fmtmsg.h>", "<poll.h>", "<sys/msg.h>", "<trace.h>",
+		"<fnmatch.h>", "<pthread.h>", "<sys/resource.h>", "<ulimit.h>",
+		"<ftw.h>", "<pwd.h>", "<sys/select.h>", "<unistd.h>",
+		"<glob.h>", "<regex.h>", "<sys/sem.h>", "<utime.h>",
+		"<grp.h>", "<sched.h>", "<sys/shm.h>", "<utmpx.h>",
+		"<iconv.h>", "<search.h>", "<sys/socket.h>", "<wchar.h>",
+		"<inttypes.h>", "<semaphore.h>", "<sys/stat.h>", "<wctype.h>",
+		"<iso646.h>", "<setjmp.h>", "<sys/statvfs.h>", "<wordexp.h>",
+		"<langinfo.h>", "<signal.h>", "<curses.h>", "<term.h>",
+		"<uncntrl.h>", "<linux/"}
+
+	for _, header := range stdHeaders {
+		if strings.Contains(fileLine, header) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func processSourceFiles(sourcesPath, appFolder, includeFolder string,
 	sourceFiles, includesFiles []string) ([]string, error) {
 
 	err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo,
 		err error) error {
-
 		if !info.IsDir() {
-
 			extension := filepath.Ext(info.Name())
 			if _, ok := srcLanguages[extension]; ok {
 				// Add source files to sourceFiles list
@@ -190,7 +456,7 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string,
 				if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil {
 					return err
 				}
-			} else if extension == ".h" {
+			} else if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
 				// Add source files to includesFiles list
 				includesFiles = append(includesFiles, info.Name())
 
@@ -202,10 +468,8 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string,
 				u.PrintWarning("Unsupported extension for file: " + info.Name())
 			}
 		}
-
 		return nil
 	})
-
 	if err != nil {
 		return nil, err
 	}

srcs/common/arguments.go

@@ -8,9 +8,10 @@ package common
 
 import (
 	"errors"
-	"github.com/akamensky/argparse"
 	"os"
 	"strings"
+
+	"github.com/akamensky/argparse"
 )
 
 // Exported constants to determine arguments type.
@@ -18,6 +19,7 @@ const (
 	INT = iota
 	BOOL
 	STRING
+	STRINGLIST
 )
 
 // Exported constants to determine which tool is used.
@@ -38,9 +40,10 @@ const (
 
 // Exported constants to represent different types of arguments.
 type Arguments struct {
-	IntArg    map[string]*int
+	IntArg        map[string]*int
-	BoolArg   map[string]*bool
+	BoolArg       map[string]*bool
-	StringArg map[string]*string
+	StringArg     map[string]*string
+	StringListArg map[string]*[]string
 }
 
 // InitArguments allows to initialize the parser in order to parse given
@@ -52,6 +55,7 @@ func (args *Arguments) InitArguments(name, description string) (*argparse.Parser
 	args.IntArg = make(map[string]*int)
 	args.BoolArg = make(map[string]*bool)
 	args.StringArg = make(map[string]*string)
+	args.StringListArg = make(map[string]*[]string)
 
 	p := argparse.NewParser(name, description)
 
@@ -130,5 +134,8 @@ func (*Arguments) InitArgParse(p *argparse.Parser, args *Arguments, typeVar int,
 	case STRING:
 		args.StringArg[long] = new(string)
 		args.StringArg[long] = p.String(short, long, options)
+	case STRINGLIST:
+		args.StringListArg[long] = new([]string)
+		args.StringListArg[long] = p.StringList(short, long, options)
 	}
 }

srcs/common/data.go

@@ -6,10 +6,11 @@
 
 package common
 
-// Exported struct that represents static and dynamic data.
+// Exported struct that represents static, dynamic and sources data.
 type Data struct {
 	StaticData  StaticData  `json:"static_data"`
 	DynamicData DynamicData `json:"dynamic_data"`
+	SourcesData SourcesData `json:"sources_data"`
 }
 
 // Exported struct that represents data for static dependency analysis.
@@ -26,3 +27,9 @@ type DynamicData struct {
 	SystemCalls map[string]int      `json:"system_calls"`
 	Symbols     map[string]string   `json:"symbols"`
 }
+
+// Exported struct that represents data for sources dependency analysis.
+type SourcesData struct {
+	SystemCalls map[string]int    `json:"system_calls"`
+	Symbols     map[string]string `json:"symbols"`
+}

srcs/common/git.go

@@ -6,7 +6,7 @@
 
 package common
 
-const branch = "RELEASE-0.7.0"
+const branch = "RELEASE-0.9.0"
 
 // GitCloneRepository clones a git repository at the the given url.
 //

srcs/dependtool/args.go

@@ -7,9 +7,10 @@
 package dependtool
 
 import (
-	"github.com/akamensky/argparse"
 	"os"
 	u "tools/srcs/common"
+
+	"github.com/akamensky/argparse"
 )
 
 const (
@@ -52,7 +53,8 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error {
 			Help: "Full static analysis (analyse shared libraries too)"})
 	args.InitArgParse(p, args, u.INT, "", typeAnalysis,
 		&argparse.Options{Required: false, Default: 0,
-			Help: "Kind of analysis (0: both; 1: static; 2: dynamic)"})
+			Help: "Kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: " +
+				"sources; 5: stripped-down app and json for buildtool)"})
 
 	return u.ParserWrapper(p, os.Args)
 }

srcs/dependtool/interdependence_graph.go

@@ -0,0 +1,354 @@
+package dependtool
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	u "tools/srcs/common"
+)
+
+// ---------------------------------Gather Data---------------------------------
+
+// findSourceFilesAndFolders puts together all the application C/C++ source files found on one hand
+// and all the application (sub-)folder paths on the other hand.
+//
+// It returns two slices: one containing the found source file paths and one containing the found
+// (sub-)folder paths, and an error if any, otherwise it returns nil.
+func findSourceFilesAndFolders(workspace string) ([]string, []string, error) {
+
+	var filenames []string
+	var foldernames []string
+
+	err := filepath.Walk(workspace,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			// Ignore hidden elements
+			if string(info.Name()[0]) == "." {
+				return nil
+			}
+
+			// Found (sub-)folder
+			if info.IsDir() {
+				foldernames = append(foldernames, "-I")
+				foldernames = append(foldernames, path)
+				return nil
+			}
+
+			// Found source file
+			ext := filepath.Ext(info.Name())
+			if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
+				ext == ".hcc" {
+				filenames = append(filenames, path)
+			}
+			return nil
+		})
+	if err != nil {
+		return nil, nil, err
+	}
+	return filenames, foldernames, nil
+}
+
+// checkIncDir checks that an inclusion directive path does not contain the two dots ".." which
+// refer to the parent directory and that can cause trouble when pruning the map and generating the
+// graph. If the path does contain these dots, it is simplified.
+//
+// It returns the simplified directive path or the path itself if it contains no dots.
+func checkIncDir(directive string) string {
+
+	// No problem
+	if !strings.Contains(directive, "../") {
+		return directive
+	}
+
+	// Must be simplified
+	splitDir := strings.Split(directive, u.SEP)
+	var i int
+	for i = 0; i < len(splitDir); i++ {
+		if splitDir[i] == ".." {
+
+			// Dots at the beginning of the path
+			if i == 0 {
+				splitDir = splitDir[1:]
+				i--
+
+				// Dots somewhere else in the path
+			} else {
+				splitDir = append(splitDir[:i-1], splitDir[i+1:]...)
+				i -= 2
+			}
+		}
+	}
+	return strings.Join(splitDir, u.SEP)
+}
+
+// sourceFileIncludesAnalysis collects all the inclusion directives from a C/C++ source file.
+//
+// It returns a slice containing all the relative paths contained in the inclusion directives.
+func sourceFileIncludesAnalysis(sourceFile string) []string {
+
+	var fileIncDir []string
+
+	fileLines, err := u.ReadLinesFile(sourceFile)
+	if err != nil {
+		u.PrintErr(err)
+	}
+
+	// Find inclusion directives using regexp
+	var re = regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`)
+
+	for lineIndex := range fileLines {
+		for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) {
+
+			// Append the relative path to the list of relative paths
+			for i := 1; i < len(match); i++ {
+				if match[i] == "\"" || match[i] == "<" {
+					fileIncDir = append(fileIncDir, checkIncDir(match[i+1]))
+					break
+				}
+			}
+		}
+	}
+	return fileIncDir
+}
+
+// TODO REPLACE
+// ExecuteCommand a single command without displaying the output.
+//
+// It returns a string which represents stdout and an error if any, otherwise
+// it returns nil.
+func executeCommand(command string, arguments []string) (string, error) {
+
+	out, err := exec.Command(command, arguments...).CombinedOutput()
+	return string(out), err
+}
+
+// gppSourceFileIncludesAnalysis collects all the inclusion directives from a C/C++ source file
+// using the gpp preprocessor.
+//
+// It returns a slice containing all the absolute paths contained in the inclusion directives.
+func gppSourceFileIncludesAnalysis(sourceFile, programPath string,
+	sourceFolders []string) ([]string, error) {
+
+	var fileIncDir []string
+
+	// g++ command
+	outputStr, err := executeCommand("g++", append([]string{"-E", sourceFile}, sourceFolders...))
+
+	// If command g++ returns an error, prune file: it contains non-standard libraries
+	if err != nil {
+		return nil, err
+	}
+
+	// Find inclusion directive paths
+	outputSlice := strings.Split(outputStr, "\n")
+	for _, line := range outputSlice {
+
+		// If warnings or errors are present, ignore their paths
+		if strings.Contains(line, ":") {
+			continue
+		}
+
+		// Only interested in file paths not coming from the standard library
+		if strings.Contains(line, programPath) {
+			includeDirective :=
+				checkIncDir(line[strings.Index(line, "\"")+1 : strings.LastIndex(line, "\"")])
+			if !u.Contains(fileIncDir, includeDirective) {
+				fileIncDir = append(fileIncDir, includeDirective)
+			}
+		}
+	}
+	return fileIncDir, nil
+}
+
+// pruneRemovableFiles prunes interdependence graph elements if the latter are unused header files.
+func pruneRemovableFiles(interdependMap *map[string][]string) {
+
+	for file := range *interdependMap {
+
+		// No removal of C/C++ source files
+		if filepath.Ext(file) != ".c" && filepath.Ext(file) != ".cpp" &&
+			filepath.Ext(file) != ".cc" {
+
+			// Lookup for files depending on the current header file
+			depends := false
+			for _, dependencies := range *interdependMap {
+				for _, dependency := range dependencies {
+					if file == dependency {
+						depends = true
+						break
+					}
+				}
+				if depends {
+					break
+				}
+			}
+
+			// Prune header file if unused
+			if !depends {
+				delete(*interdependMap, file)
+			}
+		}
+	}
+}
+
+// pruneElemFiles prunes interdependence graph elements if the latter contain the substring in
+// argument.
+func pruneElemFiles(interdependMap *map[string][]string, pruneElem string) {
+
+	// Lookup for key elements containing the substring and prune them
+	for key := range *interdependMap {
+		if strings.Contains(key, pruneElem) {
+			delete(*interdependMap, key)
+
+			// Lookup for key elements that depend on the key found above and prune them
+			for file, dependencies := range *interdependMap {
+				for _, dependency := range dependencies {
+					if dependency == key {
+						pruneElemFiles(interdependMap, file)
+					}
+				}
+			}
+		}
+	}
+}
+
+// requestUnikraftExtLibs collects all the GitHub repositories of Unikraft through the GitHub API
+// and returns the whole list of Unikraft external libraries.
+//
+// It returns a slice containing all the libraries names.
+func requestUnikraftExtLibs() []string {
+
+	var extLibsList, appsList []string
+
+	// Only 2 Web pages of repositories as for february 2023 (125 repos - 100 repos per page)
+	nbPages := 2
+
+	for i := 1; i <= nbPages; i++ {
+
+		// HTTP Get request
+		resp, err := http.Get("https://api.github.com/orgs/unikraft/repos?page=" +
+			strconv.Itoa(i) + "&per_page=100")
+		if err != nil {
+			u.PrintErr(err)
+		}
+		defer resp.Body.Close()
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			u.PrintErr(err)
+		}
+
+		// Collect libraries
+		fileLines := strings.Split(string(body), "\"name\":\"lib-")
+		for i := 1; i < len(fileLines); i++ {
+			extLibsList = append(extLibsList, fileLines[i][0:strings.Index(fileLines[i][0:],
+				"\"")])
+		}
+
+		// Collect applications
+		fileLines = strings.Split(string(body), "\"name\":\"app-")
+		for i := 1; i < len(fileLines); i++ {
+			appsList = append(appsList, fileLines[i][0:strings.Index(fileLines[i][0:],
+				"\"")])
+		}
+	}
+
+	// Avoid libs that are also apps (e.g. nginx, redis)
+	for i, lib := range extLibsList {
+		if u.Contains(appsList, lib) {
+			extLibsList = append(extLibsList[:i], extLibsList[i+1:]...)
+		}
+	}
+	return extLibsList
+}
+
+// -------------------------------------Run-------------------------------------
+
+// runInterdependAnalyser collects all the inclusion directives (i.e., dependencies) from each
+// C/C++ application source file and builds an interdependence graph (dot file and png image)
+// between the source files that it deemed compilable with Unikraft.
+//
+// It returns the path to the folder containing the source files that have been kept for generating
+// the graph.
+func interdependAnalyser(programPath, programName, outFolder string) string {
+
+	// Find all application source files and (sub-)folders
+	sourceFiles, sourceFolders, err := findSourceFilesAndFolders(programPath)
+	if err != nil {
+		u.PrintErr(err)
+	}
+	u.PrintOk(fmt.Sprint(len(sourceFiles)) + " source files found")
+	u.PrintOk(fmt.Sprint(len(sourceFolders)) + " source subfolders found")
+
+	// Collect source file inclusion directives. Source files are first analysed by g++ to make
+	// sure to avoid directives that are commented or subjected to a macro and then "by hand" to
+	// sort the include directives of the g++ analysis (i.e. to avoid inclusion directives that are
+	// not present in the source file currently analysed).
+	interdependMap := make(map[string][]string)
+	for _, sourceFile := range sourceFiles {
+		analysis := sourceFileIncludesAnalysis(sourceFile)
+		gppAnalysis, err := gppSourceFileIncludesAnalysis(sourceFile, programPath, sourceFolders)
+		if err != nil {
+			continue
+		}
+
+		// Build the interdependence map with the paths contained in the inclusion directives
+		interdependMap[sourceFile] = make([]string, 0)
+		for _, directive := range analysis {
+			for _, gppDirective := range gppAnalysis {
+				if strings.Contains(gppDirective, directive) {
+					interdependMap[sourceFile] = append(interdependMap[sourceFile], gppDirective)
+				}
+			}
+		}
+	}
+
+	// Prune the interdependence graph
+	extLibsList := requestUnikraftExtLibs()
+	extLibsList = append(extLibsList, "win32", "test", "TEST")
+	for _, extLib := range extLibsList {
+		pruneElemFiles(&interdependMap, extLib)
+	}
+	pruneRemovableFiles(&interdependMap)
+
+	// Create a folder and copy all the kept source files into it for later use with build tool
+	outAppFolder := outFolder + programName + u.SEP
+	_, err = u.CreateFolder(outAppFolder)
+	if err != nil {
+		u.PrintErr(err)
+	}
+	for _, sourceFile := range sourceFiles {
+		if _, ok := interdependMap[sourceFile]; ok {
+			if err := u.CopyFileContents(sourceFile,
+				outAppFolder+filepath.Base(sourceFile)); err != nil {
+				u.PrintErr(err)
+			}
+		}
+	}
+	u.PrintOk(fmt.Sprint(len(interdependMap)) + " source files kept and copied to " + outAppFolder)
+
+	// Change the absolute paths in the interdependence map into relative paths for more
+	// readability in the png image
+	graphMap := make(map[string][]string)
+	for appFilePath := range interdependMap {
+		appFile := strings.Split(appFilePath, programPath)[1][1:]
+		graphMap[appFile] = make([]string, 0)
+		for _, fileDepPath := range interdependMap[appFilePath] {
+			graphMap[appFile] = append(graphMap[appFile], strings.Split(fileDepPath,
+				programPath)[1][1:])
+		}
+	}
+
+	// Create dot and png files
+	u.GenerateGraph(programName, outFolder+programName, graphMap, nil)
+
+	return outAppFolder
+}

srcs/dependtool/parserClang.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+#---------------------------------------------------------------------
+# (*) Installation:
+#
+# pip3 install clang
+#
+# cd /usr/lib/x86_64-linux-gnu/
+# sudo ln -s libclang-X.Y.so.1 libclang-14.so (X.Y the version number)
+#
+# (*) Run:
+#
+# python3 parserClang.py <filepath> [includepathsfile]
+#
+# where filepath can be a repository/folder or a file (c/cpp/h/hpp)
+#
+#
+# Gaulthier Gain <gaulthier.gain@uliege.be>
+# License: BSD
+#---------------------------------------------------------------------
+
+import getopt
+import os
+import sys
+import json
+import clang.cindex
+import clang
+import platform
+from clang.cindex import CursorKind
+from collections import Counter
+
+verbose = False # Change it to verbose mode
+
+global_funcs = Counter()
+global_calls = Counter()
+
+silent_flag = False
+
+# Check if a path is a directory or a file
+def check_input_path(path, includePaths):
+    if os.path.isdir(path):
+        iterate_root_folder(path, includePaths)
+    elif os.path.isfile(path):
+        check_type_file(path, includePaths)
+    else:
+        print("Unable to analyse this file")
+        exit(1)
+
+def get_include_paths(rootdir, includepathsFile):
+    paths = []
+    with open(includepathsFile, 'r') as file:
+        for includePath in file.readlines():
+            path = '-isystem ' + rootdir + includePath.replace('\n', '')
+            paths.append(path)
+
+    return ' '.join(paths)
+
+# Check type/exenstion of a given file
+def check_type_file(filepath, includePaths):
+    cplusplusOptions = '-x c++ --std=c++11'
+    cOptions = ''
+
+    if includePaths is not None:
+        cplusplusOptions = cplusplusOptions + ' ' + includePaths
+        cOptions = cOptions + ' ' + includePaths
+    if silent_flag is False:
+        print("Gathering symbols of " + filepath)
+    if filepath.endswith(".cpp") or filepath.endswith(".hpp") or filepath.endswith(".cc") or \
+        filepath.endswith(".hcc"):
+        parse_file(filepath, cplusplusOptions)
+    elif filepath.endswith(".c") or filepath.endswith(".h"):
+        parse_file(filepath, cOptions)
+
+# Iterate through a root folder
+def iterate_root_folder(rootdir, includePaths):
+    for subdir, dirs, files in os.walk(rootdir):
+        for file in files:
+            filepath = subdir + os.sep + file
+            check_type_file(filepath, includePaths)
+
+# Print info about symbols (verbose mode)
+def display_info_function(funcs, calls):
+    for f in funcs:
+        print(fully_qualified(f), f.location)
+        for c in calls:
+            if is_function_call(f, c):
+                print('-', c.location)
+        print()
+
+# Parse a given file to generate a AST
+def parse_file(filepath, arguments):
+
+    idx = clang.cindex.Index.create()
+    args = arguments.split()
+    tu = idx.parse(filepath, args=args)
+    funcs, calls = find_funcs_and_calls(tu)
+    if verbose:
+        display_info_function(funcs, calls)
+        print(list(tu.diagnostics))
+
+
+# Retrieve a fully qualified function name (with namespaces)
+def fully_qualified(c):
+    if c is None:
+        return ''
+    elif c.kind == CursorKind.TRANSLATION_UNIT:
+        return ''
+    else:
+        res = fully_qualified(c.semantic_parent)
+        if res != '':
+            return res + '::' + c.spelling
+    return c.spelling
+
+# Determine where a call-expression cursor refers to a particular
+# function declaration
+def is_function_call(funcdecl, c):
+    defn = c.get_definition()
+    return (defn is not None) and (defn == funcdecl)
+
+# Filter name to take only the function name (remove "(args)")
+def filter_func_name(displayname):
+    if "(" in displayname:
+        funcName = displayname.split('(')[0]
+    else:
+        funcName = displayname
+    return funcName
+
+# Retrieve lists of function declarations and call expressions in a
+#translation unit
+def find_funcs_and_calls(tu):
+    filename = tu.cursor.spelling
+    calls = []
+    funcs = []
+    for c in tu.cursor.walk_preorder():
+        if c.kind == CursorKind.CALL_EXPR:
+            calls.append(c)
+            # filter name to take only the name if necessary
+            funcName = filter_func_name(c.displayname)
+            global_calls[funcName] += 1
+        elif c.kind == CursorKind.FUNCTION_DECL:
+            funcs.append(c)
+            # filter name to take only the name if necessary
+            funcName = filter_func_name(c.displayname)
+            global_funcs[funcName] += 1
+    return funcs, calls
+
+# Write data to json file
+def write_to_json(output_filename, data):
+    with open(output_filename + '.json', 'w') as fp:
+        json.dump(data, fp, indent=4, sort_keys=True)
+
+# Open data to json file
+def read_from_json(filename):
+    with open(output_filename + '.json', 'r') as fp:
+        data = json.load(fp)
+    return data
+
+# Read the list of syscalls (text file)
+def read_syscalls_list(filename):
+    syscalls = set()
+    with open(filename) as f:
+        for line in f:
+            syscalls.add(line.strip())
+    return syscalls
+
+# Check which syscall is called
+def compare_syscalls(syscalls):
+    if silent_flag is False:
+        print("Gathered syscalls from function calls:")
+
+    return [key for key in global_calls.keys() if key not in syscalls]
+
+
+
+# Main function
+def main():
+    optlist, args = getopt.getopt(sys.argv[1:], "o:qvt")
+    input_file_names = None
+    includepathsFile = None
+    output_file_name = None
+    textFormat = False
+    for opt in optlist:
+        if opt[0] == "-i":
+            includepathFile = opt[1]
+        if opt[0] == "-o":
+            output_file_name = opt[1]
+        if opt[0] == "-q":
+            global silent_flag
+            silent_flag = True
+        if opt[0] == "-v":
+            global verbose
+            verbose = True
+        if opt[0] == "-t":
+            textFormat = True
+
+    
+
+    input_file_names = args
+    if len(input_file_names) == 0:
+        if silent_flag is False:
+            print("No input files supplied")
+        exit(1)
+    if includepathsFile is not None:
+        includePaths = get_include_paths(input_file_name, includepathsFile)
+        for input_file_name in input_file_names:
+            check_input_path(input_file_name, includePaths)
+    else:
+        for input_file_name in input_file_names:
+            check_input_path(input_file_name, None)
+
+    if silent_flag is False:
+        print("---------------------------------------------------------")
+
+    if textFormat:
+        i = 0
+        for key,value in global_funcs.items():
+            if i < len(global_funcs.items())-1:
+                print(key, end=',')
+            else:
+                print(key)
+            i = i + 1
+    else:
+        # Dump function declarations and calls to json
+        output_dikt = {
+            'functions':'',
+            'calls':''
+        }
+        output_dikt['functions'] = [{'name':key, 'value':value} for key,value in
+        global_funcs.items()]
+        output_dikt['calls'] = [{'name':key, 'value':value} for key,value in global_calls.items()]
+        if includepathsFile is not None:
+            # Read syscalls from txt file
+            all_syscalls = read_syscalls_list('syscall_list.txt')
+            called_syscalls = compare_syscalls(all_syscalls)
+            output_dikt['syscalls'] = called_syscalls
+
+        output_file = sys.stdout
+        json.dump(output_dikt, output_file)
+
+
+if __name__== "__main__":
+    if platform.system() == "Darwin":
+        clang.cindex.Config.set_library_file
+        ("/Applications/Xcode.app/Contents/Frameworks/libclang.dylib")
+    main()

srcs/dependtool/run_deptool.go

@@ -4,12 +4,32 @@ import (
 	"debug/elf"
 	"errors"
 	"fmt"
-	"github.com/fatih/color"
 	"runtime"
 	"strings"
 	u "tools/srcs/common"
+
+	"github.com/fatih/color"
 )
 
+// getProgramFolder gets the folder path in which the given program is located, according to the
+// Unikraft standard (e.g., /home/.../apps/programFolder/.../program).
+//
+// It returns the folder containing the program files according to the standard described above.
+func getProgramFolder(programPath string) string {
+
+	tmp := strings.Split(programPath, u.SEP)
+	i := 2
+
+	for ; i < len(tmp); i++ {
+		if tmp[len(tmp)-i] == "apps" {
+			break
+		}
+	}
+
+	folderPath := strings.Join(tmp[:len(tmp)-i+2], u.SEP)
+	return folderPath
+}
+
 // RunAnalyserTool allows to run the dependency analyser tool.
 func RunAnalyserTool(homeDir string, data *u.Data) {
 
@@ -29,10 +49,11 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
 		u.PrintErr(err)
 	}
 
-	// Get the kind of analysis (0: both; 1: static; 2: dynamic)
+	// Get the kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: sources; 5:
+	// stripped-down app and json for buildtool)
 	typeAnalysis := *args.IntArg[typeAnalysis]
-	if typeAnalysis < 0 || typeAnalysis > 2 {
+	if typeAnalysis < 0 || typeAnalysis > 5 {
-		u.PrintErr(errors.New("analysis argument must be between [0,2]"))
+		u.PrintErr(errors.New("analysis argument must be between [0,5]"))
 	}
 
 	// Get program path
@@ -68,7 +89,8 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
 	// Run static analyser
 	if typeAnalysis == 0 || typeAnalysis == 1 {
 		u.PrintHeader1("(1.1) RUN STATIC ANALYSIS")
-		runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder, data)
+		runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder,
+			data)
 	}
 
 	// Run dynamic analyser
@@ -82,6 +104,24 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
 		}
 	}
 
+	// Run interdependence analyser
+	if typeAnalysis == 0 || typeAnalysis == 3 {
+		u.PrintHeader1("(1.3) RUN INTERDEPENDENCE ANALYSIS")
+		_ = runInterdependAnalyser(getProgramFolder(programPath), programName, outFolder)
+	}
+
+	// Run sources analyser
+	if typeAnalysis == 0 || typeAnalysis == 4 {
+		u.PrintHeader1("(1.4) RUN SOURCES ANALYSIS")
+		runSourcesAnalyser(getProgramFolder(programPath), data)
+	}
+
+	// Prepare stripped-down app for buildtool
+	if typeAnalysis == 5 {
+		u.PrintHeader1("(1.5) PREPARE STRIPPED-DOWN APP AND JSON FOR BUILDTOOL")
+		runSourcesAnalyser(runInterdependAnalyser(programPath, programName, outFolder), data)
+	}
+
 	// Save Data to JSON
 	if err = u.RecordDataJson(outFolder+programName, data); err != nil {
 		u.PrintErr(err)
@@ -156,8 +196,8 @@ func checkElf(programPath *string) (*elf.File, bool) {
 }
 
 // runStaticAnalyser runs the static analyser
-func runStaticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args *u.Arguments, programName, programPath,
+func runStaticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args *u.Arguments, programName,
-	outFolder string, data *u.Data) {
+	programPath, outFolder string, data *u.Data) {
 
 	staticAnalyser(elfFile, isDynamic, isLinux, *args, data, programPath)
 
@@ -209,6 +249,12 @@ func runDynamicAnalyser(args *u.Arguments, programName, programPath,
 	}
 }
 
+// runDynamicAnalyser runs the sources analyser.
+func runSourcesAnalyser(programPath string, data *u.Data) {
+
+	sourcesAnalyser(data, programPath)
+}
+
 // saveGraph saves dependency graphs of a given app into the output folder.
 func saveGraph(programName, outFolder string, data *u.Data) {
 
@@ -222,8 +268,14 @@ func saveGraph(programName, outFolder string, data *u.Data) {
 			programName+"_dependencies", data.StaticData.Dependencies, nil)
 	}
 
-	if len(data.StaticData.SharedLibs) > 0 {
+	if len(data.DynamicData.SharedLibs) > 0 {
 		u.GenerateGraph(programName, outFolder+"dynamic"+u.SEP+
 			programName+"_shared_libs", data.DynamicData.SharedLibs, nil)
 	}
 }
+
+// runInterdependAnalyser runs the interdependence analyser.
+func runInterdependAnalyser(programPath, programName, outFolder string) string {
+
+	return interdependAnalyser(programPath, programName, outFolder)
+}

srcs/dependtool/sources_analyser.go

@@ -0,0 +1,113 @@
+package dependtool
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	u "tools/srcs/common"
+)
+
+// ---------------------------------Gather Data---------------------------------
+
+// findSourcesFiles puts together all C/C++ source files found in a given application folder.
+//
+// It returns a slice containing the found source file names and an error if any, otherwise it
+// returns nil.
+func findSourceFiles(workspace string) ([]string, error) {
+
+	var filenames []string
+
+	err := filepath.Walk(workspace,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			ext := filepath.Ext(info.Name())
+			if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
+				ext == ".hcc" {
+				filenames = append(filenames, path)
+			}
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+	return filenames, nil
+}
+
+// addSourceFileSymbols adds all the symbols present in 'output' to the static data field in
+// 'data'.
+func addSourceFileSymbols(output string, data *u.SourcesData) {
+
+	outputTab := strings.Split(output, ",")
+
+	// Get the list of system calls
+	systemCalls := initSystemCalls()
+
+	for _, s := range outputTab {
+		if _, isSyscall := systemCalls[s]; isSyscall {
+			data.SystemCalls[s] = systemCalls[s]
+		} else {
+			data.Symbols[s] = ""
+		}
+	}
+}
+
+// extractPrototype executes the parserClang.py script on each source file to extracts all possible
+// symbols of each of these files.
+//
+// It returns an error if any, otherwise it returns nil.
+func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error {
+
+	for _, f := range sourcesFiltered {
+		script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool",
+			"parserClang.py")
+		output, err := u.ExecuteCommand("python3", []string{script, "-q", "-t", f})
+		if err != nil {
+			u.PrintWarning("Incomplete analysis with file " + f)
+			continue
+		}
+		addSourceFileSymbols(output, data)
+	}
+
+	return nil
+}
+
+// gatherSourceFileSymbols gathers symbols of source files from a given application folder.
+//
+// It returns an error if any, otherwise it returns nil.
+func gatherSourceFileSymbols(data *u.SourcesData, programPath string) error {
+
+	sourceFiles, err := findSourceFiles(programPath)
+	if err != nil {
+		u.PrintErr(err)
+	}
+	u.PrintOk(fmt.Sprint(len(sourceFiles)) + " source files found")
+
+	if err := extractPrototype(sourceFiles, data); err != nil {
+		u.PrintErr(err)
+	}
+
+	return nil
+}
+
+// -------------------------------------Run-------------------------------------
+
+// staticAnalyser runs the static analysis to get system calls and library calls of a given
+// application.
+func sourcesAnalyser(data *u.Data, programPath string) {
+
+	sourcesData := &data.SourcesData
+
+	// Init symbols members
+	sourcesData.Symbols = make(map[string]string)
+	sourcesData.SystemCalls = make(map[string]int)
+
+	// Detect symbols from source files
+	u.PrintHeader2("(*) Gathering symbols from source files")
+	if err := gatherSourceFileSymbols(sourcesData, programPath); err != nil {
+		u.PrintWarning(err)
+	}
+}

srcs/dependtool/static_analyser.go

@@ -195,8 +195,8 @@ func executeDependAptCache(programName string, data *u.StaticData,
 
 // staticAnalyser runs the static analysis to get shared libraries,
 // system calls and library calls of a given application.
-//
+func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments, data *u.Data,
-func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments, data *u.Data, programPath string) {
+	programPath string) {
 
 	programName := *args.StringArg[programArg]
 	fullDeps := *args.BoolArg[fullDepsArg]
@@ -242,7 +242,8 @@ func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments
 
 	// Detect symbols from shared libraries
 	if fullStaticAnalysis && isLinux {
-		u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary file")
+		u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary" +
+			"file")
 		for key, path := range staticData.SharedLibs {
 			if len(path) > 0 {
 				fmt.Printf("\t-> Analysing %s - %s\n", key, path[0])

srcs/extractertool/parserClang.py

@@ -5,7 +5,7 @@
 # pip3 install clang
 #
 # cd /usr/lib/x86_64-linux-gnu/
-# sudo ln -s libclang-X.Y.so.1 libclang.so (X.Y the version number)
+# sudo ln -s libclang-X.Y.so.1 libclang-14.so (X.Y the version number)
 #
 # (*) Run:
 #
@@ -64,7 +64,8 @@ def check_type_file(filepath, includePaths):
         cOptions = cOptions + ' ' + includePaths
     if silent_flag is False:
         print("Gathering symbols of " + filepath)
-    if filepath.endswith(".cpp") or filepath.endswith(".hpp"):
+    if filepath.endswith(".cpp") or filepath.endswith(".hpp") or filepath.endswith(".cc") or \
+        filepath.endswith(".hcc"):
         parse_file(filepath, cplusplusOptions)
     elif filepath.endswith(".c") or filepath.endswith(".h"):
         parse_file(filepath, cOptions)

srcs/extractertool/run_extractertool.go

@@ -6,7 +6,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -32,7 +31,7 @@ type Variables struct {
 }
 
 func getMakefileSources(content string, mapSources map[string]string) {
-	var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp`)
+	var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp|\/.*\.hpp|\/.*\.cc|\/.*\.hcc`)
 	for _, match := range re.FindAllString(content, -1) {
 		vars := strings.Split(match, "/")
 		mapSources[vars[len(vars)-1]] = match
@@ -51,7 +50,7 @@ func findVariables(content string, mapVariables map[string]*Variables) {
 				value: "",
 			}
 
-			regexVar := regexp.MustCompile("(?m)" + v.name + "=.*$")
+			regexVar := regexp.MustCompile("(?m)" + v.name + "[ \t]*=.*$")
 			for _, matchVar := range regexVar.FindAllString(content, -1) {
 				v.value = matchVar
 				break
@@ -93,7 +92,8 @@ func resolveVariables(mapVariables map[string]*Variables) {
 func detectURL(mapVariables map[string]*Variables) *string {
 	for key, value := range mapVariables {
 		if strings.Contains(key, URL) && strings.Contains(value.value, HTTP) {
-			vars := strings.Split(value.value, "=")
+			spaceDel := strings.Join(strings.Split(value.value, " "), "")
+			vars := strings.Split(spaceDel, "=")
 			if len(vars) > 1 {
 				return &vars[1]
 			}
@@ -103,7 +103,7 @@ func detectURL(mapVariables map[string]*Variables) *string {
 	return nil
 }
 
-//TODO REPLACE
+// TODO REPLACE
 func CreateFolder(path string) (bool, error) {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		if err = os.Mkdir(path, 0755); err != nil {
@@ -146,7 +146,8 @@ func findSourcesFiles(workspace string) ([]string, error) {
 			}
 
 			ext := filepath.Ext(info.Name())
-			if ext == ".c" || ext == ".cpp" {
+			if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
+				ext == ".hcc" {
 				filenames = append(filenames, path)
 			}
 			return nil
@@ -157,19 +158,6 @@ func findSourcesFiles(workspace string) ([]string, error) {
 	return filenames, nil
 }
 
-//TODO REPLACE
-// ExecuteCommand a single command without displaying the output.
-//
-// It returns a string which represents stdout and an error if any, otherwise
-// it returns nil.
-func ExecuteCommand(command string, arguments []string) (string, error) {
-	out, err := exec.Command(command, arguments...).CombinedOutput()
-	if err != nil {
-		return "", err
-	}
-	return string(out), nil
-}
-
 func saveSymbols(output string, mapSymbols map[string]string, libName string) {
 	if strings.Contains(output, "\n") {
 		output = strings.TrimSuffix(output, "\n")
@@ -179,7 +167,7 @@ func saveSymbols(output string, mapSymbols map[string]string, libName string) {
 	for _, s := range symbols {
 		if len(s) > 0 {
 			if _, ok := mapSymbols[s]; !ok {
-				if s == "main" || strings.Contains(s, "test") {
+				if s == "main" || strings.Contains(s, "test") || strings.Contains(s, "TEST") {
 					u.PrintWarning("Ignore function: " + s)
 				} else {
 					mapSymbols[s] = libName
@@ -189,11 +177,13 @@ func saveSymbols(output string, mapSymbols map[string]string, libName string) {
 	}
 }
 
-func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string, libName string) error {
+func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string,
+	libName string) error {
 
 	for _, f := range sourcesFiltered {
-		script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool", "parserClang.py")
+		script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool",
-		output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f})
+			"parserClang.py")
+		output, err := u.ExecuteCommand("python3", []string{script, "-q", "-t", f})
 		if err != nil {
 			u.PrintWarning("Incomplete analysis with file " + f)
 			continue
@@ -222,7 +212,8 @@ func RunExtracterTool(homeDir string) {
 	// Init and parse local arguments
 	args := new(u.Arguments)
 	p, err := args.InitArguments("--extracter",
-		"The extracter tool allows to extract all the symbols (functions) of an external/internal library")
+		"The extracter tool allows to extract all the symbols (functions) of an "+
+			"external/internal library")
 	if err != nil {
 		u.PrintErr(err)
 	}
@@ -252,69 +243,110 @@ func RunExtracterTool(homeDir string) {
 
 	mapSources := make(map[string]string)
 	getMakefileSources(content, mapSources)
-
 	findVariables(content, mapVariables)
 	resolveVariables(mapVariables)
 	url := detectURL(mapVariables)
-	if url == nil {
-		u.PrintErr(errors.New("url of the lib not found"))
-		return
-	}
-
-	fileExtension := filepath.Ext(*url)
 	folderName := lib + "_sources_folder"
-	created, err := CreateFolder(folderName)
+	var archiveName string
-	if err != nil {
+	var sourcesFiltered []string
-		u.PrintErr(err)
-	}
 
-	var files []string
+	if url != nil {
-	archiveName := lib + "_sources" + fileExtension
+		var fileExtension string
-	if created {
+		urlSplit := strings.Split(*url, "/")
-		u.PrintInfo(*url + " is found. Download the lib sources...")
+		if urlSplit[len(urlSplit)-1] == "download" {
-		err := DownloadFile(archiveName, *url)
+			fileExtension = filepath.Ext(urlSplit[len(urlSplit)-2])
+		} else {
+			fileExtension = filepath.Ext(*url)
+		}
+
+		created, err := CreateFolder(folderName)
 		if err != nil {
 			u.PrintErr(err)
 		}
-		u.PrintOk(*url + " successfully downloaded.")
 
-		u.PrintInfo("Extracting " + archiveName + "...")
+		var files []string
-		if fileExtension == ".zip" {
-			files, err = Unzip(archiveName, folderName)
-			if err != nil {
-				_ = os.Remove(archiveName)
-				_ = os.RemoveAll(folderName)
-				u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
-			}
 
-		} else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == ".tgz" {
+		if fileExtension == ".gz" || fileExtension == ".xz" {
-			files, err = unTarGz(archiveName, folderName)
+			archiveName = lib + "_sources.tar" + fileExtension
-			if err != nil {
-				_ = os.Remove(archiveName)
-				_ = os.RemoveAll(folderName)
-				u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
-			}
 
 		} else {
-			u.PrintErr(errors.New("unknown extension for archive"))
+			archiveName = lib + "_sources" + fileExtension
+		}
+
+		if created {
+			u.PrintInfo(*url + " is found. Download the lib sources...")
+			err := DownloadFile(archiveName, *url)
+			if err != nil {
+				_ = os.RemoveAll(folderName)
+				u.PrintErr(err)
+			}
+			u.PrintOk(*url + " successfully downloaded.")
+
+			u.PrintInfo("Extracting " + archiveName + "...")
+			if fileExtension == ".zip" {
+				files, err = Unzip(archiveName, folderName)
+				if err != nil {
+					_ = os.Remove(archiveName)
+					_ = os.RemoveAll(folderName)
+					u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
+				}
+
+			} else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension ==
+				".tgz" {
+				files, err = unTarGz(archiveName, folderName)
+				if err != nil {
+					_ = os.Remove(archiveName)
+					_ = os.RemoveAll(folderName)
+					u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
+				}
+
+			} else if fileExtension == ".tar" || fileExtension == ".xz" || fileExtension ==
+				".txz" {
+				_, err := u.ExecuteCommand("tar", []string{"-xf", archiveName, "-C", folderName})
+				if err != nil {
+					_ = os.Remove(archiveName)
+					_ = os.RemoveAll(folderName)
+					u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
+				}
+
+			} else {
+				_ = os.Remove(archiveName)
+				_ = os.RemoveAll(folderName)
+				u.PrintErr(errors.New("unknown extension for archive"))
+			}
 		}
-	}
 
-	if len(files) == 0 {
 		u.PrintInfo("Inspecting folder " + folderName + " for sources...")
-		files, err = findSourcesFiles(folderName)
+		folderFiles, err := findSourcesFiles(folderName)
 		if err != nil {
+			_ = os.Remove(archiveName)
+			_ = os.RemoveAll(folderName)
 			u.PrintErr(err)
 		}
+
+		sourcesFiltered = filterSourcesFiles(files, mapSources)
+		sourcesFiltered = append(sourcesFiltered, folderFiles...)
 	}
 
-	sourcesFiltered := filterSourcesFiles(files, mapSources)
+	libpathFiles, err := findSourcesFiles(libpath)
+	if err != nil {
+		if url != nil {
+			_ = os.Remove(archiveName)
+			_ = os.RemoveAll(folderName)
+		}
+		u.PrintErr(err)
+	}
+	sourcesFiltered = append(sourcesFiltered, libpathFiles...)
 
 	u.PrintInfo("Find " + strconv.Itoa(len(sourcesFiltered)) + " files to analyse")
 
 	mapSymbols := make(map[string]string)
 	u.PrintInfo("Extracting symbols from all sources of " + lib + ". This may take some times...")
 	if err := extractPrototype(sourcesFiltered, mapSymbols, lib); err != nil {
+		if url != nil {
+			_ = os.Remove(archiveName)
+			_ = os.RemoveAll(folderName)
+		}
 		u.PrintErr(err)
 	}
 
@@ -328,14 +360,20 @@ func RunExtracterTool(homeDir string) {
 
 	u.PrintOk(strconv.Itoa(len(mapSymbols)) + " symbols from " + lib + " have been extracted.")
 
-	filename := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib)
+	var filename string
+	if url != nil {
+		u.PrintInfo("Remove folders " + archiveName + " and " + folderName)
+		_ = os.Remove(archiveName)
+		_ = os.RemoveAll(folderName)
+		filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib)
+
+	} else {
+		filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal", lib)
+	}
+
 	if err := u.RecordDataJson(filename, mf); err != nil {
 		u.PrintErr(err)
 	} else {
 		u.PrintOk("Symbols file have been written to " + filename + ".json")
 	}
-
-	u.PrintInfo("Remove folders " + archiveName + " and " + folderName)
-	_ = os.Remove(archiveName)
-	_ = os.RemoveAll(folderName)
 }